You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2025/07/05 18:26:10 UTC
(iggy) branch rebase_master created (now af1b1ae7)
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a change to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
at af1b1ae7 add cargo lock
This branch includes the following new commits:
new af3dcac2 initial setup and migration of shards
new 5ca643c4 feat(io_uring): bootstrap the main function (#1788)
new dab0759e feat(io_uring): load server_config (#1798)
new a60c9910 begin shard init
new cee584cb todos
new 044f59fb refactors
new 69de5d8d fix state initialization with gate
new 8101ded4 temp
new d0ea2c19 begin fix cg
new 110f8f41 fix issues with RefCell in core/server/src/streaming/topics/consumer_groups* files
new 8566672c fixes
new 2ace7e8e fix iggy shard errors
new 81c29b1b finish handlers
new 7f13ce06 broken iggyrwlock import
new a718676d server compiles
new e98ed531 fix sdk
new 492ea25d fix segment writing/reading
new 05ab5a77 finish send/poll messages
new 235a7cc7 feat(io_uring): fix send/poll (#1930)
new 170b02a1 chore: reformat code on tpc branch (#1939)
new 2bcf4de7 feat(tpc): implement cooperative shutdown (#1941)
new c7b62350 feat(tpc): implement async walk_dir and remove_dir_all (#1942)
new 63c0b2f4 refactor(io_uring): refactor the send/poll method flow (#1944)
new f44adee7 feat(io_uring): fix metadata crud operations (#1950)
new e1e31ea8 feat(io_uring): fix tcp tls server (#1952)
new b37120ac feat(io_uring): move to compio runtime (#1960)
new e01de4f4 refactor(io_uring): use tcp opts from compio instead of socket config (#1962)
new 384b969b fix(io_uring): skip thread_pool_limit(0) on macOS (#1963)
new 9a634fd4 fix(io_uring): fix buffer reading with memory pool enabled (#1964)
new af1b1ae7 add cargo lock
The 30 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iggy) 14/30: broken iggyrwlock import
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 7f13ce06271a269b0e5bbf2afeb46d43d9a4b1a7
Author: numinex <g....@gmail.com>
AuthorDate: Fri Jun 27 14:01:13 2025 +0200
broken iggyrwlock import
---
Cargo.lock | 47 ++++++++++++----------
.../server/src/streaming/topics/consumer_groups.rs | 6 +--
core/server/src/streaming/topics/storage.rs | 2 +-
3 files changed, 28 insertions(+), 27 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 19be4ecb..f06db7bf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1177,9 +1177,9 @@ dependencies = [
[[package]]
name = "bumpalo"
-version = "3.18.1"
+version = "3.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee"
+checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
[[package]]
name = "byte-unit"
@@ -1479,9 +1479,12 @@ dependencies = [
[[package]]
name = "cobs"
-version = "0.2.3"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15"
+checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1"
+dependencies = [
+ "thiserror 2.0.12",
+]
[[package]]
name = "colorchoice"
@@ -1813,9 +1816,9 @@ dependencies = [
[[package]]
name = "crunchy"
-version = "0.2.3"
+version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-common"
@@ -3275,7 +3278,7 @@ dependencies = [
"futures-sink",
"futures-util",
"http 0.2.12",
- "indexmap 2.9.0",
+ "indexmap 2.10.0",
"slab",
"tokio",
"tokio-util",
@@ -3294,7 +3297,7 @@ dependencies = [
"futures-core",
"futures-sink",
"http 1.3.1",
- "indexmap 2.9.0",
+ "indexmap 2.10.0",
"slab",
"tokio",
"tokio-util",
@@ -4026,7 +4029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8a9aa791c7b5a71b636b7a68207fdebf171ddfc593d9c8506ec4cbc527b6a84"
dependencies = [
"implicit-clone-derive",
- "indexmap 2.9.0",
+ "indexmap 2.10.0",
]
[[package]]
@@ -4052,9 +4055,9 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "2.9.0"
+version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"
+checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661"
dependencies = [
"equivalent",
"hashbrown 0.15.4",
@@ -4502,9 +4505,9 @@ dependencies = [
[[package]]
name = "libredox"
-version = "0.1.3"
+version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
+checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638"
dependencies = [
"bitflags 2.9.1",
"libc",
@@ -5728,9 +5731,9 @@ dependencies = [
[[package]]
name = "postcard"
-version = "1.1.1"
+version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "170a2601f67cc9dba8edd8c4870b15f71a6a2dc196daec8c83f72b59dff628a8"
+checksum = "6c1de96e20f51df24ca73cafcc4690e044854d803259db27a00a461cb3b9d17a"
dependencies = [
"cobs",
"embedded-io 0.4.0",
@@ -6940,7 +6943,7 @@ dependencies = [
"chrono",
"hex",
"indexmap 1.9.3",
- "indexmap 2.9.0",
+ "indexmap 2.10.0",
"schemars",
"serde",
"serde_derive",
@@ -6967,7 +6970,7 @@ version = "0.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd"
dependencies = [
- "indexmap 2.9.0",
+ "indexmap 2.10.0",
"itoa",
"libyml",
"memchr",
@@ -7710,7 +7713,7 @@ version = "0.19.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
dependencies = [
- "indexmap 2.9.0",
+ "indexmap 2.10.0",
"toml_datetime",
"winnow 0.5.40",
]
@@ -7721,7 +7724,7 @@ version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
- "indexmap 2.9.0",
+ "indexmap 2.10.0",
"serde",
"serde_spanned",
"toml_datetime",
@@ -7832,7 +7835,7 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
dependencies = [
"futures-core",
"futures-util",
- "indexmap 2.9.0",
+ "indexmap 2.10.0",
"pin-project-lite",
"slab",
"sync_wrapper",
@@ -9029,7 +9032,7 @@ dependencies = [
"futures",
"gloo 0.10.0",
"implicit-clone",
- "indexmap 2.9.0",
+ "indexmap 2.10.0",
"js-sys",
"prokio",
"rustversion",
@@ -9238,7 +9241,7 @@ dependencies = [
"flate2",
"getrandom 0.3.3",
"hmac",
- "indexmap 2.9.0",
+ "indexmap 2.10.0",
"liblzma",
"memchr",
"pbkdf2",
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index a2efba95..bf249d2e 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -321,10 +321,8 @@ mod tests {
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
{
- let created_consumer_group = result.unwrap();
- assert_eq!(created_consumer_group.group_id, group_id);
- assert_eq!(created_consumer_group.name, name);
- assert_eq!(created_consumer_group.topic_id, topic_id);
+ let created_consumer_group_id = result.unwrap();
+ assert_eq!(created_consumer_group_id.get_u32_value().unwrap(), group_id);
}
assert_eq!(topic.consumer_groups.borrow().len(), 1);
diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs
index ae9435e5..bfb7ead0 100644
--- a/core/server/src/streaming/topics/storage.rs
+++ b/core/server/src/streaming/topics/storage.rs
@@ -34,7 +34,7 @@ use std::path::Path;
use std::sync::Arc;
use tokio::fs;
use tokio::fs::create_dir_all;
-use tokio::sync::{Mutex, RwLock};
+use tokio::sync::{Mutex};
use tracing::{error, info, warn};
#[derive(Debug)]
(iggy) 02/30: feat(io_uring): bootstrap the main function (#1788)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 5ca643c4712ca2611e905bacae215d3f5325ebce
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Wed May 14 20:38:55 2025 +0200
feat(io_uring): bootstrap the main function (#1788)
---
Cargo.lock | 1 +
core/server/Cargo.toml | 1 +
core/server/src/bootstrap.rs | 20 ++++++++++
core/server/src/lib.rs | 4 +-
core/server/src/main.rs | 80 ++++++++++++++++++++++++++++++++++++--
core/server/src/shard/builder.rs | 73 ++++++++++++++++++++++++++++++++++
core/server/src/shard/connector.rs | 2 +
core/server/src/shard/frame.rs | 3 +-
core/server/src/shard/mod.rs | 79 +++++++++++++++++++++++++++----------
9 files changed, 234 insertions(+), 29 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index c0fbfbea..298dd1f9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6928,6 +6928,7 @@ dependencies = [
"futures",
"human-repr",
"iggy_common",
+ "io-uring",
"jsonwebtoken",
"lending-iterator",
"mimalloc",
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 21b4ddd9..d4542eb4 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -59,6 +59,7 @@ flume = { workspace = true }
futures = { workspace = true }
human-repr = { workspace = true }
iggy_common = { workspace = true }
+io-uring = "0.6"
jsonwebtoken = "9.3.1"
lending-iterator = "0.1.7"
mimalloc = { workspace = true, optional = true }
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index e69de29b..8eacf223 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -0,0 +1,20 @@
+use crate::shard::{connector::ShardConnector, frame::ShardFrame};
+use std::ops::Range;
+
+pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<ShardFrame>> {
+ let shards_count = shards_set.len();
+ let connectors = shards_set
+ .into_iter()
+ .map(|id| ShardConnector::new(id as u16, shards_count))
+ .collect();
+
+ connectors
+}
+
+pub async fn create_directories() {
+
+}
+
+pub async fn create_root_user() {
+
+}
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index 572301e0..e64837da 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -26,18 +26,18 @@ static GLOBAL: MiMalloc = MiMalloc;
#[cfg(windows)]
compile_error!("iggy-server doesn't support windows.");
-mod bootstrap;
pub mod archiver;
pub mod args;
pub mod binary;
+pub mod bootstrap;
pub mod channels;
pub(crate) mod compat;
pub mod configs;
-pub mod shard;
pub mod http;
pub mod log;
pub mod quic;
pub mod server_error;
+pub mod shard;
pub mod state;
pub mod streaming;
pub mod tcp;
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index d7351b5f..381ac578 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -16,11 +16,15 @@
* under the License.
*/
+use std::thread::available_parallelism;
+
use anyhow::Result;
use clap::Parser;
use dotenvy::dotenv;
use figlet_rs::FIGfont;
+use monoio::Buildable;
use server::args::Args;
+use server::bootstrap::{create_directories, create_root_user, create_shard_connections};
use server::channels::commands::archive_state::ArchiveStateExecutor;
use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
use server::channels::commands::maintain_messages::MaintainMessagesExecutor;
@@ -37,15 +41,15 @@ use server::log::logger::Logging;
use server::log::tokio_console::Logging;
use server::quic::quic_server;
use server::server_error::ServerError;
+use server::shard::IggyShard;
use server::streaming::systems::system::{SharedSystem, System};
use server::streaming::utils::MemoryPool;
use server::tcp::tcp_server;
use tokio::time::Instant;
use tracing::{info, instrument};
-#[tokio::main]
#[instrument(skip_all, name = "trace_start_server")]
-async fn main() -> Result<(), ServerError> {
+fn main() -> Result<(), ServerError> {
let startup_timestamp = Instant::now();
let standard_font = FIGfont::standard().unwrap();
let figure = standard_font.convert("Iggy Server");
@@ -68,6 +72,24 @@ async fn main() -> Result<(), ServerError> {
let args = Args::parse();
let config_provider = config_provider::resolve(&args.config_provider)?;
+ let config = ServerConfig::default();
+ //TODO: Load config.
+ /*
+ let xd = std::thread::scope(|scope| {
+ let config = scope.spawn(move || {
+ let mut rt = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new().build().unwrap();
+ let config: Result<ServerConfig, ServerError> = rt.block_on(async {
+ let config = ServerConfig::load(&config_provider).await?;
+ create_directories().await;
+ create_root_user().await;
+ Ok(config)
+ });
+ config
+ }).join().unwrap();
+ config
+ });
+ */
+ /*
let config = ServerConfig::load(&config_provider).await?;
if args.fresh {
let system_path = config.system.get_system_path();
@@ -80,9 +102,56 @@ async fn main() -> Result<(), ServerError> {
}
let mut logging = Logging::new(config.telemetry.clone());
logging.early_init();
+ */
+ // TODO: Make this configurable from config as a range
+ // for example this instance of Iggy will use cores from 0..4
+ let available_cpus = available_parallelism()
+ .expect("Failed to get num of cores")
+ .into();
+ let shards_count = available_cpus;
+ let shards_set = 0..shards_count;
+
+ let connections = create_shard_connections(shards_set.clone());
+
+ for shard_id in shards_set {
+ let id = shard_id as u16;
+ let connections = connections.clone();
+ let server_config = config.clone();
+ std::thread::Builder::new()
+ .name(format!("shard-{id}"))
+ .spawn(move || {
+ monoio::utils::bind_to_cpu_set(Some(shard_id))
+ .expect(format!("Failed to set CPU affinity for shard-{id}").as_str());
+ // TODO: Figure out what else we could tweak there
+ // We for sure want to disable the userspace interrupts on new cq entry (set_coop_taskrun)
+ // let urb = io_uring::IoUring::builder();
+ // TODO: Shall we make the size of ring be configureable ?
+ let mut rt = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new()
+ //.uring_builder(urb.setup_coop_taskrun())
+ .with_entries(1024) // Default size
+ .enable_timer()
+ .build()
+ .expect(format!("Failed to build monoio runtime for shard-{id}").as_str());
+ rt.block_on(async move {
+ let builder = IggyShard::builder();
+ let shard = builder
+ .id(id)
+ .connections(connections)
+ .server_config(server_config)
+ .build_and_init()
+ .await;
+
+ shard.assert_init();
+ })
+ })
+ .expect(format!("Failed to spawn thread for shard-{id}").as_str())
+ .join()
+ .expect(format!("Failed to join thread for shard-{id}").as_str());
+ }
// From this point on, we can use tracing macros to log messages.
+ /*
logging.late_init(config.system.get_system_path(), &config.system.logging)?;
#[cfg(feature = "disable-mimalloc")]
@@ -99,13 +168,15 @@ async fn main() -> Result<(), ServerError> {
config.data_maintenance.clone(),
config.personal_access_token.clone(),
));
+ */
// Workaround to ensure that the statistics are initialized before the server
// loads streams and starts accepting connections. This is necessary to
// have the correct statistics when the server starts.
- system.write().await.get_stats().await?;
- system.write().await.init().await?;
+ //system.write().await.get_stats().await?;
+ //system.write().await.init().await?;
+ /*
let _command_handler = BackgroundServerCommandHandler::new(system.clone(), &config)
.install_handler(SaveMessagesExecutor)
.install_handler(MaintainMessagesExecutor)
@@ -181,5 +252,6 @@ async fn main() -> Result<(), ServerError> {
"Iggy server has shutdown successfully. Shutdown took {} ms.",
elapsed_time.as_millis()
);
+ */
Ok(())
}
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
new file mode 100644
index 00000000..08a02e54
--- /dev/null
+++ b/core/server/src/shard/builder.rs
@@ -0,0 +1,73 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::{configs::server::ServerConfig, shard::Shard};
+
+use super::{connector::ShardConnector, frame::ShardFrame, IggyShard};
+
+#[derive(Default)]
+pub struct IggyShardBuilder {
+ id: Option<u16>,
+ connections: Option<Vec<ShardConnector<ShardFrame>>>,
+ config: Option<ServerConfig>,
+}
+
+impl IggyShardBuilder {
+ pub fn id(mut self, id: u16) -> Self {
+ self.id = Some(id);
+ self
+ }
+
+ pub fn connections(mut self, connections: Vec<ShardConnector<ShardFrame>>) -> Self {
+ self.connections = Some(connections);
+ self
+ }
+
+ pub fn server_config(mut self, config: ServerConfig) -> Self {
+ self.config = Some(config);
+ self
+ }
+
+ pub async fn build_and_init(self) -> IggyShard {
+ let id = self.id.unwrap();
+ let config = self.config.unwrap();
+ let connections = self.connections.unwrap();
+ let (stop_sender, stop_receiver, receiver) = connections
+ .iter()
+ .filter(|c| c.id == id)
+ .map(|c| {
+ (
+ c.stop_sender.clone(),
+ c.stop_receiver.clone(),
+ c.receiver.clone(),
+ )
+ })
+ .next()
+ .expect("Failed to find connection with the specified ID");
+ let shards = connections.into_iter().map(Shard::new).collect();
+
+ IggyShard::new(
+ id,
+ shards,
+ config,
+ stop_receiver,
+ stop_sender,
+ receiver,
+ )
+ }
+}
diff --git a/core/server/src/shard/connector.rs b/core/server/src/shard/connector.rs
index 349218ba..1d762d68 100644
--- a/core/server/src/shard/connector.rs
+++ b/core/server/src/shard/connector.rs
@@ -54,6 +54,8 @@ impl<T: Clone> ShardConnector<T> {
}
}
+// TODO: I think those Arcs can be replaced with 'static lifetimes...
+// Those shards will live for the entire duration of the application.
#[derive(Clone)]
pub struct Receiver<T> {
channel: Arc<ShardedChannel<T>>,
diff --git a/core/server/src/shard/frame.rs b/core/server/src/shard/frame.rs
index b1524ad6..a2c1220f 100644
--- a/core/server/src/shard/frame.rs
+++ b/core/server/src/shard/frame.rs
@@ -27,8 +27,7 @@ pub enum ShardMessage {
}
#[derive(Debug, Clone)]
-pub enum ShardEvent {
-}
+pub enum ShardEvent {}
#[derive(Debug)]
pub enum ShardResponse {
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 6ccbcab3..bffaca45 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -15,22 +15,33 @@
* specific language governing permissions and limitations
* under the License.
*/
-mod connector;
-mod namespace;
-mod frame;
+pub mod builder;
+pub mod connector;
+pub mod frame;
+pub mod namespace;
-
-
-use std::cell::RefCell;
use ahash::HashMap;
-use connector::ShardConnector;
+use builder::IggyShardBuilder;
+use connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use frame::ShardFrame;
use namespace::IggyNamespace;
-struct Shard {
+use std::cell::{Cell, RefCell};
+
+use crate::configs::server::ServerConfig;
+pub(crate) struct Shard {
id: u16,
connection: ShardConnector<ShardFrame>,
}
+impl Shard {
+ pub fn new(connection: ShardConnector<ShardFrame>) -> Self {
+ Self {
+ id: connection.id,
+ connection,
+ }
+ }
+}
+
struct ShardInfo {
id: u16,
}
@@ -38,22 +49,48 @@ struct ShardInfo {
pub struct IggyShard {
pub id: u16,
shards: Vec<Shard>,
- shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
+ //shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
- pub(crate) permissioner: RefCell<Permissioner>,
- pub(crate) storage: Rc<SystemStorage>,
- pub(crate) streams: RwLock<HashMap<u32, Stream>>,
- pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
- pub(crate) users: RefCell<HashMap<UserId, User>>,
+ //pub(crate) permissioner: RefCell<Permissioner>,
+ //pub(crate) streams: RwLock<HashMap<u32, Stream>>,
+ //pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
+ //pub(crate) users: RefCell<HashMap<UserId, User>>,
// TODO - get rid of this dynamic dispatch.
- pub(crate) state: Rc<FileState>,
- pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
- pub(crate) config: ServerConfig,
- pub(crate) client_manager: RefCell<ClientManager>,
- pub(crate) active_sessions: RefCell<Vec<Session>>,
- pub(crate) metrics: Metrics,
+ //pub(crate) state: Rc<FileState>,
+ //pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
+ config: ServerConfig,
+ //pub(crate) client_manager: RefCell<ClientManager>,
+ //pub(crate) active_sessions: RefCell<Vec<Session>>,
+ //pub(crate) metrics: Metrics,
pub message_receiver: Cell<Option<Receiver<ShardFrame>>>,
stop_receiver: StopReceiver,
stop_sender: StopSender,
-}
\ No newline at end of file
+}
+
+impl IggyShard {
+ pub fn builder() -> IggyShardBuilder {
+ Default::default()
+ }
+
+ pub(crate) fn new(
+ id: u16,
+ shards: Vec<Shard>,
+ config: ServerConfig,
+ stop_receiver: StopReceiver,
+ stop_sender: StopSender,
+ shard_messages_receiver: Receiver<ShardFrame>,
+ ) -> Self {
+ Self {
+ id,
+ shards,
+ config,
+ stop_receiver,
+ stop_sender,
+ message_receiver: Cell::new(Some(shard_messages_receiver)),
+ }
+ }
+
+ pub fn assert_init(&self) {
+ }
+}
(iggy) 27/30: refactor(io_uring): use tcp opts from compio instead of socket config (#1962)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit e01de4f4720dee1cd689df0d6cdb38b6ca849153
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Sat Jul 5 10:31:28 2025 +0200
refactor(io_uring): use tcp opts from compio instead of socket config (#1962)
---
core/common/src/error/iggy_error.rs | 3 ++
core/server/src/shard/mod.rs | 2 +-
core/server/src/shard/transmission/event.rs | 5 ++-
core/server/src/tcp/tcp_listener.rs | 51 ++++++++++++++++++++++++-----
core/server/src/tcp/tcp_server.rs | 2 +-
5 files changed, 51 insertions(+), 12 deletions(-)
diff --git a/core/common/src/error/iggy_error.rs b/core/common/src/error/iggy_error.rs
index 334e14c7..53b4eb5b 100644
--- a/core/common/src/error/iggy_error.rs
+++ b/core/common/src/error/iggy_error.rs
@@ -464,6 +464,9 @@ pub enum IggyError {
ShardNotFound(u32, u32, u32) = 11000,
#[error("Shard communication error, shard ID: {0}")]
ShardCommunicationError(u16) = 11001,
+
+ #[error("Cannot bind to socket with addr: {0}")]
+ CannotBindToSocket(String) = 12000,
}
impl IggyError {
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 2178d2b3..16b4459c 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -535,7 +535,7 @@ impl IggyShard {
*compression_algorithm,
*max_topic_size,
*replication_factor,
- shards_assignment.clone()
+ shards_assignment.clone(),
)
.await
}
diff --git a/core/server/src/shard/transmission/event.rs b/core/server/src/shard/transmission/event.rs
index 60a28dfc..c1a34c27 100644
--- a/core/server/src/shard/transmission/event.rs
+++ b/core/server/src/shard/transmission/event.rs
@@ -2,7 +2,10 @@ use std::net::SocketAddr;
use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize};
-use crate::{shard::{namespace::IggyNamespace, ShardInfo}, streaming::clients::client_manager::Transport};
+use crate::{
+ shard::{ShardInfo, namespace::IggyNamespace},
+ streaming::clients::client_manager::Transport,
+};
#[derive(Debug)]
pub enum ShardEvent {
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index b541305b..f927cf33 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -17,33 +17,66 @@
*/
use crate::binary::sender::SenderKind;
+use crate::configs::tcp::TcpSocketConfig;
use crate::shard::IggyShard;
use crate::shard::transmission::event::ShardEvent;
use crate::streaming::clients::client_manager::Transport;
use crate::tcp::connection_handler::{handle_connection, handle_error};
+use compio::net::{TcpListener, TcpOpts};
+use error_set::ErrContext;
use futures::FutureExt;
use iggy_common::IggyError;
use socket2::Socket;
use std::net::SocketAddr;
+use std::num::TryFromIntError;
use std::rc::Rc;
use std::time::Duration;
use tracing::{error, info};
+async fn create_listener(
+ addr: SocketAddr,
+ config: &TcpSocketConfig,
+) -> Result<TcpListener, std::io::Error> {
+ // Required by the thread-per-core model...
+ // We create bunch of sockets on different threads, that bind to exactly the same address and port.
+ let opts = TcpOpts::new().set_reuse_address(true).set_reuse_port(true);
+ let opts = if config.override_defaults {
+ let recv_buffer_size = config
+ .recv_buffer_size
+ .as_bytes_u64()
+ .try_into()
+ .expect("Failed to parse recv_buffer_size for TCP socket");
+
+ let send_buffer_size = config
+ .send_buffer_size
+ .as_bytes_u64()
+ .try_into()
+ .expect("Failed to parse send_buffer_size for TCP socket");
+
+ opts.set_recv_buffer_size(recv_buffer_size)
+ .set_send_buffer_size(send_buffer_size)
+ .set_keepalive(config.keepalive)
+ .set_linger(config.linger.get_duration())
+ .set_no_delay(config.nodelay)
+ } else {
+ opts
+ };
+ TcpListener::bind_with_options(addr, opts).await
+}
+
pub async fn start(
server_name: &'static str,
addr: SocketAddr,
- socket: Socket,
+ config: &TcpSocketConfig,
shard: Rc<IggyShard>,
) -> Result<(), IggyError> {
- socket
- .bind(&addr.into())
- .expect("Failed to bind TCP listener");
- socket.listen(1024).unwrap();
- //TODO: Fix it later to use `TcpOpts` from compio.
- let listener: std::net::TcpListener = socket.into();
- let listener = compio::net::TcpListener::from_std(listener).unwrap();
+ let listener = create_listener(addr, config)
+ .await
+ .map_err(|_| IggyError::CannotBindToSocket(addr.to_string()))
+ .with_error_context(|err| {
+ format!("Failed to bind {server_name} server to address: {addr}, {err}")
+ })?;
info!("{server_name} server has started on: {:?}", addr);
-
loop {
let shutdown_check = async {
loop {
diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs
index ac019bb0..6b299bff 100644
--- a/core/server/src/tcp/tcp_server.rs
+++ b/core/server/src/tcp/tcp_server.rs
@@ -44,7 +44,7 @@ pub async fn spawn_tcp_server(shard: Rc<IggyShard>) -> Result<(), IggyError> {
// TODO: Fixme -- storing addr of the server inside of the config for integration tests...
match shard.config.tcp.tls.enabled {
true => tcp_tls_listener::start(server_name, addr, socket, shard.clone()).await?,
- false => tcp_listener::start(server_name, addr, socket, shard.clone()).await?,
+ false => tcp_listener::start(server_name, addr, socket_config, shard.clone()).await?,
};
Ok(())
(iggy) 12/30: fix iggy shard errors
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 2ace7e8e53baf35ffaa1349bf56e07f7d0f0d9ff
Author: numinex <g....@gmail.com>
AuthorDate: Thu Jun 26 16:34:52 2025 +0200
fix iggy shard errors
---
core/common/src/locking/mod.rs | 4 +-
core/common/src/locking/tokio_lock.rs | 3 +-
core/server/src/binary/command.rs | 1 -
.../create_consumer_group_handler.rs | 13 +-
core/server/src/binary/handlers/utils.rs | 15 +-
core/server/src/binary/mapper.rs | 14 +-
core/server/src/http/consumer_groups.rs | 6 +-
core/server/src/http/jwt/cleaner.rs | 8 +-
core/server/src/http/jwt/jwt_manager.rs | 6 +-
core/server/src/http/mapper.rs | 14 +-
core/server/src/http/messages.rs | 2 +-
core/server/src/http/mod.rs | 2 +
core/server/src/http/shared.rs | 5 +-
core/server/src/http/system.rs | 2 +-
core/server/src/shard/mod.rs | 2 +-
core/server/src/shard/system/clients.rs | 35 ++---
core/server/src/shard/system/consumer_groups.rs | 139 +++++++++++-------
core/server/src/shard/system/consumer_offsets.rs | 24 +++-
core/server/src/shard/system/messages.rs | 34 +++--
core/server/src/shard/system/partitions.rs | 48 +++++--
.../src/shard/system/personal_access_tokens.rs | 14 +-
core/server/src/shard/system/segments.rs | 40 +++++-
core/server/src/shard/system/snapshot/mod.rs | 62 ++++++---
core/server/src/shard/system/stats.rs | 8 +-
core/server/src/shard/system/storage.rs | 1 +
core/server/src/shard/system/streams.rs | 155 +++++++++++++--------
core/server/src/shard/system/topics.rs | 122 ++++++++--------
core/server/src/shard/system/users.rs | 154 ++++++++++++--------
.../src/streaming/partitions/consumer_offsets.rs | 5 +-
core/server/src/streaming/storage.rs | 29 ++--
core/server/src/streaming/topics/consumer_group.rs | 5 +-
.../server/src/streaming/topics/consumer_groups.rs | 22 +--
core/server/src/streaming/topics/topic.rs | 4 +-
core/server/src/streaming/users/user.rs | 2 +-
34 files changed, 595 insertions(+), 405 deletions(-)
diff --git a/core/common/src/locking/mod.rs b/core/common/src/locking/mod.rs
index c2e542c7..429b4f59 100644
--- a/core/common/src/locking/mod.rs
+++ b/core/common/src/locking/mod.rs
@@ -37,11 +37,11 @@ pub type IggyRwLock<T> = fast_async_lock::IggyFastAsyncRwLock<T>;
#[allow(async_fn_in_trait)]
pub trait IggySharedMutFn<T> {
- type ReadGuard<'a>: Deref<Target = T>
+ type ReadGuard<'a>: Deref<Target = T>
where
T: 'a,
Self: 'a;
- type WriteGuard<'a>: DerefMut<Target = T>
+ type WriteGuard<'a>: DerefMut<Target = T>
where
T: 'a,
Self: 'a;
diff --git a/core/common/src/locking/tokio_lock.rs b/core/common/src/locking/tokio_lock.rs
index ef5baaad..d6a96f06 100644
--- a/core/common/src/locking/tokio_lock.rs
+++ b/core/common/src/locking/tokio_lock.rs
@@ -24,8 +24,7 @@ use tokio::sync::{RwLock as TokioRwLock, RwLockReadGuard, RwLockWriteGuard};
#[derive(Debug)]
pub struct IggyTokioRwLock<T>(Arc<TokioRwLock<T>>);
-impl<T> IggySharedMutFn<T> for IggyTokioRwLock<T>
-{
+impl<T> IggySharedMutFn<T> for IggyTokioRwLock<T> {
type ReadGuard<'a>
= RwLockReadGuard<'a, T>
where
diff --git a/core/server/src/binary/command.rs b/core/server/src/binary/command.rs
index aa935e09..1ebac395 100644
--- a/core/server/src/binary/command.rs
+++ b/core/server/src/binary/command.rs
@@ -22,7 +22,6 @@ use crate::binary::sender::SenderKind;
use crate::define_server_command_enum;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use bytes::{BufMut, Bytes, BytesMut};
use enum_dispatch::enum_dispatch;
use iggy_common::change_password::ChangePassword;
diff --git a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
index 3d87fa17..5d695687 100644
--- a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
@@ -16,14 +16,16 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::{handlers::consumer_groups::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::state::models::CreateConsumerGroupWithId;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
@@ -39,14 +41,13 @@ impl ServerCommandHandler for CreateConsumerGroup {
async fn handle(
self,
sender: &mut SenderKind,
- _length: u32,
- session: &Session,
- system: &SharedSystem,
+ length: u32,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let mut system = system.write().await;
- let consumer_group = system
+ let consumer_group = shard
.create_consumer_group(
session,
&self.stream_id,
diff --git a/core/server/src/binary/handlers/utils.rs b/core/server/src/binary/handlers/utils.rs
index 351f6297..80bcede4 100644
--- a/core/server/src/binary/handlers/utils.rs
+++ b/core/server/src/binary/handlers/utils.rs
@@ -26,12 +26,17 @@ pub async fn receive_and_validate(
length: u32,
) -> Result<ServerCommand, IggyError> {
let mut buffer = BytesMut::with_capacity(length as usize);
- if length > 0 {
- unsafe {
- buffer.set_len(length as usize);
- }
- sender.read(&mut buffer).await?;
+ unsafe {
+ buffer.set_len(length as usize);
}
+ let buffer = if length == 0 {
+ buffer
+ } else {
+ let (result, buffer) = sender.read(buffer).await;
+ result?;
+ buffer
+ };
+
let command = ServerCommand::from_code_and_payload(code, buffer.freeze())?;
command.validate()?;
Ok(command)
diff --git a/core/server/src/binary/mapper.rs b/core/server/src/binary/mapper.rs
index 61c65235..219ceed9 100644
--- a/core/server/src/binary/mapper.rs
+++ b/core/server/src/binary/mapper.rs
@@ -24,9 +24,8 @@ use crate::streaming::topics::consumer_group::ConsumerGroup;
use crate::streaming::topics::topic::Topic;
use crate::streaming::users::user::User;
use bytes::{BufMut, Bytes, BytesMut};
-use iggy_common::locking::{IggySharedMut, IggySharedMutFn};
+use iggy_common::locking::IggySharedMutFn;
use iggy_common::{BytesSerializable, ConsumerOffsetInfo, Sizeable, Stats, UserId};
-use tokio::sync::RwLock;
pub fn map_stats(stats: &Stats) -> Bytes {
let mut bytes = BytesMut::with_capacity(104);
@@ -95,11 +94,10 @@ pub fn map_client(client: &Client) -> Bytes {
bytes.freeze()
}
-pub async fn map_clients(clients: &[IggySharedMut<Client>]) -> Bytes {
+pub async fn map_clients(clients: &[Client]) -> Bytes {
let mut bytes = BytesMut::new();
for client in clients {
- let client = client.read().await;
- extend_client(&client, &mut bytes);
+ extend_client(client, &mut bytes);
}
bytes.freeze()
}
@@ -183,12 +181,11 @@ pub async fn map_topic(topic: &Topic) -> Bytes {
bytes.freeze()
}
-pub async fn map_consumer_group(consumer_group: &ConsumerGroup) -> Bytes {
+pub fn map_consumer_group(consumer_group: &ConsumerGroup) -> Bytes {
let mut bytes = BytesMut::new();
extend_consumer_group(consumer_group, &mut bytes);
let members = consumer_group.get_members();
for member in members {
- let member = member.read().await;
bytes.put_u32_le(member.id);
let partitions = member.get_partitions();
bytes.put_u32_le(partitions.len() as u32);
@@ -199,10 +196,9 @@ pub async fn map_consumer_group(consumer_group: &ConsumerGroup) -> Bytes {
bytes.freeze()
}
-pub async fn map_consumer_groups(consumer_groups: &[&RwLock<ConsumerGroup>]) -> Bytes {
+pub fn map_consumer_groups(consumer_groups: &[ConsumerGroup]) -> Bytes {
let mut bytes = BytesMut::new();
for consumer_group in consumer_groups {
- let consumer_group = consumer_group.read().await;
extend_consumer_group(&consumer_group, &mut bytes);
}
bytes.freeze()
diff --git a/core/server/src/http/consumer_groups.rs b/core/server/src/http/consumer_groups.rs
index dd48c8c2..d58faef3 100644
--- a/core/server/src/http/consumer_groups.rs
+++ b/core/server/src/http/consumer_groups.rs
@@ -72,7 +72,7 @@ async fn get_consumer_group(
};
let consumer_group = consumer_group.read().await;
- let consumer_group = mapper::map_consumer_group(&consumer_group).await;
+ let consumer_group = mapper::map_consumer_group(&consumer_group);
Ok(Json(consumer_group))
}
@@ -89,7 +89,7 @@ async fn get_consumer_groups(
&stream_id,
&topic_id,
)?;
- let consumer_groups = mapper::map_consumer_groups(&consumer_groups).await;
+ let consumer_groups = mapper::map_consumer_groups(&consumer_groups);
Ok(Json(consumer_groups))
}
@@ -116,7 +116,7 @@ async fn create_consumer_group(
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to create consumer group, stream ID: {}, topic ID: {}, group ID: {:?}", stream_id, topic_id, command.group_id))?;
let consumer_group = consumer_group.read().await;
let group_id = consumer_group.group_id;
- let consumer_group_details = mapper::map_consumer_group(&consumer_group).await;
+ let consumer_group_details = mapper::map_consumer_group(&consumer_group);
drop(consumer_group);
let system = system.downgrade();
diff --git a/core/server/src/http/jwt/cleaner.rs b/core/server/src/http/jwt/cleaner.rs
index 40003eca..6d4329e7 100644
--- a/core/server/src/http/jwt/cleaner.rs
+++ b/core/server/src/http/jwt/cleaner.rs
@@ -19,12 +19,12 @@
use crate::http::shared::AppState;
use iggy_common::IggyTimestamp;
use std::sync::Arc;
-use std::time::Duration;
+use monoio::time::Duration;
use tracing::{error, trace};
pub fn start_expired_tokens_cleaner(app_state: Arc<AppState>) {
- tokio::spawn(async move {
- let mut interval_timer = tokio::time::interval(Duration::from_secs(300));
+ monoio::spawn(async move {
+ let mut interval_timer = monoio::time::interval(Duration::from_secs(300));
loop {
interval_timer.tick().await;
trace!("Deleting expired tokens...");
@@ -34,7 +34,7 @@ pub fn start_expired_tokens_cleaner(app_state: Arc<AppState>) {
.delete_expired_revoked_tokens(now)
.await
.unwrap_or_else(|err| {
- error!("Failed to delete expired revoked access tokens. Error: {err}",);
+ error!("Failed to delete expired revoked access tokens. Error: {err}");
});
}
});
diff --git a/core/server/src/http/jwt/jwt_manager.rs b/core/server/src/http/jwt/jwt_manager.rs
index 56e2ce9b..dfe0f66b 100644
--- a/core/server/src/http/jwt/jwt_manager.rs
+++ b/core/server/src/http/jwt/jwt_manager.rs
@@ -28,7 +28,7 @@ use iggy_common::IggyError;
use iggy_common::IggyExpiry;
use iggy_common::IggyTimestamp;
use iggy_common::UserId;
-use iggy_common::locking::IggySharedMut;
+use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation, encode};
use std::sync::Arc;
@@ -54,7 +54,7 @@ pub struct JwtManager {
issuer: IssuerOptions,
validator: ValidatorOptions,
tokens_storage: TokenStorage,
- revoked_tokens: IggySharedMut<AHashMap<String, u64>>,
+ revoked_tokens: IggyRwLock<AHashMap<String, u64>>,
validations: AHashMap<Algorithm, Validation>,
}
@@ -77,7 +77,7 @@ impl JwtManager {
issuer,
validator,
tokens_storage: TokenStorage::new(persister, path),
- revoked_tokens: IggySharedMut::new(AHashMap::new()),
+ revoked_tokens: IggyRwLock::new(AHashMap::new()),
})
}
diff --git a/core/server/src/http/mapper.rs b/core/server/src/http/mapper.rs
index ed9ebba3..d839b9d9 100644
--- a/core/server/src/http/mapper.rs
+++ b/core/server/src/http/mapper.rs
@@ -28,12 +28,11 @@ use iggy_common::PersonalAccessTokenInfo;
use iggy_common::Sizeable;
use iggy_common::StreamDetails;
use iggy_common::TopicDetails;
-use iggy_common::locking::IggySharedMut;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{ConsumerGroupDetails, ConsumerGroupMember};
use iggy_common::{IdentityInfo, TokenInfo};
use iggy_common::{UserInfo, UserInfoDetails};
-use tokio::sync::RwLock;
+
pub fn map_stream(stream: &Stream) -> StreamDetails {
let topics = map_topics(&stream.get_topics());
@@ -177,10 +176,9 @@ pub fn map_client(client: &Client) -> iggy_common::ClientInfoDetails {
}
}
-pub async fn map_clients(clients: &[IggySharedMut<Client>]) -> Vec<iggy_common::ClientInfo> {
+pub fn map_clients(clients: &[Client]) -> Vec<iggy_common::ClientInfo> {
let mut all_clients = Vec::new();
for client in clients {
- let client = client.read().await;
let client = iggy_common::ClientInfo {
client_id: client.session.client_id,
user_id: client.user_id,
@@ -195,12 +193,11 @@ pub async fn map_clients(clients: &[IggySharedMut<Client>]) -> Vec<iggy_common::
all_clients
}
-pub async fn map_consumer_groups(
- consumer_groups: &[&RwLock<ConsumerGroup>],
+pub fn map_consumer_groups(
+ consumer_groups: &[ConsumerGroup],
) -> Vec<iggy_common::ConsumerGroup> {
let mut groups = Vec::new();
for consumer_group in consumer_groups {
- let consumer_group = consumer_group.read().await;
let consumer_group = iggy_common::ConsumerGroup {
id: consumer_group.group_id,
name: consumer_group.name.clone(),
@@ -213,7 +210,7 @@ pub async fn map_consumer_groups(
groups
}
-pub async fn map_consumer_group(consumer_group: &ConsumerGroup) -> ConsumerGroupDetails {
+pub fn map_consumer_group(consumer_group: &ConsumerGroup) -> ConsumerGroupDetails {
let mut consumer_group_details = ConsumerGroupDetails {
id: consumer_group.group_id,
name: consumer_group.name.clone(),
@@ -223,7 +220,6 @@ pub async fn map_consumer_group(consumer_group: &ConsumerGroup) -> ConsumerGroup
};
let members = consumer_group.get_members();
for member in members {
- let member = member.read().await;
let partitions = member.get_partitions();
consumer_group_details.members.push(ConsumerGroupMember {
id: member.id,
diff --git a/core/server/src/http/messages.rs b/core/server/src/http/messages.rs
index 251c503b..e289adb9 100644
--- a/core/server/src/http/messages.rs
+++ b/core/server/src/http/messages.rs
@@ -20,9 +20,9 @@ use crate::http::COMPONENT;
use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::shared::AppState;
+use crate::shard::system::messages::PollingArgs;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
-use crate::streaming::systems::messages::PollingArgs;
use crate::streaming::utils::PooledBuffer;
use axum::extract::{Path, Query, State};
use axum::http::StatusCode;
diff --git a/core/server/src/http/mod.rs b/core/server/src/http/mod.rs
index 194ed74a..7b24bfff 100644
--- a/core/server/src/http/mod.rs
+++ b/core/server/src/http/mod.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+/*
pub mod consumer_groups;
pub mod consumer_offsets;
pub mod diagnostics;
@@ -32,5 +33,6 @@ pub mod streams;
pub mod system;
pub mod topics;
pub mod users;
+*/
pub const COMPONENT: &str = "HTTP";
diff --git a/core/server/src/http/shared.rs b/core/server/src/http/shared.rs
index 50703357..a21ddb1c 100644
--- a/core/server/src/http/shared.rs
+++ b/core/server/src/http/shared.rs
@@ -16,14 +16,13 @@
* under the License.
*/
-use crate::http::jwt::jwt_manager::JwtManager;
-use crate::streaming::systems::system::SharedSystem;
+use crate::{http::jwt::jwt_manager::JwtManager, shard::IggyShard};
use std::net::SocketAddr;
use ulid::Ulid;
pub struct AppState {
pub jwt_manager: JwtManager,
- pub system: SharedSystem,
+ pub shard: IggyShard,
}
#[derive(Debug, Copy, Clone)]
diff --git a/core/server/src/http/system.rs b/core/server/src/http/system.rs
index 1c7166e3..4428d36c 100644
--- a/core/server/src/http/system.rs
+++ b/core/server/src/http/system.rs
@@ -114,7 +114,7 @@ async fn get_clients(
identity.user_id
)
})?;
- let clients = mapper::map_clients(&clients).await;
+ let clients = mapper::map_clients(&clients);
Ok(Json(clients))
}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 12ba64bb..b5e48525 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -461,6 +461,6 @@ impl IggyShard {
Err(IggyError::Unauthenticated)
}
})?;
- Ok(user_id)
+ Ok(user_id)
}
}
diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs
index a2debc42..ee9e1634 100644
--- a/core/server/src/shard/system/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -16,17 +16,16 @@
* under the License.
*/
+use super::COMPONENT;
use crate::shard::IggyShard;
use crate::streaming::clients::client_manager::{Client, Transport};
use crate::streaming::session::Session;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
-use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
use std::net::SocketAddr;
use std::rc::Rc;
-use std::sync::Arc;
use tracing::{error, info};
impl IggyShard {
@@ -43,7 +42,7 @@ impl IggyShard {
{
let mut client_manager = self.client_manager.borrow_mut();
- let client = client_manager.delete_client(client_id).await;
+ let client = client_manager.delete_client(client_id);
if client.is_none() {
error!("Client with ID: {client_id} was not found in the client manager.",);
return;
@@ -51,7 +50,6 @@ impl IggyShard {
self.metrics.decrement_clients(1);
let client = client.unwrap();
- let client = client.read().await;
consumer_groups = client
.consumer_groups
.iter()
@@ -65,14 +63,12 @@ impl IggyShard {
}
for (stream_id, topic_id, consumer_group_id) in consumer_groups.into_iter() {
- _ = self
- .leave_consumer_group_by_client(
- &Identifier::numeric(stream_id).unwrap(),
- &Identifier::numeric(topic_id).unwrap(),
- &Identifier::numeric(consumer_group_id).unwrap(),
- client_id,
- )
- .await
+ _ = self.leave_consumer_group_by_client(
+ &Identifier::numeric(stream_id).unwrap(),
+ &Identifier::numeric(topic_id).unwrap(),
+ &Identifier::numeric(consumer_group_id).unwrap(),
+ client_id,
+ )
}
}
@@ -80,9 +76,10 @@ impl IggyShard {
&self,
session: &Session,
client_id: u32,
- ) -> Result<Option<IggySharedMut<Client>>, IggyError> {
+ ) -> Result<Option<Client>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.get_client(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -91,16 +88,13 @@ impl IggyShard {
)
})?;
- let client_manager = self.client_manager.read().await;
- Ok(client_manager.try_get_client(client_id))
+ Ok(self.client_manager.borrow().try_get_client(client_id))
}
- pub async fn get_clients(
- &self,
- session: &Session,
- ) -> Result<Vec<IggySharedMut<Client>>, IggyError> {
+ pub async fn get_clients(&self, session: &Session) -> Result<Vec<Client>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.get_clients(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -109,7 +103,6 @@ impl IggyShard {
)
})?;
- let client_manager = self.client_manager.read().await;
- Ok(client_manager.get_clients())
+ Ok(self.client_manager.borrow().get_clients())
}
}
diff --git a/core/server/src/shard/system/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs
index d8d8818e..768d4e5c 100644
--- a/core/server/src/shard/system/consumer_groups.rs
+++ b/core/server/src/shard/system/consumer_groups.rs
@@ -16,29 +16,39 @@
* under the License.
*/
+use std::cell::Ref;
+
+use super::COMPONENT;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
+use crate::streaming::streams::stream::Stream;
use crate::streaming::topics::consumer_group::ConsumerGroup;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
use iggy_common::locking::IggySharedMutFn;
-use tokio::sync::RwLock;
impl IggyShard {
- pub fn get_consumer_group(
+ pub fn get_consumer_group<'cg, 'stream>(
&self,
session: &Session,
- stream_id: &Identifier,
+ stream: &'stream Stream,
topic_id: &Identifier,
group_id: &Identifier,
- ) -> Result<Option<&RwLock<ConsumerGroup>>, IggyError> {
+ ) -> Result<Option<Ref<'cg, ConsumerGroup>>, IggyError>
+ where
+ 'stream: 'cg,
+ {
self.ensure_authenticated(session)?;
- let Some(topic) = self.try_find_topic(session, stream_id, topic_id)? else {
- return Ok(None);
- };
+ let stream_id = stream.stream_id;
+ let topic = stream.get_topic(topic_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - topic with ID: {topic_id} was not found in stream with ID: {stream_id}",
+ )
+ })?;
self.permissioner
+ .borrow()
.get_consumer_group(session.get_user_id(), topic.stream_id, topic.topic_id)
.with_error_context(|error| {
format!(
@@ -55,12 +65,16 @@ impl IggyShard {
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
- ) -> Result<Vec<&RwLock<ConsumerGroup>>, IggyError> {
+ ) -> Result<Vec<ConsumerGroup>, IggyError> {
self.ensure_authenticated(session)?;
- let topic = self.find_topic(session, stream_id, topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream with ID: {stream_id} was not found")
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic with ID: {topic_id} was not found in stream with ID: {stream_id}"))?;
self.permissioner
+ .borrow()
.get_consumer_groups(session.get_user_id(), topic.stream_id, topic.topic_id)
.with_error_context(|error| {
format!(
@@ -72,40 +86,45 @@ impl IggyShard {
Ok(topic.get_consumer_groups())
}
- pub async fn create_consumer_group(
- &mut self,
+ pub fn create_consumer_group(
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
group_id: Option<u32>,
name: &str,
- ) -> Result<&RwLock<ConsumerGroup>, IggyError> {
+ ) -> Result<u32, IggyError> {
self.ensure_authenticated(session)?;
{
- let topic = self.find_topic(session, stream_id, topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}"
+ )
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.create_consumer_group(
+ self.permissioner.borrow().create_consumer_group(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
).with_error_context(|error| format!("{COMPONENT} (error: {error}) - permission denied to create consumer group for user {} on stream ID: {}, topic ID: {}", session.get_user_id(), topic.stream_id, topic.topic_id))?;
}
- let topic = self.get_stream_mut(stream_id)?
- .get_topic_mut(topic_id)
+ let mut stream = self.get_stream_mut(stream_id)
+ .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}"))?;
+ let topic = stream.get_topic_mut(topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
topic
.create_consumer_group(group_id, name)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to create consumer group with name: {name}")
- })
+ }).map(|cg| cg.group_id)
}
pub async fn delete_consumer_group(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -115,10 +134,15 @@ impl IggyShard {
let stream_id_value;
let topic_id_value;
{
- let topic = self.find_topic(session, stream_id, topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}"
+ )
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.delete_consumer_group(
+ self.permissioner.borrow().delete_consumer_group(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -130,7 +154,7 @@ impl IggyShard {
let consumer_group;
{
- let stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {stream_id}"
)
@@ -142,25 +166,20 @@ impl IggyShard {
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to delete consumer group with ID: {consumer_group_id}"))?
}
- let client_manager = self.client_manager.read().await;
- let consumer_group = consumer_group.read().await;
for member in consumer_group.get_members() {
- let member = member.read().await;
- client_manager
- .leave_consumer_group(
+ self.client_manager.borrow_mut().leave_consumer_group(
member.id,
stream_id_value,
topic_id_value,
consumer_group.group_id,
)
- .await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to make client leave consumer group for client ID: {}, group ID: {}", member.id, consumer_group.group_id))?;
}
Ok(())
}
- pub async fn join_consumer_group(
+ pub fn join_consumer_group(
&self,
session: &Session,
stream_id: &Identifier,
@@ -171,15 +190,20 @@ impl IggyShard {
let stream_id_value;
let topic_id_value;
{
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}"
+ )
+ })?;
let topic = self
- .find_topic(session, stream_id, topic_id)
+ .find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}",
)
})?;
- self.permissioner.join_consumer_group(
+ self.permissioner.borrow().join_consumer_group(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -191,7 +215,14 @@ impl IggyShard {
let group_id;
{
- let topic = self.find_topic(session, stream_id, topic_id)?;
+ let mut stream = self.get_stream_mut(stream_id)
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}")
+ })?;
+ let topic = stream.get_topic_mut(topic_id)
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get mutable reference to topic with ID: {topic_id}")
+ })?;
{
let consumer_group = topic
@@ -202,13 +233,11 @@ impl IggyShard {
)
})?;
- let consumer_group = consumer_group.read().await;
group_id = consumer_group.group_id;
}
topic
.join_consumer_group(consumer_group_id, session.client_id)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to join consumer group for group ID: {group_id}"
@@ -216,10 +245,7 @@ impl IggyShard {
})?;
}
- let client_manager = self.client_manager.read().await;
- client_manager
- .join_consumer_group(session.client_id, stream_id_value, topic_id_value, group_id)
- .await
+ self.client_manager.borrow_mut().join_consumer_group(session.client_id, stream_id_value, topic_id_value, group_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to make client join consumer group for client ID: {}",
@@ -239,15 +265,23 @@ impl IggyShard {
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
{
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
let topic = self
- .find_topic(session, stream_id, topic_id)
+ .find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!(
+<<<<<<< HEAD
"{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id:?}, topic_id: {topic_id:?}"
+=======
+ "{COMPONENT} (error: {error}) - topic not found for stream ID: {:?}, topic_id: {:?}",
+ stream.stream_id, topic_id
+>>>>>>> 48107890 (fix iggy shard errors)
)
})?;
- self.permissioner.leave_consumer_group(
+ self.permissioner.borrow().leave_consumer_group(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -260,7 +294,6 @@ impl IggyShard {
consumer_group_id,
session.client_id,
)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to leave consumer group for client ID: {}",
@@ -269,7 +302,7 @@ impl IggyShard {
})
}
- pub async fn leave_consumer_group_by_client(
+ pub fn leave_consumer_group_by_client(
&self,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -281,15 +314,17 @@ impl IggyShard {
let group_id;
{
- let stream = self.get_stream(stream_id).with_error_context(|error| {
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
})?;
- let topic = stream.get_topic(topic_id)
+ let stream_id = stream.stream_id;
+ let topic = stream.get_topic_mut(topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}",
)
})?;
+ let topic_id = topic.topic_id;
{
let consumer_group = topic
.get_consumer_group(consumer_group_id)
@@ -298,23 +333,23 @@ impl IggyShard {
"{COMPONENT} (error: {error}) - consumer group not found for group_id: {consumer_group_id}",
)
})?;
- let consumer_group = consumer_group.read().await;
group_id = consumer_group.group_id;
}
- stream_id_value = stream.stream_id;
- topic_id_value = topic.topic_id;
+ stream_id_value = stream_id;
+ topic_id_value = topic_id;
topic
.leave_consumer_group(consumer_group_id, client_id)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed leave consumer group, client ID {client_id}",)
})?;
}
- let client_manager = self.client_manager.read().await;
- client_manager
- .leave_consumer_group(client_id, stream_id_value, topic_id_value, group_id)
- .await
+ self.client_manager.borrow_mut().leave_consumer_group(
+ client_id,
+ stream_id_value,
+ topic_id_value,
+ group_id,
+ )
}
}
diff --git a/core/server/src/shard/system/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs
index 0c7438a6..e5504173 100644
--- a/core/server/src/shard/system/consumer_offsets.rs
+++ b/core/server/src/shard/system/consumer_offsets.rs
@@ -16,7 +16,8 @@
* under the License.
*/
-use crate::streaming::session::Session;
+use super::COMPONENT;
+use crate::{shard::IggyShard, streaming::session::Session};
use error_set::ErrContext;
use iggy_common::{Consumer, ConsumerOffsetInfo, Identifier, IggyError};
@@ -31,9 +32,12 @@ impl IggyShard {
offset: u64,
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
- let topic = self.find_topic(session, stream_id, topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream with ID: {stream_id} was not found")
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic with ID: {topic_id} was not found in stream with ID: {stream_id}"))?;
- self.permissioner.store_consumer_offset(
+ self.permissioner.borrow().store_consumer_offset(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -53,11 +57,14 @@ impl IggyShard {
partition_id: Option<u32>,
) -> Result<Option<ConsumerOffsetInfo>, IggyError> {
self.ensure_authenticated(session)?;
- let Some(topic) = self.try_find_topic(session, stream_id, topic_id)? else {
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream with ID: {stream_id} was not found")
+ })?;
+ let Some(topic) = self.try_find_topic(session, &stream, topic_id)? else {
return Ok(None);
};
- self.permissioner.get_consumer_offset(
+ self.permissioner.borrow().get_consumer_offset(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -82,9 +89,12 @@ impl IggyShard {
partition_id: Option<u32>,
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
- let topic = self.find_topic(session, stream_id, topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream with ID: {stream_id} was not found")
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic with ID: {topic_id} was not found in stream with ID: {stream_id}"))?;
- self.permissioner.delete_consumer_offset(
+ self.permissioner.borrow().delete_consumer_offset(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index 54046dba..06499bd3 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -16,11 +16,13 @@
* under the License.
*/
+use super::COMPONENT;
use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
use crate::shard::IggyShard;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet};
use crate::streaming::session::Session;
use crate::streaming::utils::PooledBuffer;
+use async_zip::tokio::read::stream;
use error_set::ErrContext;
use iggy_common::{
BytesSerializable, Confirmation, Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier,
@@ -43,8 +45,13 @@ impl IggyShard {
return Err(IggyError::InvalidMessagesCount);
}
- let topic = self.find_topic(session, stream_id, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
self.permissioner
+ .borrow()
.poll_messages(session.get_user_id(), topic.stream_id, topic.topic_id)
.with_error_context(|error| format!(
"{COMPONENT} (error: {error}) - permission denied to poll messages for user {} on stream ID: {}, topic ID: {}",
@@ -60,7 +67,6 @@ impl IggyShard {
// There might be no partition assigned, if it's the consumer group member without any partitions.
let Some((polling_consumer, partition_id)) = topic
.resolve_consumer_with_partition_id(consumer, session.client_id, partition_id, true)
- .await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer with partition id, consumer: {consumer}, client ID: {}, partition ID: {:?}", session.client_id, partition_id))? else {
return Ok((IggyPollMetadata::new(0, 0), IggyMessagesBatchSet::empty()));
};
@@ -80,11 +86,15 @@ impl IggyShard {
topic
.store_consumer_offset_internal(polling_consumer, offset, partition_id)
.await
+<<<<<<< HEAD
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to store consumer offset internal, polling consumer: {polling_consumer}, offset: {offset}, partition ID: {partition_id}")) ?;
+=======
+ .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to store consumer offset internal, polling consumer: {}, offset: {}, partition ID: {}", polling_consumer, offset, partition_id))?;
+>>>>>>> 48107890 (fix iggy shard errors)
}
let batch_set = if let Some(encryptor) = &self.encryptor {
- self.decrypt_messages(batch_set, encryptor.as_ref()).await?
+ self.decrypt_messages(batch_set, encryptor).await?
} else {
batch_set
};
@@ -102,8 +112,12 @@ impl IggyShard {
confirmation: Option<Confirmation>,
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
- let topic = self.find_topic(session, stream_id, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream_id: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.append_messages(
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream not found for stream_id: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream_id: {stream_id}, topic_id: {topic_id}"))?;
+ self.permissioner.borrow().append_messages(
session.get_user_id(),
topic.stream_id,
topic.topic_id
@@ -117,7 +131,7 @@ impl IggyShard {
// Encrypt messages if encryptor is configured
let messages = if let Some(encryptor) = &self.encryptor {
- self.encrypt_messages(messages, encryptor.as_ref())?
+ self.encrypt_messages(messages, encryptor)?
} else {
messages
};
@@ -139,8 +153,12 @@ impl IggyShard {
fsync: bool,
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
- let topic = self.find_topic(session, &stream_id, &topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.append_messages(
+ let stream = self.get_stream(&stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let topic = self.find_topic(session, &stream, &topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic ID: {topic_id}"))?;
+ self.permissioner.borrow().append_messages(
session.get_user_id(),
topic.stream_id,
topic.topic_id
diff --git a/core/server/src/shard/system/partitions.rs b/core/server/src/shard/system/partitions.rs
index 7712f498..01e02968 100644
--- a/core/server/src/shard/system/partitions.rs
+++ b/core/server/src/shard/system/partitions.rs
@@ -16,16 +16,16 @@
* under the License.
*/
+use super::COMPONENT;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
impl IggyShard {
pub async fn create_partitions(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -33,8 +33,13 @@ impl IggyShard {
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
{
- let topic = self.find_topic(session, stream_id, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.create_partitions(
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}"
+ )
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic ID: {topic_id}"))?;
+ self.permissioner.borrow().create_partitions(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -46,28 +51,33 @@ impl IggyShard {
))?;
}
- let topic = self
- .get_stream_mut(stream_id)?
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let topic = stream
.get_topic_mut(topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {stream_id}"
)
})?;
+
+ // TODO: Make add persisted partitions to topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
topic
.add_persisted_partitions(partitions_count)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to add persisted partitions, topic: {topic}")
})?;
- topic.reassign_consumer_groups().await;
+ topic.reassign_consumer_groups();
self.metrics.increment_partitions(partitions_count);
self.metrics.increment_segments(partitions_count);
Ok(())
}
pub async fn delete_partitions(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -75,8 +85,13 @@ impl IggyShard {
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
{
- let topic = self.find_topic(session, stream_id, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.delete_partitions(
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}"
+ )
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
+ self.permissioner.borrow().delete_partitions(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -88,21 +103,26 @@ impl IggyShard {
))?;
}
- let topic = self
- .get_stream_mut(stream_id)?
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let topic = stream
.get_topic_mut(topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {stream_id}"
)
})?;
+
+ // TODO: Make delete persisted partitions from topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
let partitions = topic
.delete_persisted_partitions(partitions_count)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to delete persisted partitions for topic: {topic}")
})?;
- topic.reassign_consumer_groups().await;
+ topic.reassign_consumer_groups();
if let Some(partitions) = partitions {
self.metrics.decrement_partitions(partitions_count);
self.metrics.decrement_segments(partitions.segments_count);
diff --git a/core/server/src/shard/system/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs
index b5452e9a..082fcd93 100644
--- a/core/server/src/shard/system/personal_access_tokens.rs
+++ b/core/server/src/shard/system/personal_access_tokens.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+use super::COMPONENT;
+use crate::shard::IggyShard;
use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
use crate::streaming::session::Session;
use crate::streaming::users::user::User;
@@ -64,7 +66,7 @@ impl IggyShard {
let user = self.get_user(&identifier).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get user with id: {user_id}")
})?;
- let max_token_per_user = self.personal_access_token.max_tokens_per_user;
+ let max_token_per_user = self.config.personal_access_token.max_tokens_per_user;
if user.personal_access_tokens.len() as u32 >= max_token_per_user {
error!(
"User with ID: {user_id} has reached the maximum number of personal access tokens: {max_token_per_user}.",
@@ -76,7 +78,7 @@ impl IggyShard {
}
}
- let user = self.get_user(&identifier).with_error_context(|error| {
+ let user = self.get_user_mut(&identifier).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get mutable reference to the user with id: {user_id}")
})?;
@@ -102,7 +104,7 @@ impl IggyShard {
}
pub async fn delete_personal_access_token(
- &mut self,
+ &self,
session: &Session,
name: &str,
) -> Result<(), IggyError> {
@@ -137,10 +139,11 @@ impl IggyShard {
&self,
token: &str,
session: Option<&Session>,
- ) -> Result<&User, IggyError> {
+ ) -> Result<User, IggyError> {
let token_hash = PersonalAccessToken::hash_token(token);
+ let users = self.users.borrow();
let mut personal_access_token = None;
- for user in self.users.values() {
+ for user in users.values() {
if let Some(pat) = user.personal_access_tokens.get(&token_hash) {
personal_access_token = Some(pat);
break;
@@ -173,6 +176,5 @@ impl IggyShard {
)
})?;
self.login_user_with_credentials(&user.username, None, session)
- .await
}
}
diff --git a/core/server/src/shard/system/segments.rs b/core/server/src/shard/system/segments.rs
index 27abb76b..6a8ad68e 100644
--- a/core/server/src/shard/system/segments.rs
+++ b/core/server/src/shard/system/segments.rs
@@ -16,6 +16,7 @@ use crate::shard::IggyShard;
* specific language governing permissions and limitations
* under the License.
*/
+use super::COMPONENT;
use crate::streaming::session::Session;
use error_set::ErrContext;
use iggy_common::Identifier;
@@ -24,7 +25,7 @@ use iggy_common::locking::IggySharedMutFn;
impl IggyShard {
pub async fn delete_segments(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -35,9 +36,14 @@ impl IggyShard {
self.ensure_authenticated(session)?;
{
- let topic = self.find_topic(session, stream_id, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream_id: {stream_id}, topic_id: {topic_id}"))?;
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream_id: {stream_id}"
+ )
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream_id: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.delete_segments(
+ self.permissioner.borrow().delete_segments(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -49,9 +55,11 @@ impl IggyShard {
))?;
}
- let topic = self
- .get_stream_mut(stream_id)?
- .get_topic_mut(topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let topic = stream
+ .get_topic(topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}"
@@ -95,7 +103,25 @@ impl IggyShard {
(segments_count, messages_count)
};
- topic.reassign_consumer_groups().await;
+ drop(partition);
+ drop(topic);
+ drop(stream);
+
+ let mut stream = self
+ .get_stream_mut(stream_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}"
+ )
+ })?;
+ let topic = stream
+ .get_topic_mut(topic_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get mutable reference to topic with ID: {topic_id} in stream with ID: {stream_id}"
+ )
+ })?;
+ topic.reassign_consumer_groups();
self.metrics.decrement_segments(deleted_segments_count);
self.metrics.decrement_messages(deleted_messages_count);
diff --git a/core/server/src/shard/system/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs
index f6e07785..78c5d393 100644
--- a/core/server/src/shard/system/snapshot/mod.rs
+++ b/core/server/src/shard/system/snapshot/mod.rs
@@ -19,22 +19,23 @@
mod procdump;
use crate::configs::system::SystemConfig;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
use async_zip::tokio::write::ZipFileWriter;
use async_zip::{Compression, ZipEntryBuilder};
use iggy_common::{IggyDuration, IggyError, Snapshot, SnapshotCompression, SystemSnapshotType};
+use monoio::fs::{File, OpenOptions};
use std::io::Cursor;
use std::path::PathBuf;
-use std::sync::Arc;
+use std::rc::Rc;
use std::time::Instant;
use tempfile::NamedTempFile;
-use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::Command;
use tokio_util::compat::TokioAsyncWriteCompatExt;
use tracing::{error, info};
-impl System {
+impl IggyShard {
pub async fn get_snapshot(
&self,
session: &Session,
@@ -69,18 +70,19 @@ impl System {
let now = Instant::now();
for snapshot_type in snapshot_types {
- match get_command_result(snapshot_type, self.config.clone()).await {
+ match get_command_result(snapshot_type, self.config.system.clone()).await {
Ok(temp_file) => {
let filename = format!("{snapshot_type}.txt");
let entry = ZipEntryBuilder::new(filename.clone().into(), compression);
- let mut file = File::open(temp_file.path()).await.map_err(|e| {
+ let file = File::open(temp_file.path()).await.map_err(|e| {
error!("Failed to open temporary file: {}", e);
IggyError::SnapshotFileCompletionFailed
})?;
- let mut content = Vec::new();
- if let Err(e) = file.read_to_end(&mut content).await {
+ let content = Vec::new();
+ let (result, content) = file.read_exact_at(content, 0).await;
+ if let Err(e) = result {
error!("Failed to read temporary file: {}", e);
continue;
}
@@ -130,9 +132,10 @@ async fn write_command_output_to_temp_file(
) -> Result<NamedTempFile, std::io::Error> {
let output = command.output().await?;
let temp_file = NamedTempFile::new()?;
- let mut file = File::from_std(temp_file.as_file().try_clone()?);
- file.write_all(&output.stdout).await?;
- file.flush().await?;
+ let file = File::from_std(temp_file.as_file().try_clone()?).unwrap();
+ let (result, _) = file.write_all_at(output.stdout, 0).await;
+ result?;
+ file.sync_all().await?;
Ok(temp_file)
}
@@ -142,18 +145,35 @@ async fn get_filesystem_overview() -> Result<NamedTempFile, std::io::Error> {
async fn get_process_info() -> Result<NamedTempFile, std::io::Error> {
let temp_file = NamedTempFile::new()?;
- let mut file = File::from_std(temp_file.as_file().try_clone()?);
+ let file = File::from_std(temp_file.as_file().try_clone()?).unwrap();
+ let mut position = 0;
let ps_output = Command::new("ps").arg("aux").output().await?;
- file.write_all(b"=== Process List (ps aux) ===\n").await?;
- file.write_all(&ps_output.stdout).await?;
- file.write_all(b"\n\n").await?;
+ let (result, written) = file
+ .write_all_at(b"=== Process List (ps aux) ===\n", 0)
+ .await;
+ result?;
+ position += written.len() as u64;
+
+ let (result, written) = file.write_all_at(ps_output.stdout, position).await;
+ result?;
+ position += written.len() as u64;
+
+ let (result, written) = file.write_all_at(b"\n\n", position).await;
+ result?;
+ position += written.len() as u64;
+
+ let (result, written) = file
+ .write_all_at(b"=== Detailed Process Information ===\n", position)
+ .await;
+ result?;
+ position += written.len() as u64;
- file.write_all(b"=== Detailed Process Information ===\n")
- .await?;
let proc_info = procdump::get_proc_info().await?;
- file.write_all(proc_info.as_bytes()).await?;
- file.flush().await?;
+ let bytes = proc_info.as_bytes().to_owned();
+ let (result, _) = file.write_all_at(bytes, position).await;
+ result?;
+ file.sync_all().await?;
Ok(temp_file)
}
@@ -166,7 +186,7 @@ async fn get_test_snapshot() -> Result<NamedTempFile, std::io::Error> {
write_command_output_to_temp_file(Command::new("echo").arg("test")).await
}
-async fn get_server_logs(config: Arc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
+async fn get_server_logs(config: Rc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
let base_directory = PathBuf::from(config.get_system_path());
let logs_subdirectory = PathBuf::from(&config.logging.path);
let logs_path = base_directory.join(logs_subdirectory);
@@ -179,7 +199,7 @@ async fn get_server_logs(config: Arc<SystemConfig>) -> Result<NamedTempFile, std
write_command_output_to_temp_file(Command::new("sh").args(["-c", &list_and_cat])).await
}
-async fn get_server_config(config: Arc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
+async fn get_server_config(config: Rc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
let base_directory = PathBuf::from(config.get_system_path());
let config_path = base_directory.join("runtime").join("current_config.toml");
@@ -188,7 +208,7 @@ async fn get_server_config(config: Arc<SystemConfig>) -> Result<NamedTempFile, s
async fn get_command_result(
snapshot_type: &SystemSnapshotType,
- config: Arc<SystemConfig>,
+ config: Rc<SystemConfig>,
) -> Result<NamedTempFile, std::io::Error> {
match snapshot_type {
SystemSnapshotType::FilesystemOverview => get_filesystem_overview().await,
diff --git a/core/server/src/shard/system/stats.rs b/core/server/src/shard/system/stats.rs
index 8410cac2..53e31322 100644
--- a/core/server/src/shard/system/stats.rs
+++ b/core/server/src/shard/system/stats.rs
@@ -16,8 +16,8 @@
* under the License.
*/
-use crate::shard::IggyShard;
use crate::VERSION;
+use crate::shard::IggyShard;
use crate::versioning::SemanticVersion;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{IggyDuration, IggyError, Stats};
@@ -45,7 +45,7 @@ impl IggyShard {
let total_cpu_usage = sys.global_cpu_usage();
let total_memory = sys.total_memory().into();
let available_memory = sys.available_memory().into();
- let clients_count = self.client_manager.read().await.get_clients().len() as u32;
+ let clients_count = self.client_manager.borrow().get_clients().len() as u32;
let hostname = sysinfo::System::host_name().unwrap_or("unknown_hostname".to_string());
let os_name = sysinfo::System::name().unwrap_or("unknown_os_name".to_string());
let os_version =
@@ -90,7 +90,7 @@ impl IggyShard {
drop(sys);
- for stream in self.streams.values() {
+ for stream in self.streams.borrow().values() {
stats.messages_count += stream.get_messages_count();
stats.segments_count += stream.get_segments_count();
stats.messages_size_bytes += stream.get_size();
@@ -104,7 +104,7 @@ impl IggyShard {
stats.consumer_groups_count += stream
.topics
.values()
- .map(|t| t.consumer_groups.len() as u32)
+ .map(|t| t.consumer_groups.borrow().len() as u32)
.sum::<u32>();
}
diff --git a/core/server/src/shard/system/storage.rs b/core/server/src/shard/system/storage.rs
index 4cb41120..cc413583 100644
--- a/core/server/src/shard/system/storage.rs
+++ b/core/server/src/shard/system/storage.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use super::COMPONENT;
use crate::shard::system::info::SystemInfo;
use crate::streaming::persistence::persister::PersisterKind;
use crate::streaming::storage::SystemInfoStorage;
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index 8a0772a9..43b45246 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -16,24 +16,37 @@
* under the License.
*/
+use super::COMPONENT;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
use crate::streaming::streams::stream::Stream;
use error_set::ErrContext;
use futures::future::try_join_all;
-use std::cell::RefCell;
+use iggy_common::{IdKind, Identifier, IggyError};
+use std::cell::{Ref, RefCell, RefMut};
use std::sync::atomic::{AtomicU32, Ordering};
use tokio::fs;
use tracing::{error, info, warn};
+static CURRENT_STREAM_ID: AtomicU32 = AtomicU32::new(1);
+
impl IggyShard {
- pub fn get_streams(&self) -> Vec<&Stream> {
- self.streams.values().collect()
+ pub fn get_streams(&self) -> Vec<Ref<'_, Stream>> {
+ let len = self.streams.borrow().len();
+ let result = (0..len)
+ .map(|i| {
+ Ref::map(self.streams.borrow(), |streams| {
+ streams.values().nth(i).unwrap()
+ })
+ })
+ .collect();
+ result
}
- pub fn find_streams(&self, session: &Session) -> Result<Vec<&Stream>, IggyError> {
+ pub fn find_streams(&self, session: &Session) -> Result<Vec<Ref<'_, Stream>>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.get_streams(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -44,15 +57,16 @@ impl IggyShard {
Ok(self.get_streams())
}
- pub fn find_stream(
+ pub fn find_stream<'a>(
&self,
session: &Session,
identifier: &Identifier,
- ) -> Result<&Stream, IggyError> {
+ ) -> Result<Ref<'_, Stream>, IggyError> {
self.ensure_authenticated(session)?;
let stream = self.get_stream(identifier);
if let Ok(stream) = stream {
self.permissioner
+ .borrow()
.get_stream(session.get_user_id(), stream.stream_id)
.with_error_context(|error| {
format!(
@@ -70,13 +84,14 @@ impl IggyShard {
&self,
session: &Session,
identifier: &Identifier,
- ) -> Result<Option<&Stream>, IggyError> {
+ ) -> Result<Option<Ref<'_, Stream>>, IggyError> {
self.ensure_authenticated(session)?;
let Some(stream) = self.try_get_stream(identifier)? else {
return Ok(None);
};
self.permissioner
+ .borrow()
.get_stream(session.get_user_id(), stream.stream_id)
.with_error_context(|error| {
format!(
@@ -87,83 +102,92 @@ impl IggyShard {
Ok(Some(stream))
}
- pub fn try_get_stream(&self, identifier: &Identifier) -> Result<Option<&Stream>, IggyError> {
+ pub fn try_get_stream(
+ &self,
+ identifier: &Identifier,
+ ) -> Result<Option<Ref<'_, Stream>>, IggyError> {
match identifier.kind {
- IdKind::Numeric => Ok(self.streams.get(&identifier.get_u32_value()?)),
+ IdKind::Numeric => self.get_stream_by_id(identifier.get_u32_value()?).map(Some),
IdKind::String => Ok(self.try_get_stream_by_name(&identifier.get_cow_str_value()?)),
}
}
- fn try_get_stream_by_name(&self, name: &str) -> Option<&Stream> {
+ fn try_get_stream_by_name(&self, name: &str) -> Option<Ref<'_, Stream>> {
self.streams_ids
+ .borrow()
.get(name)
- .and_then(|id| self.streams.get(id))
+ .and_then(|id| Some(self.get_stream_ref(*id)))
}
- pub fn get_stream(&self, identifier: &Identifier) -> Result<&Stream, IggyError> {
+ pub fn get_stream(&self, identifier: &Identifier) -> Result<Ref<'_, Stream>, IggyError> {
match identifier.kind {
IdKind::Numeric => self.get_stream_by_id(identifier.get_u32_value()?),
IdKind::String => self.get_stream_by_name(&identifier.get_cow_str_value()?),
}
}
- pub fn get_stream_mut(&mut self, identifier: &Identifier) -> Result<&mut Stream, IggyError> {
+ pub fn get_stream_mut(&self, identifier: &Identifier) -> Result<RefMut<'_, Stream>, IggyError> {
match identifier.kind {
IdKind::Numeric => self.get_stream_by_id_mut(identifier.get_u32_value()?),
IdKind::String => self.get_stream_by_name_mut(&identifier.get_cow_str_value()?),
}
}
- fn get_stream_by_name(&self, name: &str) -> Result<&Stream, IggyError> {
- let stream_id = self.streams_ids.get(name);
- if stream_id.is_none() {
+ fn get_stream_by_name(&self, name: &str) -> Result<Ref<'_, Stream>, IggyError> {
+ let exists = self.streams_ids.borrow().iter().any(|s| s.0 == name);
+ if !exists {
return Err(IggyError::StreamNameNotFound(name.to_string()));
}
-
- self.get_stream_by_id(*stream_id.unwrap())
+ let stream_id = self.streams_ids.borrow().get(name).cloned().unwrap();
+ self.get_stream_by_id(stream_id)
}
- fn get_stream_by_id(&self, stream_id: u32) -> Result<&Stream, IggyError> {
- let stream = self.streams.get(&stream_id);
- if stream.is_none() {
+ fn get_stream_by_id(&self, stream_id: u32) -> Result<Ref<'_, Stream>, IggyError> {
+ let exists = self.streams.borrow().iter().any(|s| s.0 == &stream_id);
+ if !exists {
return Err(IggyError::StreamIdNotFound(stream_id));
}
-
- Ok(stream.unwrap())
+ Ok(self.get_stream_ref(stream_id))
}
- fn get_stream_by_name_mut(&mut self, name: &str) -> Result<&mut Stream, IggyError> {
- let stream_id;
- {
- let id = self.streams_ids.get_mut(name);
- if id.is_none() {
- return Err(IggyError::StreamNameNotFound(name.to_string()));
- }
+ fn get_stream_ref(&self, stream_id: u32) -> Ref<'_, Stream> {
+ Ref::map(self.streams.borrow(), |streams| {
+ streams.get(&stream_id).expect("Stream ID not found")
+ })
+ }
- stream_id = *id.unwrap();
+ fn get_stream_by_name_mut(&self, name: &str) -> Result<RefMut<'_, Stream>, IggyError> {
+ let exists = self.streams_ids.borrow().iter().any(|s| s.0 == name);
+ if !exists {
+ return Err(IggyError::StreamNameAlreadyExists(name.to_owned()));
}
-
- self.get_stream_by_id_mut(stream_id)
+ let streams_ids = self.streams_ids.borrow();
+ let id = streams_ids.get(name).cloned();
+ drop(streams_ids);
+ self.get_stream_by_id_mut(id.unwrap())
}
- fn get_stream_by_id_mut(&mut self, stream_id: u32) -> Result<&mut Stream, IggyError> {
- let stream = self.streams.get_mut(&stream_id);
- if stream.is_none() {
+ fn get_stream_by_id_mut(&self, stream_id: u32) -> Result<RefMut<'_, Stream>, IggyError> {
+ let exists = self.streams.borrow().iter().any(|s| s.0 == &stream_id);
+ if !exists {
return Err(IggyError::StreamIdNotFound(stream_id));
}
-
- Ok(stream.unwrap())
+ Ok(RefMut::map(self.streams.borrow_mut(), |s| {
+ s.get_mut(&stream_id).unwrap()
+ }))
}
pub async fn create_stream(
- &mut self,
+ &self,
session: &Session,
stream_id: Option<u32>,
name: &str,
- ) -> Result<&Stream, IggyError> {
+ ) -> Result<u32, IggyError> {
self.ensure_authenticated(session)?;
- self.permissioner.create_stream(session.get_user_id())?;
- if self.streams_ids.contains_key(name) {
+ self.permissioner
+ .borrow()
+ .create_stream(session.get_user_id())?;
+ if self.streams_ids.borrow().contains_key(name) {
return Err(IggyError::StreamNameAlreadyExists(name.to_owned()));
}
@@ -171,7 +195,7 @@ impl IggyShard {
if stream_id.is_none() {
id = CURRENT_STREAM_ID.fetch_add(1, Ordering::SeqCst);
loop {
- if self.streams.contains_key(&id) {
+ if self.streams.borrow().contains_key(&id) {
if id == u32::MAX {
return Err(IggyError::StreamIdAlreadyExists(id));
}
@@ -184,21 +208,23 @@ impl IggyShard {
id = stream_id.unwrap();
}
- if self.streams.contains_key(&id) {
+ if self.streams.borrow().contains_key(&id) {
return Err(IggyError::StreamIdAlreadyExists(id));
}
- let stream = Stream::create(id, name, self.config.clone(), self.storage.clone());
+ let stream = Stream::create(id, name, self.config.system.clone(), self.storage.clone());
stream.persist().await?;
info!("Created stream with ID: {id}, name: '{name}'.");
- self.streams_ids.insert(name.to_owned(), stream.stream_id);
- self.streams.insert(stream.stream_id, stream);
+ self.streams_ids
+ .borrow_mut()
+ .insert(name.to_owned(), stream.stream_id);
+ self.streams.borrow_mut().insert(stream.stream_id, stream);
self.metrics.increment_streams(1);
- self.get_stream_by_id(id)
+ Ok(id)
}
pub async fn update_stream(
- &mut self,
+ &self,
session: &Session,
id: &Identifier,
name: &str,
@@ -213,6 +239,7 @@ impl IggyShard {
}
self.permissioner
+ .borrow()
.update_stream(session.get_user_id(), stream_id)
.with_error_context(|error| {
format!(
@@ -223,7 +250,7 @@ impl IggyShard {
})?;
{
- if let Some(stream_id_by_name) = self.streams_ids.get(name) {
+ if let Some(stream_id_by_name) = self.streams_ids.borrow().get(name) {
if *stream_id_by_name != stream_id {
return Err(IggyError::StreamNameAlreadyExists(name.to_owned()));
}
@@ -232,7 +259,7 @@ impl IggyShard {
let old_name;
{
- let stream = self.get_stream_mut(id).with_error_context(|error| {
+ let mut stream = self.get_stream_mut(id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {id}")
})?;
old_name = stream.name.clone();
@@ -241,8 +268,10 @@ impl IggyShard {
}
{
- self.streams_ids.remove(&old_name);
- self.streams_ids.insert(name.to_owned(), stream_id);
+ self.streams_ids.borrow_mut().remove(&old_name);
+ self.streams_ids
+ .borrow_mut()
+ .insert(name.to_owned(), stream_id);
}
info!("Stream with ID '{id}' updated. Old name: '{old_name}' changed to: '{name}'.");
@@ -250,7 +279,7 @@ impl IggyShard {
}
pub async fn delete_stream(
- &mut self,
+ &self,
session: &Session,
id: &Identifier,
) -> Result<u32, IggyError> {
@@ -260,6 +289,7 @@ impl IggyShard {
})?;
let stream_id = stream.stream_id;
self.permissioner
+ .borrow()
.delete_stream(session.get_user_id(), stream_id)
.with_error_context(|error| {
format!(
@@ -279,17 +309,16 @@ impl IggyShard {
.decrement_partitions(stream.get_partitions_count());
self.metrics.decrement_messages(stream.get_messages_count());
self.metrics.decrement_segments(stream.get_segments_count());
- self.streams.remove(&stream_id);
- self.streams_ids.remove(&stream_name);
+ self.streams.borrow_mut().remove(&stream_id);
+ self.streams_ids.borrow_mut().remove(&stream_name);
let current_stream_id = CURRENT_STREAM_ID.load(Ordering::SeqCst);
if current_stream_id > stream_id {
CURRENT_STREAM_ID.store(stream_id, Ordering::SeqCst);
}
- let client_manager = self.client_manager.read().await;
- client_manager
- .delete_consumer_groups_for_stream(stream_id)
- .await;
+ self.client_manager
+ .borrow_mut()
+ .delete_consumer_groups_for_stream(stream_id);
Ok(stream_id)
}
@@ -302,6 +331,7 @@ impl IggyShard {
format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
})?;
self.permissioner
+ .borrow()
.purge_stream(session.get_user_id(), stream.stream_id)
.with_error_context(|error| {
format!(
@@ -329,10 +359,12 @@ mod tests {
sync::Arc,
};
+ //TODO: Fixme
+ /*
#[tokio::test]
async fn should_get_stream_by_id_and_name() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
@@ -378,4 +410,5 @@ mod tests {
assert_eq!(stream.stream_id, stream_id);
assert_eq!(stream.name, stream_name);
}
+ */
}
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index 145689af..d118090f 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -16,29 +16,31 @@
* under the License.
*/
+use super::COMPONENT;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
+use crate::streaming::streams::stream::Stream;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize};
+use tokio_util::io::StreamReader;
impl IggyShard {
- pub fn find_topic(
+ pub fn find_topic<'topic, 'stream>(
&self,
session: &Session,
- stream_id: &Identifier,
+ stream: &'stream Stream,
topic_id: &Identifier,
- ) -> Result<&Topic, IggyError> {
+ ) -> Result<&'topic Topic, IggyError>
+ where
+ 'stream: 'topic,
+ {
self.ensure_authenticated(session)?;
- let stream = self
- .find_stream(session, stream_id)
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to find stream with ID: {stream_id}")
- })?;
- let topic = stream.get_topic(topic_id);
- if let Ok(topic) = topic {
- self.permissioner
+ let stream_id = stream.stream_id;
+ let topic = stream.get_topic(topic_id)?;
+ self.permissioner
+ .borrow()
.get_topic(session.get_user_id(), stream.stream_id, topic.topic_id)
.with_error_context(|error| {
format!(
@@ -46,22 +48,21 @@ impl IggyShard {
session.get_user_id(),
)
})?;
- return Ok(topic);
- }
-
- topic
+ Ok(topic)
}
- pub fn find_topics(
+ pub fn find_topics<'stream, 'topic>(
&self,
session: &Session,
- stream_id: &Identifier,
- ) -> Result<Vec<&Topic>, IggyError> {
+ stream: &'stream Stream,
+ ) -> Result<Vec<&'topic Topic>, IggyError>
+ where
+ 'stream: 'topic,
+ {
self.ensure_authenticated(session)?;
- let stream = self.get_stream(stream_id).with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
- })?;
+ let stream_id = stream.stream_id;
self.permissioner
+ .borrow()
.get_topics(session.get_user_id(), stream.stream_id)
.with_error_context(|error| {
format!(
@@ -72,28 +73,24 @@ impl IggyShard {
Ok(stream.get_topics())
}
- pub fn try_find_topic(
+ pub fn try_find_topic<'stream, 'topic>(
&self,
session: &Session,
- stream_id: &Identifier,
+ stream: &'stream Stream,
topic_id: &Identifier,
- ) -> Result<Option<&Topic>, IggyError> {
+ ) -> Result<Option<&'topic Topic>, IggyError>
+ where
+ 'stream: 'topic,
+ {
self.ensure_authenticated(session)?;
- let Some(stream) = self
- .try_find_stream(session, stream_id)
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to find stream with ID: {stream_id}")
- })?
- else {
- return Ok(None);
- };
-
+ let stream_id = stream.stream_id;
let Some(topic) = stream.try_get_topic(topic_id)? else {
return Ok(None);
};
self.permissioner
- .get_topic(session.get_user_id(), stream.stream_id, topic.topic_id)
+ .borrow()
+ .get_topic(session.get_user_id(), stream_id, topic.topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - permission denied to get topic with ID: {topic_id} in stream with ID: {stream_id} for user with ID: {}",
@@ -105,7 +102,7 @@ impl IggyShard {
#[allow(clippy::too_many_arguments)]
pub async fn create_topic(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: Option<u32>,
@@ -115,13 +112,14 @@ impl IggyShard {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
- ) -> Result<&Topic, IggyError> {
+ ) -> Result<u32, IggyError> {
self.ensure_authenticated(session)?;
{
let stream = self.get_stream(stream_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
})?;
self.permissioner
+ .borrow()
.create_topic(session.get_user_id(), stream.stream_id)
.with_error_context(|error| {
format!(
@@ -131,6 +129,8 @@ impl IggyShard {
})?;
}
+ // TODO: Make create topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
let created_topic_id = self
.get_stream_mut(stream_id)?
.create_topic(
@@ -151,21 +151,12 @@ impl IggyShard {
self.metrics.increment_partitions(partitions_count);
self.metrics.increment_segments(partitions_count);
- self.get_stream(stream_id)
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
- })?
- .get_topic(&created_topic_id.try_into()?)
- .with_error_context(|error| {
- format!(
- "{COMPONENT} (error: {error}) - failed to get created topic with ID: {created_topic_id} in stream with ID: {stream_id}",
- )
- })
+ Ok(created_topic_id)
}
#[allow(clippy::too_many_arguments)]
pub async fn update_topic(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -174,17 +165,20 @@ impl IggyShard {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
- ) -> Result<&Topic, IggyError> {
+ ) -> Result<u32, IggyError> {
self.ensure_authenticated(session)?;
{
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
let topic = self
- .find_topic(session, stream_id, topic_id)
+ .find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to find topic with ID: {topic_id}"
)
})?;
- self.permissioner.update_topic(
+ self.permissioner.borrow().update_topic(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -198,6 +192,8 @@ impl IggyShard {
})?;
}
+ // TODO: Make update topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
self.get_stream_mut(stream_id)?
.update_topic(
topic_id,
@@ -224,11 +220,11 @@ impl IggyShard {
.get_topic(topic_id)
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get topic with ID: {topic_id} in stream with ID: {stream_id}")
- })
+ }).map(|topic| topic.topic_id)
}
pub async fn delete_topic(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -236,12 +232,15 @@ impl IggyShard {
self.ensure_authenticated(session)?;
let stream_id_value;
{
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
let topic = self
- .find_topic(session, stream_id, topic_id)
+ .find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to find topic with ID: {topic_id} in stream with ID: {stream_id}")
})?;
- self.permissioner.delete_topic(
+ self.permissioner.borrow().delete_topic(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -254,6 +253,8 @@ impl IggyShard {
stream_id_value = topic.stream_id;
}
+ // TODO: Make delete topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
let topic = self
.get_stream_mut(stream_id)?
.delete_topic(topic_id)
@@ -266,10 +267,9 @@ impl IggyShard {
self.metrics.decrement_messages(topic.get_messages_count());
self.metrics
.decrement_segments(topic.get_segments_count().await);
- let client_manager = self.client_manager.read().await;
- client_manager
- .delete_consumer_groups_for_topic(stream_id_value, topic.topic_id)
- .await;
+ self.client_manager
+ .borrow_mut()
+ .delete_consumer_groups_for_topic(stream_id_value, topic.topic_id);
Ok(())
}
@@ -279,12 +279,16 @@ impl IggyShard {
stream_id: &Identifier,
topic_id: &Identifier,
) -> Result<(), IggyError> {
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
let topic = self
- .find_topic(session, stream_id, topic_id)
+ .find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to find topic with ID: {topic_id} in stream with ID: {stream_id}")
})?;
self.permissioner
+ .borrow()
.purge_topic(session.get_user_id(), topic.stream_id, topic.topic_id)
.with_error_context(|error| {
format!(
diff --git a/core/server/src/shard/system/users.rs b/core/server/src/shard/system/users.rs
index f2405f1f..f3b271a7 100644
--- a/core/server/src/shard/system/users.rs
+++ b/core/server/src/shard/system/users.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use super::COMPONENT;
use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::state::models::CreateUserWithId;
@@ -33,6 +34,7 @@ use iggy_common::create_user::CreateUser;
use iggy_common::defaults::*;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{IdKind, Identifier};
+use std::cell::RefMut;
use std::env;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
@@ -46,7 +48,7 @@ impl IggyShard {
&self,
session: &Session,
user_id: &Identifier,
- ) -> Result<Option<&User>, IggyError> {
+ ) -> Result<Option<User>, IggyError> {
self.ensure_authenticated(session)?;
let Some(user) = self.try_get_user(user_id)? else {
return Ok(None);
@@ -54,7 +56,7 @@ impl IggyShard {
let session_user_id = session.get_user_id();
if user.id != session_user_id {
- self.permissioner.get_user(session_user_id).with_error_context(|error| {
+ self.permissioner.borrow().get_user(session_user_id).with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - permission denied to get user with ID: {user_id} for current user with ID: {session_user_id}"
)
@@ -64,45 +66,73 @@ impl IggyShard {
Ok(Some(user))
}
- pub fn get_user(&self, user_id: &Identifier) -> Result<&User, IggyError> {
+ pub fn get_user(&self, user_id: &Identifier) -> Result<User, IggyError> {
self.try_get_user(user_id)?
.ok_or(IggyError::ResourceNotFound(user_id.to_string()))
}
- pub fn try_get_user(&self, user_id: &Identifier) -> Result<Option<&User>, IggyError> {
+ pub fn try_get_user(&self, user_id: &Identifier) -> Result<Option<User>, IggyError> {
match user_id.kind {
- IdKind::Numeric => Ok(self.users.get(&user_id.get_u32_value()?)),
+ IdKind::Numeric => {
+ let user = self
+ .users
+ .borrow()
+ .get(&user_id.get_u32_value()?)
+ .map(|user| user.clone());
+ Ok(user)
+ }
IdKind::String => {
let username = user_id.get_cow_str_value()?;
- Ok(self
+ let user = self
.users
+ .borrow()
.iter()
.find(|(_, user)| user.username == username)
- .map(|(_, user)| user))
+ .map(|(_, user)| user.clone());
+ Ok(user)
}
}
}
- pub fn get_user_mut(&mut self, user_id: &Identifier) -> Result<&mut User, IggyError> {
+ pub fn get_user_mut(&self, user_id: &Identifier) -> Result<RefMut<'_, User>, IggyError> {
match user_id.kind {
- IdKind::Numeric => self
- .users
- .get_mut(&user_id.get_u32_value()?)
- .ok_or(IggyError::ResourceNotFound(user_id.to_string())),
+ IdKind::Numeric => {
+ let user_id = user_id.get_u32_value()?;
+ let users = self.users.borrow_mut();
+ let exists = users.contains_key(&user_id);
+ if !exists {
+ return Err(IggyError::ResourceNotFound(user_id.to_string()));
+ }
+ Ok(RefMut::map(users, |u| {
+ let user = u.get_mut(&user_id);
+ user.unwrap()
+ }))
+ }
IdKind::String => {
let username = user_id.get_cow_str_value()?;
- self.users
- .iter_mut()
+ let users = self.users.borrow_mut();
+ let exists = users
+ .iter()
.find(|(_, user)| user.username == username)
- .map(|(_, user)| user)
- .ok_or(IggyError::ResourceNotFound(user_id.to_string()))
+ .is_some();
+ if !exists {
+ return Err(IggyError::ResourceNotFound(user_id.to_string()));
+ }
+ Ok(RefMut::map(users, |u| {
+ let user = u
+ .iter_mut()
+ .find(|(_, user)| user.username == username)
+ .map(|(_, user)| user);
+ user.unwrap()
+ }))
}
}
}
- pub async fn get_users(&self, session: &Session) -> Result<Vec<&User>, IggyError> {
+ pub async fn get_users(&self, session: &Session) -> Result<Vec<User>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.get_users(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -110,19 +140,20 @@ impl IggyShard {
session.get_user_id()
)
})?;
- Ok(self.users.values().collect())
+ Ok(self.users.borrow().values().cloned().collect())
}
pub async fn create_user(
- &mut self,
+ &self,
session: &Session,
username: &str,
password: &str,
status: UserStatus,
permissions: Option<Permissions>,
- ) -> Result<&User, IggyError> {
+ ) -> Result<User, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.create_user(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -131,12 +162,17 @@ impl IggyShard {
)
})?;
- if self.users.iter().any(|(_, user)| user.username == username) {
+ if self
+ .users
+ .borrow()
+ .iter()
+ .any(|(_, user)| user.username == username)
+ {
error!("User: {username} already exists.");
return Err(IggyError::UserAlreadyExists);
}
- if self.users.len() >= MAX_USERS {
+ if self.users.borrow().len() >= MAX_USERS {
error!("Available users limit reached.");
return Err(IggyError::UsersLimitReached);
}
@@ -145,8 +181,9 @@ impl IggyShard {
info!("Creating user: {username} with ID: {user_id}...");
let user = User::new(user_id, username, password, status, permissions.clone());
self.permissioner
+ .borrow_mut()
.init_permissions_for_user(user_id, permissions);
- self.users.insert(user.id, user);
+ self.users.borrow_mut().insert(user.id, user);
info!("Created user: {username} with ID: {user_id}.");
self.metrics.increment_users(1);
self.get_user(&user_id.try_into()?)
@@ -155,16 +192,13 @@ impl IggyShard {
})
}
- pub async fn delete_user(
- &mut self,
- session: &Session,
- user_id: &Identifier,
- ) -> Result<User, IggyError> {
+ pub fn delete_user(&self, session: &Session, user_id: &Identifier) -> Result<User, IggyError> {
self.ensure_authenticated(session)?;
let existing_user_id;
let existing_username;
{
self.permissioner
+ .borrow()
.delete_user(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -187,14 +221,14 @@ impl IggyShard {
info!("Deleting user: {existing_username} with ID: {user_id}...");
let user = self
.users
+ .borrow_mut()
.remove(&existing_user_id)
.ok_or(IggyError::ResourceNotFound(user_id.to_string()))?;
self.permissioner
+ .borrow_mut()
.delete_permissions_for_user(existing_user_id);
- let mut client_manager = self.client_manager.write().await;
- client_manager
+ self.client_manager.borrow_mut()
.delete_clients_for_user(existing_user_id)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to delete clients for user with ID: {existing_user_id}"
@@ -205,15 +239,16 @@ impl IggyShard {
Ok(user)
}
- pub async fn update_user(
- &mut self,
+ pub fn update_user(
+ &self,
session: &Session,
user_id: &Identifier,
username: Option<String>,
status: Option<UserStatus>,
- ) -> Result<&User, IggyError> {
+ ) -> Result<User, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.update_user(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -231,7 +266,7 @@ impl IggyShard {
}
}
- let user = self.get_user_mut(user_id).with_error_context(|error| {
+ let mut user = self.get_user_mut(user_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get mutable reference to the user with id: {user_id}")
})?;
if let Some(username) = username {
@@ -241,13 +276,18 @@ impl IggyShard {
if let Some(status) = status {
user.status = status;
}
+ let cloned_user = user.clone();
+ drop(user);
- info!("Updated user: {} with ID: {}.", user.username, user.id);
- Ok(user)
+ info!(
+ "Updated user: {} with ID: {}.",
+ cloned_user.username, cloned_user.id
+ );
+ Ok(cloned_user)
}
- pub async fn update_permissions(
- &mut self,
+ pub fn update_permissions(
+ &self,
session: &Session,
user_id: &Identifier,
permissions: Option<Permissions>,
@@ -256,13 +296,14 @@ impl IggyShard {
{
self.permissioner
+ .borrow()
.update_permissions(session.get_user_id())
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - permission denied to update permissions for user with id: {}", session.get_user_id()
)
})?;
- let user = self.get_user(user_id).with_error_context(|error| {
+ let user: User = self.get_user(user_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get user with id: {user_id}")
})?;
if user.is_root() {
@@ -271,11 +312,12 @@ impl IggyShard {
}
self.permissioner
+ .borrow_mut()
.update_permissions_for_user(user.id, permissions.clone());
}
{
- let user = self.get_user_mut(user_id).with_error_context(|error| {
+ let mut user = self.get_user_mut(user_id).with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to the user with id: {user_id}"
)
@@ -290,8 +332,8 @@ impl IggyShard {
Ok(())
}
- pub async fn change_password(
- &mut self,
+ pub fn change_password(
+ &self,
session: &Session,
user_id: &Identifier,
current_password: &str,
@@ -305,11 +347,13 @@ impl IggyShard {
})?;
let session_user_id = session.get_user_id();
if user.id != session_user_id {
- self.permissioner.change_password(session_user_id)?;
+ self.permissioner
+ .borrow()
+ .change_password(session_user_id)?;
}
}
- let user = self.get_user_mut(user_id).with_error_context(|error| {
+ let mut user = self.get_user_mut(user_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get mutable reference to the user with id: {user_id}")
})?;
if !crypto::verify_password(current_password, &user.password) {
@@ -328,22 +372,21 @@ impl IggyShard {
Ok(())
}
- pub async fn login_user(
+ pub fn login_user(
&self,
username: &str,
password: &str,
session: Option<&Session>,
- ) -> Result<&User, IggyError> {
+ ) -> Result<User, IggyError> {
self.login_user_with_credentials(username, Some(password), session)
- .await
}
- pub async fn login_user_with_credentials(
+ pub fn login_user_with_credentials(
&self,
username: &str,
password: Option<&str>,
session: Option<&Session>,
- ) -> Result<&User, IggyError> {
+ ) -> Result<User, IggyError> {
let user = match self.get_user(&username.try_into()?) {
Ok(user) => user,
Err(_) => {
@@ -380,14 +423,13 @@ impl IggyShard {
user.username,
session.get_user_id()
);
- self.logout_user(session).await?;
+ self.logout_user(session)?;
}
session.set_user_id(user.id);
- let mut client_manager = self.client_manager.write().await;
+ let mut client_manager = self.client_manager.borrow_mut();
client_manager
.set_user_id(session.client_id, user.id)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to set user_id to client, client ID: {}, user ID: {}",
@@ -397,7 +439,7 @@ impl IggyShard {
Ok(user)
}
- pub async fn logout_user(&self, session: &Session) -> Result<(), IggyError> {
+ pub fn logout_user(&self, session: &Session) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
let user = self
.get_user(&Identifier::numeric(session.get_user_id())?)
@@ -412,8 +454,8 @@ impl IggyShard {
user.username, user.id
);
if session.client_id > 0 {
- let mut client_manager = self.client_manager.write().await;
- client_manager.clear_user_id(session.client_id).await?;
+ let mut client_manager = self.client_manager.borrow_mut();
+ client_manager.clear_user_id(session.client_id)?;
info!(
"Cleared user ID: {} for client: {}.",
user.id, session.client_id
diff --git a/core/server/src/streaming/partitions/consumer_offsets.rs b/core/server/src/streaming/partitions/consumer_offsets.rs
index 96080c03..e9f5c57d 100644
--- a/core/server/src/streaming/partitions/consumer_offsets.rs
+++ b/core/server/src/streaming/partitions/consumer_offsets.rs
@@ -26,10 +26,7 @@ use iggy_common::IggyError;
use tracing::trace;
impl Partition {
- pub fn get_consumer_offset(
- &self,
- consumer: PollingConsumer,
- ) -> Result<Option<u64>, IggyError> {
+ pub fn get_consumer_offset(&self, consumer: PollingConsumer) -> Result<Option<u64>, IggyError> {
trace!(
"Getting consumer offset for {}, partition: {}, current: {}...",
consumer, self.partition_id, self.current_offset
diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs
index 978b97aa..eaf96dc3 100644
--- a/core/server/src/streaming/storage.rs
+++ b/core/server/src/streaming/storage.rs
@@ -87,7 +87,7 @@ pub enum PartitionStorageKind {
#[cfg_attr(test, automock)]
pub trait SystemInfoStorage {
fn load(&self) -> impl Future<Output = Result<SystemInfo, IggyError>>;
- fn save(&self, system_info: &SystemInfo) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, system_info: &SystemInfo) -> impl Future<Output = Result<(), IggyError>>;
}
#[cfg_attr(test, automock)]
@@ -96,8 +96,8 @@ pub trait StreamStorage {
&self,
stream: &mut Stream,
state: StreamState,
- ) -> impl Future<Output = Result<(), IggyError>>;
- fn save(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>>;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>>;
fn delete(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>>;
}
@@ -107,9 +107,9 @@ pub trait TopicStorage {
&self,
topic: &mut Topic,
state: TopicState,
- ) -> impl Future<Output = Result<(), IggyError>>;
- fn save(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
- fn delete(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
}
#[cfg_attr(test, automock)]
@@ -119,9 +119,8 @@ pub trait PartitionStorage {
partition: &mut Partition,
state: PartitionState,
) -> impl Future<Output = Result<(), IggyError>>;
- fn save(&self, partition: &mut Partition)
- -> impl Future<Output = Result<(), IggyError>>;
- fn delete(&self, partition: &Partition) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, partition: &mut Partition) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, partition: &Partition) -> impl Future<Output = Result<(), IggyError>>;
fn save_consumer_offset(
&self,
offset: u64,
@@ -131,15 +130,9 @@ pub trait PartitionStorage {
&self,
kind: ConsumerKind,
path: &str,
- ) -> impl Future<Output = Result<Vec<ConsumerOffset>, IggyError>>;
- fn delete_consumer_offsets(
- &self,
- path: &str,
- ) -> impl Future<Output = Result<(), IggyError>>;
- fn delete_consumer_offset(
- &self,
- path: &str,
- ) -> impl Future<Output = Result<(), IggyError>>;
+ ) -> impl Future<Output = Result<Vec<ConsumerOffset>, IggyError>>;
+ fn delete_consumer_offsets(&self, path: &str) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete_consumer_offset(&self, path: &str) -> impl Future<Output = Result<(), IggyError>>;
}
#[derive(Debug)]
diff --git a/core/server/src/streaming/topics/consumer_group.rs b/core/server/src/streaming/topics/consumer_group.rs
index f000ce79..c810159c 100644
--- a/core/server/src/streaming/topics/consumer_group.rs
+++ b/core/server/src/streaming/topics/consumer_group.rs
@@ -57,10 +57,7 @@ impl ConsumerGroup {
self.assign_partitions();
}
- pub fn calculate_partition_id(
- &mut self,
- member_id: u32,
- ) -> Result<Option<u32>, IggyError> {
+ pub fn calculate_partition_id(&mut self, member_id: u32) -> Result<Option<u32>, IggyError> {
let member = self.members.get_mut(&member_id);
if let Some(member) = member {
return Ok(member.calculate_partition_id());
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index c1371096..92168738 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -17,12 +17,12 @@
*/
use crate::binary::handlers::topics::get_topic_handler;
-use crate::streaming::topics::{consumer_group, COMPONENT};
use crate::streaming::topics::consumer_group::ConsumerGroup;
use crate::streaming::topics::topic::Topic;
+use crate::streaming::topics::{COMPONENT, consumer_group};
use error_set::ErrContext;
-use iggy_common::locking::IggySharedMutFn;
use iggy_common::IggyError;
+use iggy_common::locking::IggySharedMutFn;
use iggy_common::{IdKind, Identifier};
use std::cell::{Ref, RefMut};
use std::sync::atomic::Ordering;
@@ -48,7 +48,10 @@ impl Topic {
self.consumer_groups.borrow().values().cloned().collect()
}
- pub fn get_consumer_group(&self, identifier: &Identifier) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
+ pub fn get_consumer_group(
+ &self,
+ identifier: &Identifier,
+ ) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
match identifier.kind {
IdKind::Numeric => self.get_consumer_group_by_id(identifier.get_u32_value().unwrap()),
IdKind::String => self.get_consumer_group_by_name(&identifier.get_cow_str_value()?),
@@ -93,7 +96,10 @@ impl Topic {
}))
}
- pub fn get_consumer_group_by_name(&self, name: &str) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
+ pub fn get_consumer_group_by_name(
+ &self,
+ name: &str,
+ ) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
let group_id = self.consumer_groups_ids.get(name);
if group_id.is_none() {
return Err(IggyError::ConsumerGroupNameNotFound(
@@ -199,12 +205,8 @@ impl Topic {
return Err(IggyError::ConsumerGroupIdAlreadyExists(id, self.topic_id));
}
- let consumer_group = ConsumerGroup::new(
- self.topic_id,
- id,
- name,
- self.partitions.len() as u32,
- );
+ let consumer_group =
+ ConsumerGroup::new(self.topic_id, id, name, self.partitions.len() as u32);
self.consumer_groups_ids.insert(name.to_owned(), id);
let cloned_group = consumer_group.clone();
self.consumer_groups.borrow_mut().insert(id, consumer_group);
diff --git a/core/server/src/streaming/topics/topic.rs b/core/server/src/streaming/topics/topic.rs
index 16aa324c..7e8ca4ac 100644
--- a/core/server/src/streaming/topics/topic.rs
+++ b/core/server/src/streaming/topics/topic.rs
@@ -23,13 +23,13 @@ use crate::streaming::storage::SystemStorage;
use crate::streaming::topics::consumer_group::ConsumerGroup;
use ahash::AHashMap;
use core::fmt;
-use std::cell::RefCell;
-use std::rc::Rc;
use iggy_common::locking::IggyRwLock;
use iggy_common::{
CompressionAlgorithm, Consumer, ConsumerKind, IggyByteSize, IggyError, IggyExpiry,
IggyTimestamp, MaxTopicSize, Sizeable,
};
+use std::cell::RefCell;
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
diff --git a/core/server/src/streaming/users/user.rs b/core/server/src/streaming/users/user.rs
index 01a9faa0..7e68d98f 100644
--- a/core/server/src/streaming/users/user.rs
+++ b/core/server/src/streaming/users/user.rs
@@ -24,7 +24,7 @@ use iggy_common::defaults::*;
use iggy_common::{Permissions, UserId};
use std::sync::Arc;
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct User {
pub id: UserId,
pub status: UserStatus,
(iggy) 13/30: finish handlers
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 81c29b1b7276d684a88a5d56157a33fb8602df5f
Author: numinex <g....@gmail.com>
AuthorDate: Fri Jun 27 12:26:28 2025 +0200
finish handlers
---
Cargo.lock | 16 ---------
.../create_consumer_group_handler.rs | 28 +++++++++++-----
.../delete_consumer_group_handler.rs | 13 ++++----
.../consumer_groups/get_consumer_group_handler.rs | 23 +++++++++----
.../consumer_groups/get_consumer_groups_handler.rs | 12 +++----
.../consumer_groups/join_consumer_group_handler.rs | 11 +++----
.../leave_consumer_group_handler.rs | 13 ++++----
.../delete_consumer_offset_handler.rs | 10 +++---
.../get_consumer_offset_handler.rs | 10 +++---
.../store_consumer_offset_handler.rs | 11 ++++---
.../messages/flush_unsaved_buffer_handler.rs | 10 +++---
.../handlers/messages/poll_messages_handler.rs | 23 +++++++------
.../handlers/messages/send_messages_handler.rs | 38 +++++++++++++---------
.../partitions/create_partitions_handler.rs | 13 ++++----
.../partitions/delete_partitions_handler.rs | 13 ++++----
.../create_personal_access_token_handler.rs | 14 ++++----
.../delete_personal_access_token_handler.rs | 14 ++++----
.../get_personal_access_tokens_handler.rs | 13 ++++----
.../login_with_personal_access_token_handler.rs | 11 +++----
.../handlers/segments/delete_segments_handler.rs | 13 ++++----
.../handlers/streams/create_stream_handler.rs | 21 +++++++-----
.../handlers/streams/delete_stream_handler.rs | 13 ++++----
.../binary/handlers/streams/get_stream_handler.rs | 12 +++----
.../binary/handlers/streams/get_streams_handler.rs | 12 +++----
.../handlers/streams/purge_stream_handler.rs | 12 +++----
.../handlers/streams/update_stream_handler.rs | 13 ++++----
.../binary/handlers/system/get_client_handler.rs | 12 +++----
.../binary/handlers/system/get_clients_handler.rs | 13 ++++----
.../src/binary/handlers/system/get_me_handler.rs | 13 +++-----
.../src/binary/handlers/system/get_snapshot.rs | 10 +++---
.../binary/handlers/system/get_stats_handler.rs | 10 +++---
.../src/binary/handlers/system/ping_handler.rs | 17 +++++-----
.../binary/handlers/topics/create_topic_handler.rs | 29 ++++++++++++-----
.../binary/handlers/topics/delete_topic_handler.rs | 13 ++++----
.../binary/handlers/topics/get_topic_handler.rs | 19 ++++++++---
.../binary/handlers/topics/get_topics_handler.rs | 21 ++++++++----
.../binary/handlers/topics/purge_topic_handler.rs | 12 +++----
.../binary/handlers/topics/update_topic_handler.rs | 26 ++++++++++-----
.../handlers/users/change_password_handler.rs | 16 ++++-----
.../binary/handlers/users/create_user_handler.rs | 16 ++++-----
.../binary/handlers/users/delete_user_handler.rs | 17 +++++-----
.../src/binary/handlers/users/get_user_handler.rs | 13 ++++----
.../src/binary/handlers/users/get_users_handler.rs | 20 +++++-------
.../binary/handlers/users/login_user_handler.rs | 14 ++++----
.../binary/handlers/users/logout_user_handler.rs | 20 +++++-------
.../handlers/users/update_permissions_handler.rs | 15 ++++-----
.../binary/handlers/users/update_user_handler.rs | 15 ++++-----
core/server/src/binary/mapper.rs | 22 +++++++------
core/server/src/binary/sender.rs | 23 ++++++++-----
core/server/src/lib.rs | 9 +++++
core/server/src/quic/quic_sender.rs | 3 +-
core/server/src/shard/system/clients.rs | 4 +--
core/server/src/shard/system/consumer_groups.rs | 10 ++----
core/server/src/shard/system/messages.rs | 4 ---
.../src/shard/system/personal_access_tokens.rs | 8 ++---
core/server/src/shard/system/snapshot/mod.rs | 5 +++
core/server/src/shard/system/streams.rs | 6 ++--
core/server/src/shard/system/topics.rs | 15 +++------
.../server/src/streaming/clients/client_manager.rs | 8 +++--
.../server/src/streaming/topics/consumer_groups.rs | 5 ++-
core/server/src/streaming/utils/pooled_buffer.rs | 15 +++++++++
core/server/src/tcp/sender.rs | 7 ++--
core/server/src/tcp/tcp_sender.rs | 2 +-
core/server/src/tcp/tcp_tls_sender.rs | 3 +-
64 files changed, 467 insertions(+), 415 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 3e51b287..19be4ecb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -143,7 +143,6 @@ dependencies = [
"futures-core",
"futures-util",
"mio 1.0.4",
- "mio 1.0.4",
"socket2",
"tokio",
"tracing",
@@ -653,7 +652,6 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
- "syn 2.0.104",
]
[[package]]
@@ -4806,18 +4804,6 @@ dependencies = [
"windows-sys 0.48.0",
]
-[[package]]
-name = "mio"
-version = "0.8.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
-dependencies = [
- "libc",
- "log",
- "wasi 0.11.1+wasi-snapshot-preview1",
- "windows-sys 0.48.0",
-]
-
[[package]]
name = "mio"
version = "1.0.4"
@@ -4916,7 +4902,6 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
- "syn 2.0.104",
]
[[package]]
@@ -7643,7 +7628,6 @@ dependencies = [
"bytes",
"libc",
"mio 1.0.4",
- "mio 1.0.4",
"parking_lot 0.12.4",
"pin-project-lite",
"signal-hook-registry",
diff --git a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
index 5d695687..266b46a6 100644
--- a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
@@ -16,8 +16,6 @@
* under the License.
*/
-use std::rc::Rc;
-
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
@@ -30,6 +28,7 @@ use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::create_consumer_group::CreateConsumerGroup;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for CreateConsumerGroup {
@@ -47,7 +46,7 @@ impl ServerCommandHandler for CreateConsumerGroup {
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let consumer_group = shard
+ let consumer_group_id = shard
.create_consumer_group(
session,
&self.stream_id,
@@ -55,23 +54,34 @@ impl ServerCommandHandler for CreateConsumerGroup {
self.group_id,
&self.name,
)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to create consumer group for stream_id: {}, topic_id: {}, group_id: {:?}, session: {session}",
self.stream_id, self.topic_id, self.group_id
)
})?;
- let consumer_group = consumer_group.read().await;
+
+ let stream = shard.find_stream(session, &self.stream_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to find stream with ID: {} for session: {}",
+ self.stream_id, session
+ )
+ })?;
+ let consumer_group = shard.get_consumer_group(session, &stream, &self.topic_id, &consumer_group_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get consumer group for stream_id: {}, topic_id: {}, group_id: {:?}, session: {session}",
+ self.stream_id, self.topic_id, consumer_group_id
+ )
+ })?.unwrap();
let group_id = consumer_group.group_id;
- let response = mapper::map_consumer_group(&consumer_group).await;
- drop(consumer_group);
+ let response = mapper::map_consumer_group(&consumer_group);
- let system = system.downgrade();
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
- system
+ shard
.state
.apply(
session.get_user_id(),
diff --git a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
index 828d999f..b674baf1 100644
--- a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::consumer_groups::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::delete_consumer_group::DeleteConsumerGroup;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeleteConsumerGroup {
@@ -38,13 +39,12 @@ impl ServerCommandHandler for DeleteConsumerGroup {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let mut system = system.write().await;
- system
+ shard
.delete_consumer_group(
session,
&self.stream_id,
@@ -56,12 +56,11 @@ impl ServerCommandHandler for DeleteConsumerGroup {
self.group_id, self.topic_id, self.stream_id, session
))?;
- let system = system.downgrade();
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
let group_id = self.group_id.clone();
- system
+ shard
.state
.apply(
session.get_user_id(),
diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
index 694ac38c..8f67192b 100644
--- a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
@@ -20,13 +20,17 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
+use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::get_consumer_group::GetConsumerGroup;
+use std::rc::Rc;
use tracing::debug;
+use super::COMPONENT;
+
impl ServerCommandHandler for GetConsumerGroup {
fn code(&self) -> u32 {
iggy_common::GET_CONSUMER_GROUP_CODE
@@ -36,14 +40,20 @@ impl ServerCommandHandler for GetConsumerGroup {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
+ let stream_id = &self.stream_id;
+ let stream = shard.find_stream(session, &self.stream_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get stream for stream_id: {stream_id}"
+ )
+ })?;
let Ok(consumer_group) =
- system.get_consumer_group(session, &self.stream_id, &self.topic_id, &self.group_id)
+ shard.get_consumer_group(session, &stream, &self.topic_id, &self.group_id)
else {
sender.send_empty_ok_response().await?;
return Ok(());
@@ -53,8 +63,7 @@ impl ServerCommandHandler for GetConsumerGroup {
return Ok(());
};
- let consumer_group = consumer_group.read().await;
- let consumer_group = mapper::map_consumer_group(&consumer_group).await;
+ let consumer_group = mapper::map_consumer_group(&consumer_group);
sender.send_ok_response(&consumer_group).await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
index 7ff6389f..bc6b60b4 100644
--- a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
@@ -21,12 +21,13 @@ use crate::binary::handlers::consumer_groups::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::get_consumer_groups::GetConsumerGroups;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetConsumerGroups {
@@ -38,12 +39,11 @@ impl ServerCommandHandler for GetConsumerGroups {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let consumer_groups = system
+ let consumer_groups = shard
.get_consumer_groups(session, &self.stream_id, &self.topic_id)
.with_error_context(|error| {
format!(
@@ -51,7 +51,7 @@ impl ServerCommandHandler for GetConsumerGroups {
self.stream_id, self.topic_id, session
)
})?;
- let consumer_groups = mapper::map_consumer_groups(&consumer_groups).await;
+ let consumer_groups = mapper::map_consumer_groups(consumer_groups);
sender.send_ok_response(&consumer_groups).await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs
index 3d01d41a..f72609c3 100644
--- a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs
@@ -19,12 +19,13 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::consumer_groups::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::join_consumer_group::JoinConsumerGroup;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for JoinConsumerGroup {
@@ -37,19 +38,17 @@ impl ServerCommandHandler for JoinConsumerGroup {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- system
+ shard
.join_consumer_group(
session,
&self.stream_id,
&self.topic_id,
&self.group_id,
)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to join consumer group for stream_id: {}, topic_id: {}, group_id: {}, session: {}",
diff --git a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
index e094531d..ba352bf9 100644
--- a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
@@ -18,14 +18,16 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::{handlers::consumer_groups::COMPONENT, sender::SenderKind};
+use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::leave_consumer_group::LeaveConsumerGroup;
+use std::rc::Rc;
use tracing::{debug, instrument};
+use super::COMPONENT;
impl ServerCommandHandler for LeaveConsumerGroup {
fn code(&self) -> u32 {
@@ -37,13 +39,12 @@ impl ServerCommandHandler for LeaveConsumerGroup {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- system
+ shard
.leave_consumer_group(
session,
&self.stream_id,
diff --git a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs
index f639223d..96c1e43c 100644
--- a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs
+++ b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs
@@ -20,12 +20,13 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::consumer_offsets::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::delete_consumer_offset::DeleteConsumerOffset;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for DeleteConsumerOffset {
@@ -37,12 +38,11 @@ impl ServerCommandHandler for DeleteConsumerOffset {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- system
+ shard
.delete_consumer_offset(
session,
self.consumer,
diff --git a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs
index dbe119ff..04c89852 100644
--- a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs
+++ b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs
@@ -20,11 +20,12 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::get_consumer_offset::GetConsumerOffset;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetConsumerOffset {
@@ -36,12 +37,11 @@ impl ServerCommandHandler for GetConsumerOffset {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let Ok(offset) = system
+ let Ok(offset) = shard
.get_consumer_offset(
session,
&self.consumer,
diff --git a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs
index c0343bcb..db770e4a 100644
--- a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs
+++ b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs
@@ -16,12 +16,14 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::consumer_offsets::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
@@ -37,12 +39,11 @@ impl ServerCommandHandler for StoreConsumerOffset {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- system
+ shard
.store_consumer_offset(
session,
self.consumer,
diff --git a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs
index 5a50a50d..45bc9db6 100644
--- a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs
+++ b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs
@@ -19,11 +19,12 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::messages::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::{FlushUnsavedBuffer, IggyError};
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for FlushUnsavedBuffer {
@@ -36,17 +37,16 @@ impl ServerCommandHandler for FlushUnsavedBuffer {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
let partition_id = self.partition_id;
let fsync = self.fsync;
- system
+ shard
.flush_unsaved_buffer(session, stream_id, topic_id, partition_id, fsync)
.await
.with_error_context(|error| {
diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
index b0a2ccea..a591f691 100644
--- a/core/server/src/binary/handlers/messages/poll_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
@@ -20,13 +20,15 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::messages::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::sender::SenderKind;
+use crate::shard::system::messages::PollingArgs;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::messages::PollingArgs;
-use crate::streaming::systems::system::SharedSystem;
+use crate::to_iovec;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::{IggyError, PollMessages};
use std::io::IoSlice;
+use std::rc::Rc;
use tracing::{debug, trace};
#[derive(Debug)]
@@ -53,13 +55,12 @@ impl ServerCommandHandler for PollMessages {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let (metadata, messages) = system
+ let (metadata, messages) = shard
.poll_messages(
session,
&self.consumer,
@@ -73,7 +74,6 @@ impl ServerCommandHandler for PollMessages {
"{COMPONENT} (error: {error}) - failed to poll messages for consumer: {}, stream_id: {}, topic_id: {}, partition_id: {:?}, session: {session}.",
self.consumer, self.stream_id, self.topic_id, self.partition_id
))?;
- drop(system);
// Collect all chunks first into a Vec to extend their lifetimes.
// This ensures the Bytes (in reality Arc<[u8]>) references from each IggyMessagesBatch stay alive
@@ -89,12 +89,11 @@ impl ServerCommandHandler for PollMessages {
let count = messages.count().to_le_bytes();
let mut io_slices = Vec::with_capacity(messages.containers_count() + 3);
- io_slices.push(IoSlice::new(&partition_id));
- io_slices.push(IoSlice::new(¤t_offset));
- io_slices.push(IoSlice::new(&count));
-
- io_slices.extend(messages.iter().map(|m| IoSlice::new(m)));
+ io_slices.push(to_iovec(&partition_id));
+ io_slices.push(to_iovec(¤t_offset));
+ io_slices.push(to_iovec(&count));
+ io_slices.extend(messages.iter().map(|m| to_iovec(&m)));
trace!(
"Sending {} messages to client ({} bytes) to client",
messages.count(),
diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs
index 17821a16..5603dfe9 100644
--- a/core/server/src/binary/handlers/messages/send_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs
@@ -18,15 +18,17 @@
use crate::binary::command::{BinaryServerCommand, ServerCommandHandler};
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use crate::streaming::utils::PooledBuffer;
use anyhow::Result;
+use bytes::BytesMut;
use iggy_common::INDEX_SIZE;
use iggy_common::Identifier;
use iggy_common::Sizeable;
use iggy_common::{IggyError, Partitioning, SendMessages, Validatable};
+use std::rc::Rc;
use tracing::instrument;
impl ServerCommandHandler for SendMessages {
@@ -45,36 +47,40 @@ impl ServerCommandHandler for SendMessages {
mut self,
sender: &mut SenderKind,
length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
let total_payload_size = length as usize - std::mem::size_of::<u32>();
let metadata_len_field_size = std::mem::size_of::<u32>();
- let mut metadata_length_buffer = [0u8; 4];
- sender.read(&mut metadata_length_buffer).await?;
- let metadata_size = u32::from_le_bytes(metadata_length_buffer);
+ let mut metadata_length_buffer = BytesMut::with_capacity(4);
+ unsafe { metadata_length_buffer.set_len(4) };
+ let (result, metadata_len_buf) = sender.read(metadata_length_buffer).await;
+ result?;
+ let metadata_len_buf = metadata_len_buf.freeze();
+ let metadata_size = u32::from_le_bytes(metadata_len_buf[..].try_into().unwrap());
let mut metadata_buffer = PooledBuffer::with_capacity(metadata_size as usize);
unsafe { metadata_buffer.set_len(metadata_size as usize) };
- sender.read(&mut metadata_buffer).await?;
+ let (result, metadata_buf) = sender.read(metadata_buffer).await;
+ result?;
let mut element_size = 0;
- let stream_id = Identifier::from_raw_bytes(&metadata_buffer)?;
+ let stream_id = Identifier::from_raw_bytes(&metadata_buf)?;
element_size += stream_id.get_size_bytes().as_bytes_usize();
self.stream_id = stream_id;
- let topic_id = Identifier::from_raw_bytes(&metadata_buffer[element_size..])?;
+ let topic_id = Identifier::from_raw_bytes(&metadata_buf[element_size..])?;
element_size += topic_id.get_size_bytes().as_bytes_usize();
self.topic_id = topic_id;
- let partitioning = Partitioning::from_raw_bytes(&metadata_buffer[element_size..])?;
+ let partitioning = Partitioning::from_raw_bytes(&metadata_buf[element_size..])?;
element_size += partitioning.get_size_bytes().as_bytes_usize();
self.partitioning = partitioning;
let messages_count = u32::from_le_bytes(
- metadata_buffer[element_size..element_size + 4]
+ metadata_buf[element_size..element_size + 4]
.try_into()
.unwrap(),
);
@@ -82,13 +88,15 @@ impl ServerCommandHandler for SendMessages {
let mut indexes_buffer = PooledBuffer::with_capacity(indexes_size);
unsafe { indexes_buffer.set_len(indexes_size) };
- sender.read(&mut indexes_buffer).await?;
+ let (result, indexes_buffer) = sender.read(indexes_buffer).await;
+ result?;
let messages_size =
total_payload_size - metadata_size as usize - indexes_size - metadata_len_field_size;
let mut messages_buffer = PooledBuffer::with_capacity(messages_size);
unsafe { messages_buffer.set_len(messages_size) };
- sender.read(&mut messages_buffer).await?;
+ let (result, messages_buffer) = sender.read(messages_buffer).await;
+ result?;
let indexes = IggyIndexesMut::from_bytes(indexes_buffer, 0);
let batch = IggyMessagesBatchMut::from_indexes_and_messages(
@@ -99,8 +107,7 @@ impl ServerCommandHandler for SendMessages {
batch.validate()?;
- let system = system.read().await;
- system
+ shard
.append_messages(
session,
&self.stream_id,
@@ -110,7 +117,6 @@ impl ServerCommandHandler for SendMessages {
None,
)
.await?;
- drop(system);
sender.send_empty_ok_response().await?;
Ok(())
diff --git a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
index 895c83b2..5045dbfb 100644
--- a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
+++ b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::partitions::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::create_partitions::CreatePartitions;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for CreatePartitions {
@@ -38,13 +39,12 @@ impl ServerCommandHandler for CreatePartitions {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let mut system = system.write().await;
- system
+ shard
.create_partitions(
session,
&self.stream_id,
@@ -59,11 +59,10 @@ impl ServerCommandHandler for CreatePartitions {
)
})?;
- let system = system.downgrade();
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
- system
+ shard
.state
.apply(
session.get_user_id(),
diff --git a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
index c6351403..952b6b64 100644
--- a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
+++ b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::partitions::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::delete_partitions::DeletePartitions;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeletePartitions {
@@ -38,15 +39,14 @@ impl ServerCommandHandler for DeletePartitions {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
- let mut system = system.write().await;
- system
+ shard
.delete_partitions(
session,
&self.stream_id,
@@ -60,8 +60,7 @@ impl ServerCommandHandler for DeletePartitions {
)
})?;
- let system = system.downgrade();
- system
+ shard
.state
.apply(
session.get_user_id(),
diff --git a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
index b43103f3..c28ac77b 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
@@ -20,15 +20,16 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::{handlers::personal_access_tokens::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::state::models::CreatePersonalAccessTokenWithHash;
use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::create_personal_access_token::CreatePersonalAccessToken;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for CreatePersonalAccessToken {
@@ -41,15 +42,13 @@ impl ServerCommandHandler for CreatePersonalAccessToken {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.write().await;
- let token = system
+ let token = shard
.create_personal_access_token(session, &self.name, self.expiry)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to create personal access token with name: {}, session: {session}",
@@ -59,8 +58,7 @@ impl ServerCommandHandler for CreatePersonalAccessToken {
let bytes = mapper::map_raw_pat(&token);
let token_hash = PersonalAccessToken::hash_token(&token);
- let system = system.downgrade();
- system
+ shard
.state
.apply(
session.get_user_id(),
diff --git a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
index 4ba2de04..bb8dd463 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::personal_access_tokens::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::delete_personal_access_token::DeletePersonalAccessToken;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeletePersonalAccessToken {
@@ -38,22 +39,19 @@ impl ServerCommandHandler for DeletePersonalAccessToken {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
let token_name = self.name.clone();
- let mut system = system.write().await;
- system
+ shard
.delete_personal_access_token(session, &self.name)
- .await
.with_error_context(|error| {format!(
"{COMPONENT} (error: {error}) - failed to delete personal access token with name: {token_name}, session: {session}"
)})?;
- let system = system.downgrade();
- system
+ shard
.state
.apply(
session.get_user_id(),
diff --git a/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs
index 08d7c6e6..973af58d 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs
@@ -21,11 +21,12 @@ use crate::binary::handlers::personal_access_tokens::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::get_personal_access_tokens::GetPersonalAccessTokens;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetPersonalAccessTokens {
@@ -37,18 +38,16 @@ impl ServerCommandHandler for GetPersonalAccessTokens {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let personal_access_tokens = system
+ let personal_access_tokens = shard
.get_personal_access_tokens(session)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get personal access tokens with session: {session}")
})?;
- let personal_access_tokens = mapper::map_personal_access_tokens(&personal_access_tokens);
+ let personal_access_tokens = mapper::map_personal_access_tokens(personal_access_tokens);
sender.send_ok_response(&personal_access_tokens).await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs
index 33a4dee2..f7ca5628 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs
@@ -15,13 +15,14 @@
* specific language governing permissions and limitations
* under the License.
*/
+use crate::shard::IggyShard;
+use std::rc::Rc;
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::{handlers::personal_access_tokens::COMPONENT, sender::SenderKind};
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
@@ -38,14 +39,12 @@ impl ServerCommandHandler for LoginWithPersonalAccessToken {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let user = system
+ let user = shard
.login_with_personal_access_token(&self.token, Some(session))
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to login with personal access token: {}, session: {session}",
diff --git a/core/server/src/binary/handlers/segments/delete_segments_handler.rs b/core/server/src/binary/handlers/segments/delete_segments_handler.rs
index 7a2f106c..b3eef13b 100644
--- a/core/server/src/binary/handlers/segments/delete_segments_handler.rs
+++ b/core/server/src/binary/handlers/segments/delete_segments_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::partitions::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::delete_segments::DeleteSegments;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeleteSegments {
@@ -38,16 +39,15 @@ impl ServerCommandHandler for DeleteSegments {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
let partition_id = self.partition_id;
- let mut system = system.write().await;
- system
+ shard
.delete_segments(
session,
&self.stream_id,
@@ -62,8 +62,7 @@ impl ServerCommandHandler for DeleteSegments {
)
})?;
- let system = system.downgrade();
- system
+ shard
.state
.apply(
session.get_user_id(),
diff --git a/core/server/src/binary/handlers/streams/create_stream_handler.rs b/core/server/src/binary/handlers/streams/create_stream_handler.rs
index 86a91d0c..a00afde7 100644
--- a/core/server/src/binary/handlers/streams/create_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/create_stream_handler.rs
@@ -20,14 +20,15 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::{handlers::streams::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::state::models::CreateStreamWithId;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::create_stream::CreateStream;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for CreateStream {
@@ -40,14 +41,13 @@ impl ServerCommandHandler for CreateStream {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
let stream_id = self.stream_id;
- let mut system = system.write().await;
- let stream = system
+ let created_stream_id = shard
.create_stream(session, self.stream_id, &self.name)
.await
.with_error_context(|error| {
@@ -55,11 +55,16 @@ impl ServerCommandHandler for CreateStream {
"{COMPONENT} (error: {error}) - failed to create stream with id: {stream_id:?}, session: {session}"
)
})?;
+ let stream = shard.find_stream(session, &created_stream_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to find created stream with id: {created_stream_id:?}, session: {session}"
+ )
+ })?;
let stream_id = stream.stream_id;
- let response = mapper::map_stream(stream);
+ let response = mapper::map_stream(&stream);
- let system = system.downgrade();
- system
+ shard
.state
.apply(session.get_user_id(), &EntryCommand::CreateStream(CreateStreamWithId {
stream_id,
diff --git a/core/server/src/binary/handlers/streams/delete_stream_handler.rs b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
index f480c0ea..a0e52a43 100644
--- a/core/server/src/binary/handlers/streams/delete_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::streams::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::delete_stream::DeleteStream;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeleteStream {
@@ -38,22 +39,20 @@ impl ServerCommandHandler for DeleteStream {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
let stream_id = self.stream_id.clone();
- let mut system = system.write().await;
- system
+ shard
.delete_stream(session, &self.stream_id)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to delete stream with ID: {stream_id}, session: {session}")
})?;
- let system = system.downgrade();
- system
+ shard
.state
.apply(session.get_user_id(), &EntryCommand::DeleteStream(self))
.await
diff --git a/core/server/src/binary/handlers/streams/get_stream_handler.rs b/core/server/src/binary/handlers/streams/get_stream_handler.rs
index 12e015af..fef6458c 100644
--- a/core/server/src/binary/handlers/streams/get_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/get_stream_handler.rs
@@ -20,11 +20,12 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::get_stream::GetStream;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetStream {
@@ -36,12 +37,11 @@ impl ServerCommandHandler for GetStream {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let Ok(stream) = system.try_find_stream(session, &self.stream_id) else {
+ let Ok(stream) = shard.try_find_stream(session, &self.stream_id) else {
sender.send_empty_ok_response().await?;
return Ok(());
};
@@ -51,7 +51,7 @@ impl ServerCommandHandler for GetStream {
return Ok(());
};
- let response = mapper::map_stream(stream);
+ let response = mapper::map_stream(&stream);
sender.send_ok_response(&response).await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/streams/get_streams_handler.rs b/core/server/src/binary/handlers/streams/get_streams_handler.rs
index fa08bc13..0f83a9cb 100644
--- a/core/server/src/binary/handlers/streams/get_streams_handler.rs
+++ b/core/server/src/binary/handlers/streams/get_streams_handler.rs
@@ -21,12 +21,13 @@ use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::get_streams::GetStreams;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetStreams {
@@ -38,15 +39,14 @@ impl ServerCommandHandler for GetStreams {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let streams = system.find_streams(session).with_error_context(|error| {
+ let streams = shard.find_streams(session).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to find streams for session: {session}")
})?;
- let response = mapper::map_streams(&streams);
+ let response = mapper::map_streams(streams);
sender.send_ok_response(&response).await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/streams/purge_stream_handler.rs b/core/server/src/binary/handlers/streams/purge_stream_handler.rs
index dd37e155..3a387ce9 100644
--- a/core/server/src/binary/handlers/streams/purge_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/purge_stream_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::streams::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::purge_stream::PurgeStream;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for PurgeStream {
@@ -38,21 +39,20 @@ impl ServerCommandHandler for PurgeStream {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
let stream_id = self.stream_id.clone();
- system
+ shard
.purge_stream(session, &self.stream_id)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to purge stream with id: {stream_id}, session: {session}")
})?;
- system
+ shard
.state
.apply(session.get_user_id(), &EntryCommand::PurgeStream(self))
.await
diff --git a/core/server/src/binary/handlers/streams/update_stream_handler.rs b/core/server/src/binary/handlers/streams/update_stream_handler.rs
index cdb02c91..2aa4ddc9 100644
--- a/core/server/src/binary/handlers/streams/update_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/update_stream_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::streams::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::update_stream::UpdateStream;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for UpdateStream {
@@ -38,22 +39,20 @@ impl ServerCommandHandler for UpdateStream {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
let stream_id = self.stream_id.clone();
- let mut system = system.write().await;
- system
+ shard
.update_stream(session, &self.stream_id, &self.name)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to update stream with id: {stream_id}, session: {session}")
})?;
- let system = system.downgrade();
- system
+ shard
.state
.apply(session.get_user_id(), &EntryCommand::UpdateStream(self))
.await
diff --git a/core/server/src/binary/handlers/system/get_client_handler.rs b/core/server/src/binary/handlers/system/get_client_handler.rs
index 532eddf4..9ca18a38 100644
--- a/core/server/src/binary/handlers/system/get_client_handler.rs
+++ b/core/server/src/binary/handlers/system/get_client_handler.rs
@@ -20,11 +20,11 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use iggy_common::IggyError;
use iggy_common::get_client::GetClient;
-use iggy_common::locking::IggySharedMutFn;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetClient {
@@ -36,13 +36,12 @@ impl ServerCommandHandler for GetClient {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let Ok(client) = system.get_client(session, self.client_id).await else {
+ let Ok(client) = shard.get_client(session, self.client_id) else {
sender.send_empty_ok_response().await?;
return Ok(());
};
@@ -52,7 +51,6 @@ impl ServerCommandHandler for GetClient {
return Ok(());
};
- let client = client.read().await;
let bytes = mapper::map_client(&client);
sender.send_ok_response(&bytes).await?;
diff --git a/core/server/src/binary/handlers/system/get_clients_handler.rs b/core/server/src/binary/handlers/system/get_clients_handler.rs
index bf05f7f1..c73ac987 100644
--- a/core/server/src/binary/handlers/system/get_clients_handler.rs
+++ b/core/server/src/binary/handlers/system/get_clients_handler.rs
@@ -21,11 +21,12 @@ use crate::binary::handlers::system::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::get_clients::GetClients;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetClients {
@@ -37,19 +38,17 @@ impl ServerCommandHandler for GetClients {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let clients = system
+ let clients = shard
.get_clients(session)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get clients, session: {session}")
})?;
- let clients = mapper::map_clients(&clients).await;
+ let clients = mapper::map_clients(clients).await;
sender.send_ok_response(&clients).await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/system/get_me_handler.rs b/core/server/src/binary/handlers/system/get_me_handler.rs
index b29972a9..44fdd651 100644
--- a/core/server/src/binary/handlers/system/get_me_handler.rs
+++ b/core/server/src/binary/handlers/system/get_me_handler.rs
@@ -21,12 +21,12 @@ use crate::binary::handlers::system::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::get_me::GetMe;
-use iggy_common::locking::IggySharedMutFn;
+use std::rc::Rc;
impl ServerCommandHandler for GetMe {
fn code(&self) -> u32 {
@@ -37,13 +37,11 @@ impl ServerCommandHandler for GetMe {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
- let system = system.read().await;
- let Some(client) = system
+ let Some(client) = shard
.get_client(session, session.client_id)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get current client for session: {session}")
})?
@@ -51,7 +49,6 @@ impl ServerCommandHandler for GetMe {
return Err(IggyError::ClientNotFound(session.client_id));
};
- let client = client.read().await;
let bytes = mapper::map_client(&client);
sender.send_ok_response(&bytes).await?;
diff --git a/core/server/src/binary/handlers/system/get_snapshot.rs b/core/server/src/binary/handlers/system/get_snapshot.rs
index ff850388..27b2a3a7 100644
--- a/core/server/src/binary/handlers/system/get_snapshot.rs
+++ b/core/server/src/binary/handlers/system/get_snapshot.rs
@@ -19,11 +19,12 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use bytes::Bytes;
use iggy_common::IggyError;
use iggy_common::get_snapshot::GetSnapshot;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetSnapshot {
@@ -35,13 +36,12 @@ impl ServerCommandHandler for GetSnapshot {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let snapshot = system
+ let snapshot = shard
.get_snapshot(session, self.compression, &self.snapshot_types)
.await?;
let bytes = Bytes::copy_from_slice(&snapshot.0);
diff --git a/core/server/src/binary/handlers/system/get_stats_handler.rs b/core/server/src/binary/handlers/system/get_stats_handler.rs
index 098ace33..17c87191 100644
--- a/core/server/src/binary/handlers/system/get_stats_handler.rs
+++ b/core/server/src/binary/handlers/system/get_stats_handler.rs
@@ -21,11 +21,12 @@ use crate::binary::handlers::system::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::get_stats::GetStats;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetStats {
@@ -37,13 +38,12 @@ impl ServerCommandHandler for GetStats {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let stats = system.get_stats().await.with_error_context(|error| {
+ let stats = shard.get_stats().await.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get stats, session: {session}")
})?;
let bytes = mapper::map_stats(&stats);
diff --git a/core/server/src/binary/handlers/system/ping_handler.rs b/core/server/src/binary/handlers/system/ping_handler.rs
index 396719c5..a66d91db 100644
--- a/core/server/src/binary/handlers/system/ping_handler.rs
+++ b/core/server/src/binary/handlers/system/ping_handler.rs
@@ -18,13 +18,13 @@
use crate::binary::command::{BinaryServerCommand, ServerCommandHandler};
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::IggyTimestamp;
-use iggy_common::locking::IggySharedMutFn;
use iggy_common::ping::Ping;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for Ping {
@@ -36,14 +36,15 @@ impl ServerCommandHandler for Ping {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let client_manager = system.client_manager.read().await;
- if let Some(client) = client_manager.try_get_client(session.client_id) {
- let mut client = client.write().await;
+ if let Some(client) = shard
+ .client_manager
+ .borrow_mut()
+ .try_get_client_mut(session.client_id)
+ {
let now = IggyTimestamp::now();
client.last_heartbeat = now;
debug!("Updated last heartbeat to: {now} for session: {session}");
diff --git a/core/server/src/binary/handlers/topics/create_topic_handler.rs b/core/server/src/binary/handlers/topics/create_topic_handler.rs
index 0f42616f..db09863e 100644
--- a/core/server/src/binary/handlers/topics/create_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/create_topic_handler.rs
@@ -20,14 +20,15 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::{handlers::topics::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::state::models::CreateTopicWithId;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::create_topic::CreateTopic;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for CreateTopic {
@@ -40,14 +41,13 @@ impl ServerCommandHandler for CreateTopic {
mut self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id;
- let mut system = system.write().await;
- let topic = system
+ let created_topic_id = shard
.create_topic(
session,
&self.stream_id,
@@ -62,18 +62,31 @@ impl ServerCommandHandler for CreateTopic {
.await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to create topic for stream_id: {stream_id}, topic_id: {topic_id:?}"
))?;
+ let stream = shard
+ .find_stream(session, &self.stream_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get stream for stream_id: {stream_id}"
+ )
+ })?;
+ let topic = shard.find_topic(session, &stream, &created_topic_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get topic with ID: {created_topic_id} in stream with ID: {stream_id}"
+ )
+ })?;
self.message_expiry = topic.message_expiry;
self.max_topic_size = topic.max_topic_size;
let topic_id = topic.topic_id;
let response = mapper::map_topic(topic).await;
- let system = system.downgrade();
- system
+ shard
.state
.apply(session.get_user_id(), &EntryCommand::CreateTopic(CreateTopicWithId {
topic_id,
command: self
- })) .await
+ }))
+ .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to apply create topic for stream_id: {stream_id}, topic_id: {topic_id:?}"
diff --git a/core/server/src/binary/handlers/topics/delete_topic_handler.rs b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
index e83115f4..a8978511 100644
--- a/core/server/src/binary/handlers/topics/delete_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::topics::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::delete_topic::DeleteTopic;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for DeleteTopic {
@@ -38,23 +39,21 @@ impl ServerCommandHandler for DeleteTopic {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
- let mut system = system.write().await;
- system
+ shard
.delete_topic(session, &self.stream_id, &self.topic_id)
.await
.with_error_context(|error| format!(
"{COMPONENT} (error: {error}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}, session: {session}",
))?;
- let system = system.downgrade();
- system
+ shard
.state
.apply(session.get_user_id(), &EntryCommand::DeleteTopic(self))
.await
diff --git a/core/server/src/binary/handlers/topics/get_topic_handler.rs b/core/server/src/binary/handlers/topics/get_topic_handler.rs
index 8c652eb3..597c13bf 100644
--- a/core/server/src/binary/handlers/topics/get_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/get_topic_handler.rs
@@ -20,11 +20,13 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
+use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::get_topic::GetTopic;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetTopic {
@@ -36,12 +38,19 @@ impl ServerCommandHandler for GetTopic {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let Ok(topic) = system.try_find_topic(session, &self.stream_id, &self.topic_id) else {
+ let stream = shard
+ .find_stream(session, &self.stream_id)
+ .with_error_context(|error| {
+ format!(
+ "Failed to find stream with ID: {}, session: {session} (error: {error})",
+ self.stream_id
+ )
+ })?;
+ let Ok(topic) = shard.try_find_topic(session, &stream, &self.topic_id) else {
sender.send_empty_ok_response().await?;
return Ok(());
};
diff --git a/core/server/src/binary/handlers/topics/get_topics_handler.rs b/core/server/src/binary/handlers/topics/get_topics_handler.rs
index 8d5c61b1..1606245e 100644
--- a/core/server/src/binary/handlers/topics/get_topics_handler.rs
+++ b/core/server/src/binary/handlers/topics/get_topics_handler.rs
@@ -21,12 +21,13 @@ use crate::binary::handlers::topics::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::get_topics::GetTopics;
+use std::rc::Rc;
use tracing::debug;
impl ServerCommandHandler for GetTopics {
@@ -38,20 +39,26 @@ impl ServerCommandHandler for GetTopics {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let topics = system
- .find_topics(session, &self.stream_id)
+ let stream = shard.find_stream(session, &self.stream_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get stream, stream_id: {}, session: {session}",
+ self.stream_id
+ )
+ })?;
+ let topics = shard
+ .find_topics(session, &stream)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to find topics, stream_id: {}, session: {session}",
self.stream_id
)
})?;
- let response = mapper::map_topics(&topics);
+ let response = mapper::map_topics(topics);
sender.send_ok_response(&response).await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/topics/purge_topic_handler.rs b/core/server/src/binary/handlers/topics/purge_topic_handler.rs
index d9c91470..ab7e392c 100644
--- a/core/server/src/binary/handlers/topics/purge_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/purge_topic_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::topics::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::purge_topic::PurgeTopic;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for PurgeTopic {
@@ -38,12 +39,11 @@ impl ServerCommandHandler for PurgeTopic {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- system
+ shard
.purge_topic(session, &self.stream_id, &self.topic_id)
.await
.with_error_context(|error| {
@@ -55,7 +55,7 @@ impl ServerCommandHandler for PurgeTopic {
let topic_id = self.topic_id.clone();
let stream_id = self.stream_id.clone();
- system
+ shard
.state
.apply(session.get_user_id(), &EntryCommand::PurgeTopic(self))
.await
diff --git a/core/server/src/binary/handlers/topics/update_topic_handler.rs b/core/server/src/binary/handlers/topics/update_topic_handler.rs
index f460cc24..9d75a255 100644
--- a/core/server/src/binary/handlers/topics/update_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/update_topic_handler.rs
@@ -19,13 +19,14 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::topics::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::update_topic::UpdateTopic;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for UpdateTopic {
@@ -38,14 +39,12 @@ impl ServerCommandHandler for UpdateTopic {
mut self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let mut system = system.write().await;
-
- let topic = system
+ shard
.update_topic(
session,
&self.stream_id,
@@ -61,14 +60,25 @@ impl ServerCommandHandler for UpdateTopic {
"{COMPONENT} (error: {error}) - failed to update topic with id: {}, stream_id: {}, session: {session}",
self.topic_id, self.stream_id
))?;
+
+ let stream = shard.find_stream(session, &self.stream_id)
+ .with_error_context(|error| format!(
+ "{COMPONENT} (error: {error}) - failed to find stream, stream_id: {}, session: {session}",
+ self.stream_id
+ ))?;
+ let topic = stream
+ .get_topic(&self.topic_id)
+ .with_error_context(|error| format!(
+ "{COMPONENT} (error: {error}) - failed to find topic, topic_id: {}, stream_id: {}, session: {session}",
+ self.topic_id, self.stream_id
+ ))?;
self.message_expiry = topic.message_expiry;
self.max_topic_size = topic.max_topic_size;
let topic_id = self.topic_id.clone();
let stream_id = self.stream_id.clone();
- let system = system.downgrade();
- system
+ shard
.state
.apply(session.get_user_id(), &EntryCommand::UpdateTopic(self))
.await
diff --git a/core/server/src/binary/handlers/users/change_password_handler.rs b/core/server/src/binary/handlers/users/change_password_handler.rs
index 935e8f87..32b376cd 100644
--- a/core/server/src/binary/handlers/users/change_password_handler.rs
+++ b/core/server/src/binary/handlers/users/change_password_handler.rs
@@ -19,14 +19,15 @@
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::users::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use crate::streaming::utils::crypto;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::change_password::ChangePassword;
+use std::rc::Rc;
use tracing::{debug, instrument};
impl ServerCommandHandler for ChangePassword {
@@ -38,21 +39,19 @@ impl ServerCommandHandler for ChangePassword {
async fn handle(
self,
sender: &mut SenderKind,
- _length: u32,
- session: &Session,
- system: &SharedSystem,
+ length: u32,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let mut system = system.write().await;
- system
+ shard
.change_password(
session,
&self.user_id,
&self.current_password,
&self.new_password,
)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to change password for user_id: {}, session: {session}",
@@ -61,8 +60,7 @@ impl ServerCommandHandler for ChangePassword {
})?;
// For the security of the system, we hash the password before storing it in metadata.
- let system = system.downgrade();
- system
+ shard
.state
.apply(
session.get_user_id(),
diff --git a/core/server/src/binary/handlers/users/create_user_handler.rs b/core/server/src/binary/handlers/users/create_user_handler.rs
index 17d17fdf..e51b147b 100644
--- a/core/server/src/binary/handlers/users/create_user_handler.rs
+++ b/core/server/src/binary/handlers/users/create_user_handler.rs
@@ -16,6 +16,9 @@
* under the License.
*/
+use crate::shard::IggyShard;
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
@@ -23,7 +26,6 @@ use crate::binary::{handlers::users::COMPONENT, sender::SenderKind};
use crate::state::command::EntryCommand;
use crate::state::models::CreateUserWithId;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use crate::streaming::utils::crypto;
use anyhow::Result;
use error_set::ErrContext;
@@ -41,13 +43,12 @@ impl ServerCommandHandler for CreateUser {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let mut system = system.write().await;
- let user = system
+ let user = shard
.create_user(
session,
&self.username,
@@ -63,11 +64,10 @@ impl ServerCommandHandler for CreateUser {
)
})?;
let user_id = user.id;
- let response = mapper::map_user(user);
+ let response = mapper::map_user(&user);
// For the security of the system, we hash the password before storing it in metadata.
- let system = system.downgrade();
- system
+ shard
.state
.apply(
session.get_user_id(),
diff --git a/core/server/src/binary/handlers/users/delete_user_handler.rs b/core/server/src/binary/handlers/users/delete_user_handler.rs
index b9fddfd7..10de56a4 100644
--- a/core/server/src/binary/handlers/users/delete_user_handler.rs
+++ b/core/server/src/binary/handlers/users/delete_user_handler.rs
@@ -16,12 +16,14 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::users::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
@@ -37,16 +39,14 @@ impl ServerCommandHandler for DeleteUser {
async fn handle(
self,
sender: &mut SenderKind,
- _length: u32,
- session: &Session,
- system: &SharedSystem,
+ length: u32,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let mut system = system.write().await;
- system
+ shard
.delete_user(session, &self.user_id)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to delete user with ID: {}, session: {session}",
@@ -54,9 +54,8 @@ impl ServerCommandHandler for DeleteUser {
)
})?;
- let system = system.downgrade();
let user_id = self.user_id.clone();
- system
+ shard
.state
.apply(session.get_user_id(), &EntryCommand::DeleteUser(self))
.await
diff --git a/core/server/src/binary/handlers/users/get_user_handler.rs b/core/server/src/binary/handlers/users/get_user_handler.rs
index 5523c4cc..e276c9a4 100644
--- a/core/server/src/binary/handlers/users/get_user_handler.rs
+++ b/core/server/src/binary/handlers/users/get_user_handler.rs
@@ -16,12 +16,14 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use iggy_common::IggyError;
use iggy_common::get_user::GetUser;
use tracing::debug;
@@ -35,12 +37,11 @@ impl ServerCommandHandler for GetUser {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let Ok(user) = system.find_user(session, &self.user_id) else {
+ let Ok(user) = shard.find_user(session, &self.user_id) else {
sender.send_empty_ok_response().await?;
return Ok(());
};
@@ -49,7 +50,7 @@ impl ServerCommandHandler for GetUser {
return Ok(());
};
- let bytes = mapper::map_user(user);
+ let bytes = mapper::map_user(&user);
sender.send_ok_response(&bytes).await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/users/get_users_handler.rs b/core/server/src/binary/handlers/users/get_users_handler.rs
index 7471c591..72a0ca1e 100644
--- a/core/server/src/binary/handlers/users/get_users_handler.rs
+++ b/core/server/src/binary/handlers/users/get_users_handler.rs
@@ -16,13 +16,15 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use error_set::ErrContext;
use iggy_common::IggyError;
use iggy_common::get_users::GetUsers;
@@ -37,18 +39,14 @@ impl ServerCommandHandler for GetUsers {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let users = system
- .get_users(session)
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get users, session: {session}")
- })?;
- let users = mapper::map_users(&users);
+ let users = shard.get_users(session).await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get users, session: {session}")
+ })?;
+ let users = mapper::map_users(users);
sender.send_ok_response(&users).await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/users/login_user_handler.rs b/core/server/src/binary/handlers/users/login_user_handler.rs
index 7fa67be7..21cbba49 100644
--- a/core/server/src/binary/handlers/users/login_user_handler.rs
+++ b/core/server/src/binary/handlers/users/login_user_handler.rs
@@ -16,12 +16,14 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::{handlers::users::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
@@ -37,15 +39,13 @@ impl ServerCommandHandler for LoginUser {
async fn handle(
self,
sender: &mut SenderKind,
- _length: u32,
- session: &Session,
- system: &SharedSystem,
+ length: u32,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- let user = system
+ let user = shard
.login_user(&self.username, &self.password, Some(session))
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to login user with name: {}, session: {session}",
diff --git a/core/server/src/binary/handlers/users/logout_user_handler.rs b/core/server/src/binary/handlers/users/logout_user_handler.rs
index 77fa33f4..d1ef95bf 100644
--- a/core/server/src/binary/handlers/users/logout_user_handler.rs
+++ b/core/server/src/binary/handlers/users/logout_user_handler.rs
@@ -16,11 +16,13 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::users::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
@@ -36,18 +38,14 @@ impl ServerCommandHandler for LogoutUser {
async fn handle(
self,
sender: &mut SenderKind,
- _length: u32,
- session: &Session,
- system: &SharedSystem,
+ length: u32,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let system = system.read().await;
- system
- .logout_user(session)
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to logout user, session: {session}")
- })?;
+ shard.logout_user(session).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to logout user, session: {session}")
+ })?;
session.clear_user_id();
sender.send_empty_ok_response().await?;
Ok(())
diff --git a/core/server/src/binary/handlers/users/update_permissions_handler.rs b/core/server/src/binary/handlers/users/update_permissions_handler.rs
index 3606c308..dea49a1c 100644
--- a/core/server/src/binary/handlers/users/update_permissions_handler.rs
+++ b/core/server/src/binary/handlers/users/update_permissions_handler.rs
@@ -16,12 +16,14 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::users::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
@@ -38,21 +40,18 @@ impl ServerCommandHandler for UpdatePermissions {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let mut system = system.write().await;
- system
+ shard
.update_permissions(session, &self.user_id, self.permissions.clone())
- .await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to update permissions for user_id: {}, session: {session}",
self.user_id
))?;
- let system = system.downgrade();
- system
+ shard
.state
.apply(
session.get_user_id(),
diff --git a/core/server/src/binary/handlers/users/update_user_handler.rs b/core/server/src/binary/handlers/users/update_user_handler.rs
index 873a3c6f..982c4ae1 100644
--- a/core/server/src/binary/handlers/users/update_user_handler.rs
+++ b/core/server/src/binary/handlers/users/update_user_handler.rs
@@ -16,12 +16,14 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::{handlers::users::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
@@ -38,20 +40,18 @@ impl ServerCommandHandler for UpdateUser {
self,
sender: &mut SenderKind,
_length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let mut system = system.write().await;
- system
+ shard
.update_user(
session,
&self.user_id,
self.username.clone(),
self.status,
)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to update user with user_id: {}, session: {session}",
@@ -59,10 +59,9 @@ impl ServerCommandHandler for UpdateUser {
)
})?;
- let system = system.downgrade();
let user_id = self.user_id.clone();
- system
+ shard
.state
.apply(session.get_user_id(), &EntryCommand::UpdateUser(self))
.await
diff --git a/core/server/src/binary/mapper.rs b/core/server/src/binary/mapper.rs
index 219ceed9..1ecce265 100644
--- a/core/server/src/binary/mapper.rs
+++ b/core/server/src/binary/mapper.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+use std::cell::Ref;
+
use crate::streaming::clients::client_manager::{Client, Transport};
use crate::streaming::partitions::partition::Partition;
use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
@@ -94,9 +96,9 @@ pub fn map_client(client: &Client) -> Bytes {
bytes.freeze()
}
-pub async fn map_clients(clients: &[Client]) -> Bytes {
+pub async fn map_clients(clients: Vec<Client>) -> Bytes {
let mut bytes = BytesMut::new();
- for client in clients {
+ for client in clients.iter() {
extend_client(client, &mut bytes);
}
bytes.freeze()
@@ -117,9 +119,9 @@ pub fn map_user(user: &User) -> Bytes {
bytes.freeze()
}
-pub fn map_users(users: &[&User]) -> Bytes {
+pub fn map_users(users: Vec<User>) -> Bytes {
let mut bytes = BytesMut::new();
- for user in users {
+ for user in users.iter() {
extend_user(user, &mut bytes);
}
bytes.freeze()
@@ -138,9 +140,9 @@ pub fn map_raw_pat(token: &str) -> Bytes {
bytes.freeze()
}
-pub fn map_personal_access_tokens(personal_access_tokens: &[PersonalAccessToken]) -> Bytes {
+pub fn map_personal_access_tokens(personal_access_tokens: Vec<PersonalAccessToken>) -> Bytes {
let mut bytes = BytesMut::new();
- for personal_access_token in personal_access_tokens {
+ for personal_access_token in personal_access_tokens.iter() {
extend_pat(personal_access_token, &mut bytes);
}
bytes.freeze()
@@ -155,15 +157,15 @@ pub fn map_stream(stream: &Stream) -> Bytes {
bytes.freeze()
}
-pub fn map_streams(streams: &[&Stream]) -> Bytes {
+pub fn map_streams(streams: Vec<Ref<'_, Stream>>) -> Bytes {
let mut bytes = BytesMut::new();
- for stream in streams {
+ for stream in streams.iter() {
extend_stream(stream, &mut bytes);
}
bytes.freeze()
}
-pub fn map_topics(topics: &[&Topic]) -> Bytes {
+pub fn map_topics(topics: Vec<&Topic>) -> Bytes {
let mut bytes = BytesMut::new();
for topic in topics {
extend_topic(topic, &mut bytes);
@@ -196,7 +198,7 @@ pub fn map_consumer_group(consumer_group: &ConsumerGroup) -> Bytes {
bytes.freeze()
}
-pub fn map_consumer_groups(consumer_groups: &[ConsumerGroup]) -> Bytes {
+pub fn map_consumer_groups(consumer_groups: Vec<ConsumerGroup>) -> Bytes {
let mut bytes = BytesMut::new();
for consumer_group in consumer_groups {
extend_consumer_group(&consumer_group, &mut bytes);
diff --git a/core/server/src/binary/sender.rs b/core/server/src/binary/sender.rs
index 06e9c51f..ff405eb6 100644
--- a/core/server/src/binary/sender.rs
+++ b/core/server/src/binary/sender.rs
@@ -33,28 +33,33 @@ use quinn::{RecvStream, SendStream};
macro_rules! forward_async_methods {
(
$(
- async fn $method_name:ident(
+ async fn $method_name:ident
+ $(<$($generic:ident $(: $bound:path)?),+>)?
+ (
&mut self $(, $arg:ident : $arg_ty:ty )*
) -> $ret:ty ;
)*
) => {
$(
- pub async fn $method_name(&mut self, $( $arg: $arg_ty ),* ) -> $ret {
+ pub async fn $method_name
+ $(<$($generic $(: $bound)?),+>)?
+ (&mut self, $( $arg: $arg_ty ),* ) -> $ret {
match self {
- Self::Tcp(d) => d.$method_name($( $arg ),*).await,
- Self::TcpTls(s) => s.$method_name($( $arg ),*).await,
- Self::Quic(s) => s.$method_name($( $arg ),*).await,
+ Self::Tcp(d) => d.$method_name$(::<$($generic),+>)?($( $arg ),*).await,
+ Self::TcpTls(s) => s.$method_name$(::<$($generic),+>)?($( $arg ),*).await,
+ Self::Quic(s) => s.$method_name$(::<$($generic),+>)?($( $arg ),*).await,
}
}
)*
}
}
+
pub trait Sender {
- fn read(
+ fn read<B: IoBufMut>(
&mut self,
- buffer: BytesMut,
- ) -> impl Future<Output = (Result<usize, IggyError>, BytesMut)>;
+ buffer: B
+ ) -> impl Future<Output = (Result<usize, IggyError>, B)>;
fn send_empty_ok_response(&mut self) -> impl Future<Output = Result<(), IggyError>>;
fn send_ok_response(&mut self, payload: &[u8]) -> impl Future<Output = Result<(), IggyError>>;
fn send_ok_response_vectored(
@@ -92,7 +97,7 @@ impl SenderKind {
}
forward_async_methods! {
- async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut);
+ async fn read<B: IoBufMut>(&mut self, buffer: B) -> (Result<usize, IggyError>, B);
async fn send_empty_ok_response(&mut self) -> Result<(), IggyError>;
async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError>;
async fn send_ok_response_vectored(&mut self, length: &[u8], slices: Vec<libc::iovec>) -> Result<(), IggyError>;
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index 3dcf64b2..4a5dbe53 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -18,6 +18,8 @@
#[cfg(not(feature = "disable-mimalloc"))]
use mimalloc::MiMalloc;
+use nix::libc::iovec;
+use nix::libc::c_void;
#[cfg(not(feature = "disable-mimalloc"))]
#[global_allocator]
@@ -53,3 +55,10 @@ pub fn map_toggle_str<'a>(enabled: bool) -> &'a str {
false => "disabled",
}
}
+
+pub fn to_iovec<T>(data: &[T]) -> iovec {
+ iovec {
+ iov_base: data.as_ptr() as *mut c_void,
+ iov_len: data.len() * std::mem::size_of::<T>(),
+ }
+}
\ No newline at end of file
diff --git a/core/server/src/quic/quic_sender.rs b/core/server/src/quic/quic_sender.rs
index 17b60818..c6403349 100644
--- a/core/server/src/quic/quic_sender.rs
+++ b/core/server/src/quic/quic_sender.rs
@@ -36,7 +36,8 @@ pub struct QuicSender {
}
impl Sender for QuicSender {
- async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut) {
+ async fn read<B: IoBufMut>(&mut self, buffer: B) -> (Result<usize, IggyError>, B) {
+ //TODO: Fixme
// Not-so-nice code because quinn recv stream has different API for read_exact
/*
let read_bytes = buffer.len();
diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs
index ee9e1634..4668b337 100644
--- a/core/server/src/shard/system/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -72,7 +72,7 @@ impl IggyShard {
}
}
- pub async fn get_client(
+ pub fn get_client(
&self,
session: &Session,
client_id: u32,
@@ -91,7 +91,7 @@ impl IggyShard {
Ok(self.client_manager.borrow().try_get_client(client_id))
}
- pub async fn get_clients(&self, session: &Session) -> Result<Vec<Client>, IggyError> {
+ pub fn get_clients(&self, session: &Session) -> Result<Vec<Client>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
.borrow()
diff --git a/core/server/src/shard/system/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs
index 768d4e5c..7b98205f 100644
--- a/core/server/src/shard/system/consumer_groups.rs
+++ b/core/server/src/shard/system/consumer_groups.rs
@@ -26,7 +26,6 @@ use crate::streaming::topics::consumer_group::ConsumerGroup;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
-use iggy_common::locking::IggySharedMutFn;
impl IggyShard {
pub fn get_consumer_group<'cg, 'stream>(
@@ -93,7 +92,7 @@ impl IggyShard {
topic_id: &Identifier,
group_id: Option<u32>,
name: &str,
- ) -> Result<u32, IggyError> {
+ ) -> Result<Identifier, IggyError> {
self.ensure_authenticated(session)?;
{
let stream = self.get_stream(stream_id).with_error_context(|error| {
@@ -120,7 +119,7 @@ impl IggyShard {
.create_consumer_group(group_id, name)
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to create consumer group with name: {name}")
- }).map(|cg| cg.group_id)
+ })
}
pub async fn delete_consumer_group(
@@ -272,12 +271,7 @@ impl IggyShard {
.find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!(
-<<<<<<< HEAD
"{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id:?}, topic_id: {topic_id:?}"
-=======
- "{COMPONENT} (error: {error}) - topic not found for stream ID: {:?}, topic_id: {:?}",
- stream.stream_id, topic_id
->>>>>>> 48107890 (fix iggy shard errors)
)
})?;
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index 06499bd3..7668566a 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -86,11 +86,7 @@ impl IggyShard {
topic
.store_consumer_offset_internal(polling_consumer, offset, partition_id)
.await
-<<<<<<< HEAD
- .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to store consumer offset internal, polling consumer: {polling_consumer}, offset: {offset}, partition ID: {partition_id}")) ?;
-=======
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to store consumer offset internal, polling consumer: {}, offset: {}, partition ID: {}", polling_consumer, offset, partition_id))?;
->>>>>>> 48107890 (fix iggy shard errors)
}
let batch_set = if let Some(encryptor) = &self.encryptor {
diff --git a/core/server/src/shard/system/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs
index 082fcd93..8cc89cdc 100644
--- a/core/server/src/shard/system/personal_access_tokens.rs
+++ b/core/server/src/shard/system/personal_access_tokens.rs
@@ -28,7 +28,7 @@ use iggy_common::IggyTimestamp;
use tracing::{error, info};
impl IggyShard {
- pub async fn get_personal_access_tokens(
+ pub fn get_personal_access_tokens(
&self,
session: &Session,
) -> Result<Vec<PersonalAccessToken>, IggyError> {
@@ -53,7 +53,7 @@ impl IggyShard {
Ok(personal_access_tokens)
}
- pub async fn create_personal_access_token(
+ pub fn create_personal_access_token(
&self,
session: &Session,
name: &str,
@@ -103,7 +103,7 @@ impl IggyShard {
Ok(token)
}
- pub async fn delete_personal_access_token(
+ pub fn delete_personal_access_token(
&self,
session: &Session,
name: &str,
@@ -135,7 +135,7 @@ impl IggyShard {
Ok(())
}
- pub async fn login_with_personal_access_token(
+ pub fn login_with_personal_access_token(
&self,
token: &str,
session: Option<&Session>,
diff --git a/core/server/src/shard/system/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs
index 78c5d393..b4de2992 100644
--- a/core/server/src/shard/system/snapshot/mod.rs
+++ b/core/server/src/shard/system/snapshot/mod.rs
@@ -54,6 +54,11 @@ impl IggyShard {
snapshot_types
};
+ // TODO: Replace this with
+ // https://github.com/bearcove/rc-zip
+ // and impl the monoio async writer, based on this example:
+ // https://youtu.be/RYHYiXMJdZI?si=d2roKeHn5lJrw2ri&t=1140
+ // and rc-zip-tokio crate.
let cursor = Cursor::new(Vec::new());
let mut zip_writer = ZipFileWriter::new(cursor.compat_write());
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index 43b45246..3cb0da5b 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -57,7 +57,7 @@ impl IggyShard {
Ok(self.get_streams())
}
- pub fn find_stream<'a>(
+ pub fn find_stream(
&self,
session: &Session,
identifier: &Identifier,
@@ -182,7 +182,7 @@ impl IggyShard {
session: &Session,
stream_id: Option<u32>,
name: &str,
- ) -> Result<u32, IggyError> {
+ ) -> Result<Identifier, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
.borrow()
@@ -220,7 +220,7 @@ impl IggyShard {
.insert(name.to_owned(), stream.stream_id);
self.streams.borrow_mut().insert(stream.stream_id, stream);
self.metrics.increment_streams(1);
- Ok(id)
+ Ok(Identifier::numeric(id)?)
}
pub async fn update_stream(
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index d118090f..477308bc 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -112,7 +112,7 @@ impl IggyShard {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
- ) -> Result<u32, IggyError> {
+ ) -> Result<Identifier, IggyError> {
self.ensure_authenticated(session)?;
{
let stream = self.get_stream(stream_id).with_error_context(|error| {
@@ -151,7 +151,7 @@ impl IggyShard {
self.metrics.increment_partitions(partitions_count);
self.metrics.increment_segments(partitions_count);
- Ok(created_topic_id)
+ Ok(Identifier::numeric(created_topic_id)?)
}
#[allow(clippy::too_many_arguments)]
@@ -165,7 +165,7 @@ impl IggyShard {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
- ) -> Result<u32, IggyError> {
+ ) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
{
let stream = self.get_stream(stream_id).with_error_context(|error| {
@@ -213,14 +213,7 @@ impl IggyShard {
// TODO: if message_expiry is changed, we need to check if we need to purge messages based on the new expiry
// TODO: if max_size_bytes is changed, we need to check if we need to purge messages based on the new size
// TODO: if replication_factor is changed, we need to do `something`
- self.get_stream(stream_id)
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
- })?
- .get_topic(topic_id)
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get topic with ID: {topic_id} in stream with ID: {stream_id}")
- }).map(|topic| topic.topic_id)
+ Ok(())
}
pub async fn delete_topic(
diff --git a/core/server/src/streaming/clients/client_manager.rs b/core/server/src/streaming/clients/client_manager.rs
index fc5c00cc..922f4f11 100644
--- a/core/server/src/streaming/clients/client_manager.rs
+++ b/core/server/src/streaming/clients/client_manager.rs
@@ -84,7 +84,7 @@ impl ClientManager {
return Err(IggyError::ClientNotFound(client_id));
}
- let mut client = client.unwrap();
+ let client = client.unwrap();
client.user_id = Some(user_id);
Ok(())
}
@@ -95,7 +95,7 @@ impl ClientManager {
return Err(IggyError::ClientNotFound(client_id));
}
- let mut client = client.unwrap();
+ let client = client.unwrap();
client.user_id = None;
Ok(())
}
@@ -104,6 +104,10 @@ impl ClientManager {
self.clients.get(&client_id).cloned()
}
+ pub fn try_get_client_mut(&mut self, client_id: u32) -> Option<&mut Client> {
+ self.clients.get_mut(&client_id)
+ }
+
pub fn get_clients(&self) -> Vec<Client> {
self.clients.values().cloned().collect()
}
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index 92168738..a2efba95 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -172,7 +172,7 @@ impl Topic {
&mut self,
group_id: Option<u32>,
name: &str,
- ) -> Result<ConsumerGroup, IggyError> {
+ ) -> Result<Identifier, IggyError> {
if self.consumer_groups_ids.contains_key(name) {
return Err(IggyError::ConsumerGroupNameAlreadyExists(
name.to_owned(),
@@ -208,13 +208,12 @@ impl Topic {
let consumer_group =
ConsumerGroup::new(self.topic_id, id, name, self.partitions.len() as u32);
self.consumer_groups_ids.insert(name.to_owned(), id);
- let cloned_group = consumer_group.clone();
self.consumer_groups.borrow_mut().insert(id, consumer_group);
info!(
"Created consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
id, self.topic_id, self.stream_id
);
- Ok(cloned_group)
+ Ok(Identifier::numeric(id)?)
}
pub async fn delete_consumer_group(
diff --git a/core/server/src/streaming/utils/pooled_buffer.rs b/core/server/src/streaming/utils/pooled_buffer.rs
index 9c4375d2..0aa1dd3a 100644
--- a/core/server/src/streaming/utils/pooled_buffer.rs
+++ b/core/server/src/streaming/utils/pooled_buffer.rs
@@ -18,6 +18,7 @@
use super::memory_pool::{BytesMutExt, memory_pool};
use bytes::{Buf, BufMut, BytesMut};
+use monoio::buf::IoBufMut;
use std::ops::{Deref, DerefMut};
#[derive(Debug)]
@@ -215,3 +216,17 @@ impl Buf for PooledBuffer {
self.inner.chunks_vectored(dst)
}
}
+
+unsafe impl IoBufMut for PooledBuffer {
+ fn write_ptr(&mut self) -> *mut u8 {
+ self.inner.write_ptr()
+ }
+
+ fn bytes_total(&mut self) -> usize {
+ self.inner.bytes_total()
+ }
+
+ unsafe fn set_init(&mut self, pos: usize) {
+ unsafe { self.inner.set_init(pos) }
+ }
+}
diff --git a/core/server/src/tcp/sender.rs b/core/server/src/tcp/sender.rs
index 6a10adb2..22796812 100644
--- a/core/server/src/tcp/sender.rs
+++ b/core/server/src/tcp/sender.rs
@@ -28,12 +28,13 @@ use tracing::debug;
const STATUS_OK: &[u8] = &[0; 4];
-pub(crate) async fn read<T>(
+pub(crate) async fn read<T, B>(
stream: &mut T,
- buffer: BytesMut,
-) -> (Result<usize, IggyError>, BytesMut)
+ buffer: B,
+) -> (Result<usize, IggyError>, B)
where
T: AsyncReadRent + AsyncWriteRent + Unpin,
+ B: IoBufMut,
{
match stream.read_exact(buffer).await {
(Ok(0), buffer) => (Err(IggyError::ConnectionClosed), buffer),
diff --git a/core/server/src/tcp/tcp_sender.rs b/core/server/src/tcp/tcp_sender.rs
index 81089c85..cfa25aad 100644
--- a/core/server/src/tcp/tcp_sender.rs
+++ b/core/server/src/tcp/tcp_sender.rs
@@ -33,7 +33,7 @@ pub struct TcpSender {
}
impl Sender for TcpSender {
- async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut) {
+ async fn read<B: IoBufMut>(&mut self, buffer: B) -> (Result<usize, IggyError>, B) {
sender::read(&mut self.stream, buffer).await
}
diff --git a/core/server/src/tcp/tcp_tls_sender.rs b/core/server/src/tcp/tcp_tls_sender.rs
index 7213ad64..167427c1 100644
--- a/core/server/src/tcp/tcp_tls_sender.rs
+++ b/core/server/src/tcp/tcp_tls_sender.rs
@@ -22,6 +22,7 @@ use crate::{server_error::ServerError, tcp::sender};
use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::IggyError;
+use monoio::buf::IoBufMut;
use monoio::io::AsyncWriteRent;
use monoio::net::TcpStream;
use monoio_native_tls::TlsStream;
@@ -33,7 +34,7 @@ pub struct TcpTlsSender {
}
impl Sender for TcpTlsSender {
- async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut) {
+ async fn read<B: IoBufMut>(&mut self, buffer: B) -> (Result<usize, IggyError>, B) {
sender::read(&mut self.stream, buffer).await
}
(iggy) 05/30: todos
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit cee584cbb86571bf2e265af5676872287f416e35
Author: numinex <g....@gmail.com>
AuthorDate: Thu Jun 19 10:17:42 2025 +0200
todos
---
Cargo.lock | 14 +++++++++++++
core/sdk/src/tcp/tcp_client.rs | 1 +
core/server/src/binary/command.rs | 1 +
core/server/src/bootstrap.rs | 27 ++++++++++++++++++-------
core/server/src/main.rs | 23 ++++++++--------------
core/server/src/shard/builder.rs | 40 +++++++++++++++++++++++++-------------
core/server/src/shard/connector.rs | 4 ++--
core/server/src/shard/mod.rs | 38 ++++++++++++++++++++++++++++--------
8 files changed, 103 insertions(+), 45 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index a79f4b82..40849f7d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -143,6 +143,7 @@ dependencies = [
"futures-core",
"futures-util",
"mio 1.0.4",
+ "mio 1.0.4",
"socket2",
"tokio",
"tracing",
@@ -4776,6 +4777,18 @@ dependencies = [
"windows-sys 0.48.0",
]
+[[package]]
+name = "mio"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+dependencies = [
+ "libc",
+ "log",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "mio"
version = "1.0.4"
@@ -7545,6 +7558,7 @@ dependencies = [
"bytes",
"libc",
"mio 1.0.4",
+ "mio 1.0.4",
"parking_lot 0.12.4",
"pin-project-lite",
"signal-hook-registry",
diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs
index 692d7689..2bab2efb 100644
--- a/core/sdk/src/tcp/tcp_client.rs
+++ b/core/sdk/src/tcp/tcp_client.rs
@@ -17,6 +17,7 @@
*/
use crate::prelude::Client;
+use crate::prelude::IggyConsumer;
use crate::prelude::TcpClientConfig;
use crate::tcp::tcp_connection_stream::TcpConnectionStream;
use crate::tcp::tcp_connection_stream_kind::ConnectionStreamKind;
diff --git a/core/server/src/binary/command.rs b/core/server/src/binary/command.rs
index c7802be9..0b997941 100644
--- a/core/server/src/binary/command.rs
+++ b/core/server/src/binary/command.rs
@@ -66,6 +66,7 @@ use iggy_common::update_stream::UpdateStream;
use iggy_common::update_topic::UpdateTopic;
use iggy_common::update_user::UpdateUser;
use iggy_common::*;
+use rustls::crypto::hash::Output;
use strum::EnumString;
use tracing::error;
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index d51096a2..d70d0317 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -1,22 +1,19 @@
use iggy_common::{
+ IggyError,
defaults::{
DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH,
MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH,
},
- IggyError,
-};
-use monoio::{
- fs::create_dir_all,
- Buildable, Driver, Runtime,
};
+use monoio::{fs::create_dir_all, time::TimeDriver, Buildable, Driver, Runtime};
use tracing::info;
use crate::{
+ IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
configs::{config_provider::ConfigProviderKind, server::ServerConfig, system::SystemConfig},
server_error::ServerError,
shard::{connector::ShardConnector, frame::ShardFrame},
streaming::users::user::User,
- IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
};
use std::{env, fs::remove_dir_all, ops::Range, path::Path};
@@ -109,7 +106,9 @@ pub fn create_root_user() -> User {
let username = env::var(IGGY_ROOT_USERNAME_ENV);
let password = env::var(IGGY_ROOT_PASSWORD_ENV);
if (username.is_ok() && password.is_err()) || (username.is_err() && password.is_ok()) {
- panic!("When providing the custom root user credentials, both username and password must be set.");
+ panic!(
+ "When providing the custom root user credentials, both username and password must be set."
+ );
}
if username.is_ok() && password.is_ok() {
info!("Using the custom root user credentials.");
@@ -146,3 +145,17 @@ where
let rt = Buildable::build(builder).expect("Failed to create default runtime");
rt
}
+
+pub fn create_shard_executor() -> Runtime<TimeDriver<monoio::IoUringDriver>>
+{
+ // TODO: Figure out what else we could tweak there
+ // We for sure want to disable the userspace interrupts on new cq entry (set_coop_taskrun)
+ // let urb = io_uring::IoUring::builder();
+ // TODO: Shall we make the size of ring be configureable ?
+ let builder = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new()
+ //.uring_builder(urb.setup_coop_taskrun()) // broken shit.
+ .with_entries(1024) // Default size
+ .enable_timer();
+ let rt = Buildable::build(builder).expect("Failed to create default runtime");
+ rt
+}
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index bb4ea690..06ffec35 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -23,11 +23,9 @@ use clap::Parser;
use dotenvy::dotenv;
use error_set::ErrContext;
use figlet_rs::FIGfont;
-use monoio::Buildable;
use server::args::Args;
use server::bootstrap::{
- create_default_executor, create_directories, create_root_user, create_shard_connections,
- load_config,
+ create_default_executor, create_directories, create_root_user, create_shard_connections, create_shard_executor, load_config
};
use server::channels::commands::archive_state::ArchiveStateExecutor;
use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
@@ -112,7 +110,7 @@ fn main() -> Result<(), ServerError> {
}
}
- // Create directories.
+ // Create directories.
create_directories(&config.system).await?;
Ok::<(), ServerError>(())
})
@@ -146,16 +144,7 @@ fn main() -> Result<(), ServerError> {
monoio::utils::bind_to_cpu_set(Some(shard_id))
.expect(format!("Failed to set CPU affinity for shard-{id}").as_str());
- // TODO: Figure out what else we could tweak there
- // We for sure want to disable the userspace interrupts on new cq entry (set_coop_taskrun)
- // let urb = io_uring::IoUring::builder();
- // TODO: Shall we make the size of ring be configureable ?
- let mut rt = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new()
- //.uring_builder(urb.setup_coop_taskrun()) // broken shit.
- .with_entries(1024) // Default size
- .enable_timer()
- .build()
- .expect(format!("Failed to build monoio runtime for shard-{id}").as_str());
+ let mut rt = create_shard_executor();
rt.block_on(async move {
let builder = IggyShard::builder();
let mut shard = builder
@@ -165,7 +154,11 @@ fn main() -> Result<(), ServerError> {
.build()
.await;
- shard.init().await;
+ if let Err(e) = shard.init().await {
+ //TODO: If one of the shards fails to initialize, we should crash the whole program;
+ panic!("Failed to initialize shard-{id}: {e}");
+ }
+ //TODO: If one of the shards fails to initialize, we should crash the whole program;
shard.assert_init();
})
})
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 640ffb57..06366ebf 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -21,9 +21,19 @@ use std::{cell::Cell, rc::Rc, sync::Arc};
use iggy_common::{Aes256GcmEncryptor, EncryptorKind};
use tracing::info;
-use crate::{configs::server::ServerConfig, map_toggle_str, shard::Shard, state::{file::FileState, StateKind}, streaming::{persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind}, storage::SystemStorage}, versioning::SemanticVersion};
+use crate::{
+ configs::server::ServerConfig,
+ map_toggle_str,
+ shard::Shard,
+ state::{StateKind, file::FileState},
+ streaming::{
+ persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind},
+ storage::SystemStorage,
+ },
+ versioning::SemanticVersion,
+};
-use super::{connector::ShardConnector, frame::ShardFrame, IggyShard};
+use super::{IggyShard, connector::ShardConnector, frame::ShardFrame};
#[derive(Default)]
pub struct IggyShardBuilder {
@@ -66,8 +76,10 @@ impl IggyShardBuilder {
.next()
.expect("Failed to find connection with the specified ID");
let shards = connections.into_iter().map(Shard::new).collect();
+ //TODO: This can be discrete step in the builder bootstrapped from main function.
let version = SemanticVersion::current().expect("Invalid version");
+ //TODO: This can be discrete step in the builder bootstrapped from main function.
info!(
"Server-side encryption is {}.",
map_toggle_str(config.system.encryption.enabled)
@@ -79,6 +91,7 @@ impl IggyShardBuilder {
false => None,
};
+ //TODO: This can be discrete step in the builder bootstrapped from main function.
let state_persister = Self::resolve_persister(config.system.state.enforce_fsync);
let state = Rc::new(StateKind::File(FileState::new(
&config.system.get_state_messages_file_path(),
@@ -87,20 +100,21 @@ impl IggyShardBuilder {
encryptor.clone(),
)));
+ //TODO: This can be discrete step in the builder bootstrapped from main function.
let partition_persister = Self::resolve_persister(config.system.partition.enforce_fsync);
- let storage = SystemStorage::new(config.system, partition_persister);
+ let storage = Rc::new(SystemStorage::new(config.system.clone(), partition_persister));
IggyShard {
- id: id,
- shards: shards,
- shards_table: Default::default(),
- storage: storage,
- state: state,
- config: config,
- stop_receiver: stop_receiver,
- stop_sender: stop_sender,
- frame_receiver: Cell::new(Some(frame_receiver)),
- }
+ id: id,
+ shards: shards,
+ shards_table: Default::default(),
+ storage: storage,
+ state: state,
+ config: config,
+ stop_receiver: stop_receiver,
+ stop_sender: stop_sender,
+ frame_receiver: Cell::new(Some(frame_receiver)),
+ }
}
fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
diff --git a/core/server/src/shard/connector.rs b/core/server/src/shard/connector.rs
index 1d762d68..95aedc73 100644
--- a/core/server/src/shard/connector.rs
+++ b/core/server/src/shard/connector.rs
@@ -15,10 +15,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-use futures::{task::AtomicWaker, Stream};
+use futures::{Stream, task::AtomicWaker};
use sharded_queue::ShardedQueue;
use std::{
- sync::{atomic::AtomicUsize, Arc},
+ sync::{Arc, atomic::AtomicUsize},
task::Poll,
};
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 54b88fb1..b98f6381 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -25,10 +25,19 @@ use ahash::HashMap;
use builder::IggyShardBuilder;
use connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use frame::ShardFrame;
+use iggy_common::IggyError;
use namespace::IggyNamespace;
-use std::{cell::{Cell, RefCell}, rc::Rc, sync::Arc};
+use tracing::info;
+use std::{
+ cell::{Cell, RefCell},
+ rc::Rc,
+ sync::Arc, time::Instant,
+};
-use crate::{bootstrap::create_root_user, configs::server::ServerConfig, state::file::FileState, streaming::storage::SystemStorage};
+use crate::{
+ bootstrap::create_root_user, configs::server::ServerConfig, state::{file::FileState, StateKind},
+ streaming::storage::SystemStorage,
+};
pub(crate) struct Shard {
id: u16,
connection: ShardConnector<ShardFrame>,
@@ -57,10 +66,10 @@ pub struct IggyShard {
//pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
//pub(crate) users: RefCell<HashMap<UserId, User>>,
// TODO: Refactor.
- pub(crate) storage: Arc<SystemStorage>,
+ pub(crate) storage: Rc<SystemStorage>,
// TODO - get rid of this dynamic dispatch.
- pub(crate) state: Rc<FileState>,
+ pub(crate) state: Rc<StateKind>,
//pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
config: ServerConfig,
//pub(crate) client_manager: RefCell<ClientManager>,
@@ -76,14 +85,27 @@ impl IggyShard {
Default::default()
}
- pub async fn init(&mut self) {
- let user = create_root_user();
+ pub async fn init(&mut self) -> Result<(), IggyError> {
+ let now = Instant::now();
+ //TODO: Fix this either by moving it to main function, or by using `run_once` barrier.
+ //let state_entries = self.state.init().await?;
+ //let system_state = SystemState::init(state_entries).await?;
+ //let user = create_root_user();
self.load_state().await;
self.load_users().await;
// Add default root user.
- todo!();
self.load_streams().await;
-
+ //TODO: Fix the archiver.
+ /*
+ if let Some(archiver) = self.archiver.as_ref() {
+ archiver
+ .init()
+ .await
+ .expect("Failed to initialize archiver");
+ }
+ */
+ info!("Initialized system in {} ms.", now.elapsed().as_millis());
+ Ok(())
}
async fn load_state(&self) {
(iggy) 08/30: temp
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 8101ded4ac960428ee7937d25155aa9c0c4c0d41
Author: numinex <g....@gmail.com>
AuthorDate: Tue Jun 24 11:52:06 2025 +0200
temp
---
Cargo.lock | 83 ++++++++++-
core/server/Cargo.toml | 3 +-
core/server/src/binary/sender.rs | 16 +--
core/server/src/configs/server.rs | 3 +-
core/server/src/main.rs | 8 +-
core/server/src/quic/listener.rs | 3 +
core/server/src/shard/builder.rs | 16 ++-
core/server/src/shard/gate.rs | 2 +-
core/server/src/shard/mod.rs | 158 ++++++++++++++-------
core/server/src/shard/namespace.rs | 17 ++-
.../{streaming/systems => shard/system}/clients.rs | 2 +-
.../systems => shard/system}/consumer_groups.rs | 2 +-
.../systems => shard/system}/consumer_offsets.rs | 2 +-
.../{streaming/systems => shard/system}/info.rs | 0
.../systems => shard/system}/messages.rs | 5 +-
.../src/{streaming/systems => shard/system}/mod.rs | 2 +-
.../systems => shard/system}/partitions.rs | 2 +-
.../system}/personal_access_tokens.rs | 2 +-
.../systems => shard/system}/segments.rs | 2 +-
.../systems => shard/system}/snapshot/mod.rs | 0
.../systems => shard/system}/snapshot/procdump.rs | 0
.../{streaming/systems => shard/system}/stats.rs | 2 +-
.../{streaming/systems => shard/system}/storage.rs | 2 -
.../{streaming/systems => shard/system}/streams.rs | 142 +-----------------
.../{streaming/systems => shard/system}/system.rs | 0
.../{streaming/systems => shard/system}/topics.rs | 0
.../{streaming/systems => shard/system}/users.rs | 106 +-------------
core/server/src/streaming/mod.rs | 1 -
core/server/src/streaming/storage.rs | 5 +-
core/server/src/streaming/streams/stream.rs | 9 +-
core/server/src/tcp/sender.rs | 16 +--
core/server/src/tcp/tcp_listener.rs | 58 +++-----
core/server/src/tcp/tcp_sender.rs | 3 +-
core/server/src/tcp/tcp_server.rs | 21 ++-
core/server/src/tcp/tcp_socket.rs | 1 -
core/server/src/tcp/tcp_tls_sender.rs | 5 +-
36 files changed, 295 insertions(+), 404 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 267da703..17894833 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -653,6 +653,7 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
+ "syn 2.0.104",
]
[[package]]
@@ -1684,6 +1685,16 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "core-foundation"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
[[package]]
name = "core-foundation"
version = "0.10.1"
@@ -3327,6 +3338,12 @@ dependencies = [
"byteorder",
]
+[[package]]
+name = "hash32"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e4e11d13d42fc8d55326b323c63978c75721fbbb695a6e6686765bcb8b33917"
+
[[package]]
name = "hashbrown"
version = "0.12.3"
@@ -3382,7 +3399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f"
dependencies = [
"atomic-polyfill",
- "hash32",
+ "hash32 0.2.1",
"rustc_version",
"serde",
"spin",
@@ -4881,6 +4898,15 @@ dependencies = [
"windows-sys 0.48.0",
]
+[[package]]
+name = "monoio-io-wrapper"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bcfaa76e5daf87cc4d31b4d1b6bc93c12db59c19df50b9200afdbde42077655"
+dependencies = [
+ "monoio",
+]
+
[[package]]
name = "monoio-macros"
version = "0.1.0"
@@ -4890,6 +4916,20 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
+ "syn 2.0.104",
+]
+
+[[package]]
+name = "monoio-native-tls"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9022f5aaa19f9688f97bfcfa0c4a4318d424851995badb356674ca742652cdb"
+dependencies = [
+ "bytes",
+ "monoio",
+ "monoio-io-wrapper",
+ "native-tls",
+ "thiserror 1.0.69",
]
[[package]]
@@ -4901,6 +4941,23 @@ dependencies = [
"getrandom 0.2.16",
]
+[[package]]
+name = "native-tls"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework 2.11.1",
+ "security-framework-sys",
+ "tempfile",
+]
+
[[package]]
name = "never-say-never"
version = "6.6.666"
@@ -6573,7 +6630,7 @@ dependencies = [
"openssl-probe",
"rustls-pki-types",
"schannel",
- "security-framework",
+ "security-framework 3.2.0",
]
[[package]]
@@ -6601,7 +6658,7 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1"
dependencies = [
- "core-foundation",
+ "core-foundation 0.10.1",
"core-foundation-sys",
"jni",
"log",
@@ -6610,7 +6667,7 @@ dependencies = [
"rustls-native-certs",
"rustls-platform-verifier-android",
"rustls-webpki",
- "security-framework",
+ "security-framework 3.2.0",
"security-framework-sys",
"webpki-root-certs 0.26.11",
"windows-sys 0.59.0",
@@ -6758,6 +6815,19 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "security-framework"
+version = "2.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
+dependencies = [
+ "bitflags 2.9.1",
+ "core-foundation 0.9.4",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
[[package]]
name = "security-framework"
version = "3.2.0"
@@ -6765,7 +6835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
dependencies = [
"bitflags 2.9.1",
- "core-foundation",
+ "core-foundation 0.10.1",
"core-foundation-sys",
"libc",
"security-framework-sys",
@@ -6973,15 +7043,16 @@ dependencies = [
"figment",
"flume",
"futures",
+ "hash32 1.0.0",
"human-repr",
"iggy_common",
- "io-uring",
"jsonwebtoken",
"lending-iterator",
"mimalloc",
"mockall",
"moka",
"monoio",
+ "monoio-native-tls",
"nix 0.30.1",
"once_cell",
"opentelemetry",
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 138bdc81..a493e837 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -60,12 +60,13 @@ async-channel = { workspace = true }
futures = { workspace = true }
human-repr = { workspace = true }
iggy_common = { workspace = true }
-io-uring = "0.6"
jsonwebtoken = "9.3.1"
lending-iterator = "0.1.7"
+hash32 = "1.0.0"
mimalloc = { workspace = true, optional = true }
moka = { version = "0.12.10", features = ["future"] }
monoio = { version = "0.2.4", features = ["mkdirat", "unlinkat"] }
+monoio-native-tls = "0.4.0"
nix = { version = "0.30", features = ["fs"] }
once_cell = "1.21.3"
opentelemetry = { version = "0.30.0", features = ["trace", "logs"] }
diff --git a/core/server/src/binary/sender.rs b/core/server/src/binary/sender.rs
index 7273543c..2ccf349b 100644
--- a/core/server/src/binary/sender.rs
+++ b/core/server/src/binary/sender.rs
@@ -24,8 +24,8 @@ use crate::tcp::tcp_tls_sender::TcpTlsSender;
use crate::{quic::quic_sender::QuicSender, server_error::ServerError};
use iggy_common::IggyError;
use quinn::{RecvStream, SendStream};
-use tokio::net::TcpStream;
-use tokio_rustls::server::TlsStream;
+use monoio::net::TcpStream;
+use tokio_native_tls::TlsStream;
macro_rules! forward_async_methods {
(
@@ -48,22 +48,22 @@ macro_rules! forward_async_methods {
}
pub trait Sender {
- fn read(&mut self, buffer: &mut [u8]) -> impl Future<Output = Result<usize, IggyError>> + Send;
- fn send_empty_ok_response(&mut self) -> impl Future<Output = Result<(), IggyError>> + Send;
+ fn read(&mut self, buffer: &mut [u8]) -> impl Future<Output = Result<usize, IggyError>>;
+ fn send_empty_ok_response(&mut self) -> impl Future<Output = Result<(), IggyError>>;
fn send_ok_response(
&mut self,
payload: &[u8],
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
fn send_ok_response_vectored(
&mut self,
length: &[u8],
slices: Vec<IoSlice<'_>>,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
fn send_error_response(
&mut self,
error: IggyError,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn shutdown(&mut self) -> impl Future<Output = Result<(), ServerError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn shutdown(&mut self) -> impl Future<Output = Result<(), ServerError>>;
}
#[allow(clippy::large_enum_variant)]
diff --git a/core/server/src/configs/server.rs b/core/server/src/configs/server.rs
index 252df9a7..059fd1c4 100644
--- a/core/server/src/configs/server.rs
+++ b/core/server/src/configs/server.rs
@@ -31,6 +31,7 @@ use iggy_common::Validatable;
use serde::{Deserialize, Serialize};
use serde_with::DisplayFromStr;
use serde_with::serde_as;
+use std::rc::Rc;
use std::str::FromStr;
use std::sync::Arc;
@@ -40,7 +41,7 @@ pub struct ServerConfig {
pub message_saver: MessageSaverConfig,
pub personal_access_token: PersonalAccessTokenConfig,
pub heartbeat: HeartbeatConfig,
- pub system: Arc<SystemConfig>,
+ pub system: Rc<SystemConfig>,
pub quic: QuicConfig,
pub tcp: TcpConfig,
pub http: HttpConfig,
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 3a6c1686..78b3a1e9 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -235,7 +235,7 @@ fn main() -> Result<(), ServerError> {
.await;
let builder = IggyShard::builder();
- let mut shard = builder
+ let shard: Rc<IggyShard> = builder
.id(id)
.connections(connections)
.config(config)
@@ -243,13 +243,13 @@ fn main() -> Result<(), ServerError> {
.version(version)
.state(state)
.build()
- .await;
+ .into();
//TODO: If one of the shards fails to initialize, we should crash the whole program;
- shard.init().await.expect("Failed to initialize shard-{id}: {e}");
- info!("Initiated shard with ID: {id}");
+ shard.run().await.expect("Failed to run shard");
//TODO: If one of the shards fails to initialize, we should crash the whole program;
shard.assert_init();
+ let shard = Rc::new(shard);
})
})
.expect(format!("Failed to spawn thread for shard-{id}").as_str())
diff --git a/core/server/src/quic/listener.rs b/core/server/src/quic/listener.rs
index b5eb859d..e9b329de 100644
--- a/core/server/src/quic/listener.rs
+++ b/core/server/src/quic/listener.rs
@@ -34,6 +34,8 @@ pub fn start(endpoint: Endpoint, system: SharedSystem) {
for _ in 0..LISTENERS_COUNT {
let endpoint = endpoint.clone();
let system = system.clone();
+ //TODO: Fixme
+ /*
tokio::spawn(async move {
while let Some(incoming_connection) = endpoint.accept().await {
info!(
@@ -57,6 +59,7 @@ pub fn start(endpoint: Endpoint, system: SharedSystem) {
});
}
});
+ */
}
}
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 0fd5250f..0428a483 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -26,8 +26,8 @@ use crate::{
configs::server::ServerConfig,
map_toggle_str,
shard::Shard,
- state::{StateKind, file::FileState},
- streaming::storage::SystemStorage,
+ state::{file::FileState, StateKind},
+ streaming::{diagnostics::metrics::Metrics, storage::SystemStorage},
versioning::SemanticVersion,
};
@@ -75,11 +75,12 @@ impl IggyShardBuilder {
}
// TODO: Too much happens in there, some of those bootstrapping logic should be moved outside.
- pub async fn build(self) -> IggyShard {
+ pub fn build(self) -> IggyShard {
let id = self.id.unwrap();
let config = self.config.unwrap();
let connections = self.connections.unwrap();
let state = self.state.unwrap();
+ let encryptor = self.encryptor;
let version = self.version.unwrap();
let (stop_sender, stop_receiver, frame_receiver) = connections
.iter()
@@ -106,12 +107,21 @@ impl IggyShardBuilder {
shards: shards,
shards_table: Default::default(),
storage: storage,
+ encryptor: encryptor,
state: state,
config: config,
version: version,
stop_receiver: stop_receiver,
stop_sender: stop_sender,
frame_receiver: Cell::new(Some(frame_receiver)),
+ metrics: Metrics::init(),
+
+ users: Default::default(),
+ permissioner: Default::default(),
+ streams: Default::default(),
+ streams_ids: Default::default(),
+ client_manager: Default::default(),
+ active_sessions: Default::default(),
}
}
}
diff --git a/core/server/src/shard/gate.rs b/core/server/src/shard/gate.rs
index a26950a4..2ac20445 100644
--- a/core/server/src/shard/gate.rs
+++ b/core/server/src/shard/gate.rs
@@ -1,4 +1,4 @@
-use std::sync::{Condvar, Mutex};
+use std::sync::Mutex;
#[derive(Default)]
pub struct Gate<T> {
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 23ef1e15..3e7e10f1 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -17,39 +17,37 @@
*/
pub mod builder;
+pub mod system;
pub mod gate;
pub mod namespace;
pub mod transmission;
-use ahash::HashMap;
+use ahash::{AHashMap, AHashSet, HashMap};
use builder::IggyShardBuilder;
use error_set::ErrContext;
-use iggy_common::IggyError;
+use futures::future::try_join_all;
+use iggy_common::{EncryptorKind, IggyError, UserId};
use namespace::IggyNamespace;
use std::{
- cell::{Cell, RefCell},
- rc::Rc,
- str::FromStr,
- sync::Arc,
- time::Instant,
+ cell::{Cell, RefCell}, pin::Pin, rc::Rc, str::FromStr, sync::{atomic::{AtomicU32, Ordering}, Arc, RwLock}, time::Instant
};
-use tracing::info;
+use tracing::{error, info, instrument, trace, warn};
use transmission::connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use crate::{
- bootstrap::create_root_user,
configs::server::ServerConfig,
- shard::transmission::frame::ShardFrame,
+ shard::{system::info::SystemInfo, transmission::frame::ShardFrame},
state::{
- StateKind,
- file::FileState,
- system::{SystemState, UserState},
+ file::FileState, system::{StreamState, SystemState, UserState}, StateKind
},
- streaming::{storage::SystemStorage, systems::info::SystemInfo},
+ streaming::{clients::client_manager::ClientManager, diagnostics::metrics::Metrics, personal_access_tokens::personal_access_token::PersonalAccessToken, session::Session, storage::SystemStorage, streams::stream::Stream, users::{permissioner::Permissioner, user::User}},
versioning::SemanticVersion,
};
pub const COMPONENT: &str = "SHARD";
+static USER_ID: AtomicU32 = AtomicU32::new(1);
+
+type Task = Pin<Box<dyn Future<Output = Result<(), IggyError>>>>;
pub(crate) struct Shard {
id: u16,
@@ -75,19 +73,21 @@ pub struct IggyShard {
shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
version: SemanticVersion,
- //pub(crate) permissioner: RefCell<Permissioner>,
- //pub(crate) streams: RwLock<HashMap<u32, Stream>>,
- //pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
- //pub(crate) users: RefCell<HashMap<UserId, User>>,
+ pub(crate) streams: RefCell<HashMap<u32, Stream>>,
+ pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
// TODO: Refactor.
pub(crate) storage: Rc<SystemStorage>,
pub(crate) state: StateKind,
- //pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
- config: ServerConfig,
- //pub(crate) client_manager: RefCell<ClientManager>,
- //pub(crate) active_sessions: RefCell<Vec<Session>>,
- //pub(crate) metrics: Metrics,
+ pub(crate) encryptor: Option<EncryptorKind>,
+ pub(crate) config: ServerConfig,
+ //TODO: This could be shared.
+ pub(crate) client_manager: RefCell<ClientManager>,
+ pub(crate) active_sessions: RefCell<Vec<Session>>,
+ pub(crate) permissioner: RefCell<Permissioner>,
+ pub(crate) users: RefCell<HashMap<UserId, User>>,
+
+ pub(crate) metrics: Metrics,
pub frame_receiver: Cell<Option<Receiver<ShardFrame>>>,
stop_receiver: StopReceiver,
stop_sender: StopSender,
@@ -98,16 +98,12 @@ impl IggyShard {
Default::default()
}
- pub async fn init(&mut self) -> Result<(), IggyError> {
+ pub async fn init(&self) -> Result<(), IggyError> {
let now = Instant::now();
- //TODO: Fix this either by moving it to main function, or by using `run_once` barrier.
- //let state_entries = self.state.init().await?;
- //let system_state = SystemState::init(state_entries).await?;
- //let user = create_root_user();
+
self.load_version().await?;
let SystemState { users, streams } = self.load_state().await?;
self.load_users(users.into_values().collect()).await;
- // Add default root user.
self.load_streams(streams.into_values().collect()).await;
//TODO: Fix the archiver.
@@ -123,6 +119,27 @@ impl IggyShard {
Ok(())
}
+ pub async fn run(self: &Rc<Self>) -> Result<(), IggyError> {
+ // Workaround to ensure that the statistics are initialized before the server
+ // loads streams and starts accepting connections. This is necessary to
+ // have the correct statistics when the server starts.
+ //self.get_stats().await?;
+ self.init().await?;
+ self.assert_init();
+ info!("Initiated shard with ID: {}", self.id);
+ // Create all tasks (tcp listener, http listener, command processor, in the future also the background jobs).
+ /*
+ let mut tasks: Vec<Task> = vec![Box::pin(spawn_shard_message_task(shard.clone()))];
+ if self.config.tcp.enabled {
+ tasks.push(Box::pin(spawn_tcp_server(self.clone())));
+ }
+ let result = try_join_all(tasks).await;
+ result?;
+ */
+
+ Ok(())
+ }
+
async fn load_version(&self) -> Result<(), IggyError> {
async fn update_system_info(
storage: &Rc<SystemStorage>,
@@ -181,10 +198,8 @@ impl IggyShard {
Ok(system_state)
}
- async fn load_users(&mut self, users: Vec<UserState>) -> Result<(), IggyError> {
+ async fn load_users(&self, users: Vec<UserState>) -> Result<(), IggyError> {
info!("Loading users...");
- /*
-
for user_state in users.into_iter() {
let mut user = User::with_password(
user_state.id,
@@ -210,32 +225,34 @@ impl IggyShard {
)
})
.collect();
- self.users.insert(user_state.id, user);
+ self.users.borrow_mut().insert(user_state.id, user);
}
- let users_count = self.users.len();
- let current_user_id = self.users.keys().max().unwrap_or(&1);
+ let users = self.users.borrow();
+ let users_count = users.len();
+ let current_user_id = users.keys().max().unwrap_or(&1);
USER_ID.store(current_user_id + 1, Ordering::SeqCst);
self.permissioner
- .init(&self.users.values().collect::<Vec<&User>>());
+ .borrow_mut()
+ .init(&users.values().collect::<Vec<_>>());
self.metrics.increment_users(users_count as u32);
info!("Initialized {users_count} user(s).");
- */
Ok(())
}
- async fn load_streams(&mut self, streams: Vec<StreamState>) -> Result<(), IggyError> {
- todo!();
+ async fn load_streams(&self, streams: Vec<StreamState>) -> Result<(), IggyError> {
info!("Loading streams from disk...");
let mut unloaded_streams = Vec::new();
- let mut dir_entries = read_dir(&self.config.get_streams_path())
- .await
+ // Does mononio has api for that ?
+ let mut dir_entries = std::fs::read_dir(&self.config.system.get_streams_path())
.map_err(|error| {
error!("Cannot read streams directory: {error}");
IggyError::CannotReadStreams
})?;
- while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) {
+ //TODO: User the dir walk impl from main function, once implemented.
+ while let Some(dir_entry) = dir_entries.next() {
+ let dir_entry = dir_entry.unwrap();
let name = dir_entry.file_name().into_string().unwrap();
let stream_id = name.parse::<u32>().map_err(|_| {
error!("Invalid stream ID file with name: '{name}'.");
@@ -246,7 +263,7 @@ impl IggyShard {
error!(
"Stream with ID: '{stream_id}' was not found in state, but exists on disk and will be removed."
);
- if let Err(error) = fs::remove_dir_all(&dir_entry.path()).await {
+ if let Err(error) = std::fs::remove_dir_all(&dir_entry.path()) {
error!("Cannot remove stream directory: {error}");
} else {
warn!("Stream with ID: '{stream_id}' was removed.");
@@ -258,7 +275,7 @@ impl IggyShard {
let mut stream = Stream::empty(
stream_id,
&stream_state.name,
- self.config.clone(),
+ self.config.system.clone(),
self.storage.clone(),
);
stream.created_at = stream_state.created_at;
@@ -281,7 +298,7 @@ impl IggyShard {
info!("All streams found on disk were found in state.");
} else {
warn!("Streams with IDs: '{missing_ids:?}' were not found on disk.");
- if self.config.recovery.recreate_missing_state {
+ if self.config.system.recovery.recreate_missing_state {
info!(
"Recreating missing state in recovery config is enabled, missing streams will be created."
);
@@ -291,7 +308,7 @@ impl IggyShard {
let stream = Stream::create(
stream_id,
&stream_state.name,
- self.config.clone(),
+ self.config.system.clone(),
self.storage.clone(),
);
stream.persist().await?;
@@ -327,12 +344,12 @@ impl IggyShard {
try_join_all(load_stream_tasks).await?;
for stream in loaded_streams.take() {
- if self.streams.contains_key(&stream.stream_id) {
+ if self.streams.borrow().contains_key(&stream.stream_id) {
error!("Stream with ID: '{}' already exists.", &stream.stream_id);
continue;
}
- if self.streams_ids.contains_key(&stream.name) {
+ if self.streams_ids.borrow().contains_key(&stream.name) {
error!("Stream with name: '{}' already exists.", &stream.name);
continue;
}
@@ -345,13 +362,52 @@ impl IggyShard {
self.metrics.increment_messages(stream.get_messages_count());
self.streams_ids
+ .borrow_mut()
.insert(stream.name.clone(), stream.stream_id);
- self.streams.insert(stream.stream_id, stream);
+ self.streams.borrow_mut().insert(stream.stream_id, stream);
}
- info!("Loaded {} stream(s) from disk.", self.streams.len());
+ info!("Loaded {} stream(s) from disk.", self.streams.borrow().len());
Ok(())
}
- pub fn assert_init(&self) {}
+ pub fn assert_init(&self) -> Result<(), IggyError> { Ok(())}
+
+ #[instrument(skip_all, name = "trace_shutdown")]
+ pub async fn shutdown(&mut self) -> Result<(), IggyError> {
+ self.persist_messages().await?;
+ Ok(())
+ }
+
+ #[instrument(skip_all, name = "trace_persist_messages")]
+ pub async fn persist_messages(&self) -> Result<usize, IggyError> {
+ trace!("Saving buffered messages on disk...");
+ let mut saved_messages_number = 0;
+ //TODO: Fixme
+ /*
+ for stream in self.streams.values() {
+ saved_messages_number += stream.persist_messages().await?;
+ }
+ */
+
+ Ok(saved_messages_number)
+ }
+
+ pub fn get_available_shards_count(&self) -> u32 {
+ self.shards.len() as u32
+ }
+
+ pub fn ensure_authenticated(&self, client_id: u32) -> Result<u32, IggyError> {
+ let active_sessions = self.active_sessions.borrow();
+ let session = active_sessions
+ .iter()
+ .find(|s| s.client_id == client_id)
+ .ok_or_else(|| IggyError::Unauthenticated)?;
+ if session.is_authenticated() {
+ Ok(session.get_user_id())
+ } else {
+ error!("{COMPONENT} - unauthenticated access attempt, session: {session}");
+ Err(IggyError::Unauthenticated)
+ }
+ }
}
diff --git a/core/server/src/shard/namespace.rs b/core/server/src/shard/namespace.rs
index d9aedfb7..d9d68ec9 100644
--- a/core/server/src/shard/namespace.rs
+++ b/core/server/src/shard/namespace.rs
@@ -16,15 +16,20 @@
* under the License.
*/
+use std::hash::Hasher as _;
+use iggy_common::Identifier;
+use hash32::{Hasher, Murmur3Hasher};
+
+//TODO: Will probably want to move it to separate crate so we can share it with sdk.
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct IggyNamespace {
- pub(crate) stream_id: u32,
- pub(crate) topic_id: u32,
+ pub(crate) stream_id: Identifier,
+ pub(crate) topic_id: Identifier,
pub(crate) partition_id: u32,
}
impl IggyNamespace {
- pub fn new(stream_id: u32, topic_id: u32, partition_id: u32) -> Self {
+ pub fn new(stream_id: Identifier, topic_id: Identifier, partition_id: u32) -> Self {
Self {
stream_id,
topic_id,
@@ -33,6 +38,10 @@ impl IggyNamespace {
}
pub fn generate_hash(&self) -> u32 {
- todo!();
+ let mut hasher = Murmur3Hasher::default();
+ hasher.write(&self.stream_id.value);
+ hasher.write(&self.topic_id.value);
+ hasher.write_u32(self.partition_id);
+ hasher.finish32()
}
}
diff --git a/core/server/src/streaming/systems/clients.rs b/core/server/src/shard/system/clients.rs
similarity index 99%
rename from core/server/src/streaming/systems/clients.rs
rename to core/server/src/shard/system/clients.rs
index 050a0b58..2cb0fbe8 100644
--- a/core/server/src/streaming/systems/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -29,7 +29,7 @@ use std::net::SocketAddr;
use std::sync::Arc;
use tracing::{error, info};
-impl System {
+impl IggyShard {
pub async fn add_client(&self, address: &SocketAddr, transport: Transport) -> Arc<Session> {
let mut client_manager = self.client_manager.write().await;
let session = client_manager.add_client(address, transport);
diff --git a/core/server/src/streaming/systems/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs
similarity index 99%
rename from core/server/src/streaming/systems/consumer_groups.rs
rename to core/server/src/shard/system/consumer_groups.rs
index 19f29ccd..4975a92d 100644
--- a/core/server/src/streaming/systems/consumer_groups.rs
+++ b/core/server/src/shard/system/consumer_groups.rs
@@ -26,7 +26,7 @@ use iggy_common::IggyError;
use iggy_common::locking::IggySharedMutFn;
use tokio::sync::RwLock;
-impl System {
+impl IggyShard {
pub fn get_consumer_group(
&self,
session: &Session,
diff --git a/core/server/src/streaming/systems/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs
similarity index 99%
rename from core/server/src/streaming/systems/consumer_offsets.rs
rename to core/server/src/shard/system/consumer_offsets.rs
index fdac2b0c..e63815ff 100644
--- a/core/server/src/streaming/systems/consumer_offsets.rs
+++ b/core/server/src/shard/system/consumer_offsets.rs
@@ -22,7 +22,7 @@ use crate::streaming::systems::system::System;
use error_set::ErrContext;
use iggy_common::{Consumer, ConsumerOffsetInfo, Identifier, IggyError};
-impl System {
+impl IggyShard {
pub async fn store_consumer_offset(
&self,
session: &Session,
diff --git a/core/server/src/streaming/systems/info.rs b/core/server/src/shard/system/info.rs
similarity index 100%
rename from core/server/src/streaming/systems/info.rs
rename to core/server/src/shard/system/info.rs
diff --git a/core/server/src/streaming/systems/messages.rs b/core/server/src/shard/system/messages.rs
similarity index 99%
rename from core/server/src/streaming/systems/messages.rs
rename to core/server/src/shard/system/messages.rs
index 7bb772aa..54046dba 100644
--- a/core/server/src/streaming/systems/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -17,10 +17,9 @@
*/
use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
+use crate::shard::IggyShard;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet};
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use crate::streaming::utils::PooledBuffer;
use error_set::ErrContext;
use iggy_common::{
@@ -29,7 +28,7 @@ use iggy_common::{
};
use tracing::{error, trace};
-impl System {
+impl IggyShard {
pub async fn poll_messages(
&self,
session: &Session,
diff --git a/core/server/src/streaming/systems/mod.rs b/core/server/src/shard/system/mod.rs
similarity index 95%
rename from core/server/src/streaming/systems/mod.rs
rename to core/server/src/shard/system/mod.rs
index 7039e919..1419abb9 100644
--- a/core/server/src/streaming/systems/mod.rs
+++ b/core/server/src/shard/system/mod.rs
@@ -32,4 +32,4 @@ pub mod system;
pub mod topics;
pub mod users;
-pub const COMPONENT: &str = "STREAMING_SYSTEMS";
+pub const COMPONENT: &str = "SYSTEM";
diff --git a/core/server/src/streaming/systems/partitions.rs b/core/server/src/shard/system/partitions.rs
similarity index 99%
rename from core/server/src/streaming/systems/partitions.rs
rename to core/server/src/shard/system/partitions.rs
index 5bb72875..7712f498 100644
--- a/core/server/src/streaming/systems/partitions.rs
+++ b/core/server/src/shard/system/partitions.rs
@@ -23,7 +23,7 @@ use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
-impl System {
+impl IggyShard {
pub async fn create_partitions(
&mut self,
session: &Session,
diff --git a/core/server/src/streaming/systems/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs
similarity index 99%
rename from core/server/src/streaming/systems/personal_access_tokens.rs
rename to core/server/src/shard/system/personal_access_tokens.rs
index 945c853b..18125bed 100644
--- a/core/server/src/streaming/systems/personal_access_tokens.rs
+++ b/core/server/src/shard/system/personal_access_tokens.rs
@@ -27,7 +27,7 @@ use iggy_common::IggyExpiry;
use iggy_common::IggyTimestamp;
use tracing::{error, info};
-impl System {
+impl IggyShard {
pub async fn get_personal_access_tokens(
&self,
session: &Session,
diff --git a/core/server/src/streaming/systems/segments.rs b/core/server/src/shard/system/segments.rs
similarity index 99%
rename from core/server/src/streaming/systems/segments.rs
rename to core/server/src/shard/system/segments.rs
index a15952c1..bd4043a2 100644
--- a/core/server/src/streaming/systems/segments.rs
+++ b/core/server/src/shard/system/segments.rs
@@ -23,7 +23,7 @@ use iggy_common::Identifier;
use iggy_common::IggyError;
use iggy_common::locking::IggySharedMutFn;
-impl System {
+impl IggyShard {
pub async fn delete_segments(
&mut self,
session: &Session,
diff --git a/core/server/src/streaming/systems/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs
similarity index 100%
rename from core/server/src/streaming/systems/snapshot/mod.rs
rename to core/server/src/shard/system/snapshot/mod.rs
diff --git a/core/server/src/streaming/systems/snapshot/procdump.rs b/core/server/src/shard/system/snapshot/procdump.rs
similarity index 100%
rename from core/server/src/streaming/systems/snapshot/procdump.rs
rename to core/server/src/shard/system/snapshot/procdump.rs
diff --git a/core/server/src/streaming/systems/stats.rs b/core/server/src/shard/system/stats.rs
similarity index 99%
rename from core/server/src/streaming/systems/stats.rs
rename to core/server/src/shard/system/stats.rs
index 80c588e6..2d2d2ea7 100644
--- a/core/server/src/streaming/systems/stats.rs
+++ b/core/server/src/shard/system/stats.rs
@@ -34,7 +34,7 @@ fn sysinfo() -> &'static Mutex<SysinfoSystem> {
})
}
-impl System {
+impl IggyShard {
pub async fn get_stats(&self) -> Result<Stats, IggyError> {
let mut sys = sysinfo().lock().await;
let process_id = std::process::id();
diff --git a/core/server/src/streaming/systems/storage.rs b/core/server/src/shard/system/storage.rs
similarity index 97%
rename from core/server/src/streaming/systems/storage.rs
rename to core/server/src/shard/system/storage.rs
index d4363631..9a493751 100644
--- a/core/server/src/streaming/systems/storage.rs
+++ b/core/server/src/shard/system/storage.rs
@@ -18,8 +18,6 @@
use crate::streaming::persistence::persister::PersisterKind;
use crate::streaming::storage::SystemInfoStorage;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::info::SystemInfo;
use crate::streaming::utils::PooledBuffer;
use crate::streaming::utils::file;
use anyhow::Context;
diff --git a/core/server/src/streaming/systems/streams.rs b/core/server/src/shard/system/streams.rs
similarity index 70%
rename from core/server/src/streaming/systems/streams.rs
rename to core/server/src/shard/system/streams.rs
index 18c6aa83..efcfff2b 100644
--- a/core/server/src/streaming/systems/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -16,156 +16,18 @@
* under the License.
*/
-use crate::state::system::StreamState;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
use crate::streaming::streams::stream::Stream;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
-use ahash::{AHashMap, AHashSet};
use error_set::ErrContext;
use futures::future::try_join_all;
-use iggy_common::locking::IggySharedMutFn;
-use iggy_common::{IdKind, Identifier, IggyError};
use std::cell::RefCell;
use std::sync::atomic::{AtomicU32, Ordering};
use tokio::fs;
-use tokio::fs::read_dir;
use tracing::{error, info, warn};
-static CURRENT_STREAM_ID: AtomicU32 = AtomicU32::new(1);
-
-impl System {
- pub(crate) async fn load_streams(
- &mut self,
- streams: Vec<StreamState>,
- ) -> Result<(), IggyError> {
- info!("Loading streams from disk...");
- let mut unloaded_streams = Vec::new();
- let mut dir_entries = read_dir(&self.config.get_streams_path())
- .await
- .map_err(|error| {
- error!("Cannot read streams directory: {error}");
- IggyError::CannotReadStreams
- })?;
-
- while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) {
- let name = dir_entry.file_name().into_string().unwrap();
- let stream_id = name.parse::<u32>().map_err(|_| {
- error!("Invalid stream ID file with name: '{name}'.");
- IggyError::InvalidNumberValue
- })?;
- let stream_state = streams.iter().find(|s| s.id == stream_id);
- if stream_state.is_none() {
- error!(
- "Stream with ID: '{stream_id}' was not found in state, but exists on disk and will be removed."
- );
- if let Err(error) = fs::remove_dir_all(&dir_entry.path()).await {
- error!("Cannot remove stream directory: {error}");
- } else {
- warn!("Stream with ID: '{stream_id}' was removed.");
- }
- continue;
- }
-
- let stream_state = stream_state.unwrap();
- let mut stream = Stream::empty(
- stream_id,
- &stream_state.name,
- self.config.clone(),
- self.storage.clone(),
- );
- stream.created_at = stream_state.created_at;
- unloaded_streams.push(stream);
- }
-
- let state_stream_ids = streams
- .iter()
- .map(|stream| stream.id)
- .collect::<AHashSet<u32>>();
- let unloaded_stream_ids = unloaded_streams
- .iter()
- .map(|stream| stream.stream_id)
- .collect::<AHashSet<u32>>();
- let mut missing_ids = state_stream_ids
- .difference(&unloaded_stream_ids)
- .copied()
- .collect::<AHashSet<u32>>();
- if missing_ids.is_empty() {
- info!("All streams found on disk were found in state.");
- } else {
- warn!("Streams with IDs: '{missing_ids:?}' were not found on disk.");
- if self.config.recovery.recreate_missing_state {
- info!(
- "Recreating missing state in recovery config is enabled, missing streams will be created."
- );
- for stream_id in missing_ids.iter() {
- let stream_id = *stream_id;
- let stream_state = streams.iter().find(|s| s.id == stream_id).unwrap();
- let stream = Stream::create(
- stream_id,
- &stream_state.name,
- self.config.clone(),
- self.storage.clone(),
- );
- stream.persist().await?;
- unloaded_streams.push(stream);
- info!(
- "Missing stream with ID: '{stream_id}', name: {} was recreated.",
- stream_state.name
- );
- }
- missing_ids.clear();
- } else {
- warn!(
- "Recreating missing state in recovery config is disabled, missing streams will not be created."
- );
- }
- }
-
- let mut streams_states = streams
- .into_iter()
- .filter(|s| !missing_ids.contains(&s.id))
- .map(|s| (s.id, s))
- .collect::<AHashMap<_, _>>();
- let loaded_streams = RefCell::new(Vec::new());
- let load_stream_tasks = unloaded_streams.into_iter().map(|mut stream| {
- let state = streams_states.remove(&stream.stream_id).unwrap();
-
- async {
- stream.load(state).await?;
- loaded_streams.borrow_mut().push(stream);
- Result::<(), IggyError>::Ok(())
- }
- });
- try_join_all(load_stream_tasks).await?;
-
- for stream in loaded_streams.take() {
- if self.streams.contains_key(&stream.stream_id) {
- error!("Stream with ID: '{}' already exists.", &stream.stream_id);
- continue;
- }
-
- if self.streams_ids.contains_key(&stream.name) {
- error!("Stream with name: '{}' already exists.", &stream.name);
- continue;
- }
-
- self.metrics.increment_streams(1);
- self.metrics.increment_topics(stream.get_topics_count());
- self.metrics
- .increment_partitions(stream.get_partitions_count());
- self.metrics.increment_segments(stream.get_segments_count());
- self.metrics.increment_messages(stream.get_messages_count());
-
- self.streams_ids
- .insert(stream.name.clone(), stream.stream_id);
- self.streams.insert(stream.stream_id, stream);
- }
-
- info!("Loaded {} stream(s) from disk.", self.streams.len());
- Ok(())
- }
+impl IggyShard {
pub fn get_streams(&self) -> Vec<&Stream> {
self.streams.values().collect()
}
diff --git a/core/server/src/streaming/systems/system.rs b/core/server/src/shard/system/system.rs
similarity index 100%
rename from core/server/src/streaming/systems/system.rs
rename to core/server/src/shard/system/system.rs
diff --git a/core/server/src/streaming/systems/topics.rs b/core/server/src/shard/system/topics.rs
similarity index 100%
rename from core/server/src/streaming/systems/topics.rs
rename to core/server/src/shard/system/topics.rs
diff --git a/core/server/src/streaming/systems/users.rs b/core/server/src/shard/system/users.rs
similarity index 79%
rename from core/server/src/streaming/systems/users.rs
rename to core/server/src/shard/system/users.rs
index cc27fcae..f2405f1f 100644
--- a/core/server/src/streaming/systems/users.rs
+++ b/core/server/src/shard/system/users.rs
@@ -16,13 +16,12 @@
* under the License.
*/
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::state::models::CreateUserWithId;
use crate::state::system::UserState;
use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use crate::streaming::users::user::User;
use crate::streaming::utils::crypto;
use crate::{IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV};
@@ -42,108 +41,7 @@ use tracing::{error, info, warn};
static USER_ID: AtomicU32 = AtomicU32::new(1);
const MAX_USERS: usize = u32::MAX as usize;
-impl System {
- pub(crate) async fn load_users(&mut self, users: Vec<UserState>) -> Result<(), IggyError> {
- info!("Loading users...");
- if users.is_empty() {
- info!("No users found, creating the root user...");
- let root = Self::create_root_user();
- let command = CreateUser {
- username: root.username.clone(),
- password: root.password.clone(),
- status: root.status,
- permissions: root.permissions.clone(),
- };
- self.state
- .apply(0, &EntryCommand::CreateUser(CreateUserWithId {
- user_id: root.id,
- command
- }))
- .await
- .with_error_context(|error| {
- format!(
- "{COMPONENT} (error: {error}) - failed to apply create user command, username: {}",
- root.username
- )
- })?;
-
- self.users.insert(root.id, root);
- info!("Created the root user.");
- }
-
- for user_state in users.into_iter() {
- let mut user = User::with_password(
- user_state.id,
- &user_state.username,
- user_state.password_hash,
- user_state.status,
- user_state.permissions,
- );
-
- user.created_at = user_state.created_at;
- user.personal_access_tokens = user_state
- .personal_access_tokens
- .into_values()
- .map(|token| {
- (
- Arc::new(token.token_hash.clone()),
- PersonalAccessToken::raw(
- user_state.id,
- &token.name,
- &token.token_hash,
- token.expiry_at,
- ),
- )
- })
- .collect();
- self.users.insert(user_state.id, user);
- }
-
- let users_count = self.users.len();
- let current_user_id = self.users.keys().max().unwrap_or(&1);
- USER_ID.store(current_user_id + 1, Ordering::SeqCst);
- self.permissioner
- .init(&self.users.values().collect::<Vec<&User>>());
- self.metrics.increment_users(users_count as u32);
- info!("Initialized {users_count} user(s).");
- Ok(())
- }
-
- fn create_root_user() -> User {
- let username = env::var(IGGY_ROOT_USERNAME_ENV);
- let password = env::var(IGGY_ROOT_PASSWORD_ENV);
- if (username.is_ok() && password.is_err()) || (username.is_err() && password.is_ok()) {
- panic!(
- "When providing the custom root user credentials, both username and password must be set."
- );
- }
- if username.is_ok() && password.is_ok() {
- info!("Using the custom root user credentials.");
- } else {
- info!("Using the default root user credentials.");
- }
-
- let username = username.unwrap_or(DEFAULT_ROOT_USERNAME.to_string());
- let password = password.unwrap_or(DEFAULT_ROOT_PASSWORD.to_string());
- if username.is_empty() || password.is_empty() {
- panic!("Root user credentials are not set.");
- }
- if username.len() < MIN_USERNAME_LENGTH {
- panic!("Root username is too short.");
- }
- if username.len() > MAX_USERNAME_LENGTH {
- panic!("Root username is too long.");
- }
- if password.len() < MIN_PASSWORD_LENGTH {
- panic!("Root password is too short.");
- }
- if password.len() > MAX_PASSWORD_LENGTH {
- panic!("Root password is too long.");
- }
-
- User::root(&username, &password)
- }
-
+impl IggyShard {
pub fn find_user(
&self,
session: &Session,
diff --git a/core/server/src/streaming/mod.rs b/core/server/src/streaming/mod.rs
index 5009cb9a..f9d40af8 100644
--- a/core/server/src/streaming/mod.rs
+++ b/core/server/src/streaming/mod.rs
@@ -28,7 +28,6 @@ pub mod segments;
pub mod session;
pub mod storage;
pub mod streams;
-pub mod systems;
pub mod topics;
pub mod users;
pub mod utils;
diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs
index fc0a752d..f423b842 100644
--- a/core/server/src/streaming/storage.rs
+++ b/core/server/src/streaming/storage.rs
@@ -33,6 +33,7 @@ use iggy_common::IggyError;
use mockall::automock;
use std::fmt::Debug;
use std::future::Future;
+use std::rc::Rc;
use std::sync::Arc;
macro_rules! forward_async_methods {
@@ -143,7 +144,7 @@ pub trait PartitionStorage: Send {
#[derive(Debug)]
pub struct SystemStorage {
- pub info: Arc<SystemInfoStorageKind>,
+ pub info: Rc<SystemInfoStorageKind>,
pub stream: Arc<StreamStorageKind>,
pub topic: Arc<TopicStorageKind>,
pub partition: Arc<PartitionStorageKind>,
@@ -151,7 +152,7 @@ pub struct SystemStorage {
}
impl SystemStorage {
- pub fn new(config: Arc<SystemConfig>, persister: Arc<PersisterKind>) -> Self {
+ pub fn new(config: Rc<SystemConfig>, persister: Arc<PersisterKind>) -> Self {
Self {
info: Arc::new(SystemInfoStorageKind::File(FileSystemInfoStorage::new(
config.get_state_info_path(),
diff --git a/core/server/src/streaming/streams/stream.rs b/core/server/src/streaming/streams/stream.rs
index ef528197..339fc08a 100644
--- a/core/server/src/streaming/streams/stream.rs
+++ b/core/server/src/streaming/streams/stream.rs
@@ -23,6 +23,7 @@ use ahash::AHashMap;
use iggy_common::IggyByteSize;
use iggy_common::IggyTimestamp;
use std::fmt::Display;
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
@@ -47,8 +48,8 @@ impl Stream {
pub fn empty(
id: u32,
name: &str,
- config: Arc<SystemConfig>,
- storage: Arc<SystemStorage>,
+ config: Rc<SystemConfig>,
+ storage: Rc<SystemStorage>,
) -> Self {
Stream::create(id, name, config, storage)
}
@@ -56,8 +57,8 @@ impl Stream {
pub fn create(
id: u32,
name: &str,
- config: Arc<SystemConfig>,
- storage: Arc<SystemStorage>,
+ config: Rc<SystemConfig>,
+ storage: Rc<SystemStorage>,
) -> Self {
let path = config.get_stream_path(id);
let topics_path = config.get_topics_path(id);
diff --git a/core/server/src/tcp/sender.rs b/core/server/src/tcp/sender.rs
index 2b6b6d17..4dd11725 100644
--- a/core/server/src/tcp/sender.rs
+++ b/core/server/src/tcp/sender.rs
@@ -17,15 +17,15 @@
*/
use iggy_common::IggyError;
+use monoio::io::{AsyncReadRent, AsyncReadRentExt, AsyncWriteRent, AsyncWriteRentExt};
use std::io::IoSlice;
-use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tracing::debug;
const STATUS_OK: &[u8] = &[0; 4];
pub(crate) async fn read<T>(stream: &mut T, buffer: &mut [u8]) -> Result<usize, IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRent + AsyncWriteRent + Unpin,
{
match stream.read_exact(buffer).await {
Ok(0) => Err(IggyError::ConnectionClosed),
@@ -42,14 +42,14 @@ where
pub(crate) async fn send_empty_ok_response<T>(stream: &mut T) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRent + AsyncWriteRent + Unpin,
{
send_ok_response(stream, &[]).await
}
pub(crate) async fn send_ok_response<T>(stream: &mut T, payload: &[u8]) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRent + AsyncWriteRent + Unpin,
{
send_response(stream, STATUS_OK, payload).await
}
@@ -60,7 +60,7 @@ pub(crate) async fn send_ok_response_vectored<T>(
slices: Vec<IoSlice<'_>>,
) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRentExt + AsyncWriteRentExt + Unpin,
{
send_response_vectored(stream, STATUS_OK, length, slices).await
}
@@ -70,7 +70,7 @@ pub(crate) async fn send_error_response<T>(
error: IggyError,
) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRent + AsyncWriteRent + Unpin,
{
send_response(stream, &error.as_code().to_le_bytes(), &[]).await
}
@@ -81,7 +81,7 @@ pub(crate) async fn send_response<T>(
payload: &[u8],
) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRent + AsyncWriteRent + Unpin,
{
debug!(
"Sending response of len: {} with status: {:?}...",
@@ -104,7 +104,7 @@ pub(crate) async fn send_response_vectored<T>(
mut slices: Vec<IoSlice<'_>>,
) -> Result<(), IggyError>
where
- T: AsyncReadExt + AsyncWriteExt + Unpin,
+ T: AsyncReadRentExt + AsyncWriteRentExt + Unpin,
{
debug!(
"Sending vectored response of len: {} with status: {:?}...",
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index 635960d0..2e4f25be 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -17,57 +17,44 @@
*/
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::clients::client_manager::Transport;
-use crate::streaming::systems::system::SharedSystem;
use crate::tcp::connection_handler::{handle_connection, handle_error};
use std::net::SocketAddr;
+use std::rc::Rc;
+use monoio::net::TcpListener;
+use rustls::pki_types::Ipv4Addr;
use tokio::net::TcpSocket;
use tokio::sync::oneshot;
use tracing::{error, info};
-pub async fn start(address: &str, socket: TcpSocket, system: SharedSystem) -> SocketAddr {
- let address = address.to_string();
- let (tx, rx) = oneshot::channel();
- tokio::spawn(async move {
- let addr = address.parse();
- if addr.is_err() {
- panic!("Unable to parse address {address:?}");
- }
-
- socket
- .bind(addr.unwrap())
- .expect("Unable to bind socket to address");
-
- let listener = socket.listen(1024).expect("Unable to start TCP server.");
-
- let local_addr = listener
- .local_addr()
- .expect("Failed to get local address for TCP listener");
-
- tx.send(local_addr).unwrap_or_else(|_| {
- panic!("Failed to send the local address {local_addr:?} for TCP listener")
- });
-
+pub async fn start(server_name: &str, shard: Rc<IggyShard>) {
+ let addr: SocketAddr = if shard.config.tcp.ipv6 {
+ shard.config.tcp.address.parse().expect("Unable to parse IPv6 address")
+ } else {
+ shard.config.tcp.address.parse().expect("Unable to parse IPv4 address")
+ };
+ monoio::spawn(async move {
+ let listener = TcpListener::bind(addr).expect(format!("Unable to start {server_name}.").as_ref());
loop {
match listener.accept().await {
Ok((stream, address)) => {
+ let shard = shard.clone();
info!("Accepted new TCP connection: {address}");
- let session = system
- .read()
- .await
- .add_client(&address, Transport::Tcp)
- .await;
+ let session = shard
+ .add_client(&address, Transport::Tcp);
let client_id = session.client_id;
info!("Created new session: {session}");
- let system = system.clone();
let mut sender = SenderKind::get_tcp_sender(stream);
- tokio::spawn(async move {
+ monoio::spawn(async move {
if let Err(error) =
- handle_connection(session, &mut sender, system.clone()).await
+ handle_connection(session, &mut sender, shard.clone()).await
{
handle_error(error);
- system.read().await.delete_client(client_id).await;
+ //TODO: Fixme
+ /*
+ //system.read().await.delete_client(client_id).await;
if let Err(error) = sender.shutdown().await {
error!(
"Failed to shutdown TCP stream for client: {client_id}, address: {address}. {error}"
@@ -77,6 +64,7 @@ pub async fn start(address: &str, socket: TcpSocket, system: SharedSystem) -> So
"Successfully closed TCP stream for client: {client_id}, address: {address}."
);
}
+ */
}
});
}
@@ -84,8 +72,4 @@ pub async fn start(address: &str, socket: TcpSocket, system: SharedSystem) -> So
}
}
});
- match rx.await {
- Ok(addr) => addr,
- Err(_) => panic!("Failed to get the local address for TCP listener."),
- }
}
diff --git a/core/server/src/tcp/tcp_sender.rs b/core/server/src/tcp/tcp_sender.rs
index 837c8f06..a157dead 100644
--- a/core/server/src/tcp/tcp_sender.rs
+++ b/core/server/src/tcp/tcp_sender.rs
@@ -21,7 +21,8 @@ use crate::tcp::COMPONENT;
use crate::{server_error::ServerError, tcp::sender};
use error_set::ErrContext;
use iggy_common::IggyError;
-use tokio::{io::AsyncWriteExt, net::TcpStream};
+use tokio::{io::AsyncWriteExt};
+use monoio::net::TcpStream;
#[derive(Debug)]
pub struct TcpSender {
diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs
index a57b7c3f..82bbe0c1 100644
--- a/core/server/src/tcp/tcp_server.rs
+++ b/core/server/src/tcp/tcp_server.rs
@@ -16,26 +16,25 @@
* under the License.
*/
-use crate::configs::tcp::TcpConfig;
-use crate::streaming::systems::system::SharedSystem;
-use crate::tcp::{tcp_listener, tcp_socket, tcp_tls_listener};
-use std::net::SocketAddr;
+use crate::shard::IggyShard;
+use crate::tcp::{tcp_listener};
+use std::rc::Rc;
+use iggy_common::IggyError;
use tracing::info;
/// Starts the TCP server.
/// Returns the address the server is listening on.
-pub async fn start(config: TcpConfig, system: SharedSystem) -> SocketAddr {
- let server_name = if config.tls.enabled {
+pub async fn start(shard: Rc<IggyShard>) -> Result<(), IggyError> {
+ let server_name = if shard.config.tcp.tls.enabled {
"Iggy TCP TLS"
} else {
"Iggy TCP"
};
info!("Initializing {server_name} server...");
- let socket = tcp_socket::build(config.ipv6, config.socket);
- let addr = match config.tls.enabled {
- true => tcp_tls_listener::start(&config.address, config.tls, socket, system).await,
- false => tcp_listener::start(&config.address, socket, system).await,
+ let addr = match shard.config.tcp.tls.enabled {
+ true => unimplemented!("TLS support is not implemented yet"),
+ false => tcp_listener::start(server_name, shard).await,
};
info!("{server_name} server has started on: {:?}", addr);
- addr
+ Ok(())
}
diff --git a/core/server/src/tcp/tcp_socket.rs b/core/server/src/tcp/tcp_socket.rs
index 91ed105a..69c1d408 100644
--- a/core/server/src/tcp/tcp_socket.rs
+++ b/core/server/src/tcp/tcp_socket.rs
@@ -18,7 +18,6 @@
use std::num::TryFromIntError;
-use tokio::net::TcpSocket;
use crate::configs::tcp::TcpSocketConfig;
diff --git a/core/server/src/tcp/tcp_tls_sender.rs b/core/server/src/tcp/tcp_tls_sender.rs
index 6e3056a5..6ba62315 100644
--- a/core/server/src/tcp/tcp_tls_sender.rs
+++ b/core/server/src/tcp/tcp_tls_sender.rs
@@ -21,9 +21,8 @@ use crate::tcp::COMPONENT;
use crate::{server_error::ServerError, tcp::sender};
use error_set::ErrContext;
use iggy_common::IggyError;
-use tokio::io::AsyncWriteExt;
-use tokio::net::TcpStream;
-use tokio_rustls::server::TlsStream;
+use monoio::net::TcpStream;
+use monoio_native_tls::TlsStream;
#[derive(Debug)]
pub struct TcpTlsSender {
(iggy) 03/30: feat(io_uring): load server_config (#1798)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit dab0759eee1e33482254e35a30d71ab2248759c8
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Fri May 16 21:37:23 2025 +0200
feat(io_uring): load server_config (#1798)
---
Cargo.lock | 31 +++++++++++---
core/server/src/bootstrap.rs | 13 +++++-
core/server/src/main.rs | 87 ++++++++++++++++++++++++++--------------
core/server/src/shard/builder.rs | 9 +----
core/server/src/shard/mod.rs | 3 +-
5 files changed, 96 insertions(+), 47 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 298dd1f9..a79f4b82 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -142,7 +142,7 @@ dependencies = [
"actix-utils",
"futures-core",
"futures-util",
- "mio 1.0.3",
+ "mio 1.0.4",
"socket2",
"tokio",
"tracing",
@@ -639,7 +639,7 @@ checksum = "fd73835ad7deb4bd2b389e6f10333b143f025d607c55ca04c66a0bcc6bb2fc6d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.101",
+ "syn 2.0.104",
]
[[package]]
@@ -4685,6 +4685,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
[[package]]
+<<<<<<< HEAD
name = "miette"
version = "7.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4704,6 +4705,14 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
+=======
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+>>>>>>> 4d172830 (feat(io_uring): load server_config (#1798))
]
[[package]]
@@ -4755,6 +4764,18 @@ dependencies = [
"adler2",
]
+[[package]]
+name = "mio"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+dependencies = [
+ "libc",
+ "log",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "mio"
version = "1.0.4"
@@ -4843,7 +4864,7 @@ checksum = "176a5f5e69613d9e88337cf2a65e11135332b4efbcc628404a7c555e4452084c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.101",
+ "syn 2.0.104",
]
[[package]]
@@ -4938,7 +4959,7 @@ dependencies = [
"kqueue",
"libc",
"log",
- "mio 1.0.3",
+ "mio 1.0.4",
"notify-types",
"walkdir",
"windows-sys 0.59.0",
@@ -7523,7 +7544,7 @@ dependencies = [
"backtrace",
"bytes",
"libc",
- "mio",
+ "mio 1.0.4",
"parking_lot 0.12.4",
"pin-project-lite",
"signal-hook-registry",
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index 8eacf223..fd6a5aa0 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -1,3 +1,5 @@
+use monoio::{Buildable, Driver, Runtime};
+
use crate::shard::{connector::ShardConnector, frame::ShardFrame};
use std::ops::Range;
@@ -12,9 +14,18 @@ pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<
}
pub async fn create_directories() {
-
+ todo!();
}
pub async fn create_root_user() {
+ todo!();
+}
+pub fn create_default_executor<D>() -> Runtime<D>
+where
+ D: Driver + Buildable,
+{
+ let builder = monoio::RuntimeBuilder::<D>::new();
+ let rt = Buildable::build(builder).expect("Failed to create default runtime");
+ rt
}
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 381ac578..3b38700f 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -24,7 +24,9 @@ use dotenvy::dotenv;
use figlet_rs::FIGfont;
use monoio::Buildable;
use server::args::Args;
-use server::bootstrap::{create_directories, create_root_user, create_shard_connections};
+use server::bootstrap::{
+ create_default_executor, create_directories, create_root_user, create_shard_connections,
+};
use server::channels::commands::archive_state::ArchiveStateExecutor;
use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
use server::channels::commands::maintain_messages::MaintainMessagesExecutor;
@@ -32,7 +34,7 @@ use server::channels::commands::print_sysinfo::SysInfoPrintExecutor;
use server::channels::commands::save_messages::SaveMessagesExecutor;
use server::channels::commands::verify_heartbeats::VerifyHeartbeatsExecutor;
use server::channels::handler::BackgroundServerCommandHandler;
-use server::configs::config_provider;
+use server::configs::config_provider::{self, ConfigProviderKind};
use server::configs::server::ServerConfig;
use server::http::http_server;
#[cfg(not(feature = "tokio-console"))]
@@ -72,37 +74,60 @@ fn main() -> Result<(), ServerError> {
let args = Args::parse();
let config_provider = config_provider::resolve(&args.config_provider)?;
- let config = ServerConfig::default();
- //TODO: Load config.
- /*
- let xd = std::thread::scope(|scope| {
- let config = scope.spawn(move || {
- let mut rt = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new().build().unwrap();
- let config: Result<ServerConfig, ServerError> = rt.block_on(async {
- let config = ServerConfig::load(&config_provider).await?;
- create_directories().await;
- create_root_user().await;
- Ok(config)
- });
- config
- }).join().unwrap();
+ let config = std::thread::scope(|scope| {
+ let config = scope
+ .spawn(move || {
+ async fn load_config(
+ config_provider: &ConfigProviderKind,
+ ) -> Result<ServerConfig, ServerError> {
+ let config = ServerConfig::load(config_provider).await?;
+ Ok(config)
+ }
+
+ let mut rt = create_default_executor::<monoio::IoUringDriver>();
+ rt.block_on(load_config(&config_provider))
+ })
+ .join()
+ .expect("Failed to load config");
config
+ })?;
+
+ // Create directories and root user.
+ // Remove `local_data` directory if run with `--fresh` flag.
+ std::thread::scope(|scope| {
+ scope
+ .spawn(|| {
+ let mut rt = create_default_executor::<monoio::IoUringDriver>();
+ rt.block_on(async {
+ if args.fresh {
+ let system_path = config.system.get_system_path();
+ if monoio::fs::metadata(&system_path).await.is_ok() {
+ println!(
+ "Removing system path at: {} because `--fresh` flag was set",
+ system_path
+ );
+ //TODO: Impl dir walk and remove the files
+ /*
+ if let Err(e) = tokio::fs::remove_dir_all(&system_path).await {
+ eprintln!("Failed to remove system path at {}: {}", system_path, e);
+ }
+ */
+ }
+ }
+
+ // Create directories and root user
+ create_directories().await;
+ create_root_user().await;
+ });
+ })
+ .join()
+ .expect("Failed to create directories and root user");
});
- */
- /*
- let config = ServerConfig::load(&config_provider).await?;
- if args.fresh {
- let system_path = config.system.get_system_path();
- if tokio::fs::metadata(&system_path).await.is_ok() {
- println!("Removing system path at: {system_path} because `--fresh` flag was set");
- if let Err(e) = tokio::fs::remove_dir_all(&system_path).await {
- eprintln!("Failed to remove system path at {system_path}: {e}");
- }
- }
- }
+
+ // Initialize logging
let mut logging = Logging::new(config.telemetry.clone());
logging.early_init();
- */
+
// TODO: Make this configurable from config as a range
// for example this instance of Iggy will use cores from 0..4
let available_cpus = available_parallelism()
@@ -149,11 +174,11 @@ fn main() -> Result<(), ServerError> {
.join()
.expect(format!("Failed to join thread for shard-{id}").as_str());
}
- // From this point on, we can use tracing macros to log messages.
- /*
+ // From this point on, we can use tracing macros to log messages.
logging.late_init(config.system.get_system_path(), &config.system.logging)?;
+ /*
#[cfg(feature = "disable-mimalloc")]
tracing::warn!(
"Using default system allocator because code was build with `disable-mimalloc` feature"
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 08a02e54..750e2b71 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -61,13 +61,6 @@ impl IggyShardBuilder {
.expect("Failed to find connection with the specified ID");
let shards = connections.into_iter().map(Shard::new).collect();
- IggyShard::new(
- id,
- shards,
- config,
- stop_receiver,
- stop_sender,
- receiver,
- )
+ IggyShard::new(id, shards, config, stop_receiver, stop_sender, receiver)
}
}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index bffaca45..dcb089d9 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -91,6 +91,5 @@ impl IggyShard {
}
}
- pub fn assert_init(&self) {
- }
+ pub fn assert_init(&self) {}
}
(iggy) 10/30: fix issues with RefCell in core/server/src/streaming/topics/consumer_groups* files
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 110f8f417fb0649954314920df875fca4e4e3e78
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Tue Jun 24 18:55:03 2025 +0200
fix issues with RefCell in core/server/src/streaming/topics/consumer_groups* files
---
core/server/src/streaming/topics/consumer_group.rs | 25 ++--
.../server/src/streaming/topics/consumer_groups.rs | 152 ++++++++++++---------
2 files changed, 94 insertions(+), 83 deletions(-)
diff --git a/core/server/src/streaming/topics/consumer_group.rs b/core/server/src/streaming/topics/consumer_group.rs
index 43d9611b..cc28ef6f 100644
--- a/core/server/src/streaming/topics/consumer_group.rs
+++ b/core/server/src/streaming/topics/consumer_group.rs
@@ -18,7 +18,6 @@
use ahash::AHashMap;
use iggy_common::IggyError;
-use tokio::sync::RwLock;
use tracing::trace;
#[derive(Debug, Clone)]
@@ -58,10 +57,13 @@ impl ConsumerGroup {
self.assign_partitions().await;
}
- pub async fn calculate_partition_id(&self, member_id: u32) -> Result<Option<u32>, IggyError> {
- let member = self.members.get(&member_id);
+ pub async fn calculate_partition_id(
+ &mut self,
+ member_id: u32,
+ ) -> Result<Option<u32>, IggyError> {
+ let member = self.members.get_mut(&member_id);
if let Some(member) = member {
- return Ok(member.await.calculate_partition_id());
+ return Ok(member.calculate_partition_id());
}
Err(IggyError::ConsumerGroupMemberNotFound(
member_id,
@@ -73,7 +75,7 @@ impl ConsumerGroup {
pub async fn get_current_partition_id(&self, member_id: u32) -> Result<Option<u32>, IggyError> {
let member = self.members.get(&member_id);
if let Some(member) = member {
- return Ok(member.read().await.current_partition_id);
+ return Ok(member.current_partition_id);
}
Err(IggyError::ConsumerGroupMemberNotFound(
member_id,
@@ -85,12 +87,12 @@ impl ConsumerGroup {
pub async fn add_member(&mut self, member_id: u32) {
self.members.insert(
member_id,
- RwLock::new(ConsumerGroupMember {
+ ConsumerGroupMember {
id: member_id,
partitions: AHashMap::new(),
current_partition_index: None,
current_partition_id: None,
- }),
+ },
);
trace!(
"Added member with ID: {} to consumer group: {} for topic with ID: {}",
@@ -117,7 +119,6 @@ impl ConsumerGroup {
let members_count = members.len() as u32;
for member in members.iter_mut() {
- let mut member = member.write().await;
member.current_partition_index = None;
member.current_partition_id = None;
member.partitions.clear();
@@ -126,8 +127,7 @@ impl ConsumerGroup {
for partition_index in 0..self.partitions_count {
let partition_id = partition_index + 1;
let member_index = partition_index % members_count;
- let member = members.get(member_index as usize).unwrap();
- let mut member = member.write().await;
+ let member = members.get_mut(member_index as usize).unwrap();
let member_partition_index = member.partitions.len() as u32;
member
.partitions
@@ -213,7 +213,6 @@ mod tests {
consumer_group.add_member(member_id).await;
let member = consumer_group.members.get(&member_id).unwrap();
- let member = member.read().await;
assert_eq!(
member.partitions.len() as u32,
consumer_group.partitions_count
@@ -240,8 +239,6 @@ mod tests {
consumer_group.add_member(member2_id).await;
let member1 = consumer_group.members.get(&member1_id).unwrap();
let member2 = consumer_group.members.get(&member2_id).unwrap();
- let member1 = member1.read().await;
- let member2 = member2.read().await;
assert_eq!(
member1.partitions.len() + member2.partitions.len(),
consumer_group.partitions_count as usize
@@ -277,8 +274,6 @@ mod tests {
consumer_group.add_member(member2_id).await;
let member1 = consumer_group.members.get(&member1_id).unwrap();
let member2 = consumer_group.members.get(&member2_id).unwrap();
- let member1 = member1.read().await;
- let member2 = member2.read().await;
if member1.partitions.len() == 1 {
assert_eq!(member2.partitions.len(), 0);
} else {
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index 26505355..1f50a1bb 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -21,37 +21,31 @@ use crate::streaming::topics::consumer_group::ConsumerGroup;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
use iggy_common::IggyError;
-use iggy_common::locking::IggySharedMutFn;
use iggy_common::{IdKind, Identifier};
use std::sync::atomic::Ordering;
-use tokio::sync::RwLock;
use tracing::info;
impl Topic {
- pub async fn reassign_consumer_groups(&mut self) {
+ pub fn reassign_consumer_groups(&mut self) {
if self.consumer_groups.is_empty() {
return;
}
- let partitions_count = self.partitions.len() as u32;
+ let partitions_count = self.partitions.borrow().len() as u32;
info!(
"Reassigning consumer groups for topic with ID: {} for stream with ID with {}, partitions count: {}",
self.topic_id, self.stream_id, partitions_count
);
for (_, consumer_group) in self.consumer_groups.iter_mut() {
- let mut consumer_group = consumer_group.write().await;
- consumer_group.reassign_partitions(partitions_count).await;
+ consumer_group.reassign_partitions(partitions_count);
}
}
- pub fn get_consumer_groups(&self) -> Vec<&RwLock<ConsumerGroup>> {
+ pub fn get_consumer_groups(&self) -> Vec<&ConsumerGroup> {
self.consumer_groups.values().collect()
}
- pub fn get_consumer_group(
- &self,
- identifier: &Identifier,
- ) -> Result<&RwLock<ConsumerGroup>, IggyError> {
+ pub fn get_consumer_group(&self, identifier: &Identifier) -> Result<&ConsumerGroup, IggyError> {
match identifier.kind {
IdKind::Numeric => self.get_consumer_group_by_id(identifier.get_u32_value().unwrap()),
IdKind::String => self.get_consumer_group_by_name(&identifier.get_cow_str_value()?),
@@ -61,7 +55,7 @@ impl Topic {
pub fn try_get_consumer_group(
&self,
identifier: &Identifier,
- ) -> Result<Option<&RwLock<ConsumerGroup>>, IggyError> {
+ ) -> Result<Option<&ConsumerGroup>, IggyError> {
match identifier.kind {
IdKind::Numeric => Ok(self.consumer_groups.get(&identifier.get_u32_value()?)),
IdKind::String => {
@@ -70,16 +64,13 @@ impl Topic {
}
}
- fn try_get_consumer_group_by_name(&self, name: &str) -> Option<&RwLock<ConsumerGroup>> {
+ fn try_get_consumer_group_by_name(&self, name: &str) -> Option<&ConsumerGroup> {
self.consumer_groups_ids
.get(name)
.and_then(|id| self.consumer_groups.get(id))
}
- pub fn get_consumer_group_by_name(
- &self,
- name: &str,
- ) -> Result<&RwLock<ConsumerGroup>, IggyError> {
+ pub fn get_consumer_group_by_name(&self, name: &str) -> Result<&ConsumerGroup, IggyError> {
let group_id = self.consumer_groups_ids.get(name);
if group_id.is_none() {
return Err(IggyError::ConsumerGroupNameNotFound(
@@ -91,7 +82,7 @@ impl Topic {
self.get_consumer_group_by_id(*group_id.unwrap())
}
- pub fn get_consumer_group_by_id(&self, id: u32) -> Result<ConsumerGroup, IggyError> {
+ pub fn get_consumer_group_by_id(&self, id: u32) -> Result<&ConsumerGroup, IggyError> {
let consumer_group = self.consumer_groups.get(&id);
if consumer_group.is_none() {
return Err(IggyError::ConsumerGroupIdNotFound(id, self.topic_id));
@@ -100,7 +91,46 @@ impl Topic {
Ok(consumer_group.unwrap())
}
- pub async fn create_consumer_group(
+ pub fn get_consumer_group_mut(
+ &mut self,
+ identifier: &Identifier,
+ ) -> Result<&mut ConsumerGroup, IggyError> {
+ match identifier.kind {
+ IdKind::Numeric => {
+ self.get_consumer_group_by_id_mut(identifier.get_u32_value().unwrap())
+ }
+ IdKind::String => self.get_consumer_group_by_name_mut(&identifier.get_cow_str_value()?),
+ }
+ }
+
+ pub fn get_consumer_group_by_id_mut(
+ &mut self,
+ id: u32,
+ ) -> Result<&mut ConsumerGroup, IggyError> {
+ let consumer_group = self.consumer_groups.get_mut(&id);
+ if consumer_group.is_none() {
+ return Err(IggyError::ConsumerGroupIdNotFound(id, self.topic_id));
+ }
+
+ Ok(consumer_group.unwrap())
+ }
+
+ pub fn get_consumer_group_by_name_mut(
+ &mut self,
+ name: &str,
+ ) -> Result<&mut ConsumerGroup, IggyError> {
+ let group_id = self.consumer_groups_ids.get(name).copied();
+ if group_id.is_none() {
+ return Err(IggyError::ConsumerGroupNameNotFound(
+ name.to_string(),
+ self.name.to_owned(),
+ ));
+ }
+
+ self.get_consumer_group_by_id_mut(group_id.unwrap())
+ }
+
+ pub fn create_consumer_group(
&mut self,
group_id: Option<u32>,
name: &str,
@@ -137,15 +167,20 @@ impl Topic {
return Err(IggyError::ConsumerGroupIdAlreadyExists(id, self.topic_id));
}
- let consumer_group =
- ConsumerGroup::new(self.topic_id, id, name, self.partitions.borrow().len() as u32);
- self.consumer_groups.insert(id, consumer_group.clone());
+ let consumer_group = ConsumerGroup::new(
+ self.topic_id,
+ id,
+ name,
+ self.partitions.borrow().len() as u32,
+ );
self.consumer_groups_ids.insert(name.to_owned(), id);
+ let cloned_group = consumer_group.clone();
+ self.consumer_groups.insert(id, consumer_group);
info!(
"Created consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
id, self.topic_id, self.stream_id
);
- consumer_group
+ Ok(cloned_group)
}
pub async fn delete_consumer_group(
@@ -157,7 +192,6 @@ impl Topic {
let consumer_group = self.get_consumer_group(id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get consumer group with id: {id}")
})?;
- let consumer_group = consumer_group.read().await;
group_id = consumer_group.group_id;
}
@@ -167,7 +201,6 @@ impl Topic {
}
let consumer_group = consumer_group.unwrap();
{
- let group_id = consumer_group.group_id;
self.consumer_groups_ids.remove(&consumer_group.name);
let current_group_id = self.current_consumer_group_id.load(Ordering::SeqCst);
if current_group_id > group_id {
@@ -193,16 +226,15 @@ impl Topic {
Ok(consumer_group)
}
- pub async fn join_consumer_group(
- &self,
+ pub fn join_consumer_group(
+ &mut self,
group_id: &Identifier,
member_id: u32,
) -> Result<(), IggyError> {
- let consumer_group = self.get_consumer_group(group_id).with_error_context(|error| {
+ let consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get consumer group with id: {group_id}")
})?;
- let mut consumer_group = consumer_group.write().await;
- consumer_group.add_member(member_id).await;
+ consumer_group.add_member(member_id);
info!(
"Member with ID: {} has joined consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
member_id, group_id, self.topic_id, self.stream_id
@@ -210,16 +242,15 @@ impl Topic {
Ok(())
}
- pub async fn leave_consumer_group(
- &self,
+ pub fn leave_consumer_group(
+ &mut self,
group_id: &Identifier,
member_id: u32,
) -> Result<(), IggyError> {
- let consumer_group = self.get_consumer_group(group_id).with_error_context(|error| {
+ let consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get consumer group with id: {group_id}")
})?;
- let mut consumer_group = consumer_group.write().await;
- consumer_group.delete_member(member_id).await;
+ consumer_group.delete_member(member_id);
info!(
"Member with ID: {} has left consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
member_id, group_id, self.topic_id, self.stream_id
@@ -236,6 +267,7 @@ mod tests {
use crate::streaming::storage::SystemStorage;
use crate::streaming::utils::MemoryPool;
use iggy_common::{CompressionAlgorithm, IggyExpiry, MaxTopicSize};
+ use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64};
@@ -245,10 +277,10 @@ mod tests {
let name = "test";
let mut topic = get_topic().await;
let topic_id = topic.topic_id;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
{
- let created_consumer_group = result.unwrap().read().await;
+ let created_consumer_group = result.unwrap();
assert_eq!(created_consumer_group.group_id, group_id);
assert_eq!(created_consumer_group.name, name);
assert_eq!(created_consumer_group.topic_id, topic_id);
@@ -258,13 +290,12 @@ mod tests {
let consumer_group = topic
.get_consumer_group(&Identifier::numeric(group_id).unwrap())
.unwrap();
- let consumer_group = consumer_group.read().await;
assert_eq!(consumer_group.group_id, group_id);
assert_eq!(consumer_group.name, name);
assert_eq!(consumer_group.topic_id, topic_id);
assert_eq!(
consumer_group.partitions_count,
- topic.partitions.len() as u32
+ topic.partitions.borrow().len() as u32
);
}
@@ -273,10 +304,10 @@ mod tests {
let group_id = 1;
let name = "test";
let mut topic = get_topic().await;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
assert_eq!(topic.consumer_groups.len(), 1);
- let result = topic.create_consumer_group(Some(group_id), "test2").await;
+ let result = topic.create_consumer_group(Some(group_id), "test2");
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, IggyError::ConsumerGroupIdAlreadyExists(_, _)));
@@ -288,11 +319,11 @@ mod tests {
let group_id = 1;
let name = "test";
let mut topic = get_topic().await;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
assert_eq!(topic.consumer_groups.len(), 1);
let group_id = group_id + 1;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(
@@ -307,7 +338,7 @@ mod tests {
let group_id = 1;
let name = "test";
let mut topic = get_topic().await;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
assert_eq!(topic.consumer_groups.len(), 1);
let result = topic
@@ -322,7 +353,7 @@ mod tests {
let group_id = 1;
let name = "test";
let mut topic = get_topic().await;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
assert_eq!(topic.consumer_groups.len(), 1);
let group_id = group_id + 1;
@@ -339,19 +370,12 @@ mod tests {
let name = "test";
let member_id = 1;
let mut topic = get_topic().await;
- topic
- .create_consumer_group(Some(group_id), name)
- .await
- .unwrap();
- let result = topic
- .join_consumer_group(&Identifier::numeric(group_id).unwrap(), member_id)
- .await;
+ topic.create_consumer_group(Some(group_id), name).unwrap();
+ let result = topic.join_consumer_group(&Identifier::numeric(group_id).unwrap(), member_id);
assert!(result.is_ok());
let consumer_group = topic
.get_consumer_group(&Identifier::numeric(group_id).unwrap())
- .unwrap()
- .read()
- .await;
+ .unwrap();
let members = consumer_group.get_members();
assert_eq!(members.len(), 1);
}
@@ -362,34 +386,26 @@ mod tests {
let name = "test";
let member_id = 1;
let mut topic = get_topic().await;
- topic
- .create_consumer_group(Some(group_id), name)
- .await
- .unwrap();
+ topic.create_consumer_group(Some(group_id), name).unwrap();
topic
.join_consumer_group(&Identifier::numeric(group_id).unwrap(), member_id)
- .await
.unwrap();
- let result = topic
- .leave_consumer_group(&Identifier::numeric(group_id).unwrap(), member_id)
- .await;
+ let result = topic.leave_consumer_group(&Identifier::numeric(group_id).unwrap(), member_id);
assert!(result.is_ok());
let consumer_group = topic
.get_consumer_group(&Identifier::numeric(group_id).unwrap())
- .unwrap()
- .read()
- .await;
+ .unwrap();
let members = consumer_group.get_members();
assert!(members.is_empty())
}
async fn get_topic() -> Topic {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
(iggy) 09/30: begin fix cg
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit d0ea2c19eb9e0e2e2314e3510675a2ca808dd11e
Author: numinex <g....@gmail.com>
AuthorDate: Tue Jun 24 18:24:30 2025 +0200
begin fix cg
---
Cargo.lock | 1 +
core/server/Cargo.toml | 1 +
core/server/src/binary/command.rs | 7 +-
core/server/src/binary/sender.rs | 28 ++++---
core/server/src/channels/commands/archive_state.rs | 4 +
.../commands/clean_personal_access_tokens.rs | 3 +
.../src/channels/commands/maintain_messages.rs | 5 ++
core/server/src/channels/commands/print_sysinfo.rs | 4 +
core/server/src/channels/commands/save_messages.rs | 4 +
.../src/channels/commands/verify_heartbeats.rs | 3 +
core/server/src/channels/handler.rs | 4 +
core/server/src/channels/server_command.rs | 4 +
core/server/src/configs/defaults.rs | 3 +-
core/server/src/http/http_server.rs | 3 +
core/server/src/quic/listener.rs | 9 +-
core/server/src/quic/quic_sender.rs | 15 +++-
core/server/src/quic/quic_server.rs | 4 +-
core/server/src/shard/builder.rs | 2 +-
core/server/src/shard/mod.rs | 95 +++++++++++++++++-----
core/server/src/shard/namespace.rs | 4 +-
core/server/src/shard/system/clients.rs | 10 +--
core/server/src/shard/system/snapshot/mod.rs | 1 -
core/server/src/shard/system/storage.rs | 1 +
core/server/src/shard/system/streams.rs | 1 -
core/server/src/shard/transmission/message.rs | 14 ++--
.../server/src/streaming/clients/client_manager.rs | 65 +++++++--------
core/server/src/streaming/partitions/messages.rs | 5 +-
core/server/src/streaming/partitions/partition.rs | 20 +++--
core/server/src/streaming/segments/segment.rs | 9 +-
core/server/src/streaming/session.rs | 21 +++--
core/server/src/streaming/storage.rs | 42 +++++-----
core/server/src/streaming/streams/storage.rs | 2 +-
core/server/src/streaming/streams/stream.rs | 10 +--
core/server/src/streaming/streams/topics.rs | 9 +-
core/server/src/streaming/topics/consumer_group.rs | 12 +--
.../server/src/streaming/topics/consumer_groups.rs | 16 ++--
core/server/src/streaming/topics/topic.rs | 18 ++--
core/server/src/streaming/utils/memory_pool.rs | 5 +-
core/server/src/tcp/connection_handler.rs | 32 +++++---
core/server/src/tcp/sender.rs | 53 +++++++-----
core/server/src/tcp/tcp_listener.rs | 45 ++++++----
core/server/src/tcp/tcp_sender.rs | 9 +-
core/server/src/tcp/tcp_server.rs | 4 +-
core/server/src/tcp/tcp_socket.rs | 24 ++++--
core/server/src/tcp/tcp_tls_listener.rs | 11 ++-
core/server/src/tcp/tcp_tls_sender.rs | 7 +-
46 files changed, 408 insertions(+), 241 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 17894833..3e51b287 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7071,6 +7071,7 @@ dependencies = [
"serde_with",
"serial_test",
"sharded_queue",
+ "socket2",
"static-toml",
"strum",
"sysinfo 0.35.2",
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index a493e837..70f6a7fd 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -61,6 +61,7 @@ futures = { workspace = true }
human-repr = { workspace = true }
iggy_common = { workspace = true }
jsonwebtoken = "9.3.1"
+socket2 = "0.5.10"
lending-iterator = "0.1.7"
hash32 = "1.0.0"
mimalloc = { workspace = true, optional = true }
diff --git a/core/server/src/binary/command.rs b/core/server/src/binary/command.rs
index 0b997941..aa935e09 100644
--- a/core/server/src/binary/command.rs
+++ b/core/server/src/binary/command.rs
@@ -16,8 +16,11 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::sender::SenderKind;
use crate::define_server_command_enum;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
use crate::streaming::systems::system::SharedSystem;
use bytes::{BufMut, Bytes, BytesMut};
@@ -130,8 +133,8 @@ pub trait ServerCommandHandler {
self,
sender: &mut SenderKind,
length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError>;
}
diff --git a/core/server/src/binary/sender.rs b/core/server/src/binary/sender.rs
index 2ccf349b..06e9c51f 100644
--- a/core/server/src/binary/sender.rs
+++ b/core/server/src/binary/sender.rs
@@ -22,10 +22,13 @@ use std::io::IoSlice;
use crate::tcp::tcp_sender::TcpSender;
use crate::tcp::tcp_tls_sender::TcpTlsSender;
use crate::{quic::quic_sender::QuicSender, server_error::ServerError};
+use bytes::BytesMut;
use iggy_common::IggyError;
-use quinn::{RecvStream, SendStream};
+use monoio::buf::IoBufMut;
use monoio::net::TcpStream;
-use tokio_native_tls::TlsStream;
+use monoio_native_tls::TlsStream;
+use nix::libc;
+use quinn::{RecvStream, SendStream};
macro_rules! forward_async_methods {
(
@@ -48,25 +51,24 @@ macro_rules! forward_async_methods {
}
pub trait Sender {
- fn read(&mut self, buffer: &mut [u8]) -> impl Future<Output = Result<usize, IggyError>>;
- fn send_empty_ok_response(&mut self) -> impl Future<Output = Result<(), IggyError>>;
- fn send_ok_response(
+ fn read(
&mut self,
- payload: &[u8],
- ) -> impl Future<Output = Result<(), IggyError>>;
+ buffer: BytesMut,
+ ) -> impl Future<Output = (Result<usize, IggyError>, BytesMut)>;
+ fn send_empty_ok_response(&mut self) -> impl Future<Output = Result<(), IggyError>>;
+ fn send_ok_response(&mut self, payload: &[u8]) -> impl Future<Output = Result<(), IggyError>>;
fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<IoSlice<'_>>,
+ slices: Vec<libc::iovec>,
) -> impl Future<Output = Result<(), IggyError>>;
fn send_error_response(
&mut self,
error: IggyError,
- ) -> impl Future<Output = Result<(), IggyError>>;
- fn shutdown(&mut self) -> impl Future<Output = Result<(), ServerError>>;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn shutdown(&mut self) -> impl Future<Output = Result<(), ServerError>>;
}
-#[allow(clippy::large_enum_variant)]
pub enum SenderKind {
Tcp(TcpSender),
TcpTls(TcpTlsSender),
@@ -90,10 +92,10 @@ impl SenderKind {
}
forward_async_methods! {
- async fn read(&mut self, buffer: &mut [u8]) -> Result<usize, IggyError>;
+ async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut);
async fn send_empty_ok_response(&mut self) -> Result<(), IggyError>;
async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError>;
- async fn send_ok_response_vectored(&mut self, length: &[u8], slices: Vec<IoSlice<'_>>) -> Result<(), IggyError>;
+ async fn send_ok_response_vectored(&mut self, length: &[u8], slices: Vec<libc::iovec>) -> Result<(), IggyError>;
async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError>;
async fn shutdown(&mut self) -> Result<(), ServerError>;
}
diff --git a/core/server/src/channels/commands/archive_state.rs b/core/server/src/channels/commands/archive_state.rs
index e80d17d3..f42393ae 100644
--- a/core/server/src/channels/commands/archive_state.rs
+++ b/core/server/src/channels/commands/archive_state.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//TODO: Fixme
+/*
use crate::channels::server_command::BackgroundServerCommand;
use crate::configs::server::StateMaintenanceConfig;
use crate::streaming::systems::system::SharedSystem;
@@ -137,3 +139,5 @@ impl BackgroundServerCommand<ArchiveStateCommand> for ArchiveStateExecutor {
});
}
}
+
+*/
diff --git a/core/server/src/channels/commands/clean_personal_access_tokens.rs b/core/server/src/channels/commands/clean_personal_access_tokens.rs
index c815a18f..fd2eaf11 100644
--- a/core/server/src/channels/commands/clean_personal_access_tokens.rs
+++ b/core/server/src/channels/commands/clean_personal_access_tokens.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+/*
use crate::channels::server_command::BackgroundServerCommand;
use crate::configs::server::PersonalAccessTokenCleanerConfig;
use crate::streaming::systems::system::SharedSystem;
@@ -135,3 +136,5 @@ impl BackgroundServerCommand<CleanPersonalAccessTokensCommand>
});
}
}
+
+*/
diff --git a/core/server/src/channels/commands/maintain_messages.rs b/core/server/src/channels/commands/maintain_messages.rs
index 16ef3647..2dc526e2 100644
--- a/core/server/src/channels/commands/maintain_messages.rs
+++ b/core/server/src/channels/commands/maintain_messages.rs
@@ -16,6 +16,9 @@
* under the License.
*/
+//TODO: Fixme
+/*
+
use crate::archiver::ArchiverKind;
use crate::channels::server_command::BackgroundServerCommand;
use crate::configs::server::MessagesMaintenanceConfig;
@@ -553,3 +556,5 @@ async fn delete_segments(
messages_count,
})
}
+
+*/
diff --git a/core/server/src/channels/commands/print_sysinfo.rs b/core/server/src/channels/commands/print_sysinfo.rs
index c130b58c..c0f2b1a2 100644
--- a/core/server/src/channels/commands/print_sysinfo.rs
+++ b/core/server/src/channels/commands/print_sysinfo.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+// TODO: Fixme
+/*
use crate::{
channels::server_command::BackgroundServerCommand,
configs::server::ServerConfig,
@@ -121,3 +123,5 @@ impl BackgroundServerCommand<SysInfoPrintCommand> for SysInfoPrintExecutor {
});
}
}
+
+*/
diff --git a/core/server/src/channels/commands/save_messages.rs b/core/server/src/channels/commands/save_messages.rs
index 586fb8b0..bf832762 100644
--- a/core/server/src/channels/commands/save_messages.rs
+++ b/core/server/src/channels/commands/save_messages.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//Todo: Fixme
+/*
use crate::channels::server_command::BackgroundServerCommand;
use crate::configs::server::MessageSaverConfig;
use crate::configs::server::ServerConfig;
@@ -116,3 +118,5 @@ impl BackgroundServerCommand<SaveMessagesCommand> for SaveMessagesExecutor {
});
}
}
+
+*/
diff --git a/core/server/src/channels/commands/verify_heartbeats.rs b/core/server/src/channels/commands/verify_heartbeats.rs
index ba48c69e..b2b01c49 100644
--- a/core/server/src/channels/commands/verify_heartbeats.rs
+++ b/core/server/src/channels/commands/verify_heartbeats.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//TODO: Fixme
+/*
use crate::channels::server_command::BackgroundServerCommand;
use crate::configs::server::HeartbeatConfig;
use crate::streaming::systems::system::SharedSystem;
@@ -148,3 +150,4 @@ impl BackgroundServerCommand<VerifyHeartbeatsCommand> for VerifyHeartbeatsExecut
});
}
}
+*/
diff --git a/core/server/src/channels/handler.rs b/core/server/src/channels/handler.rs
index 8bd82e0d..7612a259 100644
--- a/core/server/src/channels/handler.rs
+++ b/core/server/src/channels/handler.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//TODO: Fixme
+/*
use super::server_command::BackgroundServerCommand;
use crate::configs::server::ServerConfig;
use crate::streaming::systems::system::SharedSystem;
@@ -44,3 +46,5 @@ impl<'a> BackgroundServerCommandHandler<'a> {
}
}
}
+
+*/
diff --git a/core/server/src/channels/server_command.rs b/core/server/src/channels/server_command.rs
index 326b3d77..16b0dc2f 100644
--- a/core/server/src/channels/server_command.rs
+++ b/core/server/src/channels/server_command.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//TODO: Fixme
+/*
use crate::configs::server::ServerConfig;
use crate::streaming::systems::system::SharedSystem;
use flume::{Receiver, Sender};
@@ -38,3 +40,5 @@ pub trait BackgroundServerCommand<C> {
receiver: Receiver<C>,
);
}
+
+*/
diff --git a/core/server/src/configs/defaults.rs b/core/server/src/configs/defaults.rs
index 82b560cf..fc333ec7 100644
--- a/core/server/src/configs/defaults.rs
+++ b/core/server/src/configs/defaults.rs
@@ -36,6 +36,7 @@ use crate::configs::system::{
use crate::configs::tcp::{TcpConfig, TcpTlsConfig};
use iggy_common::IggyByteSize;
use iggy_common::IggyDuration;
+use std::rc::Rc;
use std::sync::Arc;
use std::time::Duration;
@@ -51,7 +52,7 @@ impl Default for ServerConfig {
heartbeat: HeartbeatConfig::default(),
message_saver: MessageSaverConfig::default(),
personal_access_token: PersonalAccessTokenConfig::default(),
- system: Arc::new(SystemConfig::default()),
+ system: Rc::new(SystemConfig::default()),
quic: QuicConfig::default(),
tcp: TcpConfig::default(),
http: HttpConfig::default(),
diff --git a/core/server/src/http/http_server.rs b/core/server/src/http/http_server.rs
index b91857f5..7dec1a7f 100644
--- a/core/server/src/http/http_server.rs
+++ b/core/server/src/http/http_server.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//TODO: Fixme
+/*
use crate::configs::http::{HttpConfig, HttpCorsConfig};
use crate::http::diagnostics::request_diagnostics;
use crate::http::jwt::cleaner::start_expired_tokens_cleaner;
@@ -191,3 +193,4 @@ fn configure_cors(config: HttpCorsConfig) -> CorsLayer {
.allow_credentials(config.allow_credentials)
.allow_private_network(config.allow_private_network)
}
+*/
diff --git a/core/server/src/quic/listener.rs b/core/server/src/quic/listener.rs
index e9b329de..f5cfc429 100644
--- a/core/server/src/quic/listener.rs
+++ b/core/server/src/quic/listener.rs
@@ -16,12 +16,14 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{ServerCommand, ServerCommandHandler};
use crate::binary::sender::SenderKind;
use crate::server_error::ConnectionError;
+use crate::shard::IggyShard;
use crate::streaming::clients::client_manager::Transport;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::anyhow;
use iggy_common::IggyError;
use quinn::{Connection, Endpoint, RecvStream, SendStream};
@@ -29,6 +31,8 @@ use tracing::{error, info, trace};
const LISTENERS_COUNT: u32 = 10;
const INITIAL_BYTES_LENGTH: usize = 4;
+//TODO: Fixme
+/*
pub fn start(endpoint: Endpoint, system: SharedSystem) {
for _ in 0..LISTENERS_COUNT {
@@ -65,7 +69,7 @@ pub fn start(endpoint: Endpoint, system: SharedSystem) {
async fn handle_connection(
incoming_connection: quinn::Connecting,
- system: SharedSystem,
+ shard: Rc<IggyShard>,
) -> Result<(), ConnectionError> {
let connection = incoming_connection.await?;
let address = connection.remote_address();
@@ -179,3 +183,4 @@ async fn handle_stream(
}
}
}
+*/
diff --git a/core/server/src/quic/quic_sender.rs b/core/server/src/quic/quic_sender.rs
index f1631cb2..17b60818 100644
--- a/core/server/src/quic/quic_sender.rs
+++ b/core/server/src/quic/quic_sender.rs
@@ -18,8 +18,11 @@
use crate::quic::COMPONENT;
use crate::{binary::sender::Sender, server_error::ServerError};
+use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::IggyError;
+use monoio::buf::IoBufMut;
+use nix::libc;
use quinn::{RecvStream, SendStream};
use std::io::IoSlice;
use tracing::{debug, error};
@@ -33,15 +36,18 @@ pub struct QuicSender {
}
impl Sender for QuicSender {
- async fn read(&mut self, buffer: &mut [u8]) -> Result<usize, IggyError> {
+ async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut) {
// Not-so-nice code because quinn recv stream has different API for read_exact
+ /*
let read_bytes = buffer.len();
self.recv.read_exact(buffer).await.map_err(|error| {
error!("Failed to read from the stream: {:?}", error);
IggyError::QuicError
})?;
+ */
- Ok(read_bytes)
+ //Ok(read_bytes)
+ todo!();
}
async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> {
@@ -64,7 +70,7 @@ impl Sender for QuicSender {
async fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<IoSlice<'_>>,
+ slices: Vec<libc::iovec>,
) -> Result<(), IggyError> {
debug!("Sending vectored response with status: {:?}...", STATUS_OK);
@@ -79,6 +85,8 @@ impl Sender for QuicSender {
let mut total_bytes_written = 0;
+ //TODO: Fixme
+ /*
for slice in slices {
let slice_data = &*slice;
if !slice_data.is_empty() {
@@ -93,6 +101,7 @@ impl Sender for QuicSender {
total_bytes_written += slice_data.len();
}
}
+ */
debug!(
"Sent vectored response: {} bytes of payload",
diff --git a/core/server/src/quic/quic_server.rs b/core/server/src/quic/quic_server.rs
index 7b8edf21..0a8020b8 100644
--- a/core/server/src/quic/quic_server.rs
+++ b/core/server/src/quic/quic_server.rs
@@ -31,8 +31,9 @@ use crate::configs::quic::QuicConfig;
use crate::quic::COMPONENT;
use crate::quic::listener;
use crate::server_error::QuicError;
-use crate::streaming::systems::system::SharedSystem;
+//TODO: Fixme
+/*
/// Starts the QUIC server.
/// Returns the address the server is listening on.
pub fn start(config: QuicConfig, system: SharedSystem) -> SocketAddr {
@@ -136,3 +137,4 @@ fn load_certificates(
let key = keys.remove(0);
Ok((certs, key))
}
+ */
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 0428a483..a2a3fdfb 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -26,7 +26,7 @@ use crate::{
configs::server::ServerConfig,
map_toggle_str,
shard::Shard,
- state::{file::FileState, StateKind},
+ state::{StateKind, file::FileState},
streaming::{diagnostics::metrics::Metrics, storage::SystemStorage},
versioning::SemanticVersion,
};
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 3e7e10f1..12ba64bb 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -17,9 +17,9 @@
*/
pub mod builder;
-pub mod system;
pub mod gate;
pub mod namespace;
+pub mod system;
pub mod transmission;
use ahash::{AHashMap, AHashSet, HashMap};
@@ -29,18 +29,42 @@ use futures::future::try_join_all;
use iggy_common::{EncryptorKind, IggyError, UserId};
use namespace::IggyNamespace;
use std::{
- cell::{Cell, RefCell}, pin::Pin, rc::Rc, str::FromStr, sync::{atomic::{AtomicU32, Ordering}, Arc, RwLock}, time::Instant
+ cell::{Cell, RefCell},
+ pin::Pin,
+ rc::Rc,
+ str::FromStr,
+ sync::{
+ Arc, RwLock,
+ atomic::{AtomicU32, Ordering},
+ },
+ time::Instant,
};
use tracing::{error, info, instrument, trace, warn};
use transmission::connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use crate::{
configs::server::ServerConfig,
- shard::{system::info::SystemInfo, transmission::frame::ShardFrame},
+ shard::{
+ system::info::SystemInfo,
+ transmission::{
+ frame::ShardFrame,
+ message::{ShardEvent, ShardMessage},
+ },
+ },
state::{
- file::FileState, system::{StreamState, SystemState, UserState}, StateKind
+ StateKind,
+ file::FileState,
+ system::{StreamState, SystemState, UserState},
+ },
+ streaming::{
+ clients::client_manager::ClientManager,
+ diagnostics::metrics::Metrics,
+ personal_access_tokens::personal_access_token::PersonalAccessToken,
+ session::Session,
+ storage::SystemStorage,
+ streams::stream::Stream,
+ users::{permissioner::Permissioner, user::User},
},
- streaming::{clients::client_manager::ClientManager, diagnostics::metrics::Metrics, personal_access_tokens::personal_access_token::PersonalAccessToken, session::Session, storage::SystemStorage, streams::stream::Stream, users::{permissioner::Permissioner, user::User}},
versioning::SemanticVersion,
};
@@ -83,7 +107,7 @@ pub struct IggyShard {
pub(crate) config: ServerConfig,
//TODO: This could be shared.
pub(crate) client_manager: RefCell<ClientManager>,
- pub(crate) active_sessions: RefCell<Vec<Session>>,
+ pub(crate) active_sessions: RefCell<Vec<Rc<Session>>>,
pub(crate) permissioner: RefCell<Permissioner>,
pub(crate) users: RefCell<HashMap<UserId, User>>,
@@ -244,8 +268,8 @@ impl IggyShard {
info!("Loading streams from disk...");
let mut unloaded_streams = Vec::new();
// Does mononio has api for that ?
- let mut dir_entries = std::fs::read_dir(&self.config.system.get_streams_path())
- .map_err(|error| {
+ let mut dir_entries =
+ std::fs::read_dir(&self.config.system.get_streams_path()).map_err(|error| {
error!("Cannot read streams directory: {error}");
IggyError::CannotReadStreams
})?;
@@ -362,16 +386,21 @@ impl IggyShard {
self.metrics.increment_messages(stream.get_messages_count());
self.streams_ids
- .borrow_mut()
+ .borrow_mut()
.insert(stream.name.clone(), stream.stream_id);
self.streams.borrow_mut().insert(stream.stream_id, stream);
}
- info!("Loaded {} stream(s) from disk.", self.streams.borrow().len());
+ info!(
+ "Loaded {} stream(s) from disk.",
+ self.streams.borrow().len()
+ );
Ok(())
}
- pub fn assert_init(&self) -> Result<(), IggyError> { Ok(())}
+ pub fn assert_init(&self) -> Result<(), IggyError> {
+ Ok(())
+ }
#[instrument(skip_all, name = "trace_shutdown")]
pub async fn shutdown(&mut self) -> Result<(), IggyError> {
@@ -397,17 +426,41 @@ impl IggyShard {
self.shards.len() as u32
}
- pub fn ensure_authenticated(&self, client_id: u32) -> Result<u32, IggyError> {
+ pub fn broadcast_event_to_all_shards(&self, client_id: u32, event: Rc<ShardEvent>) {
+ self.shards
+ .iter()
+ .filter_map(|shard| {
+ if shard.id != self.id {
+ Some(shard.connection.clone())
+ } else {
+ None
+ }
+ })
+ .map(|conn| {
+ let message = ShardMessage::Event(event.clone());
+ conn.send(ShardFrame::new(client_id, message, None));
+ })
+ .collect::<Vec<_>>();
+ }
+
+ pub fn add_active_session(&self, session: Rc<Session>) {
+ self.active_sessions.borrow_mut().push(session);
+ }
+
+ pub fn ensure_authenticated(&self, session: &Session) -> Result<u32, IggyError> {
let active_sessions = self.active_sessions.borrow();
- let session = active_sessions
+ let user_id = active_sessions
.iter()
- .find(|s| s.client_id == client_id)
- .ok_or_else(|| IggyError::Unauthenticated)?;
- if session.is_authenticated() {
- Ok(session.get_user_id())
- } else {
- error!("{COMPONENT} - unauthenticated access attempt, session: {session}");
- Err(IggyError::Unauthenticated)
- }
+ .find(|s| s.get_user_id() == session.get_user_id())
+ .ok_or_else(|| IggyError::Unauthenticated)
+ .and_then(|session| {
+ if session.is_authenticated() {
+ Ok(session.get_user_id())
+ } else {
+ error!("{COMPONENT} - unauthenticated access attempt, session: {session}");
+ Err(IggyError::Unauthenticated)
+ }
+ })?;
+ Ok(user_id)
}
}
diff --git a/core/server/src/shard/namespace.rs b/core/server/src/shard/namespace.rs
index d9d68ec9..a7c899aa 100644
--- a/core/server/src/shard/namespace.rs
+++ b/core/server/src/shard/namespace.rs
@@ -16,9 +16,9 @@
* under the License.
*/
-use std::hash::Hasher as _;
-use iggy_common::Identifier;
use hash32::{Hasher, Murmur3Hasher};
+use iggy_common::Identifier;
+use std::hash::Hasher as _;
//TODO: Will probably want to move it to separate crate so we can share it with sdk.
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs
index 2cb0fbe8..fde182ce 100644
--- a/core/server/src/shard/system/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -16,22 +16,22 @@
* under the License.
*/
+use crate::shard::IggyShard;
use crate::streaming::clients::client_manager::{Client, Transport};
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
use iggy_common::locking::IggySharedMut;
use iggy_common::locking::IggySharedMutFn;
use std::net::SocketAddr;
+use std::rc::Rc;
use std::sync::Arc;
use tracing::{error, info};
impl IggyShard {
- pub async fn add_client(&self, address: &SocketAddr, transport: Transport) -> Arc<Session> {
- let mut client_manager = self.client_manager.write().await;
+ pub fn add_client(&self, address: &SocketAddr, transport: Transport) -> Rc<Session> {
+ let mut client_manager = self.client_manager.borrow_mut();
let session = client_manager.add_client(address, transport);
info!("Added {transport} client with session: {session} for IP address: {address}");
self.metrics.increment_clients(1);
@@ -42,7 +42,7 @@ impl IggyShard {
let consumer_groups: Vec<(u32, u32, u32)>;
{
- let mut client_manager = self.client_manager.write().await;
+ let mut client_manager = self.client_manager.borrow_mut();
let client = client_manager.delete_client(client_id).await;
if client.is_none() {
error!("Client with ID: {client_id} was not found in the client manager.",);
diff --git a/core/server/src/shard/system/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs
index 92b0c400..f6e07785 100644
--- a/core/server/src/shard/system/snapshot/mod.rs
+++ b/core/server/src/shard/system/snapshot/mod.rs
@@ -20,7 +20,6 @@ mod procdump;
use crate::configs::system::SystemConfig;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::System;
use async_zip::tokio::write::ZipFileWriter;
use async_zip::{Compression, ZipEntryBuilder};
use iggy_common::{IggyDuration, IggyError, Snapshot, SnapshotCompression, SystemSnapshotType};
diff --git a/core/server/src/shard/system/storage.rs b/core/server/src/shard/system/storage.rs
index 9a493751..4cb41120 100644
--- a/core/server/src/shard/system/storage.rs
+++ b/core/server/src/shard/system/storage.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use crate::shard::system::info::SystemInfo;
use crate::streaming::persistence::persister::PersisterKind;
use crate::streaming::storage::SystemInfoStorage;
use crate::streaming::utils::PooledBuffer;
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index efcfff2b..8a0772a9 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -26,7 +26,6 @@ use std::sync::atomic::{AtomicU32, Ordering};
use tokio::fs;
use tracing::{error, info, warn};
-
impl IggyShard {
pub fn get_streams(&self) -> Vec<&Stream> {
self.streams.values().collect()
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index 3572c0ec..8d1e66eb 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -1,3 +1,5 @@
+use std::rc::Rc;
+
/* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,16 +17,18 @@
* specific language governing permissions and limitations
* under the License.
*/
-use crate::binary::command::ServerCommand;
+use crate::{binary::command::ServerCommand, streaming::session::Session};
#[derive(Debug)]
pub enum ShardMessage {
Command(ServerCommand),
- Event(ShardEvent),
+ Event(Rc<ShardEvent>),
}
#[derive(Debug)]
-pub enum ShardEvent {}
+pub enum ShardEvent {
+ NewSession(Rc<Session>),
+}
impl From<ServerCommand> for ShardMessage {
fn from(command: ServerCommand) -> Self {
@@ -32,8 +36,8 @@ impl From<ServerCommand> for ShardMessage {
}
}
-impl From<ShardEvent> for ShardMessage {
- fn from(event: ShardEvent) -> Self {
+impl From<Rc<ShardEvent>> for ShardMessage {
+ fn from(event: Rc<ShardEvent>) -> Self {
ShardMessage::Event(event)
}
}
diff --git a/core/server/src/streaming/clients/client_manager.rs b/core/server/src/streaming/clients/client_manager.rs
index 4a84a58c..fc5c00cc 100644
--- a/core/server/src/streaming/clients/client_manager.rs
+++ b/core/server/src/streaming/clients/client_manager.rs
@@ -22,27 +22,26 @@ use ahash::AHashMap;
use iggy_common::IggyError;
use iggy_common::IggyTimestamp;
use iggy_common::UserId;
-use iggy_common::locking::IggySharedMut;
use iggy_common::locking::IggySharedMutFn;
use std::fmt::{Display, Formatter};
use std::net::SocketAddr;
-use std::sync::Arc;
+use std::rc::Rc;
#[derive(Debug, Default)]
pub struct ClientManager {
- clients: AHashMap<u32, IggySharedMut<Client>>,
+ clients: AHashMap<u32, Client>,
}
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct Client {
pub user_id: Option<u32>,
- pub session: Arc<Session>,
+ pub session: Rc<Session>,
pub transport: Transport,
pub consumer_groups: Vec<ConsumerGroup>,
pub last_heartbeat: IggyTimestamp,
}
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct ConsumerGroup {
pub stream_id: u32,
pub topic_id: u32,
@@ -65,9 +64,9 @@ impl Display for Transport {
}
impl ClientManager {
- pub fn add_client(&mut self, address: &SocketAddr, transport: Transport) -> Arc<Session> {
+ pub fn add_client(&mut self, address: &SocketAddr, transport: Transport) -> Rc<Session> {
let client_id = hash::calculate_32(address.to_string().as_bytes());
- let session = Arc::new(Session::from_client_id(client_id, *address));
+ let session = Rc::new(Session::from_client_id(client_id, *address));
let client = Client {
user_id: None,
session: session.clone(),
@@ -75,44 +74,43 @@ impl ClientManager {
consumer_groups: Vec::new(),
last_heartbeat: IggyTimestamp::now(),
};
- self.clients.insert(client_id, IggySharedMut::new(client));
+ self.clients.insert(client_id, client);
session
}
- pub async fn set_user_id(&mut self, client_id: u32, user_id: UserId) -> Result<(), IggyError> {
- let client = self.clients.get(&client_id);
+ pub fn set_user_id(&mut self, client_id: u32, user_id: UserId) -> Result<(), IggyError> {
+ let client = self.clients.get_mut(&client_id);
if client.is_none() {
return Err(IggyError::ClientNotFound(client_id));
}
- let mut client = client.unwrap().write().await;
+ let mut client = client.unwrap();
client.user_id = Some(user_id);
Ok(())
}
- pub async fn clear_user_id(&mut self, client_id: u32) -> Result<(), IggyError> {
- let client = self.clients.get(&client_id);
+ pub fn clear_user_id(&mut self, client_id: u32) -> Result<(), IggyError> {
+ let client = self.clients.get_mut(&client_id);
if client.is_none() {
return Err(IggyError::ClientNotFound(client_id));
}
- let mut client = client.unwrap().write().await;
+ let mut client = client.unwrap();
client.user_id = None;
Ok(())
}
- pub fn try_get_client(&self, client_id: u32) -> Option<IggySharedMut<Client>> {
+ pub fn try_get_client(&self, client_id: u32) -> Option<Client> {
self.clients.get(&client_id).cloned()
}
- pub fn get_clients(&self) -> Vec<IggySharedMut<Client>> {
+ pub fn get_clients(&self) -> Vec<Client> {
self.clients.values().cloned().collect()
}
- pub async fn delete_clients_for_user(&mut self, user_id: UserId) -> Result<(), IggyError> {
+ pub fn delete_clients_for_user(&mut self, user_id: UserId) -> Result<(), IggyError> {
let mut clients_to_remove = Vec::new();
for client in self.clients.values() {
- let client = client.read().await;
if let Some(client_user_id) = client.user_id {
if client_user_id == user_id {
clients_to_remove.push(client.session.client_id);
@@ -127,28 +125,27 @@ impl ClientManager {
Ok(())
}
- pub async fn delete_client(&mut self, client_id: u32) -> Option<IggySharedMut<Client>> {
+ pub fn delete_client(&mut self, client_id: u32) -> Option<Client> {
let client = self.clients.remove(&client_id);
if let Some(client) = client.as_ref() {
- let client = client.read().await;
client.session.clear_user_id();
}
client
}
- pub async fn join_consumer_group(
- &self,
+ pub fn join_consumer_group(
+ &mut self,
client_id: u32,
stream_id: u32,
topic_id: u32,
group_id: u32,
) -> Result<(), IggyError> {
- let client = self.clients.get(&client_id);
+ let client = self.clients.get_mut(&client_id);
if client.is_none() {
return Err(IggyError::ClientNotFound(client_id));
}
- let mut client = client.unwrap().write().await;
+ let client = client.unwrap();
if client.consumer_groups.iter().any(|consumer_group| {
consumer_group.group_id == group_id
&& consumer_group.topic_id == topic_id
@@ -165,18 +162,18 @@ impl ClientManager {
Ok(())
}
- pub async fn leave_consumer_group(
- &self,
+ pub fn leave_consumer_group(
+ &mut self,
client_id: u32,
stream_id: u32,
topic_id: u32,
consumer_group_id: u32,
) -> Result<(), IggyError> {
- let client = self.clients.get(&client_id);
+ let client = self.clients.get_mut(&client_id);
if client.is_none() {
return Err(IggyError::ClientNotFound(client_id));
}
- let mut client = client.unwrap().write().await;
+ let mut client = client.unwrap();
for (index, consumer_group) in client.consumer_groups.iter().enumerate() {
if consumer_group.stream_id == stream_id
&& consumer_group.topic_id == topic_id
@@ -189,9 +186,8 @@ impl ClientManager {
Ok(())
}
- pub async fn delete_consumer_groups_for_stream(&self, stream_id: u32) {
- for client in self.clients.values() {
- let mut client = client.write().await;
+ pub fn delete_consumer_groups_for_stream(&mut self, stream_id: u32) {
+ for client in self.clients.values_mut() {
let indexes_to_remove = client
.consumer_groups
.iter()
@@ -210,9 +206,8 @@ impl ClientManager {
}
}
- pub async fn delete_consumer_groups_for_topic(&self, stream_id: u32, topic_id: u32) {
- for client in self.clients.values() {
- let mut client = client.write().await;
+ pub fn delete_consumer_groups_for_topic(&mut self, stream_id: u32, topic_id: u32) {
+ for client in self.clients.values_mut() {
let indexes_to_remove = client
.consumer_groups
.iter()
diff --git a/core/server/src/streaming/partitions/messages.rs b/core/server/src/streaming/partitions/messages.rs
index a00b371f..a2d71c8e 100644
--- a/core/server/src/streaming/partitions/messages.rs
+++ b/core/server/src/streaming/partitions/messages.rs
@@ -368,6 +368,7 @@ mod tests {
use crate::streaming::utils::MemoryPool;
use bytes::Bytes;
use iggy_common::{IggyExpiry, IggyMessage};
+ use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64};
use tempfile::TempDir;
@@ -710,7 +711,7 @@ mod tests {
let partition_id = 3;
let with_segment = true;
let temp_dir = TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: temp_dir.path().to_path_buf().to_str().unwrap().to_string(),
message_deduplication: MessageDeduplicationConfig {
enabled: deduplication_enabled,
@@ -718,7 +719,7 @@ mod tests {
},
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
diff --git a/core/server/src/streaming/partitions/partition.rs b/core/server/src/streaming/partitions/partition.rs
index 73c190cc..4a00ca85 100644
--- a/core/server/src/streaming/partitions/partition.rs
+++ b/core/server/src/streaming/partitions/partition.rs
@@ -28,6 +28,7 @@ use iggy_common::IggyExpiry;
use iggy_common::IggyTimestamp;
use iggy_common::Sizeable;
use std::fmt;
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
@@ -58,8 +59,8 @@ pub struct Partition {
pub(crate) consumer_offsets: DashMap<u32, ConsumerOffset>,
pub(crate) consumer_group_offsets: DashMap<u32, ConsumerOffset>,
pub(crate) segments: Vec<Segment>,
- pub(crate) config: Arc<SystemConfig>,
- pub(crate) storage: Arc<SystemStorage>,
+ pub(crate) config: Rc<SystemConfig>,
+ pub(crate) storage: Rc<SystemStorage>,
}
#[derive(Debug, PartialEq, Clone)]
@@ -88,8 +89,8 @@ impl Partition {
topic_id: u32,
partition_id: u32,
with_segment: bool,
- config: Arc<SystemConfig>,
- storage: Arc<SystemStorage>,
+ config: Rc<SystemConfig>,
+ storage: Rc<SystemStorage>,
message_expiry: IggyExpiry,
messages_count_of_parent_stream: Arc<AtomicU64>,
messages_count_of_parent_topic: Arc<AtomicU64>,
@@ -209,17 +210,18 @@ mod tests {
use iggy_common::IggyDuration;
use iggy_common::IggyExpiry;
use iggy_common::IggyTimestamp;
+ use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64};
#[tokio::test]
async fn should_be_created_with_a_single_segment_given_valid_parameters() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
@@ -264,11 +266,11 @@ mod tests {
#[tokio::test]
async fn should_not_initialize_segments_given_false_with_segment_parameter() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
@@ -279,7 +281,7 @@ mod tests {
topic_id,
1,
false,
- Arc::new(SystemConfig::default()),
+ config,
storage,
IggyExpiry::NeverExpire,
Arc::new(AtomicU64::new(0)),
diff --git a/core/server/src/streaming/segments/segment.rs b/core/server/src/streaming/segments/segment.rs
index acfc98f4..252fe7f1 100644
--- a/core/server/src/streaming/segments/segment.rs
+++ b/core/server/src/streaming/segments/segment.rs
@@ -27,6 +27,7 @@ use iggy_common::IggyByteSize;
use iggy_common::IggyError;
use iggy_common::IggyExpiry;
use iggy_common::IggyTimestamp;
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::fs::remove_file;
@@ -60,7 +61,7 @@ pub struct Segment {
pub(super) index_reader: Option<IndexReader>,
pub(super) message_expiry: IggyExpiry,
pub(super) accumulator: MessagesAccumulator,
- pub(super) config: Arc<SystemConfig>,
+ pub(super) config: Rc<SystemConfig>,
pub(super) indexes: IggyIndexesMut,
pub(super) messages_size: Arc<AtomicU64>,
pub(super) indexes_size: Arc<AtomicU64>,
@@ -73,7 +74,7 @@ impl Segment {
topic_id: u32,
partition_id: u32,
start_offset: u64,
- config: Arc<SystemConfig>,
+ config: Rc<SystemConfig>,
message_expiry: IggyExpiry,
size_of_parent_stream: Arc<AtomicU64>,
size_of_parent_topic: Arc<AtomicU64>,
@@ -450,7 +451,7 @@ mod tests {
let topic_id = 2;
let partition_id = 3;
let start_offset = 0;
- let config = Arc::new(SystemConfig::default());
+ let config = Rc::new(SystemConfig::default());
let path = config.get_segment_path(stream_id, topic_id, partition_id, start_offset);
let messages_file_path = Segment::get_messages_file_path(&path);
let index_path = Segment::get_index_path(&path);
@@ -499,7 +500,7 @@ mod tests {
let topic_id = 2;
let partition_id = 3;
let start_offset = 0;
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
segment: SegmentConfig {
cache_indexes: CacheIndexesConfig::None,
..Default::default()
diff --git a/core/server/src/streaming/session.rs b/core/server/src/streaming/session.rs
index 1eae4eb0..3f412bce 100644
--- a/core/server/src/streaming/session.rs
+++ b/core/server/src/streaming/session.rs
@@ -19,13 +19,16 @@
use iggy_common::{AtomicUserId, UserId};
use std::fmt::Display;
use std::net::SocketAddr;
+use std::rc::Rc;
use std::sync::atomic::{AtomicBool, Ordering};
// This might be extended with more fields in the future e.g. custom name, permissions etc.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct Session {
- user_id: AtomicUserId,
- active: AtomicBool,
+ // TODO: Fixme, those don't have to be atomics anymore, simple integers will suffice
+ // Once the atomics are removed, we can impl Copy trait aswell.
+ user_id: Rc<AtomicUserId>,
+ active: Rc<AtomicBool>,
pub client_id: u32,
pub ip_address: SocketAddr,
}
@@ -34,8 +37,8 @@ impl Session {
pub fn new(client_id: u32, user_id: UserId, ip_address: SocketAddr) -> Self {
Self {
client_id,
- active: AtomicBool::new(true),
- user_id: AtomicUserId::new(user_id),
+ active: Rc::new(AtomicBool::new(true)),
+ user_id: Rc::new(AtomicUserId::new(user_id)),
ip_address,
}
}
@@ -49,15 +52,15 @@ impl Session {
}
pub fn get_user_id(&self) -> UserId {
- self.user_id.load(Ordering::Acquire)
+ self.user_id.load(Ordering::Relaxed)
}
pub fn set_user_id(&self, user_id: UserId) {
- self.user_id.store(user_id, Ordering::Release)
+ self.user_id.store(user_id, Ordering::Relaxed)
}
pub fn set_stale(&self) {
- self.active.store(false, Ordering::Release)
+ self.active.store(false, Ordering::Relaxed)
}
pub fn clear_user_id(&self) {
@@ -65,7 +68,7 @@ impl Session {
}
pub fn is_active(&self) -> bool {
- self.active.load(Ordering::Acquire)
+ self.active.load(Ordering::Relaxed)
}
pub fn is_authenticated(&self) -> bool {
diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs
index f423b842..51e2565e 100644
--- a/core/server/src/streaming/storage.rs
+++ b/core/server/src/streaming/storage.rs
@@ -18,13 +18,13 @@
use super::persistence::persister::PersisterKind;
use crate::configs::system::SystemConfig;
+use crate::shard::system::info::SystemInfo;
+use crate::shard::system::storage::FileSystemInfoStorage;
use crate::state::system::{PartitionState, StreamState, TopicState};
use crate::streaming::partitions::partition::{ConsumerOffset, Partition};
use crate::streaming::partitions::storage::FilePartitionStorage;
use crate::streaming::streams::storage::FileStreamStorage;
use crate::streaming::streams::stream::Stream;
-use crate::streaming::systems::info::SystemInfo;
-use crate::streaming::systems::storage::FileSystemInfoStorage;
use crate::streaming::topics::storage::FileTopicStorage;
use crate::streaming::topics::topic::Topic;
use iggy_common::ConsumerKind;
@@ -85,61 +85,61 @@ pub enum PartitionStorageKind {
}
#[cfg_attr(test, automock)]
-pub trait SystemInfoStorage: Send {
- fn load(&self) -> impl Future<Output = Result<SystemInfo, IggyError>> + Send;
- fn save(&self, system_info: &SystemInfo) -> impl Future<Output = Result<(), IggyError>> + Send;
+pub trait SystemInfoStorage {
+ fn load(&self) -> impl Future<Output = Result<SystemInfo, IggyError>>;
+ fn save(&self, system_info: &SystemInfo) -> impl Future<Output = Result<(), IggyError>>;
}
#[cfg_attr(test, automock)]
-pub trait StreamStorage: Send {
+pub trait StreamStorage {
fn load(
&self,
stream: &mut Stream,
state: StreamState,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn save(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn delete(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>>;
}
#[cfg_attr(test, automock)]
-pub trait TopicStorage: Send {
+pub trait TopicStorage {
fn load(
&self,
topic: &mut Topic,
state: TopicState,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn save(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn delete(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
}
#[cfg_attr(test, automock)]
-pub trait PartitionStorage: Send {
+pub trait PartitionStorage {
fn load(
&self,
partition: &mut Partition,
state: PartitionState,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
fn save(&self, partition: &mut Partition)
- -> impl Future<Output = Result<(), IggyError>> + Send;
- fn delete(&self, partition: &Partition) -> impl Future<Output = Result<(), IggyError>> + Send;
+ -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, partition: &Partition) -> impl Future<Output = Result<(), IggyError>>;
fn save_consumer_offset(
&self,
offset: u64,
path: &str,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
fn load_consumer_offsets(
&self,
kind: ConsumerKind,
path: &str,
- ) -> impl Future<Output = Result<Vec<ConsumerOffset>, IggyError>> + Send;
+ ) -> impl Future<Output = Result<Vec<ConsumerOffset>, IggyError>>;
fn delete_consumer_offsets(
&self,
path: &str,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
fn delete_consumer_offset(
&self,
path: &str,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
}
#[derive(Debug)]
diff --git a/core/server/src/streaming/streams/storage.rs b/core/server/src/streaming/streams/storage.rs
index 3e3e5e76..53639de9 100644
--- a/core/server/src/streaming/streams/storage.rs
+++ b/core/server/src/streaming/streams/storage.rs
@@ -156,7 +156,7 @@ impl StreamStorage for FileStreamStorage {
for mut topic in unloaded_topics {
let loaded_topics = loaded_topics.clone();
let topic_state = state.topics.remove(&topic.topic_id).unwrap();
- let load_topic = tokio::spawn(async move {
+ let load_topic = monoio::spawn(async move {
match topic.load(topic_state).await {
Ok(_) => loaded_topics.lock().await.push(topic),
Err(error) => error!(
diff --git a/core/server/src/streaming/streams/stream.rs b/core/server/src/streaming/streams/stream.rs
index 339fc08a..03681b75 100644
--- a/core/server/src/streaming/streams/stream.rs
+++ b/core/server/src/streaming/streams/stream.rs
@@ -40,8 +40,8 @@ pub struct Stream {
pub segments_count: Arc<AtomicU32>,
pub(crate) topics: AHashMap<u32, Topic>,
pub(crate) topics_ids: AHashMap<String, u32>,
- pub(crate) config: Arc<SystemConfig>,
- pub(crate) storage: Arc<SystemStorage>,
+ pub(crate) config: Rc<SystemConfig>,
+ pub(crate) storage: Rc<SystemStorage>,
}
impl Stream {
@@ -106,18 +106,18 @@ mod tests {
#[test]
fn should_be_created_given_valid_parameters() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
MemoryPool::init_pool(config.clone());
let id = 1;
let name = "test";
- let config = Arc::new(SystemConfig::default());
+ let config = Rc::new(SystemConfig::default());
let path = config.get_stream_path(id);
let topics_path = config.get_topics_path(id);
diff --git a/core/server/src/streaming/streams/topics.rs b/core/server/src/streaming/streams/topics.rs
index f8c742be..b975ee83 100644
--- a/core/server/src/streaming/streams/topics.rs
+++ b/core/server/src/streaming/streams/topics.rs
@@ -146,8 +146,7 @@ impl Stream {
topic.name = name.to_owned();
topic.message_expiry = message_expiry;
topic.compression_algorithm = compression_algorithm;
- for partition in topic.partitions.values_mut() {
- let mut partition = partition.write().await;
+ for partition in topic.partitions.borrow_mut().values_mut() {
partition.message_expiry = message_expiry;
for segment in partition.segments.iter_mut() {
segment.update_message_expiry(message_expiry);
@@ -282,16 +281,16 @@ mod tests {
},
};
use iggy_common::IggyByteSize;
- use std::sync::Arc;
+ use std::{rc::Rc, sync::Arc};
#[tokio::test]
async fn should_get_topic_by_id_and_name() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
diff --git a/core/server/src/streaming/topics/consumer_group.rs b/core/server/src/streaming/topics/consumer_group.rs
index dedb29c7..43d9611b 100644
--- a/core/server/src/streaming/topics/consumer_group.rs
+++ b/core/server/src/streaming/topics/consumer_group.rs
@@ -21,16 +21,16 @@ use iggy_common::IggyError;
use tokio::sync::RwLock;
use tracing::trace;
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct ConsumerGroup {
pub topic_id: u32,
pub group_id: u32,
pub name: String,
pub partitions_count: u32,
- members: AHashMap<u32, RwLock<ConsumerGroupMember>>,
+ members: AHashMap<u32, ConsumerGroupMember>,
}
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct ConsumerGroupMember {
pub id: u32,
partitions: AHashMap<u32, u32>,
@@ -49,8 +49,8 @@ impl ConsumerGroup {
}
}
- pub fn get_members(&self) -> Vec<&RwLock<ConsumerGroupMember>> {
- self.members.values().collect()
+ pub fn get_members(&self) -> Vec<ConsumerGroupMember> {
+ self.members.values().cloned().collect()
}
pub async fn reassign_partitions(&mut self, partitions_count: u32) {
@@ -61,7 +61,7 @@ impl ConsumerGroup {
pub async fn calculate_partition_id(&self, member_id: u32) -> Result<Option<u32>, IggyError> {
let member = self.members.get(&member_id);
if let Some(member) = member {
- return Ok(member.write().await.calculate_partition_id());
+ return Ok(member.await.calculate_partition_id());
}
Err(IggyError::ConsumerGroupMemberNotFound(
member_id,
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index dec2f163..26505355 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -91,7 +91,7 @@ impl Topic {
self.get_consumer_group_by_id(*group_id.unwrap())
}
- pub fn get_consumer_group_by_id(&self, id: u32) -> Result<&RwLock<ConsumerGroup>, IggyError> {
+ pub fn get_consumer_group_by_id(&self, id: u32) -> Result<ConsumerGroup, IggyError> {
let consumer_group = self.consumer_groups.get(&id);
if consumer_group.is_none() {
return Err(IggyError::ConsumerGroupIdNotFound(id, self.topic_id));
@@ -104,7 +104,7 @@ impl Topic {
&mut self,
group_id: Option<u32>,
name: &str,
- ) -> Result<&RwLock<ConsumerGroup>, IggyError> {
+ ) -> Result<ConsumerGroup, IggyError> {
if self.consumer_groups_ids.contains_key(name) {
return Err(IggyError::ConsumerGroupNameAlreadyExists(
name.to_owned(),
@@ -138,20 +138,20 @@ impl Topic {
}
let consumer_group =
- ConsumerGroup::new(self.topic_id, id, name, self.partitions.len() as u32);
- self.consumer_groups.insert(id, RwLock::new(consumer_group));
+ ConsumerGroup::new(self.topic_id, id, name, self.partitions.borrow().len() as u32);
+ self.consumer_groups.insert(id, consumer_group.clone());
self.consumer_groups_ids.insert(name.to_owned(), id);
info!(
"Created consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
id, self.topic_id, self.stream_id
);
- self.get_consumer_group_by_id(id)
+ consumer_group
}
pub async fn delete_consumer_group(
&mut self,
id: &Identifier,
- ) -> Result<RwLock<ConsumerGroup>, IggyError> {
+ ) -> Result<ConsumerGroup, IggyError> {
let group_id;
{
let consumer_group = self.get_consumer_group(id).with_error_context(|error| {
@@ -167,7 +167,6 @@ impl Topic {
}
let consumer_group = consumer_group.unwrap();
{
- let consumer_group = consumer_group.read().await;
let group_id = consumer_group.group_id;
self.consumer_groups_ids.remove(&consumer_group.name);
let current_group_id = self.current_consumer_group_id.load(Ordering::SeqCst);
@@ -176,8 +175,7 @@ impl Topic {
.store(group_id, Ordering::SeqCst);
}
- for (_, partition) in self.partitions.iter() {
- let partition = partition.read().await;
+ for (_, partition) in self.partitions.borrow().iter() {
if let Some((_, offset)) = partition.consumer_group_offsets.remove(&group_id) {
self.storage
.partition
diff --git a/core/server/src/streaming/topics/topic.rs b/core/server/src/streaming/topics/topic.rs
index b0913856..ad3e25ef 100644
--- a/core/server/src/streaming/topics/topic.rs
+++ b/core/server/src/streaming/topics/topic.rs
@@ -23,6 +23,8 @@ use crate::streaming::storage::SystemStorage;
use crate::streaming::topics::consumer_group::ConsumerGroup;
use ahash::AHashMap;
use core::fmt;
+use std::cell::RefCell;
+use std::rc::Rc;
use iggy_common::locking::IggySharedMut;
use iggy_common::{
CompressionAlgorithm, Consumer, ConsumerKind, IggyByteSize, IggyError, IggyExpiry,
@@ -48,10 +50,10 @@ pub struct Topic {
pub(crate) messages_count_of_parent_stream: Arc<AtomicU64>,
pub(crate) messages_count: Arc<AtomicU64>,
pub(crate) segments_count_of_parent_stream: Arc<AtomicU32>,
- pub(crate) config: Arc<SystemConfig>,
- pub(crate) partitions: AHashMap<u32, IggySharedMut<Partition>>,
- pub(crate) storage: Arc<SystemStorage>,
- pub(crate) consumer_groups: AHashMap<u32, RwLock<ConsumerGroup>>,
+ pub(crate) config: Rc<SystemConfig>,
+ pub(crate) partitions: RefCell<AHashMap<u32, Partition>>,
+ pub(crate) storage: Rc<SystemStorage>,
+ pub(crate) consumer_groups: AHashMap<u32, ConsumerGroup>,
pub(crate) consumer_groups_ids: AHashMap<String, u32>,
pub(crate) current_consumer_group_id: AtomicU32,
pub(crate) current_partition_id: AtomicU32,
@@ -71,8 +73,8 @@ impl Topic {
size_of_parent_stream: Arc<AtomicU64>,
messages_count_of_parent_stream: Arc<AtomicU64>,
segments_count_of_parent_stream: Arc<AtomicU32>,
- config: Arc<SystemConfig>,
- storage: Arc<SystemStorage>,
+ config: Rc<SystemConfig>,
+ storage: Rc<SystemStorage>,
) -> Topic {
Topic::create(
stream_id,
@@ -99,8 +101,8 @@ impl Topic {
topic_id: u32,
name: &str,
partitions_count: u32,
- config: Arc<SystemConfig>,
- storage: Arc<SystemStorage>,
+ config: Rc<SystemConfig>,
+ storage: Rc<SystemStorage>,
size_of_parent_stream: Arc<AtomicU64>,
messages_count_of_parent_stream: Arc<AtomicU64>,
segments_count_of_parent_stream: Arc<AtomicU32>,
diff --git a/core/server/src/streaming/utils/memory_pool.rs b/core/server/src/streaming/utils/memory_pool.rs
index 61962f0a..e05b7359 100644
--- a/core/server/src/streaming/utils/memory_pool.rs
+++ b/core/server/src/streaming/utils/memory_pool.rs
@@ -21,6 +21,7 @@ use bytes::BytesMut;
use crossbeam::queue::ArrayQueue;
use human_repr::HumanCount;
use once_cell::sync::OnceCell;
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use tracing::{info, trace, warn};
@@ -153,7 +154,7 @@ impl MemoryPool {
}
/// Initialize the global pool from the given config.
- pub fn init_pool(config: Arc<SystemConfig>) {
+ pub fn init_pool(config: Rc<SystemConfig>) {
let is_enabled = config.memory_pool.enabled;
let memory_limit = config.memory_pool.size.as_bytes_usize();
let bucket_capacity = config.memory_pool.bucket_capacity as usize;
@@ -467,7 +468,7 @@ mod tests {
fn initialize_pool_for_tests() {
TEST_INIT.call_once(|| {
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
memory_pool: MemoryPoolConfig {
enabled: true,
size: IggyByteSize::from_str("4GiB").unwrap(),
diff --git a/core/server/src/tcp/connection_handler.rs b/core/server/src/tcp/connection_handler.rs
index ec61b23b..9c92d27d 100644
--- a/core/server/src/tcp/connection_handler.rs
+++ b/core/server/src/tcp/connection_handler.rs
@@ -19,27 +19,28 @@
use crate::binary::command::ServerCommandHandler;
use crate::binary::{command, sender::SenderKind};
use crate::server_error::ConnectionError;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use crate::tcp::connection_handler::command::ServerCommand;
+use bytes::BytesMut;
use iggy_common::IggyError;
use std::io::ErrorKind;
-use std::sync::Arc;
+use std::rc::Rc;
use tracing::{debug, error, info};
const INITIAL_BYTES_LENGTH: usize = 4;
pub(crate) async fn handle_connection(
- session: Arc<Session>,
+ session: &Rc<Session>,
sender: &mut SenderKind,
- system: SharedSystem,
+ shard: &Rc<IggyShard>,
) -> Result<(), ConnectionError> {
- let mut length_buffer = [0u8; INITIAL_BYTES_LENGTH];
- let mut code_buffer = [0u8; INITIAL_BYTES_LENGTH];
+ let length_buffer = BytesMut::with_capacity(INITIAL_BYTES_LENGTH);
+ let code_buffer = BytesMut::with_capacity(INITIAL_BYTES_LENGTH);
loop {
- let read_length = match sender.read(&mut length_buffer).await {
- Ok(read_length) => read_length,
- Err(error) => {
+ let (read_length, initial_buffer) = match sender.read(length_buffer.clone()).await {
+ (Ok(read_length), initial_buffer) => (read_length, initial_buffer),
+ (Err(error), _) => {
if error.as_code() == IggyError::ConnectionClosed.as_code() {
return Err(ConnectionError::from(error));
} else {
@@ -56,13 +57,18 @@ pub(crate) async fn handle_connection(
continue;
}
- let length = u32::from_le_bytes(length_buffer);
- sender.read(&mut code_buffer).await?;
- let code = u32::from_le_bytes(code_buffer);
+ let initial_buffer = initial_buffer.freeze();
+ let length =
+ u32::from_le_bytes(initial_buffer[0..INITIAL_BYTES_LENGTH].try_into().unwrap());
+ let (res, code_buffer) = sender.read(code_buffer.clone()).await;
+ let _ = res?;
+ let code_buffer = code_buffer.freeze();
+ let code: u32 =
+ u32::from_le_bytes(code_buffer[0..INITIAL_BYTES_LENGTH].try_into().unwrap());
debug!("Received a TCP request, length: {length}, code: {code}");
let command = ServerCommand::from_code_and_reader(code, sender, length - 4).await?;
debug!("Received a TCP command: {command}, payload size: {length}");
- match command.handle(sender, length, &session, &system).await {
+ match command.handle(sender, length, session, shard).await {
Ok(_) => {
debug!(
"Command was handled successfully, session: {session}. TCP response was sent."
diff --git a/core/server/src/tcp/sender.rs b/core/server/src/tcp/sender.rs
index 4dd11725..6a10adb2 100644
--- a/core/server/src/tcp/sender.rs
+++ b/core/server/src/tcp/sender.rs
@@ -16,25 +16,33 @@
* under the License.
*/
+use bytes::BytesMut;
use iggy_common::IggyError;
-use monoio::io::{AsyncReadRent, AsyncReadRentExt, AsyncWriteRent, AsyncWriteRentExt};
+use monoio::{
+ buf::IoBufMut,
+ io::{AsyncReadRent, AsyncReadRentExt, AsyncWriteRent, AsyncWriteRentExt},
+};
+use nix::libc;
use std::io::IoSlice;
use tracing::debug;
const STATUS_OK: &[u8] = &[0; 4];
-pub(crate) async fn read<T>(stream: &mut T, buffer: &mut [u8]) -> Result<usize, IggyError>
+pub(crate) async fn read<T>(
+ stream: &mut T,
+ buffer: BytesMut,
+) -> (Result<usize, IggyError>, BytesMut)
where
T: AsyncReadRent + AsyncWriteRent + Unpin,
{
match stream.read_exact(buffer).await {
- Ok(0) => Err(IggyError::ConnectionClosed),
- Ok(read_bytes) => Ok(read_bytes),
- Err(error) => {
+ (Ok(0), buffer) => (Err(IggyError::ConnectionClosed), buffer),
+ (Ok(read_bytes), buffer) => (Ok(read_bytes), buffer),
+ (Err(error), buffer) => {
if error.kind() == std::io::ErrorKind::UnexpectedEof {
- Err(IggyError::ConnectionClosed)
+ (Err(IggyError::ConnectionClosed), buffer)
} else {
- Err(IggyError::TcpError)
+ (Err(IggyError::TcpError), buffer)
}
}
}
@@ -57,7 +65,7 @@ where
pub(crate) async fn send_ok_response_vectored<T>(
stream: &mut T,
length: &[u8],
- slices: Vec<IoSlice<'_>>,
+ slices: Vec<libc::iovec>,
) -> Result<(), IggyError>
where
T: AsyncReadRentExt + AsyncWriteRentExt + Unpin,
@@ -90,8 +98,9 @@ where
);
let length = (payload.len() as u32).to_le_bytes();
stream
- .write_all(&[status, &length, payload].as_slice().concat())
+ .write_all([status, &length, payload].concat())
.await
+ .0
.map_err(|_| IggyError::TcpError)?;
debug!("Sent response with status: {:?}", status);
Ok(())
@@ -101,7 +110,7 @@ pub(crate) async fn send_response_vectored<T>(
stream: &mut T,
status: &[u8],
length: &[u8],
- mut slices: Vec<IoSlice<'_>>,
+ mut slices: Vec<libc::iovec>,
) -> Result<(), IggyError>
where
T: AsyncReadRentExt + AsyncWriteRentExt + Unpin,
@@ -111,16 +120,22 @@ where
slices.len(),
status
);
- let prefix = [IoSlice::new(status), IoSlice::new(length)];
+ let prefix = [
+ libc::iovec {
+ iov_base: length.as_ptr() as _,
+ iov_len: length.len(),
+ },
+ libc::iovec {
+ iov_base: status.as_ptr() as _,
+ iov_len: status.len(),
+ },
+ ];
slices.splice(0..0, prefix);
- let mut slice_refs = slices.as_mut_slice();
- while !slice_refs.is_empty() {
- let bytes_written = stream
- .write_vectored(slice_refs)
- .await
- .map_err(|_| IggyError::TcpError)?;
- IoSlice::advance_slices(&mut slice_refs, bytes_written);
- }
+ stream
+ .write_vectored_all(slices)
+ .await
+ .0
+ .map_err(|_| IggyError::TcpError)?;
debug!("Sent response with status: {:?}", status);
Ok(())
}
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index 2e4f25be..ffa01eb1 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -18,39 +18,50 @@
use crate::binary::sender::SenderKind;
use crate::shard::IggyShard;
+use crate::shard::transmission::message::ShardEvent;
use crate::streaming::clients::client_manager::Transport;
use crate::tcp::connection_handler::{handle_connection, handle_error};
+use crate::tcp::tcp_socket;
use std::net::SocketAddr;
use std::rc::Rc;
-use monoio::net::TcpListener;
-use rustls::pki_types::Ipv4Addr;
-use tokio::net::TcpSocket;
-use tokio::sync::oneshot;
use tracing::{error, info};
-pub async fn start(server_name: &str, shard: Rc<IggyShard>) {
- let addr: SocketAddr = if shard.config.tcp.ipv6 {
- shard.config.tcp.address.parse().expect("Unable to parse IPv6 address")
- } else {
- shard.config.tcp.address.parse().expect("Unable to parse IPv4 address")
- };
+pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) {
+ let ip_v6 = shard.config.tcp.ipv6;
+ let socket_config = &shard.config.tcp.socket;
+ let addr: SocketAddr = shard
+ .config
+ .tcp
+ .address
+ .parse()
+ .expect("Failed to parse TCP address");
+
+ let socket = tcp_socket::build(ip_v6, socket_config);
monoio::spawn(async move {
- let listener = TcpListener::bind(addr).expect(format!("Unable to start {server_name}.").as_ref());
+ socket
+ .bind(&addr.into())
+ .expect("Failed to bind eTCP listener");
+ socket.listen(1024);
+ let listener: std::net::TcpListener = socket.into();
+ let listener = monoio::net::TcpListener::from_std(listener).unwrap();
+ info!("{server_name} server has started on: {:?}", addr);
loop {
match listener.accept().await {
Ok((stream, address)) => {
let shard = shard.clone();
info!("Accepted new TCP connection: {address}");
- let session = shard
- .add_client(&address, Transport::Tcp);
+ let session = shard.add_client(&address, Transport::Tcp);
+ //TODO: Those can be shared with other shards.
+ shard.add_active_session(session.clone());
+ // Broadcast session to all shards.
+ let event = Rc::new(ShardEvent::NewSession(session.clone()));
+ shard.broadcast_event_to_all_shards(session.client_id, event);
- let client_id = session.client_id;
+ let _client_id = session.client_id;
info!("Created new session: {session}");
let mut sender = SenderKind::get_tcp_sender(stream);
monoio::spawn(async move {
- if let Err(error) =
- handle_connection(session, &mut sender, shard.clone()).await
- {
+ if let Err(error) = handle_connection(&session, &mut sender, &shard).await {
handle_error(error);
//TODO: Fixme
/*
diff --git a/core/server/src/tcp/tcp_sender.rs b/core/server/src/tcp/tcp_sender.rs
index a157dead..81089c85 100644
--- a/core/server/src/tcp/tcp_sender.rs
+++ b/core/server/src/tcp/tcp_sender.rs
@@ -19,10 +19,13 @@
use crate::binary::sender::Sender;
use crate::tcp::COMPONENT;
use crate::{server_error::ServerError, tcp::sender};
+use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::IggyError;
-use tokio::{io::AsyncWriteExt};
+use monoio::buf::IoBufMut;
+use monoio::io::AsyncWriteRent;
use monoio::net::TcpStream;
+use nix::libc;
#[derive(Debug)]
pub struct TcpSender {
@@ -30,7 +33,7 @@ pub struct TcpSender {
}
impl Sender for TcpSender {
- async fn read(&mut self, buffer: &mut [u8]) -> Result<usize, IggyError> {
+ async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut) {
sender::read(&mut self.stream, buffer).await
}
@@ -59,7 +62,7 @@ impl Sender for TcpSender {
async fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<std::io::IoSlice<'_>>,
+ slices: Vec<libc::iovec>,
) -> Result<(), IggyError> {
sender::send_ok_response_vectored(&mut self.stream, length, slices).await
}
diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs
index 82bbe0c1..9b8e1c04 100644
--- a/core/server/src/tcp/tcp_server.rs
+++ b/core/server/src/tcp/tcp_server.rs
@@ -17,9 +17,9 @@
*/
use crate::shard::IggyShard;
-use crate::tcp::{tcp_listener};
-use std::rc::Rc;
+use crate::tcp::tcp_listener;
use iggy_common::IggyError;
+use std::rc::Rc;
use tracing::info;
/// Starts the TCP server.
diff --git a/core/server/src/tcp/tcp_socket.rs b/core/server/src/tcp/tcp_socket.rs
index 69c1d408..747540fa 100644
--- a/core/server/src/tcp/tcp_socket.rs
+++ b/core/server/src/tcp/tcp_socket.rs
@@ -16,16 +16,18 @@
* under the License.
*/
+use socket2::{Domain, Protocol, Socket, Type};
use std::num::TryFromIntError;
-
use crate::configs::tcp::TcpSocketConfig;
-pub fn build(ipv6: bool, config: TcpSocketConfig) -> TcpSocket {
+pub fn build(ipv6: bool, config: &TcpSocketConfig) -> Socket {
let socket = if ipv6 {
- TcpSocket::new_v6().expect("Unable to create an ipv6 socket")
+ Socket::new(Domain::IPV6, Type::STREAM, Some(Protocol::TCP))
+ .expect("Unable to create an ipv6 socket")
} else {
- TcpSocket::new_v4().expect("Unable to create an ipv4 socket")
+ Socket::new(Domain::IPV4, Type::STREAM, Some(Protocol::TCP))
+ .expect("Unable to create an ipv4 socket")
};
if config.override_defaults {
@@ -34,7 +36,7 @@ pub fn build(ipv6: bool, config: TcpSocketConfig) -> TcpSocket {
.as_bytes_u64()
.try_into()
.map_err(|e: TryFromIntError| std::io::Error::other(e.to_string()))
- .and_then(|size: u32| socket.set_recv_buffer_size(size))
+ .and_then(|size| socket.set_recv_buffer_size(size))
.expect("Unable to set SO_RCVBUF on socket");
config
.send_buffer_size
@@ -52,6 +54,12 @@ pub fn build(ipv6: bool, config: TcpSocketConfig) -> TcpSocket {
socket
.set_linger(Some(config.linger.get_duration()))
.expect("Unable to set SO_LINGER on socket");
+ socket
+ .set_reuse_address(true)
+ .expect("Unable to set SO_REUSEADDR on socket");
+ socket
+ .set_reuse_port(true)
+ .expect("Unable to set SO_REUSEPORT on socket");
}
socket
@@ -77,9 +85,9 @@ mod tests {
nodelay: true,
linger: IggyDuration::new(linger_dur),
};
- let socket = build(false, config);
- assert!(socket.recv_buffer_size().unwrap() >= buffer_size as u32);
- assert!(socket.send_buffer_size().unwrap() >= buffer_size as u32);
+ let socket = build(false, &config);
+ assert!(socket.recv_buffer_size().unwrap() >= buffer_size as usize);
+ assert!(socket.send_buffer_size().unwrap() >= buffer_size as usize);
assert!(socket.keepalive().unwrap());
assert!(socket.nodelay().unwrap());
assert_eq!(socket.linger().unwrap(), Some(linger_dur));
diff --git a/core/server/src/tcp/tcp_tls_listener.rs b/core/server/src/tcp/tcp_tls_listener.rs
index d2bbf8da..53df4b0b 100644
--- a/core/server/src/tcp/tcp_tls_listener.rs
+++ b/core/server/src/tcp/tcp_tls_listener.rs
@@ -18,26 +18,30 @@
use crate::binary::sender::SenderKind;
use crate::configs::tcp::TcpTlsConfig;
+use crate::shard::IggyShard;
use crate::streaming::clients::client_manager::Transport;
-use crate::streaming::systems::system::SharedSystem;
use crate::tcp::connection_handler::{handle_connection, handle_error};
+use monoio_native_tls::TlsAcceptor;
use rustls::ServerConfig;
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
use rustls_pemfile::{certs, private_key};
use std::io::BufReader;
use std::net::SocketAddr;
+use std::rc::Rc;
use std::sync::Arc;
use tokio::net::TcpSocket;
use tokio::sync::oneshot;
-use tokio_rustls::{TlsAcceptor, rustls};
use tracing::{error, info};
pub(crate) async fn start(
address: &str,
config: TcpTlsConfig,
socket: TcpSocket,
- system: SharedSystem,
+ system: Rc<IggyShard>,
) -> SocketAddr {
+ //TODO: Fixme
+ todo!();
+ /*
let address = address.to_string();
let (tx, rx) = oneshot::channel();
tokio::spawn(async move {
@@ -131,6 +135,7 @@ pub(crate) async fn start(
Ok(addr) => addr,
Err(_) => panic!("Failed to get the local address for TCP TLS listener."),
}
+ */
}
fn generate_self_signed_cert()
diff --git a/core/server/src/tcp/tcp_tls_sender.rs b/core/server/src/tcp/tcp_tls_sender.rs
index 6ba62315..7213ad64 100644
--- a/core/server/src/tcp/tcp_tls_sender.rs
+++ b/core/server/src/tcp/tcp_tls_sender.rs
@@ -19,10 +19,13 @@
use crate::binary::sender::Sender;
use crate::tcp::COMPONENT;
use crate::{server_error::ServerError, tcp::sender};
+use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::IggyError;
+use monoio::io::AsyncWriteRent;
use monoio::net::TcpStream;
use monoio_native_tls::TlsStream;
+use nix::libc;
#[derive(Debug)]
pub struct TcpTlsSender {
@@ -30,7 +33,7 @@ pub struct TcpTlsSender {
}
impl Sender for TcpTlsSender {
- async fn read(&mut self, buffer: &mut [u8]) -> Result<usize, IggyError> {
+ async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut) {
sender::read(&mut self.stream, buffer).await
}
@@ -59,7 +62,7 @@ impl Sender for TcpTlsSender {
async fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<std::io::IoSlice<'_>>,
+ slices: Vec<libc::iovec>,
) -> Result<(), IggyError> {
sender::send_ok_response_vectored(&mut self.stream, length, slices).await
}
(iggy) 07/30: fix state initialization with gate
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 69de5d8d377f9e463bac377ac992a314f7729dfc
Author: numinex <g....@gmail.com>
AuthorDate: Sat Jun 21 13:51:27 2025 +0200
fix state initialization with gate
---
core/common/src/utils/crypto.rs | 3 +-
core/server/src/bootstrap.rs | 48 ++---
core/server/src/lib.rs | 8 +-
core/server/src/main.rs | 151 ++++++++++++---
core/server/src/shard/builder.rs | 65 +++----
core/server/src/shard/gate.rs | 42 +++++
core/server/src/shard/mod.rs | 256 ++++++++++++++++++++++++--
core/server/src/shard/transmission/frame.rs | 17 ++
core/server/src/shard/transmission/message.rs | 17 ++
core/server/src/shard/transmission/mod.rs | 18 ++
core/server/src/state/file.rs | 4 +-
core/server/src/streaming/systems/info.rs | 55 ------
core/server/src/streaming/systems/system.rs | 21 ++-
13 files changed, 524 insertions(+), 181 deletions(-)
diff --git a/core/common/src/utils/crypto.rs b/core/common/src/utils/crypto.rs
index 34cd72c7..11d51cb7 100644
--- a/core/common/src/utils/crypto.rs
+++ b/core/common/src/utils/crypto.rs
@@ -23,7 +23,7 @@ use aes_gcm::aead::{Aead, OsRng};
use aes_gcm::{AeadCore, Aes256Gcm, KeyInit};
use std::fmt::Debug;
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub enum EncryptorKind {
Aes256Gcm(Aes256GcmEncryptor),
}
@@ -46,6 +46,7 @@ pub trait Encryptor {
fn decrypt(&self, data: &[u8]) -> Result<Vec<u8>, IggyError>;
}
+#[derive(Clone)]
pub struct Aes256GcmEncryptor {
cipher: Aes256Gcm,
}
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index c6108568..4f356a9b 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -12,10 +12,13 @@ use crate::{
IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
configs::{config_provider::ConfigProviderKind, server::ServerConfig, system::SystemConfig},
server_error::ServerError,
- shard::{transmission::connector::ShardConnector, transmission::frame::ShardFrame},
- streaming::users::user::User,
+ shard::transmission::{connector::ShardConnector, frame::ShardFrame},
+ streaming::{
+ persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind},
+ users::user::User,
+ },
};
-use std::{env, fs::remove_dir_all, ops::Range, path::Path};
+use std::{env, fs::remove_dir_all, ops::Range, path::Path, sync::Arc};
pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<ShardFrame>> {
let shards_count = shards_set.len();
@@ -68,37 +71,7 @@ pub async fn create_directories(config: &SystemConfig) -> Result<(), IggyError>
// TODO: Move this to individual shard level
/*
- let state_entries = self.state.init().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to initialize state entries")
- })?;
- let system_state = SystemState::init(state_entries)
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to initialize system state")
- })?;
- let now = Instant::now();
- self.load_version().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load version")
- })?;
- self.load_users(system_state.users.into_values().collect())
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load users")
- })?;
- self.load_streams(system_state.streams.into_values().collect())
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load streams")
- })?;
- if let Some(archiver) = self.archiver.as_ref() {
- archiver
- .init()
- .await
- .expect("Failed to initialize archiver");
- }
- info!("Initialized system in {} ms.", now.elapsed().as_millis());
- Ok(())
- */
+ */
}
pub fn create_root_user() -> User {
@@ -158,3 +131,10 @@ pub fn create_shard_executor() -> Runtime<TimeDriver<monoio::IoUringDriver>> {
let rt = Buildable::build(builder).expect("Failed to create default runtime");
rt
}
+
+pub fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
+ match enforce_fsync {
+ true => Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister)),
+ false => Arc::new(PersisterKind::File(FilePersister)),
+ }
+}
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index e64837da..3dcf64b2 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -43,11 +43,11 @@ pub mod streaming;
pub mod tcp;
pub mod versioning;
-const VERSION: &str = env!("CARGO_PKG_VERSION");
-const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME";
-const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD";
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+pub const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME";
+pub const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD";
-pub(crate) fn map_toggle_str<'a>(enabled: bool) -> &'a str {
+pub fn map_toggle_str<'a>(enabled: bool) -> &'a str {
match enabled {
true => "enabled",
false => "disabled",
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 871ff2f1..3a6c1686 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use std::sync::Arc;
use std::thread::available_parallelism;
use anyhow::Result;
@@ -23,33 +24,32 @@ use clap::Parser;
use dotenvy::dotenv;
use error_set::ErrContext;
use figlet_rs::FIGfont;
+use iggy_common::create_user::CreateUser;
+use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError};
use server::args::Args;
use server::bootstrap::{
create_default_executor, create_directories, create_root_user, create_shard_connections,
- create_shard_executor, load_config,
+ create_shard_executor, load_config, resolve_persister,
};
-use server::channels::commands::archive_state::ArchiveStateExecutor;
-use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
-use server::channels::commands::maintain_messages::MaintainMessagesExecutor;
-use server::channels::commands::print_sysinfo::SysInfoPrintExecutor;
-use server::channels::commands::save_messages::SaveMessagesExecutor;
-use server::channels::commands::verify_heartbeats::VerifyHeartbeatsExecutor;
-use server::channels::handler::BackgroundServerCommandHandler;
-use server::configs::config_provider::{self, ConfigProviderKind};
-use server::configs::server::ServerConfig;
-use server::http::http_server;
+use server::configs::config_provider::{self};
#[cfg(not(feature = "tokio-console"))]
use server::log::logger::Logging;
#[cfg(feature = "tokio-console")]
use server::log::tokio_console::Logging;
-use server::quic::quic_server;
use server::server_error::ServerError;
use server::shard::IggyShard;
-use server::streaming::systems::system::{SharedSystem, System};
-use server::streaming::utils::MemoryPool;
-use server::tcp::tcp_server;
+use server::shard::gate::Gate;
+use server::state::StateKind;
+use server::state::command::EntryCommand;
+use server::state::file::FileState;
+use server::state::models::CreateUserWithId;
+use server::state::system::SystemState;
+use server::versioning::SemanticVersion;
+use server::{IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV, map_toggle_str};
use tokio::time::Instant;
-use tracing::{info, instrument};
+use tracing::{error, info, instrument};
+
+const COMPONENT: &str = "MAIN";
#[instrument(skip_all, name = "trace_start_server")]
fn main() -> Result<(), ServerError> {
@@ -88,7 +88,11 @@ fn main() -> Result<(), ServerError> {
config
})?;
- // Create directories and root user.
+ // Initialize logging
+ let mut logging = Logging::new(config.telemetry.clone());
+ logging.early_init();
+
+ // Create directories.
// Remove `local_data` directory if run with `--fresh` flag.
std::thread::scope(|scope| {
scope
@@ -117,13 +121,9 @@ fn main() -> Result<(), ServerError> {
})
})
.join()
- .expect("Failed to create directories and root user")
+ .expect("Failed join thread")
})
- .with_error_context(|err| format!("Failed to create directories, err: {err}"))?;
-
- // Initialize logging
- let mut logging = Logging::new(config.telemetry.clone());
- logging.early_init();
+ .with_error_context(|err| format!("Failed to init server: {err}"))?;
// TODO: Make this configurable from config as a range
// for example this instance of Iggy will use cores from 0..4
@@ -132,9 +132,11 @@ fn main() -> Result<(), ServerError> {
let shards_set = 0..shards_count;
let connections = create_shard_connections(shards_set.clone());
for shard_id in shards_set {
+ let gate: Arc<Gate<()>> = Arc::new(Gate::new());
let id = shard_id as u16;
let connections = connections.clone();
- let server_config = config.clone();
+ let config = config.clone();
+ let state_persister = resolve_persister(config.system.state.enforce_fsync);
std::thread::Builder::new()
.name(format!("shard-{id}"))
.spawn(move || {
@@ -143,18 +145,109 @@ fn main() -> Result<(), ServerError> {
let mut rt = create_shard_executor();
rt.block_on(async move {
+ let version = SemanticVersion::current().expect("Invalid version");
+ info!(
+ "Server-side encryption is {}.",
+ map_toggle_str(config.system.encryption.enabled)
+ );
+ let encryptor: Option<EncryptorKind> = match config.system.encryption.enabled {
+ true => Some(EncryptorKind::Aes256Gcm(
+ Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key)
+ .unwrap(),
+ )),
+ false => None,
+ };
+
+ let state = StateKind::File(FileState::new(
+ &config.system.get_state_messages_file_path(),
+ &version,
+ state_persister,
+ encryptor.clone(),
+ ));
+
+ // We can't use std::sync::Once because it doesn't support async.
+ // Trait bound on the closure is FnOnce.
+ let gate = gate.clone();
+ // Peak into the state to check if the root user exists.
+ // If it does not exist, create it.
+ gate.with_async::<Result<(), IggyError>>(async |gate_state| {
+ // A thread already initialized state
+ // Thus, we can skip it.
+ if let Some(_) = gate_state.inner() {
+ return Ok(());
+ }
+
+ let state_entries = state.load_entries().await.with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to load state entries"
+ )
+ })?;
+ let root_exists = state_entries
+ .into_iter()
+ .find(|entry| {
+ entry
+ .command()
+ .and_then(|command| match command {
+ EntryCommand::CreateUser(payload)
+ if payload.command.username
+ == IGGY_ROOT_USERNAME_ENV && payload.command.password == IGGY_ROOT_PASSWORD_ENV =>
+ {
+ Ok(true)
+ }
+ _ => Ok(false),
+ })
+ .map_or_else(
+ |err| {
+ error!("Failed to check if root user exists: {err}");
+ false
+ },
+ |v| v,
+ )
+ })
+ .is_some();
+
+ if !root_exists {
+ info!("No users found, creating the root user...");
+ let root = create_root_user();
+ let command = CreateUser {
+ username: root.username.clone(),
+ password: root.password.clone(),
+ status: root.status,
+ permissions: root.permissions.clone(),
+ };
+ state
+ .apply(0, &EntryCommand::CreateUser(CreateUserWithId {
+ user_id: root.id,
+ command
+ }))
+ .await
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to apply create user command, username: {}",
+ root.username
+ )
+ })?;
+ }
+
+ gate_state.set_result(());
+ Ok(())
+ })
+ .await;
+
let builder = IggyShard::builder();
let mut shard = builder
.id(id)
.connections(connections)
- .server_config(server_config)
+ .config(config)
+ .encryptor(encryptor)
+ .version(version)
+ .state(state)
.build()
.await;
- if let Err(e) = shard.init().await {
- //TODO: If one of the shards fails to initialize, we should crash the whole program;
- panic!("Failed to initialize shard-{id}: {e}");
- }
+ //TODO: If one of the shards fails to initialize, we should crash the whole program;
+ shard.init().await.expect("Failed to initialize shard-{id}: {e}");
+ info!("Initiated shard with ID: {id}");
//TODO: If one of the shards fails to initialize, we should crash the whole program;
shard.assert_init();
})
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 36fe0e54..0fd5250f 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -22,14 +22,12 @@ use iggy_common::{Aes256GcmEncryptor, EncryptorKind};
use tracing::info;
use crate::{
+ bootstrap::resolve_persister,
configs::server::ServerConfig,
map_toggle_str,
shard::Shard,
state::{StateKind, file::FileState},
- streaming::{
- persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind},
- storage::SystemStorage,
- },
+ streaming::storage::SystemStorage,
versioning::SemanticVersion,
};
@@ -40,6 +38,9 @@ pub struct IggyShardBuilder {
id: Option<u16>,
connections: Option<Vec<ShardConnector<ShardFrame>>>,
config: Option<ServerConfig>,
+ encryptor: Option<EncryptorKind>,
+ version: Option<SemanticVersion>,
+ state: Option<StateKind>,
}
impl IggyShardBuilder {
@@ -53,16 +54,33 @@ impl IggyShardBuilder {
self
}
- pub fn server_config(mut self, config: ServerConfig) -> Self {
+ pub fn config(mut self, config: ServerConfig) -> Self {
self.config = Some(config);
self
}
+ pub fn encryptor(mut self, encryptor: Option<EncryptorKind>) -> Self {
+ self.encryptor = encryptor;
+ self
+ }
+
+ pub fn version(mut self, version: SemanticVersion) -> Self {
+ self.version = Some(version);
+ self
+ }
+
+ pub fn state(mut self, state: StateKind) -> Self {
+ self.state = Some(state);
+ self
+ }
+
// TODO: Too much happens in there, some of those bootstrapping logic should be moved outside.
pub async fn build(self) -> IggyShard {
let id = self.id.unwrap();
let config = self.config.unwrap();
let connections = self.connections.unwrap();
+ let state = self.state.unwrap();
+ let version = self.version.unwrap();
let (stop_sender, stop_receiver, frame_receiver) = connections
.iter()
.filter(|c| c.id == id)
@@ -76,33 +94,8 @@ impl IggyShardBuilder {
.next()
.expect("Failed to find connection with the specified ID");
let shards = connections.into_iter().map(Shard::new).collect();
-
- //TODO: This can be discrete step in the builder bootstrapped from main function.
- let version = SemanticVersion::current().expect("Invalid version");
-
- //TODO: This can be discrete step in the builder bootstrapped from main function.
- info!(
- "Server-side encryption is {}.",
- map_toggle_str(config.system.encryption.enabled)
- );
- let encryptor: Option<Arc<EncryptorKind>> = match config.system.encryption.enabled {
- true => Some(Arc::new(EncryptorKind::Aes256Gcm(
- Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key).unwrap(),
- ))),
- false => None,
- };
-
- //TODO: This can be discrete step in the builder bootstrapped from main function.
- let state_persister = Self::resolve_persister(config.system.state.enforce_fsync);
- let state = Rc::new(StateKind::File(FileState::new(
- &config.system.get_state_messages_file_path(),
- &version,
- state_persister,
- encryptor.clone(),
- )));
-
- //TODO: This can be discrete step in the builder bootstrapped from main function.
- let partition_persister = Self::resolve_persister(config.system.partition.enforce_fsync);
+ //TODO: Eghhhh.......
+ let partition_persister = resolve_persister(config.system.partition.enforce_fsync);
let storage = Rc::new(SystemStorage::new(
config.system.clone(),
partition_persister,
@@ -115,16 +108,10 @@ impl IggyShardBuilder {
storage: storage,
state: state,
config: config,
+ version: version,
stop_receiver: stop_receiver,
stop_sender: stop_sender,
frame_receiver: Cell::new(Some(frame_receiver)),
}
}
-
- fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
- match enforce_fsync {
- true => Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister)),
- false => Arc::new(PersisterKind::File(FilePersister)),
- }
- }
}
diff --git a/core/server/src/shard/gate.rs b/core/server/src/shard/gate.rs
new file mode 100644
index 00000000..a26950a4
--- /dev/null
+++ b/core/server/src/shard/gate.rs
@@ -0,0 +1,42 @@
+use std::sync::{Condvar, Mutex};
+
+#[derive(Default)]
+pub struct Gate<T> {
+ state: Mutex<GateState<T>>,
+}
+
+#[derive(Default)]
+pub struct GateState<T> {
+ result: Option<T>,
+}
+
+impl<T> GateState<T> {
+ pub fn set_result(&mut self, result: T) {
+ self.result = Some(result);
+ }
+
+ pub fn inner(&self) -> &Option<T> {
+ &self.result
+ }
+}
+
+impl<T> Gate<T>
+where
+ T: Default,
+{
+ pub fn new() -> Self {
+ Gate {
+ state: Default::default(),
+ }
+ }
+
+ pub async fn with_async<R>(&self, f: impl AsyncFnOnce(&mut GateState<T>) -> R) {
+ let mut guard = self.state.lock().unwrap();
+ f(&mut guard).await;
+ }
+
+ pub async fn with<R>(&self, f: impl FnOnce(&mut GateState<T>) -> R) {
+ let mut guard = self.state.lock().unwrap();
+ f(&mut guard);
+ }
+}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 6a6d35b8..23ef1e15 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -17,16 +17,19 @@
*/
pub mod builder;
+pub mod gate;
pub mod namespace;
pub mod transmission;
use ahash::HashMap;
use builder::IggyShardBuilder;
+use error_set::ErrContext;
use iggy_common::IggyError;
use namespace::IggyNamespace;
use std::{
cell::{Cell, RefCell},
rc::Rc,
+ str::FromStr,
sync::Arc,
time::Instant,
};
@@ -37,9 +40,17 @@ use crate::{
bootstrap::create_root_user,
configs::server::ServerConfig,
shard::transmission::frame::ShardFrame,
- state::{StateKind, file::FileState},
- streaming::storage::SystemStorage,
+ state::{
+ StateKind,
+ file::FileState,
+ system::{SystemState, UserState},
+ },
+ streaming::{storage::SystemStorage, systems::info::SystemInfo},
+ versioning::SemanticVersion,
};
+
+pub const COMPONENT: &str = "SHARD";
+
pub(crate) struct Shard {
id: u16,
connection: ShardConnector<ShardFrame>,
@@ -62,6 +73,7 @@ pub struct IggyShard {
pub id: u16,
shards: Vec<Shard>,
shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
+ version: SemanticVersion,
//pub(crate) permissioner: RefCell<Permissioner>,
//pub(crate) streams: RwLock<HashMap<u32, Stream>>,
@@ -70,7 +82,7 @@ pub struct IggyShard {
// TODO: Refactor.
pub(crate) storage: Rc<SystemStorage>,
- pub(crate) state: Rc<StateKind>,
+ pub(crate) state: StateKind,
//pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
config: ServerConfig,
//pub(crate) client_manager: RefCell<ClientManager>,
@@ -92,10 +104,12 @@ impl IggyShard {
//let state_entries = self.state.init().await?;
//let system_state = SystemState::init(state_entries).await?;
//let user = create_root_user();
- self.load_state().await;
- self.load_users().await;
+ self.load_version().await?;
+ let SystemState { users, streams } = self.load_state().await?;
+ self.load_users(users.into_values().collect()).await;
// Add default root user.
- self.load_streams().await;
+ self.load_streams(streams.into_values().collect()).await;
+
//TODO: Fix the archiver.
/*
if let Some(archiver) = self.archiver.as_ref() {
@@ -109,16 +123,234 @@ impl IggyShard {
Ok(())
}
- async fn load_state(&self) {
- todo!()
+ async fn load_version(&self) -> Result<(), IggyError> {
+ async fn update_system_info(
+ storage: &Rc<SystemStorage>,
+ system_info: &mut SystemInfo,
+ version: &SemanticVersion,
+ ) -> Result<(), IggyError> {
+ system_info.update_version(version);
+ storage.info.save(system_info).await?;
+ Ok(())
+ }
+
+ let current_version = &self.version;
+ let mut system_info;
+ let load_system_info = self.storage.info.load().await;
+ if load_system_info.is_err() {
+ let error = load_system_info.err().unwrap();
+ if let IggyError::ResourceNotFound(_) = error {
+ info!("System info not found, creating...");
+ system_info = SystemInfo::default();
+ update_system_info(&self.storage, &mut system_info, current_version).await?;
+ } else {
+ return Err(error);
+ }
+ } else {
+ system_info = load_system_info.unwrap();
+ }
+
+ info!("Loaded {system_info}.");
+ let loaded_version = SemanticVersion::from_str(&system_info.version.version)?;
+ if current_version.is_equal_to(&loaded_version) {
+ info!("System version {current_version} is up to date.");
+ } else if current_version.is_greater_than(&loaded_version) {
+ info!(
+ "System version {current_version} is greater than {loaded_version}, checking the available migrations..."
+ );
+ update_system_info(&self.storage, &mut system_info, current_version).await?;
+ } else {
+ info!(
+ "System version {current_version} is lower than {loaded_version}, possible downgrade."
+ );
+ update_system_info(&self.storage, &mut system_info, current_version).await?;
+ }
+
+ Ok(())
}
- async fn load_users(&self) {
- todo!()
+ async fn load_state(&self) -> Result<SystemState, IggyError> {
+ let state_entries = self.state.init().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to initialize state entries")
+ })?;
+ let system_state = SystemState::init(state_entries)
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to initialize system state")
+ })?;
+ Ok(system_state)
}
- async fn load_streams(&self) {
- todo!()
+ async fn load_users(&mut self, users: Vec<UserState>) -> Result<(), IggyError> {
+ info!("Loading users...");
+ /*
+
+ for user_state in users.into_iter() {
+ let mut user = User::with_password(
+ user_state.id,
+ &user_state.username,
+ user_state.password_hash,
+ user_state.status,
+ user_state.permissions,
+ );
+
+ user.created_at = user_state.created_at;
+ user.personal_access_tokens = user_state
+ .personal_access_tokens
+ .into_values()
+ .map(|token| {
+ (
+ Arc::new(token.token_hash.clone()),
+ PersonalAccessToken::raw(
+ user_state.id,
+ &token.name,
+ &token.token_hash,
+ token.expiry_at,
+ ),
+ )
+ })
+ .collect();
+ self.users.insert(user_state.id, user);
+ }
+
+ let users_count = self.users.len();
+ let current_user_id = self.users.keys().max().unwrap_or(&1);
+ USER_ID.store(current_user_id + 1, Ordering::SeqCst);
+ self.permissioner
+ .init(&self.users.values().collect::<Vec<&User>>());
+ self.metrics.increment_users(users_count as u32);
+ info!("Initialized {users_count} user(s).");
+ */
+ Ok(())
+ }
+
+ async fn load_streams(&mut self, streams: Vec<StreamState>) -> Result<(), IggyError> {
+ todo!();
+ info!("Loading streams from disk...");
+ let mut unloaded_streams = Vec::new();
+ let mut dir_entries = read_dir(&self.config.get_streams_path())
+ .await
+ .map_err(|error| {
+ error!("Cannot read streams directory: {error}");
+ IggyError::CannotReadStreams
+ })?;
+
+ while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) {
+ let name = dir_entry.file_name().into_string().unwrap();
+ let stream_id = name.parse::<u32>().map_err(|_| {
+ error!("Invalid stream ID file with name: '{name}'.");
+ IggyError::InvalidNumberValue
+ })?;
+ let stream_state = streams.iter().find(|s| s.id == stream_id);
+ if stream_state.is_none() {
+ error!(
+ "Stream with ID: '{stream_id}' was not found in state, but exists on disk and will be removed."
+ );
+ if let Err(error) = fs::remove_dir_all(&dir_entry.path()).await {
+ error!("Cannot remove stream directory: {error}");
+ } else {
+ warn!("Stream with ID: '{stream_id}' was removed.");
+ }
+ continue;
+ }
+
+ let stream_state = stream_state.unwrap();
+ let mut stream = Stream::empty(
+ stream_id,
+ &stream_state.name,
+ self.config.clone(),
+ self.storage.clone(),
+ );
+ stream.created_at = stream_state.created_at;
+ unloaded_streams.push(stream);
+ }
+
+ let state_stream_ids = streams
+ .iter()
+ .map(|stream| stream.id)
+ .collect::<AHashSet<u32>>();
+ let unloaded_stream_ids = unloaded_streams
+ .iter()
+ .map(|stream| stream.stream_id)
+ .collect::<AHashSet<u32>>();
+ let mut missing_ids = state_stream_ids
+ .difference(&unloaded_stream_ids)
+ .copied()
+ .collect::<AHashSet<u32>>();
+ if missing_ids.is_empty() {
+ info!("All streams found on disk were found in state.");
+ } else {
+ warn!("Streams with IDs: '{missing_ids:?}' were not found on disk.");
+ if self.config.recovery.recreate_missing_state {
+ info!(
+ "Recreating missing state in recovery config is enabled, missing streams will be created."
+ );
+ for stream_id in missing_ids.iter() {
+ let stream_id = *stream_id;
+ let stream_state = streams.iter().find(|s| s.id == stream_id).unwrap();
+ let stream = Stream::create(
+ stream_id,
+ &stream_state.name,
+ self.config.clone(),
+ self.storage.clone(),
+ );
+ stream.persist().await?;
+ unloaded_streams.push(stream);
+ info!(
+ "Missing stream with ID: '{stream_id}', name: {} was recreated.",
+ stream_state.name
+ );
+ }
+ missing_ids.clear();
+ } else {
+ warn!(
+ "Recreating missing state in recovery config is disabled, missing streams will not be created."
+ );
+ }
+ }
+
+ let mut streams_states = streams
+ .into_iter()
+ .filter(|s| !missing_ids.contains(&s.id))
+ .map(|s| (s.id, s))
+ .collect::<AHashMap<_, _>>();
+ let loaded_streams = RefCell::new(Vec::new());
+ let load_stream_tasks = unloaded_streams.into_iter().map(|mut stream| {
+ let state = streams_states.remove(&stream.stream_id).unwrap();
+
+ async {
+ stream.load(state).await?;
+ loaded_streams.borrow_mut().push(stream);
+ Result::<(), IggyError>::Ok(())
+ }
+ });
+ try_join_all(load_stream_tasks).await?;
+
+ for stream in loaded_streams.take() {
+ if self.streams.contains_key(&stream.stream_id) {
+ error!("Stream with ID: '{}' already exists.", &stream.stream_id);
+ continue;
+ }
+
+ if self.streams_ids.contains_key(&stream.name) {
+ error!("Stream with name: '{}' already exists.", &stream.name);
+ continue;
+ }
+
+ self.metrics.increment_streams(1);
+ self.metrics.increment_topics(stream.get_topics_count());
+ self.metrics
+ .increment_partitions(stream.get_partitions_count());
+ self.metrics.increment_segments(stream.get_segments_count());
+ self.metrics.increment_messages(stream.get_messages_count());
+
+ self.streams_ids
+ .insert(stream.name.clone(), stream.stream_id);
+ self.streams.insert(stream.stream_id, stream);
+ }
+
+ info!("Loaded {} stream(s) from disk.", self.streams.len());
+ Ok(())
}
pub fn assert_init(&self) {}
diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs
index ac60863d..ec63daf0 100644
--- a/core/server/src/shard/transmission/frame.rs
+++ b/core/server/src/shard/transmission/frame.rs
@@ -1,3 +1,20 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
use async_channel::Sender;
use bytes::Bytes;
use iggy_common::IggyError;
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index e9f1dd4b..3572c0ec 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -1,3 +1,20 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
use crate::binary::command::ServerCommand;
#[derive(Debug)]
diff --git a/core/server/src/shard/transmission/mod.rs b/core/server/src/shard/transmission/mod.rs
index b871152a..1f9a79ed 100644
--- a/core/server/src/shard/transmission/mod.rs
+++ b/core/server/src/shard/transmission/mod.rs
@@ -1,3 +1,21 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
pub mod connector;
pub mod frame;
pub mod message;
diff --git a/core/server/src/state/file.rs b/core/server/src/state/file.rs
index 591b8072..41117c3a 100644
--- a/core/server/src/state/file.rs
+++ b/core/server/src/state/file.rs
@@ -47,7 +47,7 @@ pub struct FileState {
version: u32,
path: String,
persister: Arc<PersisterKind>,
- encryptor: Option<Arc<EncryptorKind>>,
+ encryptor: Option<EncryptorKind>,
}
impl FileState {
@@ -55,7 +55,7 @@ impl FileState {
path: &str,
version: &SemanticVersion,
persister: Arc<PersisterKind>,
- encryptor: Option<Arc<EncryptorKind>>,
+ encryptor: Option<EncryptorKind>,
) -> Self {
Self {
current_index: AtomicU64::new(0),
diff --git a/core/server/src/streaming/systems/info.rs b/core/server/src/streaming/systems/info.rs
index 61c1bcbb..3c20fd63 100644
--- a/core/server/src/streaming/systems/info.rs
+++ b/core/server/src/streaming/systems/info.rs
@@ -16,15 +16,11 @@
* under the License.
*/
-use crate::streaming::systems::system::System;
use crate::versioning::SemanticVersion;
-use iggy_common::IggyError;
use serde::{Deserialize, Serialize};
use std::collections::hash_map::DefaultHasher;
use std::fmt::Display;
use std::hash::{Hash, Hasher};
-use std::str::FromStr;
-use tracing::info;
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct SystemInfo {
@@ -46,57 +42,6 @@ pub struct Migration {
pub applied_at: u64,
}
-impl System {
- pub(crate) async fn load_version(&mut self) -> Result<(), IggyError> {
- let current_version = SemanticVersion::current()?;
- let mut system_info;
- let load_system_info = self.storage.info.load().await;
- if load_system_info.is_err() {
- let error = load_system_info.err().unwrap();
- if let IggyError::ResourceNotFound(_) = error {
- info!("System info not found, creating...");
- system_info = SystemInfo::default();
- self.update_system_info(&mut system_info, ¤t_version)
- .await?;
- } else {
- return Err(error);
- }
- } else {
- system_info = load_system_info.unwrap();
- }
-
- info!("Loaded {system_info}.");
- let loaded_version = SemanticVersion::from_str(&system_info.version.version)?;
- if current_version.is_equal_to(&loaded_version) {
- info!("System version {current_version} is up to date.");
- } else if current_version.is_greater_than(&loaded_version) {
- info!(
- "System version {current_version} is greater than {loaded_version}, checking the available migrations..."
- );
- self.update_system_info(&mut system_info, ¤t_version)
- .await?;
- } else {
- info!(
- "System version {current_version} is lower than {loaded_version}, possible downgrade."
- );
- self.update_system_info(&mut system_info, ¤t_version)
- .await?;
- }
-
- Ok(())
- }
-
- async fn update_system_info(
- &self,
- system_info: &mut SystemInfo,
- version: &SemanticVersion,
- ) -> Result<(), IggyError> {
- system_info.update_version(version);
- self.storage.info.save(system_info).await?;
- Ok(())
- }
-}
-
impl SystemInfo {
pub fn update_version(&mut self, version: &SemanticVersion) {
self.version.version = version.to_string();
diff --git a/core/server/src/streaming/systems/system.rs b/core/server/src/streaming/systems/system.rs
index 761f6d1b..5d830d71 100644
--- a/core/server/src/streaming/systems/system.rs
+++ b/core/server/src/streaming/systems/system.rs
@@ -102,10 +102,10 @@ impl System {
map_toggle_str(config.encryption.enabled)
);
- let encryptor: Option<Arc<EncryptorKind>> = match config.encryption.enabled {
- true => Some(Arc::new(EncryptorKind::Aes256Gcm(
+ let encryptor: Option<EncryptorKind> = match config.encryption.enabled {
+ true => Some(EncryptorKind::Aes256Gcm(
Aes256GcmEncryptor::from_base64_key(&config.encryption.key).unwrap(),
- ))),
+ )),
false => None,
};
@@ -116,13 +116,21 @@ impl System {
&config.get_state_messages_file_path(),
&version,
state_persister,
- encryptor.clone(),
+ encryptor,
)));
+
+ //TODO: Just shut the fuck up rust-analyzer.
+ let encryptor: Option<EncryptorKind> = match config.encryption.enabled {
+ true => Some(EncryptorKind::Aes256Gcm(
+ Aes256GcmEncryptor::from_base64_key(&config.encryption.key).unwrap(),
+ )),
+ false => None,
+ };
Self::create(
config.clone(),
SystemStorage::new(config, partition_persister),
state,
- encryptor,
+ encryptor.map(Arc::new),
data_maintenance_config,
pat_config,
)
@@ -224,9 +232,12 @@ impl System {
format!("{COMPONENT} (error: {error}) - failed to initialize system state")
})?;
let now = Instant::now();
+ //DONE
+ /*
self.load_version().await.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to load version")
})?;
+ */
self.load_users(system_state.users.into_values().collect())
.await
.with_error_context(|error| {
(iggy) 26/30: feat(io_uring): move to compio runtime (#1960)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit b37120acef9d51588dc0a5b70465210c87c1d7b4
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Fri Jul 4 15:27:28 2025 +0200
feat(io_uring): move to compio runtime (#1960)
---
Cargo.lock | 412 +++++++++++++++------
core/server/Cargo.toml | 4 +-
.../handlers/messages/poll_messages_handler.rs | 25 +-
.../binary/handlers/topics/create_topic_handler.rs | 3 +-
core/server/src/binary/sender.rs | 17 +-
core/server/src/bootstrap.rs | 35 +-
.../src/compat/index_rebuilding/index_rebuilder.rs | 42 ++-
core/server/src/io/file.rs | 117 ------
core/server/src/io/fs_utils.rs | 2 +-
core/server/src/io/mod.rs | 3 -
core/server/src/io/reader.rs | 44 ---
core/server/src/io/writer.rs | 57 ---
core/server/src/log/runtime.rs | 48 +++
core/server/src/main.rs | 52 ++-
core/server/src/quic/quic_sender.rs | 5 +-
core/server/src/shard/mod.rs | 15 +-
core/server/src/shard/system/snapshot/mod.rs | 50 ++-
core/server/src/shard/system/storage.rs | 11 +-
core/server/src/shard/system/streams.rs | 2 +-
core/server/src/shard/system/topics.rs | 35 +-
core/server/src/shard/task_registry.rs | 8 +-
core/server/src/shard/tasks/messages.rs | 2 +-
core/server/src/shard/transmission/event.rs | 3 +-
core/server/src/state/file.rs | 36 +-
core/server/src/streaming/partitions/storage.rs | 27 +-
core/server/src/streaming/persistence/persister.rs | 24 +-
core/server/src/streaming/persistence/task.rs | 10 +-
.../src/streaming/segments/indexes/index_reader.rs | 16 +-
.../src/streaming/segments/indexes/index_writer.rs | 15 +-
.../streaming/segments/messages/messages_reader.rs | 20 +-
.../streaming/segments/messages/messages_writer.rs | 23 +-
core/server/src/streaming/segments/messages/mod.rs | 32 +-
core/server/src/streaming/segments/segment.rs | 6 +-
core/server/src/streaming/streams/storage.rs | 6 +-
core/server/src/streaming/topics/storage.rs | 6 +-
core/server/src/streaming/utils/file.rs | 14 +-
core/server/src/streaming/utils/pooled_buffer.rs | 34 +-
core/server/src/tcp/connection_handler.rs | 1 +
core/server/src/tcp/sender.rs | 62 ++--
core/server/src/tcp/tcp_listener.rs | 12 +-
core/server/src/tcp/tcp_sender.rs | 9 +-
core/server/src/tcp/tcp_tls_listener.rs | 185 ++++-----
core/server/src/tcp/tcp_tls_sender.rs | 18 +-
43 files changed, 812 insertions(+), 736 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index ca282813..cc838a3a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -142,7 +142,7 @@ dependencies = [
"actix-utils",
"futures-core",
"futures-util",
- "mio 1.0.4",
+ "mio",
"socket2",
"tokio",
"tracing",
@@ -307,6 +307,15 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "aligned-array"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e05c92d086290f52938013f6242ac62bf7d401fab8ad36798a609faa65c3fd2c"
+dependencies = [
+ "generic-array",
+]
+
[[package]]
name = "alloc-no-stdlib"
version = "2.0.4"
@@ -578,6 +587,12 @@ dependencies = [
"syn 2.0.104",
]
+[[package]]
+name = "async-task"
+version = "4.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
+
[[package]]
name = "async-trait"
version = "0.1.88"
@@ -643,17 +658,6 @@ dependencies = [
"webpki-roots 0.26.11",
]
-[[package]]
-name = "auto-const-array"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd73835ad7deb4bd2b389e6f10333b143f025d607c55ca04c66a0bcc6bb2fc6d"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.104",
-]
-
[[package]]
name = "autocfg"
version = "1.5.0"
@@ -1535,6 +1539,216 @@ dependencies = [
"static_assertions",
]
+[[package]]
+name = "compio"
+version = "0.15.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "compio-buf",
+ "compio-dispatcher",
+ "compio-driver",
+ "compio-fs",
+ "compio-io",
+ "compio-log",
+ "compio-macros",
+ "compio-net",
+ "compio-process",
+ "compio-quic",
+ "compio-runtime",
+ "compio-signal",
+ "compio-tls",
+]
+
+[[package]]
+name = "compio-buf"
+version = "0.6.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "arrayvec",
+ "bytes",
+ "libc",
+]
+
+[[package]]
+name = "compio-dispatcher"
+version = "0.7.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "compio-driver",
+ "compio-runtime",
+ "flume",
+ "futures-channel",
+]
+
+[[package]]
+name = "compio-driver"
+version = "0.8.1"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "aligned-array",
+ "cfg-if",
+ "cfg_aliases",
+ "compio-buf",
+ "compio-log",
+ "crossbeam-channel",
+ "crossbeam-queue",
+ "futures-util",
+ "io-uring",
+ "io_uring_buf_ring",
+ "libc",
+ "once_cell",
+ "paste",
+ "polling",
+ "slab",
+ "socket2",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "compio-fs"
+version = "0.8.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "cfg-if",
+ "cfg_aliases",
+ "compio-buf",
+ "compio-driver",
+ "compio-io",
+ "compio-runtime",
+ "libc",
+ "os_pipe",
+ "widestring",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "compio-io"
+version = "0.7.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "compio-buf",
+ "futures-util",
+ "paste",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "compio-log"
+version = "0.1.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "tracing",
+]
+
+[[package]]
+name = "compio-macros"
+version = "0.1.2"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "proc-macro-crate 3.3.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+]
+
+[[package]]
+name = "compio-net"
+version = "0.8.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "cfg-if",
+ "compio-buf",
+ "compio-driver",
+ "compio-io",
+ "compio-runtime",
+ "either",
+ "libc",
+ "once_cell",
+ "socket2",
+ "widestring",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "compio-process"
+version = "0.5.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "cfg-if",
+ "compio-buf",
+ "compio-driver",
+ "compio-io",
+ "compio-runtime",
+ "futures-util",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "compio-quic"
+version = "0.4.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "cfg_aliases",
+ "compio-buf",
+ "compio-io",
+ "compio-log",
+ "compio-net",
+ "compio-runtime",
+ "flume",
+ "futures-util",
+ "libc",
+ "quinn-proto",
+ "rustc-hash 2.1.1",
+ "rustls",
+ "thiserror 2.0.12",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "compio-runtime"
+version = "0.8.1"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "async-task",
+ "cfg-if",
+ "compio-buf",
+ "compio-driver",
+ "compio-log",
+ "crossbeam-queue",
+ "futures-util",
+ "libc",
+ "once_cell",
+ "scoped-tls",
+ "slab",
+ "socket2",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "compio-signal"
+version = "0.6.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "compio-buf",
+ "compio-driver",
+ "compio-runtime",
+ "libc",
+ "once_cell",
+ "os_pipe",
+ "slab",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "compio-tls"
+version = "0.6.0"
+source = "git+https://github.com/compio-rs/compio.git?rev=5504e02ec2e821e9b024006224c34c06646410cc#5504e02ec2e821e9b024006224c34c06646410cc"
+dependencies = [
+ "compio-buf",
+ "compio-io",
+ "rustls",
+]
+
[[package]]
name = "concurrent-queue"
version = "2.5.0"
@@ -1702,6 +1916,17 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+[[package]]
+name = "core_affinity"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342"
+dependencies = [
+ "libc",
+ "num_cpus",
+ "winapi",
+]
+
[[package]]
name = "cpufeatures"
version = "0.2.17"
@@ -1852,7 +2077,7 @@ version = "3.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73"
dependencies = [
- "nix 0.30.1",
+ "nix",
"windows-sys 0.59.0",
]
@@ -2698,15 +2923,6 @@ dependencies = [
"slab",
]
-[[package]]
-name = "fxhash"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
-dependencies = [
- "byteorder",
-]
-
[[package]]
name = "generator"
version = "0.8.5"
@@ -4167,14 +4383,26 @@ dependencies = [
[[package]]
name = "io-uring"
-version = "0.6.4"
+version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "595a0399f411a508feb2ec1e970a4a30c249351e30208960d58298de8660b0e5"
+checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.9.1",
+ "cfg-if",
"libc",
]
+[[package]]
+name = "io_uring_buf_ring"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a8867874ff5758b47c1dac069e6e86541432f9da8be9111c5e94154134f07d0"
+dependencies = [
+ "bytes",
+ "io-uring",
+ "rustix 1.0.7",
+]
+
[[package]]
name = "ipnet"
version = "2.11.0"
@@ -4717,6 +4945,7 @@ checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
[[package]]
<<<<<<< HEAD
+<<<<<<< HEAD
name = "miette"
version = "7.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4747,6 +4976,8 @@ dependencies = [
]
[[package]]
+=======
+>>>>>>> 4f07d68c (feat(io_uring): move to compio runtime (#1960))
name = "mimalloc"
version = "0.1.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4795,18 +5026,6 @@ dependencies = [
"adler2",
]
-[[package]]
-name = "mio"
-version = "0.8.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
-dependencies = [
- "libc",
- "log",
- "wasi 0.11.1+wasi-snapshot-preview1",
- "windows-sys 0.48.0",
-]
-
[[package]]
name = "mio"
version = "1.0.4"
@@ -4867,62 +5086,6 @@ dependencies = [
"uuid",
]
-[[package]]
-name = "monoio"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3bd0f8bcde87b1949f95338b547543fcab187bc7e7a5024247e359a5e828ba6a"
-dependencies = [
- "auto-const-array",
- "bytes",
- "flume",
- "fxhash",
- "io-uring",
- "libc",
- "memchr",
- "mio 0.8.11",
- "monoio-macros",
- "nix 0.26.4",
- "once_cell",
- "pin-project-lite",
- "socket2",
- "threadpool",
- "windows-sys 0.48.0",
-]
-
-[[package]]
-name = "monoio-io-wrapper"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bcfaa76e5daf87cc4d31b4d1b6bc93c12db59c19df50b9200afdbde42077655"
-dependencies = [
- "monoio",
-]
-
-[[package]]
-name = "monoio-macros"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "176a5f5e69613d9e88337cf2a65e11135332b4efbcc628404a7c555e4452084c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.104",
-]
-
-[[package]]
-name = "monoio-rustls"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e31f422825bd7fb19957af6eaf89d7234ba143fcc0e515f5a2f526e332d1875"
-dependencies = [
- "bytes",
- "monoio",
- "monoio-io-wrapper",
- "rustls",
- "thiserror 1.0.69",
-]
-
[[package]]
name = "nanorand"
version = "0.7.0"
@@ -4938,19 +5101,6 @@ version = "6.6.666"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf5a574dadd7941adeaa71823ecba5e28331b8313fb2e1c6a5c7e5981ea53ad6"
-[[package]]
-name = "nix"
-version = "0.26.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
-dependencies = [
- "bitflags 1.3.2",
- "cfg-if",
- "libc",
- "memoffset",
- "pin-utils",
-]
-
[[package]]
name = "nix"
version = "0.30.1"
@@ -5015,7 +5165,7 @@ dependencies = [
"kqueue",
"libc",
"log",
- "mio 1.0.4",
+ "mio",
"notify-types",
"walkdir",
"windows-sys 0.59.0",
@@ -5431,6 +5581,16 @@ dependencies = [
"hashbrown 0.14.5",
]
+[[package]]
+name = "os_pipe"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db335f4760b14ead6290116f2427bf33a14d4f0617d49f78a246de10c1831224"
+dependencies = [
+ "libc",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "overload"
version = "0.1.1"
@@ -5682,6 +5842,21 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+[[package]]
+name = "polling"
+version = "3.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b53a684391ad002dd6a596ceb6c74fd004fdce75f4be2e3f615068abbea5fd50"
+dependencies = [
+ "cfg-if",
+ "concurrent-queue",
+ "hermit-abi",
+ "pin-project-lite",
+ "rustix 1.0.7",
+ "tracing",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "polonius-the-crab"
version = "0.2.1"
@@ -6993,7 +7168,9 @@ dependencies = [
"bytes",
"chrono",
"clap",
+ "compio",
"console-subscriber",
+ "core_affinity",
"crossbeam",
"ctrlc",
"dashmap",
@@ -7013,9 +7190,7 @@ dependencies = [
"mimalloc",
"mockall",
"moka",
- "monoio",
- "monoio-rustls",
- "nix 0.30.1",
+ "nix",
"once_cell",
"opentelemetry",
"opentelemetry-appender-tracing",
@@ -7528,15 +7703,6 @@ dependencies = [
"cfg-if",
]
-[[package]]
-name = "threadpool"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
-dependencies = [
- "num_cpus",
-]
-
[[package]]
name = "time"
version = "0.3.41"
@@ -7613,7 +7779,7 @@ dependencies = [
"backtrace",
"bytes",
"libc",
- "mio 1.0.4",
+ "mio",
"parking_lot 0.12.4",
"pin-project-lite",
"signal-hook-registry",
@@ -8455,6 +8621,12 @@ dependencies = [
"safe_arch",
]
+[[package]]
+name = "widestring"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d"
+
[[package]]
name = "winapi"
version = "0.3.9"
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 4b330383..630ea0e1 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -63,12 +63,12 @@ human-repr = { workspace = true }
iggy_common = { workspace = true }
jsonwebtoken = "9.3.1"
socket2 = "0.5.10"
+core_affinity = "0.8.0"
lending-iterator = "0.1.7"
hash32 = "1.0.0"
mimalloc = { workspace = true, optional = true }
moka = { version = "0.12.10", features = ["future"] }
-monoio = { version = "0.2.4", features = ["mkdirat", "unlinkat", "renameat", "sync"] }
-monoio-rustls = "0.4.0"
+compio = { git = "https://github.com/compio-rs/compio.git", rev = "5504e02ec2e821e9b024006224c34c06646410cc", features = ["runtime", "macros", "io-uring", "time", "rustls"] }
nix = { version = "0.30", features = ["fs"] }
once_cell = "1.21.3"
opentelemetry = { version = "0.30.0", features = ["trace", "logs"] }
diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
index dfc4a89f..2f02a039 100644
--- a/core/server/src/binary/handlers/messages/poll_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
@@ -27,8 +27,10 @@ use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
use crate::streaming::segments::IggyMessagesBatchSet;
use crate::streaming::session::Session;
+use crate::streaming::utils::PooledBuffer;
use crate::to_iovec;
use anyhow::Result;
+use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::{IggyError, PollMessages};
use std::io::IoSlice;
@@ -76,7 +78,7 @@ impl ServerCommandHandler for PollMessages {
let user_id = session.get_user_id();
let client_id = session.client_id;
- let (metadata, batch) = shard
+ let (metadata, mut batch) = shard
.poll_messages(
client_id,
user_id,
@@ -96,16 +98,19 @@ impl ServerCommandHandler for PollMessages {
let response_length = 4 + 8 + 4 + batch.size();
let response_length_bytes = response_length.to_le_bytes();
- let partition_id = metadata.partition_id.to_le_bytes();
- let current_offset = metadata.current_offset.to_le_bytes();
- let count = batch.count().to_le_bytes();
+ let mut bufs = Vec::with_capacity(batch.containers_count() + 5);
+ let mut partition_id_buf = PooledBuffer::with_capacity(4);
+ let mut current_offset_buf = PooledBuffer::with_capacity(8);
+ let mut count_buf = PooledBuffer::with_capacity(4);
+ partition_id_buf.put_u32_le(metadata.partition_id);
+ current_offset_buf.put_u64_le(metadata.current_offset);
+ count_buf.put_u32_le(batch.count());
- let mut iovecs = Vec::with_capacity(batch.containers_count() + 3);
- iovecs.push(to_iovec(&partition_id));
- iovecs.push(to_iovec(¤t_offset));
- iovecs.push(to_iovec(&count));
+ bufs.push(partition_id_buf);
+ bufs.push(current_offset_buf);
+ bufs.push(count_buf);
- iovecs.extend(batch.iter().map(|m| to_iovec(&m)));
+ batch.iter_mut().for_each(|m| bufs.push(m.take_messages()));
trace!(
"Sending {} messages to client ({} bytes) to client",
batch.count(),
@@ -113,7 +118,7 @@ impl ServerCommandHandler for PollMessages {
);
sender
- .send_ok_response_vectored(&response_length_bytes, iovecs)
+ .send_ok_response_vectored(&response_length_bytes, bufs)
.await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/topics/create_topic_handler.rs b/core/server/src/binary/handlers/topics/create_topic_handler.rs
index 6785e4f0..51088b55 100644
--- a/core/server/src/binary/handlers/topics/create_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/create_topic_handler.rs
@@ -48,7 +48,7 @@ impl ServerCommandHandler for CreateTopic {
debug!("session: {session}, command: {self}");
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id;
- let created_topic_id = shard
+ let (shards_assignment, created_topic_id) = shard
.create_topic(
session,
&self.stream_id,
@@ -72,6 +72,7 @@ impl ServerCommandHandler for CreateTopic {
compression_algorithm: self.compression_algorithm,
max_topic_size: self.max_topic_size,
replication_factor: self.replication_factor,
+ shards_assignment,
};
// Broadcast the event to all shards.
let _responses = shard.broadcast_event_to_all_shards(event.into());
diff --git a/core/server/src/binary/sender.rs b/core/server/src/binary/sender.rs
index 45066c75..96bed3d5 100644
--- a/core/server/src/binary/sender.rs
+++ b/core/server/src/binary/sender.rs
@@ -19,17 +19,17 @@
use std::future::Future;
use std::io::IoSlice;
+use crate::streaming::utils::PooledBuffer;
use crate::tcp::tcp_sender::TcpSender;
use crate::tcp::tcp_tls_sender::TcpTlsSender;
use crate::{quic::quic_sender::QuicSender, server_error::ServerError};
use bytes::BytesMut;
+use compio::buf::{IoBuf, IoBufMut};
+use compio::io::{AsyncReadExt, AsyncWriteExt};
+use compio::net::TcpStream;
use iggy_common::IggyError;
-use monoio::buf::IoBufMut;
-use monoio::io::{AsyncReadRent, AsyncWriteRent};
-use monoio::net::TcpStream;
use nix::libc;
use quinn::{RecvStream, SendStream};
-use monoio_rustls::{ServerTlsStream, TlsStream};
macro_rules! forward_async_methods {
(
@@ -65,7 +65,7 @@ pub trait Sender {
fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<libc::iovec>,
+ slices: Vec<PooledBuffer>,
) -> impl Future<Output = Result<(), IggyError>>;
fn send_error_response(
&mut self,
@@ -85,8 +85,9 @@ impl SenderKind {
Self::Tcp(TcpSender { stream })
}
- pub fn get_tcp_tls_sender(stream: ServerTlsStream<TcpStream>) -> Self {
- Self::TcpTls(TcpTlsSender { stream })
+ pub fn get_tcp_tls_sender(stream: ()) -> Self {
+ todo!();
+ //Self::TcpTls(TcpTlsSender { stream })
}
pub fn get_quic_sender(send_stream: SendStream, recv_stream: RecvStream) -> Self {
@@ -100,7 +101,7 @@ impl SenderKind {
async fn read<B: IoBufMut>(&mut self, buffer: B) -> (Result<usize, IggyError>, B);
async fn send_empty_ok_response(&mut self) -> Result<(), IggyError>;
async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError>;
- async fn send_ok_response_vectored(&mut self, length: &[u8], slices: Vec<libc::iovec>) -> Result<(), IggyError>;
+ async fn send_ok_response_vectored(&mut self, length: &[u8], slices: Vec<PooledBuffer>) -> Result<(), IggyError>;
async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError>;
async fn shutdown(&mut self) -> Result<(), ServerError>;
}
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index ddd0599c..6a2eb8af 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -1,3 +1,4 @@
+use compio::{fs::create_dir_all, runtime::Runtime};
use iggy_common::{
IggyError,
defaults::{
@@ -5,7 +6,6 @@ use iggy_common::{
MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH,
},
};
-use monoio::{Buildable, Driver, Runtime, fs::create_dir_all, time::TimeDriver};
use tracing::info;
use crate::{
@@ -121,24 +121,21 @@ pub fn create_root_user() -> User {
user
}
-pub fn create_default_executor<D>() -> Runtime<D>
-where
- D: Driver + Buildable,
-{
- let builder = monoio::RuntimeBuilder::<D>::new();
- let rt = Buildable::build(builder).expect("Failed to create default runtime");
- rt
-}
-
-pub fn create_shard_executor() -> Runtime<TimeDriver<monoio::IoUringDriver>> {
- // TODO: Figure out what else we could tweak there
- // We for sure want to disable the userspace interrupts on new cq entry (set_coop_taskrun)
- // TODO: Shall we make the size of ring be configureable ?
- let builder = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new()
- //.uring_builder(urb.setup_coop_taskrun()) // broken shit.
- .with_entries(1024) // Default size
- .enable_timer();
- let rt = Buildable::build(builder).expect("Failed to create default runtime");
+pub fn create_shard_executor() -> Runtime {
+ //TODO: The event intererval tick, could be configureed based on the fact
+ // How many clients we expect to have connected.
+ // This roughly estimates the number of tasks we will create.
+ let proactor = compio::driver::ProactorBuilder::new()
+ .capacity(4096)
+ .coop_taskrun(true)
+ .taskrun_flag(false) //TODO: Try enabling this.
+ .thread_pool_limit(0)
+ .to_owned();
+ let rt = compio::runtime::RuntimeBuilder::new()
+ .with_proactor(proactor)
+ .event_interval(69)
+ .build()
+ .unwrap();
rt
}
diff --git a/core/server/src/compat/index_rebuilding/index_rebuilder.rs b/core/server/src/compat/index_rebuilding/index_rebuilder.rs
index 1eebdacd..dbc93ebe 100644
--- a/core/server/src/compat/index_rebuilding/index_rebuilder.rs
+++ b/core/server/src/compat/index_rebuilding/index_rebuilder.rs
@@ -16,12 +16,14 @@
* under the License.
*/
-use crate::io::file::IggyFile;
-use crate::io::writer::IggyWriter;
+use crate::server_error::CompatError;
use crate::streaming::utils::file;
-use crate::{io::reader::IggyReader, server_error::CompatError};
+use async_zip::tokio::write;
+use compio::{
+ fs::File,
+ io::{AsyncBufRead, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, BufWriter},
+};
use iggy_common::{IGGY_MESSAGE_HEADER_SIZE, IggyMessageHeader};
-use monoio::io::{AsyncReadRent, AsyncReadRentExt, AsyncWriteRent, AsyncWriteRentExt};
use std::io::{Seek, SeekFrom};
pub struct IndexRebuilder {
@@ -40,17 +42,17 @@ impl IndexRebuilder {
}
async fn read_message_header(
- reader: &mut IggyReader<IggyFile>,
+ reader: &mut BufReader<std::io::Cursor<File>>,
) -> Result<IggyMessageHeader, std::io::Error> {
let buf = [0u8; IGGY_MESSAGE_HEADER_SIZE];
- let (result, buf) = reader.read_exact(Box::new(buf)).await;
+ let (result, buf) = reader.read_exact(Box::new(buf)).await.into();
result?;
IggyMessageHeader::from_raw_bytes(&*buf)
.map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidData))
}
async fn write_index_entry(
- writer: &mut IggyWriter<IggyFile>,
+ writer: &mut BufWriter<std::io::Cursor<File>>,
header: &IggyMessageHeader,
position: usize,
start_offset: u64,
@@ -58,26 +60,34 @@ impl IndexRebuilder {
// Write offset (4 bytes) - base_offset + last_offset_delta - start_offset
let offset = start_offset - header.offset;
debug_assert!(offset <= u32::MAX as u64);
- let (result, _) = writer.write_all(Box::new(offset.to_le_bytes())).await;
+ let (result, _) = writer
+ .write_all(Box::new(offset.to_le_bytes()))
+ .await
+ .into();
result?;
// Write position (4 bytes)
- let (result, _) = writer.write_all(Box::new(position.to_le_bytes())).await;
+ let (result, _) = writer
+ .write_all(Box::new(position.to_le_bytes()))
+ .await
+ .into();
result?;
// Write timestamp (8 bytes)
let (result, _) = writer
.write_all(Box::new(header.timestamp.to_le_bytes()))
- .await;
+ .await
+ .into();
result?;
Ok(())
}
pub async fn rebuild(&self) -> Result<(), CompatError> {
- let mut reader =
- IggyReader::new(IggyFile::new(file::open(&self.messages_file_path).await?));
- let mut writer = IggyWriter::new(IggyFile::new(file::overwrite(&self.index_path).await?));
+ let read_cursor = std::io::Cursor::new(file::open(&self.messages_file_path).await?);
+ let write_cursor = std::io::Cursor::new(file::overwrite(&self.index_path).await?);
+ let mut reader = BufReader::new(read_cursor);
+ let mut writer = BufWriter::new(write_cursor);
let mut position = 0;
let mut next_position;
@@ -93,9 +103,9 @@ impl IndexRebuilder {
.await?;
// Skip message payload and headers
- reader.seek(SeekFrom::Current(
- header.payload_length as i64 + header.user_headers_length as i64,
- ))?;
+ reader.consume(
+ header.payload_length as usize + header.user_headers_length as usize,
+ );
// Update position for next iteration
position = next_position;
diff --git a/core/server/src/io/file.rs b/core/server/src/io/file.rs
deleted file mode 100644
index 95129aa8..00000000
--- a/core/server/src/io/file.rs
+++ /dev/null
@@ -1,117 +0,0 @@
-use std::io::SeekFrom;
-
-use monoio::{
- BufResult,
- buf::{IoBuf, IoBufMut, IoVecBuf, IoVecBufMut, IoVecWrapper},
- fs::{File, Metadata},
- io::{AsyncReadRent, AsyncWriteRent},
-};
-
-/// Wrapper around `monoio::fs::File` to provide a consistent API for reading and writing files
-/// in an asynchronous context. This struct maintains the current position in the file and provides
-/// methods to read and write data at specific positions.
-#[derive(Debug)]
-pub struct IggyFile {
- file: File,
- position: u64,
-}
-
-impl From<File> for IggyFile {
- fn from(file: File) -> Self {
- Self { file, position: 0 }
- }
-}
-
-impl std::io::Seek for IggyFile {
- /// This method doesn't do bound checking aswell as, the `End` variant is not supported.
- fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
- self.position = match pos {
- SeekFrom::Start(n) => n as i64,
- SeekFrom::End(_) => unreachable!("End variant is not supported in IggyFile"),
- SeekFrom::Current(n) => self.position as i64 + n,
- } as u64;
- Ok(self.position)
- }
-}
-
-impl IggyFile {
- pub fn new(file: File) -> Self {
- Self { file, position: 0 }
- }
-
- pub async fn metadata(&self) -> std::io::Result<Metadata> {
- self.file.metadata().await
- }
-
- pub async fn write_at<T: IoBuf>(&self, buf: T, pos: usize) -> BufResult<usize, T> {
- self.file.write_at(buf, pos as u64).await
- }
-
- pub async fn write_all_at<T: IoBuf>(&self, buf: T, pos: u64) -> BufResult<(), T> {
- self.file.write_all_at(buf, pos).await
- }
-
- pub async fn read_exact_at<T: IoBufMut>(&self, buf: T, pos: u64) -> BufResult<(), T> {
- self.file.read_exact_at(buf, pos).await
- }
-
- pub async fn read_at<T: IoBufMut>(&self, buf: T, pos: usize) -> BufResult<usize, T> {
- self.file.read_at(buf, pos as u64).await
- }
-
- pub async fn sync_all(&self) -> std::io::Result<()> {
- self.file.sync_all().await
- }
-}
-
-impl AsyncReadRent for IggyFile {
- /// Reads `exactly` `buf.len()` bytes into the buffer.
- async fn read<T: IoBufMut>(&mut self, buf: T) -> BufResult<usize, T> {
- let (res, buf) = self.file.read_at(buf, self.position).await;
- let n = match res {
- Ok(n) => n,
- Err(e) => return (Err(e), buf),
- };
- self.position += n as u64;
- (Ok(n), buf)
- }
-
- //TODO(numinex) - maybe implement this ?
- fn readv<T: IoVecBufMut>(&mut self, buf: T) -> impl Future<Output = BufResult<usize, T>> {
- async move { (Ok(0), buf) }
- }
-}
-
-impl AsyncWriteRent for IggyFile {
- /// Writes entire buffer to the file at the current position.
- async fn write<B: IoBuf>(&mut self, buf: B) -> BufResult<usize, B> {
- let (res, buf) = self.file.write_at(buf, self.position).await;
- let n = match res {
- Ok(n) => n,
- Err(e) => return (Err(e), buf),
- };
- self.position += n as u64;
- (Ok(n), buf)
- }
-
- // This is bait!!!!
- async fn writev<T: IoVecBuf>(&mut self, buf_vec: T) -> BufResult<usize, T> {
- unimplemented!("writev is not implemented for IggyFile");
- }
-
- async fn flush(&mut self) -> std::io::Result<()> {
- self.file.sync_all().await
- }
-
- //TODO(numinex) - How to implement this ?
- async fn shutdown(&mut self) -> std::io::Result<()> {
- panic!("shutdown is not supported for IggyFile");
- /*
- self.file.sync_all().await;
- unsafe {
- let file = *self.file;
- file.close().await
- }
- */
- }
-}
diff --git a/core/server/src/io/fs_utils.rs b/core/server/src/io/fs_utils.rs
index d9be21e7..6bbc1b7f 100644
--- a/core/server/src/io/fs_utils.rs
+++ b/core/server/src/io/fs_utils.rs
@@ -16,7 +16,7 @@
* under the License.
*/
-use monoio::fs;
+use compio::fs;
use std::io;
use std::path::{Path, PathBuf};
diff --git a/core/server/src/io/mod.rs b/core/server/src/io/mod.rs
index 686717cd..c3140c9e 100644
--- a/core/server/src/io/mod.rs
+++ b/core/server/src/io/mod.rs
@@ -1,4 +1 @@
-pub mod file;
pub mod fs_utils;
-pub mod reader;
-pub mod writer;
diff --git a/core/server/src/io/reader.rs b/core/server/src/io/reader.rs
deleted file mode 100644
index 63443214..00000000
--- a/core/server/src/io/reader.rs
+++ /dev/null
@@ -1,44 +0,0 @@
-use std::io::{Seek, SeekFrom};
-
-use monoio::{
- BufResult,
- buf::{IoBufMut, IoVecBufMut},
- io::{AsyncReadRent, BufReader},
-};
-
-pub struct IggyReader<R: AsyncReadRent + Seek> {
- inner: BufReader<R>,
-}
-
-impl<R: AsyncReadRent + Seek> IggyReader<R> {
- pub fn new(reader: R) -> Self {
- Self {
- inner: BufReader::new(reader),
- }
- }
-
- pub fn with_capacity(capacity: usize, reader: R) -> Self {
- Self {
- inner: BufReader::with_capacity(capacity, reader),
- }
- }
-}
-
-impl<R: AsyncReadRent + Seek> Seek for IggyReader<R> {
- fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
- self.inner.get_mut().seek(pos)
- }
-}
-
-impl<R> AsyncReadRent for IggyReader<R>
-where
- R: AsyncReadRent + Seek,
-{
- fn read<T: IoBufMut>(&mut self, buf: T) -> impl Future<Output = BufResult<usize, T>> {
- self.inner.read(buf)
- }
-
- fn readv<T: IoVecBufMut>(&mut self, buf: T) -> impl Future<Output = BufResult<usize, T>> {
- self.inner.readv(buf)
- }
-}
diff --git a/core/server/src/io/writer.rs b/core/server/src/io/writer.rs
deleted file mode 100644
index e249e405..00000000
--- a/core/server/src/io/writer.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-use std::io::{Seek, SeekFrom};
-
-use monoio::{
- BufResult,
- io::{AsyncWriteRent, BufWriter},
-};
-
-pub struct IggyWriter<W: AsyncWriteRent + Seek> {
- inner: BufWriter<W>,
-}
-
-impl<W: AsyncWriteRent + Seek> IggyWriter<W> {
- pub fn new(writer: W) -> Self {
- Self {
- inner: BufWriter::new(writer),
- }
- }
-
- pub fn with_capacity(capacity: usize, writer: W) -> Self {
- Self {
- inner: BufWriter::with_capacity(capacity, writer),
- }
- }
-}
-
-impl<W: AsyncWriteRent + Seek> Seek for IggyWriter<W> {
- fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
- self.inner.get_mut().seek(pos)
- }
-}
-
-impl<W> AsyncWriteRent for IggyWriter<W>
-where
- W: AsyncWriteRent + Seek,
-{
- fn write<T: monoio::buf::IoBuf>(
- &mut self,
- buf: T,
- ) -> impl Future<Output = BufResult<usize, T>> {
- self.inner.write(buf)
- }
-
- fn writev<T: monoio::buf::IoVecBuf>(
- &mut self,
- buf_vec: T,
- ) -> impl Future<Output = BufResult<usize, T>> {
- self.inner.writev(buf_vec)
- }
-
- fn flush(&mut self) -> impl Future<Output = std::io::Result<()>> {
- self.inner.flush()
- }
-
- fn shutdown(&mut self) -> impl Future<Output = std::io::Result<()>> {
- self.inner.shutdown()
- }
-}
diff --git a/core/server/src/log/runtime.rs b/core/server/src/log/runtime.rs
new file mode 100644
index 00000000..36bbdd80
--- /dev/null
+++ b/core/server/src/log/runtime.rs
@@ -0,0 +1,48 @@
+use std::{pin::Pin, time::Duration};
+
+use futures::{FutureExt, SinkExt, Stream};
+use opentelemetry_sdk::runtime::{Runtime, RuntimeChannel, TrySend};
+
+#[derive(Clone)]
+pub struct MonoioRuntime;
+
+impl Runtime for MonoioRuntime {
+ fn spawn<F>(&self, future: F)
+ where
+ F: Future<Output = ()> + Send + 'static,
+ {
+ // TODO: This wont' work, we init Opentelemetry in the main thread, when there is no instance of monoio runtime
+ // running yet....
+ monoio::spawn(future);
+ }
+
+ fn delay(&self, duration: Duration) -> impl Future<Output = ()> + Send + 'static {
+ let sleep = Sleep::new(duration);
+ sleep
+ }
+}
+
+
+pub struct Sleep {
+ pub inner: Pin<Box<monoio::time::Sleep>>,
+}
+
+impl Sleep {
+ pub fn new(duration: Duration) -> Self {
+ Self {
+ inner: Box::pin(monoio::time::sleep(duration)),
+ }
+ }
+}
+
+impl Future for Sleep {
+ type Output = ();
+
+ fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<Self::Output> {
+ self.inner.as_mut().poll_unpin(cx)
+ }
+}
+
+// Safety: There is no way for `Sleep` future to be flipped like a burger to another thread,
+// because we create instance of OpenTelemetry SDK runtime in the main thread, and monoio futures don't require Send & Sync bounds.
+unsafe impl Send for Sleep {}
\ No newline at end of file
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 53fa2d7b..5d5f8698 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -30,8 +30,8 @@ use iggy_common::defaults::DEFAULT_ROOT_USER_ID;
use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError};
use server::args::Args;
use server::bootstrap::{
- create_default_executor, create_directories, create_root_user, create_shard_connections,
- create_shard_executor, load_config, resolve_persister,
+ create_directories, create_root_user, create_shard_connections, create_shard_executor,
+ load_config, resolve_persister,
};
use server::configs::config_provider::{self};
use server::configs::server::ServerConfig;
@@ -84,7 +84,8 @@ fn main() -> Result<(), ServerError> {
let config_provider = config_provider::resolve(&args.config_provider)?;
// Load config and create directories.
// Remove `local_data` directory if run with `--fresh` flag.
- let mut rt = create_default_executor::<monoio::IoUringDriver>();
+ // TODO: Replace this once we use `monoio::main` macro.
+ let rt = create_shard_executor();
let config = rt
.block_on(async {
let config = load_config(&config_provider)
@@ -94,7 +95,7 @@ fn main() -> Result<(), ServerError> {
})?;
if args.fresh {
let system_path = config.system.get_system_path();
- if monoio::fs::metadata(&system_path).await.is_ok() {
+ if compio::fs::metadata(&system_path).await.is_ok() {
println!(
"Removing system path at: {} because `--fresh` flag was set",
system_path
@@ -122,6 +123,7 @@ fn main() -> Result<(), ServerError> {
// TODO: Make this configurable from config as a range
// for example this instance of Iggy will use cores from 0..4
+ let core_ids = core_affinity::get_core_ids().unwrap();
let available_cpus = available_parallelism().expect("Failed to get num of cores");
let shards_count = available_cpus.into();
let shards_set = 0..shards_count;
@@ -136,14 +138,24 @@ fn main() -> Result<(), ServerError> {
let config = config.clone();
let state_persister = resolve_persister(config.system.state.enforce_fsync);
+ let core_ids = core_ids.clone();
let handle = std::thread::Builder::new()
.name(format!("shard-{id}"))
.spawn(move || {
+ let core_ids = core_ids.clone();
MemoryPool::init_pool(config.system.clone());
- monoio::utils::bind_to_cpu_set(Some(shard_id))
+ if core_ids.len() > shard_id {
+ core_affinity::set_for_current(core_ids[shard_id]);
+ }
+
+ // TODO: Fix this once this PR gets resolved.
+ // https://github.com/compio-rs/compio/pull/445
+ /*
+ compio::utils::bind_to_cpu_set(Some(shard_id))
.unwrap_or_else(|e| panic!("Failed to set CPU affinity for shard-{id}: {e}"));
+ */
- let mut rt = create_shard_executor();
+ let rt = create_shard_executor();
rt.block_on(async move {
let version = SemanticVersion::current().expect("Invalid version");
info!(
@@ -253,20 +265,22 @@ fn main() -> Result<(), ServerError> {
}
let shutdown_handles_for_signal = shutdown_handles.clone();
- ctrlc::set_handler(move || {
- info!("Received shutdown signal (SIGTERM/SIGINT), initiating graceful shutdown...");
-
- for (shard_id, stop_sender) in &shutdown_handles_for_signal {
- info!("Sending shutdown signal to shard {}", shard_id);
- if let Err(e) = stop_sender.send_blocking(()) {
- error!(
- "Failed to send shutdown signal to shard {}: {}",
- shard_id, e
- );
+ /*
+ ::set_handler(move || {
+ info!("Received shutdown signal (SIGTERM/SIGINT), initiating graceful shutdown...");
+
+ for (shard_id, stop_sender) in &shutdown_handles_for_signal {
+ info!("Sending shutdown signal to shard {}", shard_id);
+ if let Err(e) = stop_sender.send_blocking(()) {
+ error!(
+ "Failed to send shutdown signal to shard {}: {}",
+ shard_id, e
+ );
+ }
}
- }
- })
- .expect("Error setting Ctrl-C handler");
+ })
+ .expect("Error setting Ctrl-C handler");
+ */
info!("Iggy server is running. Press Ctrl+C or send SIGTERM to shutdown.");
for (idx, handle) in handles.into_iter().enumerate() {
diff --git a/core/server/src/quic/quic_sender.rs b/core/server/src/quic/quic_sender.rs
index c6403349..54bda604 100644
--- a/core/server/src/quic/quic_sender.rs
+++ b/core/server/src/quic/quic_sender.rs
@@ -17,11 +17,12 @@
*/
use crate::quic::COMPONENT;
+use crate::streaming::utils::PooledBuffer;
use crate::{binary::sender::Sender, server_error::ServerError};
use bytes::BytesMut;
+use compio::buf::{IoBuf, IoBufMut};
use error_set::ErrContext;
use iggy_common::IggyError;
-use monoio::buf::IoBufMut;
use nix::libc;
use quinn::{RecvStream, SendStream};
use std::io::IoSlice;
@@ -71,7 +72,7 @@ impl Sender for QuicSender {
async fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<libc::iovec>,
+ slices: Vec<PooledBuffer>,
) -> Result<(), IggyError> {
debug!("Sending vectored response with status: {:?}...", STATUS_OK);
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 8ffc6560..2178d2b3 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -110,6 +110,7 @@ impl Shard {
}
}
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct ShardInfo {
id: u16,
}
@@ -118,6 +119,10 @@ impl ShardInfo {
pub fn new(id: u16) -> Self {
Self { id }
}
+
+ pub fn id(&self) -> u16 {
+ self.id
+ }
}
pub struct IggyShard {
@@ -193,7 +198,8 @@ impl IggyShard {
let stop_receiver = self.get_stop_receiver();
let shard_for_shutdown = self.clone();
- monoio::spawn(async move {
+ /*
+ compio::runtime::spawn(async move {
let _ = stop_receiver.recv().await;
info!("Shard {} received shutdown signal", shard_for_shutdown.id);
@@ -207,6 +213,7 @@ impl IggyShard {
);
}
});
+ */
let result = try_join_all(tasks).await;
result?;
@@ -506,7 +513,7 @@ impl IggyShard {
async fn handle_event(&self, event: Arc<ShardEvent>) -> Result<(), IggyError> {
match &*event {
ShardEvent::CreatedStream { stream_id, name } => {
- self.create_stream_bypass_auth(*stream_id, name).await
+ self.create_stream_bypass_auth(*stream_id, name)
}
ShardEvent::CreatedTopic {
stream_id,
@@ -517,6 +524,7 @@ impl IggyShard {
compression_algorithm,
max_topic_size,
replication_factor,
+ shards_assignment,
} => {
self.create_topic_bypass_auth(
stream_id,
@@ -527,6 +535,7 @@ impl IggyShard {
*compression_algorithm,
*max_topic_size,
*replication_factor,
+ shards_assignment.clone()
)
.await
}
@@ -604,7 +613,7 @@ impl IggyShard {
pub fn insert_shard_table_records(
&self,
- records: impl Iterator<Item = (IggyNamespace, ShardInfo)>,
+ records: impl IntoIterator<Item = (IggyNamespace, ShardInfo)>,
) {
self.shards_table.borrow_mut().extend(records);
}
diff --git a/core/server/src/shard/system/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs
index eb726509..f6bf7857 100644
--- a/core/server/src/shard/system/snapshot/mod.rs
+++ b/core/server/src/shard/system/snapshot/mod.rs
@@ -23,14 +23,14 @@ use crate::shard::IggyShard;
use crate::streaming::session::Session;
use async_zip::tokio::write::ZipFileWriter;
use async_zip::{Compression, ZipEntryBuilder};
+use compio::fs::{File, OpenOptions};
+use compio::io::{AsyncReadAtExt, AsyncWriteAtExt};
use iggy_common::{IggyDuration, IggyError, Snapshot, SnapshotCompression, SystemSnapshotType};
-use monoio::fs::{File, OpenOptions};
use std::io::Cursor;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;
use tempfile::NamedTempFile;
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::Command;
use tokio_util::compat::TokioAsyncWriteCompatExt;
use tracing::{error, info};
@@ -59,6 +59,8 @@ impl IggyShard {
// and impl the monoio async writer, based on this example:
// https://youtu.be/RYHYiXMJdZI?si=d2roKeHn5lJrw2ri&t=1140
// and rc-zip-tokio crate.
+
+ /*
let cursor = Cursor::new(Vec::new());
let mut zip_writer = ZipFileWriter::new(cursor.compat_write());
@@ -80,13 +82,17 @@ impl IggyShard {
let filename = format!("{snapshot_type}.txt");
let entry = ZipEntryBuilder::new(filename.clone().into(), compression);
- let file = File::open(temp_file.path()).await.map_err(|e| {
- error!("Failed to open temporary file: {}", e);
- IggyError::SnapshotFileCompletionFailed
- })?;
+ let file = OpenOptions::new()
+ .read(true)
+ .open(temp_file.path())
+ .await
+ .map_err(|e| {
+ error!("Failed to open temporary file: {}", e);
+ IggyError::SnapshotFileCompletionFailed
+ })?;
let content = Vec::new();
- let (result, content) = file.read_exact_at(content, 0).await;
+ let (result, content) = file.read_exact_at(content, 0).await.into();
if let Err(e) = result {
error!("Failed to read temporary file: {}", e);
continue;
@@ -112,7 +118,9 @@ impl IggyShard {
}
}
}
+ */
+ /*
info!(
"Snapshot commands {:?} finished in {}",
snapshot_types,
@@ -128,7 +136,8 @@ impl IggyShard {
let zip_data = cursor.into_inner();
info!("Final zip size: {} bytes", zip_data.len());
- Ok(Snapshot::new(zip_data))
+ */
+ Ok(Snapshot::new(vec![]))
}
}
@@ -137,8 +146,11 @@ async fn write_command_output_to_temp_file(
) -> Result<NamedTempFile, std::io::Error> {
let output = command.output().await?;
let temp_file = NamedTempFile::new()?;
- let file = File::from_std(temp_file.as_file().try_clone()?).unwrap();
- let (result, _) = file.write_all_at(output.stdout, 0).await;
+ let mut file = OpenOptions::new()
+ .write(true)
+ .open(temp_file.path())
+ .await?;
+ let (result, _) = file.write_all_at(output.stdout, 0).await.into();
result?;
file.sync_all().await?;
Ok(temp_file)
@@ -150,33 +162,39 @@ async fn get_filesystem_overview() -> Result<NamedTempFile, std::io::Error> {
async fn get_process_info() -> Result<NamedTempFile, std::io::Error> {
let temp_file = NamedTempFile::new()?;
- let file = File::from_std(temp_file.as_file().try_clone()?).unwrap();
+ let mut file = OpenOptions::new()
+ .create(true)
+ .write(true)
+ .open(temp_file.path())
+ .await?;
let mut position = 0;
let ps_output = Command::new("ps").arg("aux").output().await?;
let (result, written) = file
.write_all_at(b"=== Process List (ps aux) ===\n", 0)
- .await;
+ .await
+ .into();
result?;
position += written.len() as u64;
- let (result, written) = file.write_all_at(ps_output.stdout, position).await;
+ let (result, written) = file.write_all_at(ps_output.stdout, position).await.into();
result?;
position += written.len() as u64;
- let (result, written) = file.write_all_at(b"\n\n", position).await;
+ let (result, written) = file.write_all_at(b"\n\n", position).await.into();
result?;
position += written.len() as u64;
let (result, written) = file
.write_all_at(b"=== Detailed Process Information ===\n", position)
- .await;
+ .await
+ .into();
result?;
position += written.len() as u64;
let proc_info = procdump::get_proc_info().await?;
let bytes = proc_info.as_bytes().to_owned();
- let (result, _) = file.write_all_at(bytes, position).await;
+ let (result, _) = file.write_all_at(bytes, position).await.into();
result?;
file.sync_all().await?;
diff --git a/core/server/src/shard/system/storage.rs b/core/server/src/shard/system/storage.rs
index 3d895d4c..e1e7cf84 100644
--- a/core/server/src/shard/system/storage.rs
+++ b/core/server/src/shard/system/storage.rs
@@ -17,18 +17,17 @@
*/
use super::COMPONENT;
-use crate::io::file::IggyFile;
use crate::shard::system::info::SystemInfo;
use crate::streaming::persistence::persister::PersisterKind;
use crate::streaming::storage::SystemInfoStorage;
use crate::streaming::utils::PooledBuffer;
use crate::streaming::utils::file;
use anyhow::Context;
+use compio::io::AsyncReadAtExt;
+use compio::io::AsyncReadExt;
use error_set::ErrContext;
use iggy_common::IggyError;
-use monoio::io::AsyncReadRentExt;
use std::sync::Arc;
-use tokio::io::AsyncReadExt;
use tracing::info;
#[derive(Debug)]
@@ -63,10 +62,12 @@ impl SystemInfoStorage for FileSystemInfoStorage {
.map_err(|_| IggyError::CannotReadFileMetadata)?
.len() as usize;
- let mut file = IggyFile::new(file);
+ let file = file::open(&self.path)
+ .await
+ .map_err(|_| IggyError::CannotReadFile)?;
let mut buffer = PooledBuffer::with_capacity(file_size);
buffer.put_bytes(0, file_size);
- let (result, buffer) = file.read_exact(buffer).await;
+ let (result, buffer) = file.read_exact_at(buffer, 0).await.into();
result
.with_error_context(|error| {
format!(
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index b4921066..bc8fc2bb 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -186,7 +186,7 @@ impl IggyShard {
}))
}
- pub async fn create_stream_bypass_auth(
+ pub fn create_stream_bypass_auth(
&self,
stream_id: Option<u32>,
name: &str,
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index 0f8155fb..d71b0f1d 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -112,8 +112,9 @@ impl IggyShard {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
+ shards_assignment: Vec<(IggyNamespace, ShardInfo)>,
) -> Result<(), IggyError> {
- let (topic_id, partition_ids) = self.create_topic_base(
+ let (topic_id, _) = self.create_topic_base(
stream_id,
topic_id,
name,
@@ -132,22 +133,16 @@ impl IggyShard {
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get topic with ID: {topic_id} in stream with ID: {stream_id}")
})?;
- topic.persist().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to persist topic: {topic}")
- })?;
- // TODO: Figure out a way how to distribute the shards table among different shards,
- // without the need to do code from below, everytime we handle a `ShardEvent`.
- // I think we shouldn't be sharing it tho and still maintain a single shard table per shard,
- // but find a way how to distribute it smarter (maybe move the broadcast inside of the `insert_shard_table_records`) method.
- let records = partition_ids.into_iter().map(|partition_id| {
- let namespace = IggyNamespace::new(stream_id, topic_id, partition_id);
- let hash = namespace.generate_hash();
- let shard_id = hash % self.get_available_shards_count();
- let shard_info = ShardInfo::new(shard_id as u16);
- (namespace, shard_info)
- });
- self.insert_shard_table_records(records);
+ for (_, shard_info) in &shards_assignment {
+ if shard_info.id() == self.id {
+ topic.persist().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to persist topic: {topic}")
+ })?;
+ }
+ }
+
+ self.insert_shard_table_records(shards_assignment);
self.metrics.increment_topics(1);
self.metrics.increment_partitions(partitions_count);
@@ -167,7 +162,7 @@ impl IggyShard {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
- ) -> Result<Identifier, IggyError> {
+ ) -> Result<(Vec<(IggyNamespace, ShardInfo)>, Identifier), IggyError> {
self.ensure_authenticated(session)?;
{
let stream = self.get_stream(stream_id).with_error_context(|error| {
@@ -208,6 +203,7 @@ impl IggyShard {
format!("{COMPONENT} (error: {error}) - failed to persist topic: {topic}")
})?;
+ // TODO: Refactor
let records = partition_ids.into_iter().map(|partition_id| {
let namespace = IggyNamespace::new(stream_id, topic_id, partition_id);
let hash = namespace.generate_hash();
@@ -215,13 +211,14 @@ impl IggyShard {
let shard_info = ShardInfo::new(shard_id as u16);
(namespace, shard_info)
});
- self.insert_shard_table_records(records);
+ let records = records.collect::<Vec<_>>();
+ self.insert_shard_table_records(records.clone());
self.metrics.increment_topics(1);
self.metrics.increment_partitions(partitions_count);
self.metrics.increment_segments(partitions_count);
- Ok(Identifier::numeric(topic_id)?)
+ Ok((records, Identifier::numeric(topic_id)?))
}
fn create_topic_base(
diff --git a/core/server/src/shard/task_registry.rs b/core/server/src/shard/task_registry.rs
index 1a201d3b..ff674a8e 100644
--- a/core/server/src/shard/task_registry.rs
+++ b/core/server/src/shard/task_registry.rs
@@ -17,8 +17,8 @@
*/
use async_channel::{Receiver, Sender, bounded};
+use compio::runtime::JoinHandle;
use futures::future::join_all;
-use monoio::task::JoinHandle;
use std::cell::RefCell;
use std::collections::HashMap;
use std::future::Future;
@@ -42,7 +42,7 @@ impl TaskRegistry {
where
F: Future<Output = ()> + 'static,
{
- let handle = monoio::spawn(future);
+ let handle = compio::runtime::spawn(future);
self.tasks.borrow_mut().push(handle);
}
@@ -85,8 +85,8 @@ impl TaskRegistry {
.into_iter()
.enumerate()
.map(|(idx, handle)| async move {
- match monoio::time::timeout(timeout, handle).await {
- Ok(()) => (idx, true),
+ match compio::time::timeout(timeout, handle).await {
+ Ok(_) => (idx, true),
Err(_) => {
warn!("Task {} did not complete within timeout", idx);
(idx, false)
diff --git a/core/server/src/shard/tasks/messages.rs b/core/server/src/shard/tasks/messages.rs
index ec4018c0..d573c058 100644
--- a/core/server/src/shard/tasks/messages.rs
+++ b/core/server/src/shard/tasks/messages.rs
@@ -14,7 +14,7 @@ async fn run_shard_messages_receiver(shard: Rc<IggyShard>) -> Result<(), IggyErr
if shard.is_shutting_down() {
return;
}
- monoio::time::sleep(Duration::from_millis(100)).await;
+ compio::time::sleep(Duration::from_millis(100)).await;
}
};
diff --git a/core/server/src/shard/transmission/event.rs b/core/server/src/shard/transmission/event.rs
index 49794c3f..60a28dfc 100644
--- a/core/server/src/shard/transmission/event.rs
+++ b/core/server/src/shard/transmission/event.rs
@@ -2,7 +2,7 @@ use std::net::SocketAddr;
use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize};
-use crate::streaming::clients::client_manager::Transport;
+use crate::{shard::{namespace::IggyNamespace, ShardInfo}, streaming::clients::client_manager::Transport};
#[derive(Debug)]
pub enum ShardEvent {
@@ -24,6 +24,7 @@ pub enum ShardEvent {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
+ shards_assignment: Vec<(IggyNamespace, ShardInfo)>,
},
//CreatedConsumerGroup(Identifier, Identifier, Option<u32>, String),
//DeletedConsumerGroup(Identifier, Identifier, Identifier),
diff --git a/core/server/src/state/file.rs b/core/server/src/state/file.rs
index e8eb6ee6..5716d7bb 100644
--- a/core/server/src/state/file.rs
+++ b/core/server/src/state/file.rs
@@ -16,28 +16,27 @@
* under the License.
*/
-use crate::io::file::IggyFile;
-use crate::io::reader::IggyReader;
use crate::state::command::EntryCommand;
use crate::state::{COMPONENT, State, StateEntry};
use crate::streaming::persistence::persister::PersisterKind;
use crate::streaming::utils::file;
use crate::versioning::SemanticVersion;
use bytes::{Buf, BufMut, Bytes, BytesMut};
+use compio::fs::File;
+use compio::io::{AsyncReadExt, AsyncWriteExt};
use error_set::ErrContext;
use iggy_common::BytesSerializable;
use iggy_common::EncryptorKind;
use iggy_common::IggyByteSize;
use iggy_common::IggyError;
use iggy_common::IggyTimestamp;
-use monoio::io::AsyncReadRentExt;
use std::fmt::Debug;
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use tracing::{debug, error, info};
-pub const BUF_READER_CAPACITY_BYTES: usize = 512 * 1000;
+pub const BUF_cursor_CAPACITY_BYTES: usize = 512 * 1000;
const FILE_STATE_PARSE_ERROR: &str = "STATE - failed to parse file state";
#[derive(Debug)]
@@ -129,7 +128,6 @@ impl State for FileState {
.map_err(|_| IggyError::CannotReadFileMetadata)?
.len();
- let file = IggyFile::new(file);
if file_size == 0 {
info!("State file is empty");
return Ok(Vec::new());
@@ -141,11 +139,11 @@ impl State for FileState {
);
let mut entries = Vec::new();
let mut total_size: u64 = 0;
- let mut reader = IggyReader::with_capacity(BUF_READER_CAPACITY_BYTES, file);
+ let mut cursor = std::io::Cursor::new(file);
let mut current_index = 0;
let mut entries_count = 0;
loop {
- let index = reader
+ let index = cursor
.read_u64_le()
.await
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} index. {error}"))
@@ -163,32 +161,32 @@ impl State for FileState {
current_index = index;
entries_count += 1;
- let term = reader
+ let term = cursor
.read_u64_le()
.await
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} term. {error}"))
.map_err(|_| IggyError::InvalidNumberEncoding)?;
total_size += 8;
- let leader_id = reader
+ let leader_id = cursor
.read_u32_le()
.await
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} leader_id. {error}"))
.map_err(|_| IggyError::InvalidNumberEncoding)?;
total_size += 4;
- let version = reader
+ let version = cursor
.read_u32_le()
.await
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} version. {error}"))
.map_err(|_| IggyError::InvalidNumberEncoding)?;
total_size += 4;
- let flags = reader
+ let flags = cursor
.read_u64_le()
.await
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} flags. {error}"))
.map_err(|_| IggyError::InvalidNumberEncoding)?;
total_size += 8;
let timestamp = IggyTimestamp::from(
- reader
+ cursor
.read_u64_le()
.await
.with_error_context(|error| {
@@ -197,19 +195,19 @@ impl State for FileState {
.map_err(|_| IggyError::InvalidNumberEncoding)?,
);
total_size += 8;
- let user_id = reader
+ let user_id = cursor
.read_u32_le()
.await
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} user_id. {error}"))
.map_err(|_| IggyError::InvalidNumberEncoding)?;
total_size += 4;
- let checksum = reader
+ let checksum = cursor
.read_u32_le()
.await
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} checksum. {error}"))
.map_err(|_| IggyError::InvalidNumberEncoding)?;
total_size += 4;
- let context_length = reader
+ let context_length = cursor
.read_u32_le()
.await
.with_error_context(|error| {
@@ -220,20 +218,20 @@ impl State for FileState {
total_size += 4;
let mut context = BytesMut::with_capacity(context_length);
context.put_bytes(0, context_length);
- let (result, context) = reader.read_exact(context).await;
+ let (result, context) = cursor.read_exact(context).await.into();
result
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} code. {error}"))
.map_err(|_| IggyError::CannotReadFile)?;
let context = context.freeze();
total_size += context_length as u64;
- let code = reader
+ let code = cursor
.read_u32_le()
.await
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} code. {error}"))
.map_err(|_| IggyError::InvalidNumberEncoding)?;
total_size += 4;
- let mut command_length = reader
+ let mut command_length = cursor
.read_u32_le()
.await
.with_error_context(|error| {
@@ -244,7 +242,7 @@ impl State for FileState {
total_size += 4;
let mut command = BytesMut::with_capacity(command_length);
command.put_bytes(0, command_length);
- let (result, command) = reader.read_exact(command).await;
+ let (result, command) = cursor.read_exact(command).await.into();
result
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} command. {error}"))
.map_err(|_| IggyError::CannotReadFile)?;
diff --git a/core/server/src/streaming/partitions/storage.rs b/core/server/src/streaming/partitions/storage.rs
index 6621ea19..67e3d3f4 100644
--- a/core/server/src/streaming/partitions/storage.rs
+++ b/core/server/src/streaming/partitions/storage.rs
@@ -18,7 +18,6 @@
use crate::compat::index_rebuilding::index_rebuilder::IndexRebuilder;
use crate::configs::cache_indexes::CacheIndexesConfig;
-use crate::io::file::IggyFile;
use crate::io::fs_utils;
use crate::state::system::PartitionState;
use crate::streaming::partitions::COMPONENT;
@@ -27,12 +26,12 @@ use crate::streaming::persistence::persister::PersisterKind;
use crate::streaming::segments::*;
use crate::streaming::storage::PartitionStorage;
use crate::streaming::utils::file;
+use compio::fs;
+use compio::fs::create_dir_all;
+use compio::io::AsyncReadExt;
use error_set::ErrContext;
use iggy_common::ConsumerKind;
use iggy_common::IggyError;
-use monoio::fs;
-use monoio::fs::create_dir_all;
-use monoio::io::AsyncReadRentExt;
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::Ordering;
@@ -117,8 +116,18 @@ impl PartitionStorage for FilePartitionStorage {
let messages_file_path = segment.messages_file_path().to_owned();
let time_index_path = index_path.replace(INDEX_EXTENSION, "timeindex");
- let index_path_exists = tokio::fs::try_exists(&index_path).await.unwrap();
- let time_index_path_exists = tokio::fs::try_exists(&time_index_path).await.unwrap();
+ // TODO: Move to fs_utils
+ async fn try_exists(index_path: &str) -> Result<bool, std::io::Error> {
+ match compio::fs::metadata(index_path).await {
+ Ok(_) => Ok(true),
+ Err(err) => match err.kind() {
+ std::io::ErrorKind::NotFound => Ok(false),
+ _ => Err(err),
+ },
+ }
+ }
+ let index_path_exists = try_exists(&index_path).await.unwrap();
+ let time_index_path_exists = try_exists(&time_index_path).await.unwrap();
let index_cache_enabled = matches!(
partition.config.segment.cache_indexes,
CacheIndexesConfig::All | CacheIndexesConfig::OpenSegment
@@ -416,6 +425,7 @@ impl PartitionStorage for FilePartitionStorage {
let mut consumer_offsets = Vec::new();
let mut dir_entries = dir_entries.unwrap();
+ // TODO: reimplement using the compio dir walk
while let Some(dir_entry) = dir_entries.next() {
let dir_entry = dir_entry.unwrap();
let metadata = dir_entry.metadata();
@@ -451,8 +461,9 @@ impl PartitionStorage for FilePartitionStorage {
)
})
.map_err(|_| IggyError::CannotReadFile)?;
- let mut file = IggyFile::new(file);
- let offset = file
+ // TODO: This is like awfull.
+ let mut cursor = std::io::Cursor::new(file);
+ let offset = cursor
.read_u64_le()
.await
.with_error_context(|error| {
diff --git a/core/server/src/streaming/persistence/persister.rs b/core/server/src/streaming/persistence/persister.rs
index 6ba5c87b..ca522b85 100644
--- a/core/server/src/streaming/persistence/persister.rs
+++ b/core/server/src/streaming/persistence/persister.rs
@@ -18,12 +18,13 @@
use crate::streaming::persistence::COMPONENT;
use crate::streaming::utils::file;
+use compio::buf::IoBuf;
+use compio::fs::remove_file;
+use compio::io::AsyncWriteAtExt;
use error_set::ErrContext;
use iggy_common::IggyError;
-use monoio::buf::IoBuf;
use std::fmt::Debug;
use std::future::Future;
-use tokio::fs;
#[cfg(test)]
use mockall::automock;
@@ -85,13 +86,13 @@ pub struct FileWithSyncPersister;
impl Persister for FilePersister {
async fn append<B: IoBuf>(&self, path: &str, bytes: B) -> Result<(), IggyError> {
- let file = file::append(path)
+ let (mut file, position) = file::append(path)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to append to file: {path}")
})
.map_err(|_| IggyError::CannotAppendToFile)?;
- file.write_all_at(bytes, 0)
+ file.write_all_at(bytes, position)
.await
.0
.with_error_context(|error| {
@@ -102,7 +103,7 @@ impl Persister for FilePersister {
}
async fn overwrite<B: IoBuf>(&self, path: &str, bytes: B) -> Result<(), IggyError> {
- let file = file::overwrite(path)
+ let mut file = file::overwrite(path)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to overwrite file: {path}")
@@ -120,7 +121,7 @@ impl Persister for FilePersister {
}
async fn delete(&self, path: &str) -> Result<(), IggyError> {
- fs::remove_file(path)
+ remove_file(path)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to delete file: {path}")
@@ -132,17 +133,12 @@ impl Persister for FilePersister {
impl Persister for FileWithSyncPersister {
async fn append<B: IoBuf>(&self, path: &str, bytes: B) -> Result<(), IggyError> {
- let file = file::append(path)
+ let (mut file, position) = file::append(path)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to append to file: {path}")
})
.map_err(|_| IggyError::CannotAppendToFile)?;
- let position = file
- .metadata()
- .await
- .map_err(|_| IggyError::CannotReadFileMetadata)?
- .len();
file.write_all_at(bytes, position)
.await
.0
@@ -162,7 +158,7 @@ impl Persister for FileWithSyncPersister {
}
async fn overwrite<B: IoBuf>(&self, path: &str, bytes: B) -> Result<(), IggyError> {
- let file = file::overwrite(path)
+ let mut file = file::overwrite(path)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to overwrite file: {path}")
@@ -188,7 +184,7 @@ impl Persister for FileWithSyncPersister {
}
async fn delete(&self, path: &str) -> Result<(), IggyError> {
- fs::remove_file(path)
+ remove_file(path)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to delete file: {path}")
diff --git a/core/server/src/streaming/persistence/task.rs b/core/server/src/streaming/persistence/task.rs
index d1c716aa..1b359ef7 100644
--- a/core/server/src/streaming/persistence/task.rs
+++ b/core/server/src/streaming/persistence/task.rs
@@ -18,18 +18,18 @@
use crate::streaming::persistence::COMPONENT;
use bytes::Bytes;
+use compio::runtime::Task;
use error_set::ErrContext;
use flume::{Receiver, Sender, unbounded};
use iggy_common::IggyError;
-use monoio::task;
-use std::{sync::Arc, time::Duration};
+use std::{any::Any, sync::Arc, time::Duration};
use tracing::error;
use super::persister::PersisterKind;
pub struct LogPersisterTask {
_sender: Option<Sender<Bytes>>,
- _task_handle: Option<task::JoinHandle<()>>,
+ _task_handle: Option<Task<Result<(), Box<dyn Any + Send>>>>,
}
impl std::fmt::Debug for LogPersisterTask {
@@ -50,7 +50,7 @@ impl LogPersisterTask {
) -> Self {
let (sender, receiver): (Sender<Bytes>, Receiver<Bytes>) = unbounded();
- let task_handle = monoio::spawn(async move {
+ let task_handle = compio::runtime::spawn(async move {
loop {
match receiver.recv_async().await {
Ok(data) => {
@@ -129,7 +129,7 @@ impl Drop for LogPersisterTask {
self._sender.take();
if let Some(handle) = self._task_handle.take() {
- monoio::spawn(async move { handle.await });
+ compio::runtime::spawn(async move { handle.await });
}
}
}
diff --git a/core/server/src/streaming/segments/indexes/index_reader.rs b/core/server/src/streaming/segments/indexes/index_reader.rs
index 8b09666a..ce235ef4 100644
--- a/core/server/src/streaming/segments/indexes/index_reader.rs
+++ b/core/server/src/streaming/segments/indexes/index_reader.rs
@@ -17,13 +17,16 @@
*/
use super::IggyIndexesMut;
-use crate::{io::file::IggyFile, streaming::utils::PooledBuffer};
+use crate::streaming::utils::PooledBuffer;
use bytes::BytesMut;
+use compio::{
+ BufResult,
+ fs::{File, OpenOptions},
+ io::AsyncReadAtExt,
+};
use error_set::ErrContext;
use iggy_common::{INDEX_SIZE, IggyError, IggyIndex, IggyIndexView};
-use monoio::fs::OpenOptions;
use std::{
- fs::File as StdFile,
io::ErrorKind,
os::unix::fs::FileExt,
sync::{
@@ -37,7 +40,7 @@ use tracing::{error, trace};
#[derive(Debug)]
pub struct IndexReader {
file_path: String,
- file: IggyFile,
+ file: File,
index_size_bytes: Arc<AtomicU64>,
}
@@ -51,7 +54,6 @@ impl IndexReader {
.with_error_context(|error| format!("Failed to open index file: {file_path}. {error}"))
.map_err(|_| IggyError::CannotReadFile)?;
- let file = IggyFile::new(file);
trace!(
"Opened index file for reading: {file_path}, size: {}",
index_size_bytes.load(Ordering::Acquire)
@@ -337,13 +339,13 @@ impl IndexReader {
if use_pool {
let mut buf = PooledBuffer::with_capacity(len as usize);
unsafe { buf.set_len(len as usize) };
- let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await.into();
result?;
Ok(buf)
} else {
let mut buf = BytesMut::with_capacity(len as usize);
unsafe { buf.set_len(len as usize) };
- let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await.into();
result?;
Ok(PooledBuffer::from_existing(buf))
}
diff --git a/core/server/src/streaming/segments/indexes/index_writer.rs b/core/server/src/streaming/segments/indexes/index_writer.rs
index 4b79f9d2..416cde21 100644
--- a/core/server/src/streaming/segments/indexes/index_writer.rs
+++ b/core/server/src/streaming/segments/indexes/index_writer.rs
@@ -16,25 +16,25 @@
* under the License.
*/
+use compio::fs::File;
+use compio::fs::OpenOptions;
+use compio::io::AsyncWriteAtExt;
use error_set::ErrContext;
use iggy_common::INDEX_SIZE;
use iggy_common::IggyError;
-use monoio::fs::OpenOptions;
-use monoio::io::AsyncWriteRentExt;
use std::sync::{
Arc,
atomic::{AtomicU64, Ordering},
};
use tracing::trace;
-use crate::io::file::IggyFile;
use crate::streaming::utils::PooledBuffer;
/// A dedicated struct for writing to the index file.
#[derive(Debug)]
pub struct IndexWriter {
file_path: String,
- file: IggyFile,
+ file: File,
index_size_bytes: Arc<AtomicU64>,
fsync: bool,
}
@@ -48,15 +48,13 @@ impl IndexWriter {
file_exists: bool,
) -> Result<Self, IggyError> {
let file = OpenOptions::new()
- .write(true)
- .append(true)
.create(true)
+ .write(true)
.open(file_path)
.await
.with_error_context(|error| format!("Failed to open index file: {file_path}. {error}"))
.map_err(|_| IggyError::CannotReadFile)?;
- let file = IggyFile::from(file);
if file_exists {
let _ = file.sync_all().await.with_error_context(|error| {
format!("Failed to fsync index file after creation: {file_path}. {error}",)
@@ -96,8 +94,9 @@ impl IndexWriter {
let count = indexes.len() / INDEX_SIZE;
let len = indexes.len();
+ let position = self.index_size_bytes.load(Ordering::Relaxed);
self.file
- .write_all(indexes)
+ .write_all_at(indexes, position)
.await
.0
.with_error_context(|error| {
diff --git a/core/server/src/streaming/segments/messages/messages_reader.rs b/core/server/src/streaming/segments/messages/messages_reader.rs
index 4091e8e3..e81eb793 100644
--- a/core/server/src/streaming/segments/messages/messages_reader.rs
+++ b/core/server/src/streaming/segments/messages/messages_reader.rs
@@ -16,13 +16,13 @@
* under the License.
*/
-use crate::io::file::IggyFile;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
-use crate::streaming::utils::{PooledBuffer, file};
+use crate::streaming::utils::PooledBuffer;
use bytes::BytesMut;
+use compio::fs::{File, OpenOptions};
+use compio::io::AsyncReadAtExt;
use error_set::ErrContext;
use iggy_common::IggyError;
-use monoio::fs::File;
use std::{
io::ErrorKind,
sync::{
@@ -36,7 +36,7 @@ use tracing::{error, trace};
#[derive(Debug)]
pub struct MessagesReader {
file_path: String,
- file: IggyFile,
+ file: File,
messages_size_bytes: Arc<AtomicU64>,
}
@@ -46,9 +46,12 @@ impl MessagesReader {
file_path: &str,
messages_size_bytes: Arc<AtomicU64>,
) -> Result<Self, IggyError> {
- let file = file::open_std(file_path)
+ let file = OpenOptions::new()
+ .read(true)
+ .open(file_path)
+ .await
.with_error_context(|error| {
- format!("Failed to open messages file: {file_path}, error: {error}")
+ format!("Failed to open messages file: {file_path}. {error}")
})
.map_err(|_| IggyError::CannotReadFile)?;
@@ -67,7 +70,6 @@ impl MessagesReader {
)
});
}
- let file = File::from_std(file).unwrap();
trace!(
"Opened messages file for reading: {file_path}, size: {}",
@@ -177,13 +179,13 @@ impl MessagesReader {
if use_pool {
let mut buf = PooledBuffer::with_capacity(len as usize);
unsafe { buf.set_len(len as usize) };
- let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await.into();
result?;
Ok(buf)
} else {
let mut buf = BytesMut::with_capacity(len as usize);
unsafe { buf.set_len(len as usize) };
- let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await.into();
result?;
Ok(PooledBuffer::from_existing(buf))
}
diff --git a/core/server/src/streaming/segments/messages/messages_writer.rs b/core/server/src/streaming/segments/messages/messages_writer.rs
index 63e424a9..f3d43d1e 100644
--- a/core/server/src/streaming/segments/messages/messages_writer.rs
+++ b/core/server/src/streaming/segments/messages/messages_writer.rs
@@ -16,13 +16,10 @@
* under the License.
*/
-use crate::{
- io::file::IggyFile,
- streaming::segments::{IggyMessagesBatchSet, messages::write_batch},
-};
+use crate::streaming::segments::{IggyMessagesBatchSet, messages::write_batch};
+use compio::fs::{File, OpenOptions};
use error_set::ErrContext;
use iggy_common::{IggyByteSize, IggyError};
-use monoio::fs::{File, OpenOptions};
use std::sync::{
Arc,
atomic::{AtomicU64, Ordering},
@@ -34,7 +31,7 @@ use tracing::{error, trace};
pub struct MessagesWriter {
file_path: String,
/// Holds the file for synchronous writes; when asynchronous persistence is enabled, this will be None.
- file: Option<IggyFile>,
+ file: Option<File>,
/// When set, asynchronous writes are handled by this persister task.
messages_size_bytes: Arc<AtomicU64>,
fsync: bool,
@@ -53,11 +50,13 @@ impl MessagesWriter {
file_exists: bool,
) -> Result<Self, IggyError> {
let file = OpenOptions::new()
- .write(true)
.create(true)
- .append(true)
+ .write(true)
.open(file_path)
.await
+ .with_error_context(|err| {
+ format!("Failed to open messages file: {file_path}, error: {err}")
+ })
.map_err(|_| IggyError::CannotReadFile)?;
if file_exists {
@@ -82,7 +81,7 @@ impl MessagesWriter {
messages_size_bytes.load(Ordering::Acquire)
);
- let file = Some(IggyFile::new(file));
+ let file = Some(file);
Ok(Self {
file_path: file_path.to_string(),
file,
@@ -103,16 +102,16 @@ impl MessagesWriter {
"Saving batch set of size {messages_size} bytes ({containers_count} containers, {messages_count} messages) to messages file: {}",
self.file_path
);
+ let position = self.messages_size_bytes.load(Ordering::Relaxed);
if let Some(ref mut file) = self.file {
- write_batch(file, &self.file_path, batch_set)
+ write_batch(file, position, batch_set)
.await
.with_error_context(|error| {
format!(
"Failed to write batch to messages file: {}. {error}",
self.file_path
)
- })
- .map_err(|_| IggyError::CannotWriteToFile)?;
+ })?;
} else {
error!("File handle is not available for synchronous write.");
return Err(IggyError::CannotWriteToFile);
diff --git a/core/server/src/streaming/segments/messages/mod.rs b/core/server/src/streaming/segments/messages/mod.rs
index 9799608f..a3989a2e 100644
--- a/core/server/src/streaming/segments/messages/mod.rs
+++ b/core/server/src/streaming/segments/messages/mod.rs
@@ -19,37 +19,25 @@
mod messages_reader;
mod messages_writer;
-use crate::io::file::IggyFile;
-
use super::IggyMessagesBatchSet;
-use error_set::ErrContext;
+use compio::{fs::File, io::AsyncWriteAtExt};
use iggy_common::IggyError;
-use monoio::io::AsyncWriteRentExt;
pub use messages_reader::MessagesReader;
pub use messages_writer::MessagesWriter;
/// Vectored write a batches of messages to file
async fn write_batch(
- file: &mut IggyFile,
- file_path: &str,
+ file: &mut File,
+ position: u64,
mut batches: IggyMessagesBatchSet,
) -> Result<usize, IggyError> {
- //let mut slices = batches.iter().map(|b| to_iovec(&b)).collect::<Vec<iovec>>();
- let mut total_written = 0;
- // TODO: Fork monoio, piece of shit runtime.
- for batch in batches.iter_mut() {
- let messages = batch.take_messages();
- let writen = file
- .write_all(messages)
- .await
- .0
- .with_error_context(|error| {
- format!("Failed to write messages to file: {file_path}, error: {error}",)
- // TODO: Better error variant.
- })
- .map_err(|_| IggyError::CannotAppendMessage)?;
- total_written += writen;
- }
+ let total_written = batches.iter().map(|b| b.size() as usize).sum();
+ let batches = batches
+ .iter_mut()
+ .map(|b| b.take_messages())
+ .collect::<Vec<_>>();
+ let (result, _) = file.write_vectored_all_at(batches, position).await.into();
+ result.map_err(|_| IggyError::CannotWriteToFile)?;
Ok(total_written)
}
diff --git a/core/server/src/streaming/segments/segment.rs b/core/server/src/streaming/segments/segment.rs
index 9887df90..c13ce07a 100644
--- a/core/server/src/streaming/segments/segment.rs
+++ b/core/server/src/streaming/segments/segment.rs
@@ -21,6 +21,7 @@ use super::messages::*;
use super::messages_accumulator::MessagesAccumulator;
use crate::configs::system::SystemConfig;
use crate::streaming::segments::*;
+use compio::fs::remove_file;
use error_set::ErrContext;
use iggy_common::INDEX_SIZE;
use iggy_common::IggyByteSize;
@@ -30,7 +31,6 @@ use iggy_common::IggyTimestamp;
use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
-use tokio::fs::remove_file;
use tracing::{info, warn};
const SIZE_16MB: usize = 16 * 1024 * 1024;
@@ -302,7 +302,7 @@ impl Segment {
pub async fn shutdown_writing(&mut self) {
if let Some(log_writer) = self.messages_writer.take() {
//TODO: Fixme not sure whether we should spawn a task here.
- monoio::spawn(async move {
+ compio::runtime::spawn(async move {
let _ = log_writer.fsync().await;
});
} else {
@@ -314,7 +314,7 @@ impl Segment {
if let Some(index_writer) = self.index_writer.take() {
//TODO: Fixme not sure whether we should spawn a task here.
- monoio::spawn(async move {
+ compio::runtime::spawn(async move {
let _ = index_writer.fsync().await;
drop(index_writer)
});
diff --git a/core/server/src/streaming/streams/storage.rs b/core/server/src/streaming/streams/storage.rs
index d9848f03..b2b9672c 100644
--- a/core/server/src/streaming/streams/storage.rs
+++ b/core/server/src/streaming/streams/storage.rs
@@ -23,12 +23,12 @@ use crate::streaming::streams::COMPONENT;
use crate::streaming::streams::stream::Stream;
use crate::streaming::topics::topic::Topic;
use ahash::AHashSet;
+use compio::fs;
+use compio::fs::create_dir_all;
use error_set::ErrContext;
use futures::future::join_all;
use iggy_common::IggyError;
use iggy_common::IggyTimestamp;
-use monoio::fs;
-use monoio::fs::create_dir_all;
use serde::{Deserialize, Serialize};
use std::path::Path;
use std::sync::Arc;
@@ -157,7 +157,7 @@ impl StreamStorage for FileStreamStorage {
for mut topic in unloaded_topics {
let loaded_topics = loaded_topics.clone();
let topic_state = state.topics.remove(&topic.topic_id).unwrap();
- let load_topic = monoio::spawn(async move {
+ let load_topic = compio::runtime::spawn(async move {
match topic.load(topic_state).await {
Ok(_) => loaded_topics.lock().await.push(topic),
Err(error) => error!(
diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs
index 8d92b9d1..f7a1fe38 100644
--- a/core/server/src/streaming/topics/storage.rs
+++ b/core/server/src/streaming/topics/storage.rs
@@ -25,13 +25,13 @@ use crate::streaming::topics::consumer_group::ConsumerGroup;
use crate::streaming::topics::topic::Topic;
use ahash::AHashSet;
use anyhow::Context;
+use compio::fs;
+use compio::fs::create_dir_all;
use error_set::ErrContext;
use futures::future::join_all;
use iggy_common::IggyError;
use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
-use monoio::fs;
-use monoio::fs::create_dir_all;
use serde::{Deserialize, Serialize};
use std::path::Path;
use std::sync::Arc;
@@ -188,7 +188,7 @@ impl TopicStorage for FileTopicStorage {
for mut partition in unloaded_partitions {
let loaded_partitions = loaded_partitions.clone();
let partition_state = state.partitions.remove(&partition.partition_id).unwrap();
- let load_partition = monoio::spawn(async move {
+ let load_partition = compio::runtime::spawn(async move {
match partition.load(partition_state).await {
Ok(_) => {
loaded_partitions.lock().await.push(partition);
diff --git a/core/server/src/streaming/utils/file.rs b/core/server/src/streaming/utils/file.rs
index 7a5bb524..de00aa5e 100644
--- a/core/server/src/streaming/utils/file.rs
+++ b/core/server/src/streaming/utils/file.rs
@@ -16,7 +16,7 @@
* under the License.
*/
-use monoio::fs::{File, OpenOptions, remove_file};
+use compio::fs::{File, OpenOptions, remove_file};
use std::path::Path;
pub fn open_std(path: &str) -> Result<std::fs::File, std::io::Error> {
@@ -27,8 +27,14 @@ pub async fn open(path: &str) -> Result<File, std::io::Error> {
OpenOptions::new().read(true).open(path).await
}
-pub async fn append(path: &str) -> Result<File, std::io::Error> {
- OpenOptions::new().read(true).append(true).open(path).await
+pub async fn append(path: &str) -> Result<(File, u64), std::io::Error> {
+ let file = OpenOptions::new()
+ .create(true)
+ .write(true)
+ .open(path)
+ .await?;
+ let position = file.metadata().await?.len();
+ Ok((file, position))
}
pub async fn overwrite(path: &str) -> Result<File, std::io::Error> {
@@ -45,7 +51,7 @@ pub async fn remove(path: &str) -> Result<(), std::io::Error> {
}
pub async fn rename(old_path: &str, new_path: &str) -> Result<(), std::io::Error> {
- monoio::fs::rename(Path::new(old_path), Path::new(new_path)).await
+ compio::fs::rename(Path::new(old_path), Path::new(new_path)).await
}
pub async fn exists(path: &str) -> Result<bool, std::io::Error> {
diff --git a/core/server/src/streaming/utils/pooled_buffer.rs b/core/server/src/streaming/utils/pooled_buffer.rs
index a8664325..39ef25be 100644
--- a/core/server/src/streaming/utils/pooled_buffer.rs
+++ b/core/server/src/streaming/utils/pooled_buffer.rs
@@ -18,7 +18,7 @@
use super::memory_pool::{BytesMutExt, memory_pool};
use bytes::{Buf, BufMut, BytesMut};
-use monoio::buf::{IoBuf, IoBufMut};
+use compio::buf::{IoBuf, IoBufMut, SetBufInit};
use std::ops::{Deref, DerefMut};
#[derive(Debug)]
@@ -217,26 +217,32 @@ impl Buf for PooledBuffer {
}
}
-unsafe impl IoBufMut for PooledBuffer {
- fn write_ptr(&mut self) -> *mut u8 {
- self.inner.write_ptr()
- }
-
- fn bytes_total(&mut self) -> usize {
- self.inner.bytes_total()
+impl SetBufInit for PooledBuffer {
+ unsafe fn set_buf_init(&mut self, len: usize) {
+ if self.inner.len() <= len {
+ unsafe {
+ self.inner.set_len(len);
+ }
+ }
}
+}
- unsafe fn set_init(&mut self, pos: usize) {
- unsafe { self.inner.set_init(pos) }
+unsafe impl IoBufMut for PooledBuffer {
+ fn as_buf_mut_ptr(&mut self) -> *mut u8 {
+ self.inner.as_buf_mut_ptr()
}
}
unsafe impl IoBuf for PooledBuffer {
- fn read_ptr(&self) -> *const u8 {
- self.inner.read_ptr()
+ fn as_buf_ptr(&self) -> *const u8 {
+ self.inner.as_buf_ptr()
+ }
+
+ fn buf_len(&self) -> usize {
+ self.inner.len()
}
- fn bytes_init(&self) -> usize {
- self.inner.bytes_init()
+ fn buf_capacity(&self) -> usize {
+ self.inner.capacity()
}
}
diff --git a/core/server/src/tcp/connection_handler.rs b/core/server/src/tcp/connection_handler.rs
index 418af387..9c6803b8 100644
--- a/core/server/src/tcp/connection_handler.rs
+++ b/core/server/src/tcp/connection_handler.rs
@@ -62,6 +62,7 @@ pub(crate) async fn handle_connection(
if error.as_code() == IggyError::ConnectionClosed.as_code() {
return Err(ConnectionError::from(error));
} else {
+ error!("got error: {:?}", error);
sender.send_error_response(error).await?;
continue;
}
diff --git a/core/server/src/tcp/sender.rs b/core/server/src/tcp/sender.rs
index 2f445d4f..2c0ecfa9 100644
--- a/core/server/src/tcp/sender.rs
+++ b/core/server/src/tcp/sender.rs
@@ -16,30 +16,37 @@
* under the License.
*/
-use bytes::BytesMut;
-use iggy_common::IggyError;
-use monoio::{
- buf::IoBufMut,
- io::{AsyncReadRent, AsyncReadRentExt, AsyncWriteRent, AsyncWriteRentExt},
+use bytes::{Bytes, BytesMut};
+use compio::{
+ BufResult,
+ buf::{IoBuf, IoBufMut},
+ io::{AsyncRead, AsyncReadAtExt, AsyncReadExt, AsyncWriteExt},
};
+use iggy_common::IggyError;
use nix::libc;
use std::io::IoSlice;
-use tracing::debug;
+use tracing::{debug, error};
+
+use crate::streaming::utils::PooledBuffer;
const STATUS_OK: &[u8] = &[0; 4];
pub(crate) async fn read<T, B>(stream: &mut T, buffer: B) -> (Result<usize, IggyError>, B)
where
- T: AsyncReadRent + AsyncWriteRent + Unpin,
+ T: AsyncReadExt + AsyncWriteExt + Unpin,
B: IoBufMut,
{
- match stream.read_exact(buffer).await {
- (Ok(0), buffer) => (Err(IggyError::ConnectionClosed), buffer),
- (Ok(read_bytes), buffer) => (Ok(read_bytes), buffer),
+ let BufResult(result, buffer) = stream.read_exact(buffer).await;
+ match (result, buffer) {
+ (Ok(_), buffer) => (Ok(buffer.buf_len()), buffer),
+ // TODO: How to handle this ?(Ok(0), buffer) => (Err(IggyError::ConnectionClosed), buffer),
+ // `read_exact` from compio doesn't return how many bytes it read.
(Err(error), buffer) => {
if error.kind() == std::io::ErrorKind::UnexpectedEof {
+ //error!("Got some error tho.. {}", error);
(Err(IggyError::ConnectionClosed), buffer)
} else {
+ //error!("Got some other error tho.. {}", error);
(Err(IggyError::TcpError), buffer)
}
}
@@ -48,14 +55,14 @@ where
pub(crate) async fn send_empty_ok_response<T>(stream: &mut T) -> Result<(), IggyError>
where
- T: AsyncReadRent + AsyncWriteRent + Unpin,
+ T: AsyncReadExt + AsyncWriteExt + Unpin,
{
send_ok_response(stream, &[]).await
}
pub(crate) async fn send_ok_response<T>(stream: &mut T, payload: &[u8]) -> Result<(), IggyError>
where
- T: AsyncReadRent + AsyncWriteRent + Unpin,
+ T: AsyncReadExt + AsyncWriteExt + Unpin,
{
send_response(stream, STATUS_OK, payload).await
}
@@ -63,10 +70,10 @@ where
pub(crate) async fn send_ok_response_vectored<T>(
stream: &mut T,
length: &[u8],
- slices: Vec<libc::iovec>,
+ slices: Vec<PooledBuffer>,
) -> Result<(), IggyError>
where
- T: AsyncReadRentExt + AsyncWriteRentExt + Unpin,
+ T: AsyncReadExt + AsyncWriteExt + Unpin,
{
send_response_vectored(stream, STATUS_OK, length, slices).await
}
@@ -76,7 +83,7 @@ pub(crate) async fn send_error_response<T>(
error: IggyError,
) -> Result<(), IggyError>
where
- T: AsyncReadRent + AsyncWriteRent + Unpin,
+ T: AsyncReadExt + AsyncWriteExt + Unpin,
{
send_response(stream, &error.as_code().to_le_bytes(), &[]).await
}
@@ -87,7 +94,7 @@ pub(crate) async fn send_response<T>(
payload: &[u8],
) -> Result<(), IggyError>
where
- T: AsyncReadRent + AsyncWriteRent + Unpin,
+ T: AsyncReadExt + AsyncWriteExt + Unpin,
{
debug!(
"Sending response of len: {} with status: {:?}...",
@@ -108,32 +115,25 @@ pub(crate) async fn send_response_vectored<T>(
stream: &mut T,
status: &[u8],
length: &[u8],
- mut slices: Vec<libc::iovec>,
+ mut slices: Vec<PooledBuffer>,
) -> Result<(), IggyError>
where
- T: AsyncReadRentExt + AsyncWriteRentExt + Unpin,
+ T: AsyncReadExt + AsyncWriteExt + Unpin,
{
+ let resp_status = u32::from_le_bytes(status.try_into().unwrap());
debug!(
"Sending vectored response of len: {} with status: {:?}...",
slices.len(),
- status
+ resp_status
);
- let prefix = [
- libc::iovec {
- iov_base: status.as_ptr() as _,
- iov_len: status.len(),
- },
- libc::iovec {
- iov_base: length.as_ptr() as _,
- iov_len: length.len(),
- },
- ];
- slices.splice(0..0, prefix);
+ let status = PooledBuffer::from(status);
+ let length = PooledBuffer::from(length);
+ slices.splice(0..0, [status, length]);
stream
.write_vectored_all(slices)
.await
.0
.map_err(|_| IggyError::TcpError)?;
- debug!("Sent response with status: {:?}", status);
+ debug!("Sent response with status: {:?}", resp_status);
Ok(())
}
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index 4f528016..b541305b 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -29,13 +29,19 @@ use std::rc::Rc;
use std::time::Duration;
use tracing::{error, info};
-pub async fn start(server_name: &'static str, addr: SocketAddr, socket: Socket, shard: Rc<IggyShard>) -> Result<(), IggyError> {
+pub async fn start(
+ server_name: &'static str,
+ addr: SocketAddr,
+ socket: Socket,
+ shard: Rc<IggyShard>,
+) -> Result<(), IggyError> {
socket
.bind(&addr.into())
.expect("Failed to bind TCP listener");
socket.listen(1024).unwrap();
+ //TODO: Fix it later to use `TcpOpts` from compio.
let listener: std::net::TcpListener = socket.into();
- let listener = monoio::net::TcpListener::from_std(listener).unwrap();
+ let listener = compio::net::TcpListener::from_std(listener).unwrap();
info!("{server_name} server has started on: {:?}", addr);
loop {
@@ -44,7 +50,7 @@ pub async fn start(server_name: &'static str, addr: SocketAddr, socket: Socket,
if shard.is_shutting_down() {
return;
}
- monoio::time::sleep(Duration::from_millis(100)).await;
+ compio::time::sleep(Duration::from_millis(100)).await;
}
};
diff --git a/core/server/src/tcp/tcp_sender.rs b/core/server/src/tcp/tcp_sender.rs
index cfa25aad..d0a1c27c 100644
--- a/core/server/src/tcp/tcp_sender.rs
+++ b/core/server/src/tcp/tcp_sender.rs
@@ -17,14 +17,15 @@
*/
use crate::binary::sender::Sender;
+use crate::streaming::utils::PooledBuffer;
use crate::tcp::COMPONENT;
use crate::{server_error::ServerError, tcp::sender};
use bytes::BytesMut;
+use compio::buf::{IoBuf, IoBufMut};
+use compio::io::AsyncWrite;
+use compio::net::TcpStream;
use error_set::ErrContext;
use iggy_common::IggyError;
-use monoio::buf::IoBufMut;
-use monoio::io::AsyncWriteRent;
-use monoio::net::TcpStream;
use nix::libc;
#[derive(Debug)]
@@ -62,7 +63,7 @@ impl Sender for TcpSender {
async fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<libc::iovec>,
+ slices: Vec<PooledBuffer>,
) -> Result<(), IggyError> {
sender::send_ok_response_vectored(&mut self.stream, length, slices).await
}
diff --git a/core/server/src/tcp/tcp_tls_listener.rs b/core/server/src/tcp/tcp_tls_listener.rs
index 2c81eb60..00d7b5c4 100644
--- a/core/server/src/tcp/tcp_tls_listener.rs
+++ b/core/server/src/tcp/tcp_tls_listener.rs
@@ -24,7 +24,6 @@ use crate::streaming::clients::client_manager::Transport;
use crate::tcp::connection_handler::{handle_connection, handle_error};
use futures::FutureExt;
use iggy_common::IggyError;
-use monoio_rustls::TlsAcceptor;
use rustls::ServerConfig;
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
use rustls_pemfile::{certs, private_key};
@@ -42,106 +41,108 @@ pub(crate) async fn start(
socket: Socket,
shard: Rc<IggyShard>,
) -> Result<(), IggyError> {
- let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
- let config = &shard.config.tcp.tls;
-
- let (certs, key) =
- if config.self_signed && !std::path::Path::new(&config.cert_file).exists() {
- info!("Generating self-signed certificate for TCP TLS server");
- generate_self_signed_cert()
- .unwrap_or_else(|e| panic!("Failed to generate self-signed certificate: {e}"))
- } else {
- load_certificates(&config.cert_file, &config.key_file)
- .unwrap_or_else(|e| panic!("Failed to load certificates: {e}"))
- };
-
- let server_config = ServerConfig::builder()
- .with_no_client_auth()
- .with_single_cert(certs, key)
- .unwrap_or_else(|e| panic!("Unable to create TLS server config: {e}"));
-
- let acceptor = TlsAcceptor::from(Arc::new(server_config));
-
- socket
- .bind(&addr.into())
- .unwrap_or_else(|e| panic!("Unable to bind socket to address '{addr}': {e}",));
-
- let listener: std::net::TcpListener = socket.into();
- let listener = monoio::net::TcpListener::from_std(listener).unwrap();
- info!("{server_name} server has started on: {:?}", addr);
-
- loop {
- let shutdown_check = async {
- loop {
- if shard.is_shutting_down() {
- return;
- }
- monoio::time::sleep(Duration::from_millis(100)).await;
+ /*
+ let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+ let config = &shard.config.tcp.tls;
+
+ let (certs, key) =
+ if config.self_signed && !std::path::Path::new(&config.cert_file).exists() {
+ info!("Generating self-signed certificate for TCP TLS server");
+ generate_self_signed_cert()
+ .unwrap_or_else(|e| panic!("Failed to generate self-signed certificate: {e}"))
+ } else {
+ load_certificates(&config.cert_file, &config.key_file)
+ .unwrap_or_else(|e| panic!("Failed to load certificates: {e}"))
+ };
+
+ let server_config = ServerConfig::builder()
+ .with_no_client_auth()
+ .with_single_cert(certs, key)
+ .unwrap_or_else(|e| panic!("Unable to create TLS server config: {e}"));
+
+ let acceptor = TlsAcceptor::from(Arc::new(server_config));
+
+ socket
+ .bind(&addr.into())
+ .unwrap_or_else(|e| panic!("Unable to bind socket to address '{addr}': {e}",));
+
+ let listener: std::net::TcpListener = socket.into();
+ let listener = monoio::net::TcpListener::from_std(listener).unwrap();
+ info!("{server_name} server has started on: {:?}", addr);
+
+ loop {
+ let shutdown_check = async {
+ loop {
+ if shard.is_shutting_down() {
+ return;
}
- };
+ monoio::time::sleep(Duration::from_millis(100)).await;
+ }
+ };
- let accept_future = listener.accept();
- futures::select! {
- _ = shutdown_check.fuse() => {
- info!("TCP TLS server detected shutdown flag, no longer accepting connections");
- break;
- }
- result = accept_future.fuse() => {
- match result {
- Ok((stream, address)) => {
- if shard.is_shutting_down() {
- info!("Rejecting new TLS connection from {} during shutdown", address);
- continue;
- }
- let shard_clone = shard.clone();
- info!("Accepted new TCP TLS connection: {}", address);
- let transport = Transport::Tcp;
- let session = shard_clone.add_client(&address, transport);
- //TODO: Those can be shared with other shards.
- shard_clone.add_active_session(session.clone());
- // Broadcast session to all shards.
- let event = ShardEvent::NewSession { address, transport };
- // TODO: Fixme look inside of broadcast_event_to_all_shards method.
- let _responses = shard_clone.broadcast_event_to_all_shards(event.into());
-
- let client_id = session.client_id;
- info!("Created new session: {session}");
- let acceptor = acceptor.clone();
-
- let conn_stop_receiver = shard_clone.task_registry.add_connection(client_id);
-
- let shard_for_conn = shard_clone.clone();
- shard_clone.task_registry.spawn_tracked(async move {
- match acceptor.accept(stream).await {
- Ok(tls_stream) => {
- let mut sender = SenderKind::get_tcp_tls_sender(tls_stream.into());
- if let Err(error) = handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver).await {
- handle_error(error);
- }
- shard_for_conn.task_registry.remove_connection(&client_id);
-
- if let Err(error) = sender.shutdown().await {
- error!(
- "Failed to shutdown TCP TLS stream for client: {client_id}, address: {address}. {error}"
- );
- } else {
- info!(
- "Successfully closed TCP TLS stream for client: {client_id}, address: {address}."
- );
- }
+ let accept_future = listener.accept();
+ futures::select! {
+ _ = shutdown_check.fuse() => {
+ info!("TCP TLS server detected shutdown flag, no longer accepting connections");
+ break;
+ }
+ result = accept_future.fuse() => {
+ match result {
+ Ok((stream, address)) => {
+ if shard.is_shutting_down() {
+ info!("Rejecting new TLS connection from {} during shutdown", address);
+ continue;
+ }
+ let shard_clone = shard.clone();
+ info!("Accepted new TCP TLS connection: {}", address);
+ let transport = Transport::Tcp;
+ let session = shard_clone.add_client(&address, transport);
+ //TODO: Those can be shared with other shards.
+ shard_clone.add_active_session(session.clone());
+ // Broadcast session to all shards.
+ let event = ShardEvent::NewSession { address, transport };
+ // TODO: Fixme look inside of broadcast_event_to_all_shards method.
+ let _responses = shard_clone.broadcast_event_to_all_shards(event.into());
+
+ let client_id = session.client_id;
+ info!("Created new session: {session}");
+ let acceptor = acceptor.clone();
+
+ let conn_stop_receiver = shard_clone.task_registry.add_connection(client_id);
+
+ let shard_for_conn = shard_clone.clone();
+ shard_clone.task_registry.spawn_tracked(async move {
+ match acceptor.accept(stream).await {
+ Ok(tls_stream) => {
+ let mut sender = SenderKind::get_tcp_tls_sender(tls_stream.into());
+ if let Err(error) = handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver).await {
+ handle_error(error);
}
- Err(e) => {
- error!("Failed to accept TLS connection from '{address}': {e}");
- shard_for_conn.task_registry.remove_connection(&client_id);
+ shard_for_conn.task_registry.remove_connection(&client_id);
+
+ if let Err(error) = sender.shutdown().await {
+ error!(
+ "Failed to shutdown TCP TLS stream for client: {client_id}, address: {address}. {error}"
+ );
+ } else {
+ info!(
+ "Successfully closed TCP TLS stream for client: {client_id}, address: {address}."
+ );
}
}
- });
- }
- Err(error) => error!("Unable to accept TCP TLS socket. {error}"),
+ Err(e) => {
+ error!("Failed to accept TLS connection from '{address}': {e}");
+ shard_for_conn.task_registry.remove_connection(&client_id);
+ }
+ }
+ });
}
+ Err(error) => error!("Unable to accept TCP TLS socket. {error}"),
}
}
}
+ }
+ */
Ok(())
}
diff --git a/core/server/src/tcp/tcp_tls_sender.rs b/core/server/src/tcp/tcp_tls_sender.rs
index b214e5a6..8dbb3972 100644
--- a/core/server/src/tcp/tcp_tls_sender.rs
+++ b/core/server/src/tcp/tcp_tls_sender.rs
@@ -17,41 +17,46 @@
*/
use crate::binary::sender::Sender;
+use crate::streaming::utils::PooledBuffer;
use crate::tcp::COMPONENT;
use crate::{server_error::ServerError, tcp::sender};
use bytes::BytesMut;
+use compio::buf::IoBufMut;
+use compio::io::AsyncWrite;
+use compio::net::TcpStream;
use error_set::ErrContext;
use iggy_common::IggyError;
-use monoio::buf::IoBufMut;
-use monoio::io::AsyncWriteRent;
-use monoio::net::TcpStream;
-use monoio_rustls::ServerTlsStream;
//use tokio_rustls::server::TlsStream;
use nix::libc;
#[derive(Debug)]
pub struct TcpTlsSender {
- pub(crate) stream: ServerTlsStream<TcpStream>,
+ pub(crate) stream: TcpStream,
}
impl Sender for TcpTlsSender {
async fn read<B: IoBufMut>(&mut self, buffer: B) -> (Result<usize, IggyError>, B) {
+ todo!();
sender::read(&mut self.stream, buffer).await
}
async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> {
+ todo!();
sender::send_empty_ok_response(&mut self.stream).await
}
async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> {
+ todo!();
sender::send_ok_response(&mut self.stream, payload).await
}
async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> {
+ todo!();
sender::send_error_response(&mut self.stream, error).await
}
async fn shutdown(&mut self) -> Result<(), ServerError> {
+ todo!();
self.stream
.shutdown()
.await
@@ -64,8 +69,9 @@ impl Sender for TcpTlsSender {
async fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<libc::iovec>,
+ slices: Vec<PooledBuffer>,
) -> Result<(), IggyError> {
+ todo!();
sender::send_ok_response_vectored(&mut self.stream, length, slices).await
}
}
(iggy) 15/30: server compiles
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit a718676d9679955a2b2bb1449498dfa7dc4ddbfa
Author: numinex <g....@gmail.com>
AuthorDate: Fri Jun 27 15:09:01 2025 +0200
server compiles
---
core/server/src/binary/command.rs | 1 -
.../create_consumer_group_handler.rs | 2 +-
.../consumer_groups/get_consumer_group_handler.rs | 3 +-
.../leave_consumer_group_handler.rs | 2 +-
.../handlers/messages/poll_messages_handler.rs | 2 +-
.../binary/handlers/system/get_clients_handler.rs | 8 +--
core/server/src/binary/sender.rs | 3 +-
core/server/src/configs/defaults.rs | 3 +-
core/server/src/configs/server.rs | 3 +-
core/server/src/lib.rs | 4 +-
core/server/src/main.rs | 78 ++++++++++------------
core/server/src/shard/mod.rs | 8 ++-
core/server/src/shard/system/snapshot/mod.rs | 8 +--
core/server/src/shard/transmission/message.rs | 8 +--
core/server/src/streaming/partitions/messages.rs | 2 +-
core/server/src/streaming/partitions/partition.rs | 8 +--
core/server/src/streaming/segments/segment.rs | 8 +--
core/server/src/streaming/storage.rs | 7 +-
core/server/src/streaming/streams/stream.rs | 10 +--
core/server/src/streaming/streams/topics.rs | 2 +-
.../server/src/streaming/topics/consumer_groups.rs | 2 +-
core/server/src/streaming/topics/messages.rs | 2 +-
core/server/src/streaming/topics/storage.rs | 2 +-
core/server/src/streaming/topics/topic.rs | 8 +--
core/server/src/streaming/utils/memory_pool.rs | 4 +-
core/server/src/tcp/sender.rs | 5 +-
core/server/src/tcp/tcp_listener.rs | 3 +
27 files changed, 93 insertions(+), 103 deletions(-)
diff --git a/core/server/src/binary/command.rs b/core/server/src/binary/command.rs
index 1ebac395..aa826590 100644
--- a/core/server/src/binary/command.rs
+++ b/core/server/src/binary/command.rs
@@ -68,7 +68,6 @@ use iggy_common::update_stream::UpdateStream;
use iggy_common::update_topic::UpdateTopic;
use iggy_common::update_user::UpdateUser;
use iggy_common::*;
-use rustls::crypto::hash::Output;
use strum::EnumString;
use tracing::error;
diff --git a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
index 266b46a6..e6e1f239 100644
--- a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
@@ -60,7 +60,7 @@ impl ServerCommandHandler for CreateConsumerGroup {
self.stream_id, self.topic_id, self.group_id
)
})?;
-
+
let stream = shard.find_stream(session, &self.stream_id)
.with_error_context(|error| {
format!(
diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
index 8f67192b..dab89c59 100644
--- a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
@@ -46,7 +46,8 @@ impl ServerCommandHandler for GetConsumerGroup {
debug!("session: {session}, command: {self}");
let stream_id = &self.stream_id;
- let stream = shard.find_stream(session, &self.stream_id)
+ let stream = shard
+ .find_stream(session, &self.stream_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get stream for stream_id: {stream_id}"
diff --git a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
index ba352bf9..18ee8b8c 100644
--- a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use super::COMPONENT;
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::sender::SenderKind;
@@ -27,7 +28,6 @@ use iggy_common::IggyError;
use iggy_common::leave_consumer_group::LeaveConsumerGroup;
use std::rc::Rc;
use tracing::{debug, instrument};
-use super::COMPONENT;
impl ServerCommandHandler for LeaveConsumerGroup {
fn code(&self) -> u32 {
diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
index a591f691..9d6d526b 100644
--- a/core/server/src/binary/handlers/messages/poll_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
@@ -20,8 +20,8 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::messages::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::sender::SenderKind;
-use crate::shard::system::messages::PollingArgs;
use crate::shard::IggyShard;
+use crate::shard::system::messages::PollingArgs;
use crate::streaming::session::Session;
use crate::to_iovec;
use anyhow::Result;
diff --git a/core/server/src/binary/handlers/system/get_clients_handler.rs b/core/server/src/binary/handlers/system/get_clients_handler.rs
index c73ac987..9d5241f4 100644
--- a/core/server/src/binary/handlers/system/get_clients_handler.rs
+++ b/core/server/src/binary/handlers/system/get_clients_handler.rs
@@ -43,11 +43,9 @@ impl ServerCommandHandler for GetClients {
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let clients = shard
- .get_clients(session)
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get clients, session: {session}")
- })?;
+ let clients = shard.get_clients(session).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get clients, session: {session}")
+ })?;
let clients = mapper::map_clients(clients).await;
sender.send_ok_response(&clients).await?;
Ok(())
diff --git a/core/server/src/binary/sender.rs b/core/server/src/binary/sender.rs
index ff405eb6..adbf05ee 100644
--- a/core/server/src/binary/sender.rs
+++ b/core/server/src/binary/sender.rs
@@ -54,11 +54,10 @@ macro_rules! forward_async_methods {
}
}
-
pub trait Sender {
fn read<B: IoBufMut>(
&mut self,
- buffer: B
+ buffer: B,
) -> impl Future<Output = (Result<usize, IggyError>, B)>;
fn send_empty_ok_response(&mut self) -> impl Future<Output = Result<(), IggyError>>;
fn send_ok_response(&mut self, payload: &[u8]) -> impl Future<Output = Result<(), IggyError>>;
diff --git a/core/server/src/configs/defaults.rs b/core/server/src/configs/defaults.rs
index fc333ec7..82b560cf 100644
--- a/core/server/src/configs/defaults.rs
+++ b/core/server/src/configs/defaults.rs
@@ -36,7 +36,6 @@ use crate::configs::system::{
use crate::configs::tcp::{TcpConfig, TcpTlsConfig};
use iggy_common::IggyByteSize;
use iggy_common::IggyDuration;
-use std::rc::Rc;
use std::sync::Arc;
use std::time::Duration;
@@ -52,7 +51,7 @@ impl Default for ServerConfig {
heartbeat: HeartbeatConfig::default(),
message_saver: MessageSaverConfig::default(),
personal_access_token: PersonalAccessTokenConfig::default(),
- system: Rc::new(SystemConfig::default()),
+ system: Arc::new(SystemConfig::default()),
quic: QuicConfig::default(),
tcp: TcpConfig::default(),
http: HttpConfig::default(),
diff --git a/core/server/src/configs/server.rs b/core/server/src/configs/server.rs
index 059fd1c4..252df9a7 100644
--- a/core/server/src/configs/server.rs
+++ b/core/server/src/configs/server.rs
@@ -31,7 +31,6 @@ use iggy_common::Validatable;
use serde::{Deserialize, Serialize};
use serde_with::DisplayFromStr;
use serde_with::serde_as;
-use std::rc::Rc;
use std::str::FromStr;
use std::sync::Arc;
@@ -41,7 +40,7 @@ pub struct ServerConfig {
pub message_saver: MessageSaverConfig,
pub personal_access_token: PersonalAccessTokenConfig,
pub heartbeat: HeartbeatConfig,
- pub system: Rc<SystemConfig>,
+ pub system: Arc<SystemConfig>,
pub quic: QuicConfig,
pub tcp: TcpConfig,
pub http: HttpConfig,
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index 4a5dbe53..f618d3e0 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -18,8 +18,8 @@
#[cfg(not(feature = "disable-mimalloc"))]
use mimalloc::MiMalloc;
-use nix::libc::iovec;
use nix::libc::c_void;
+use nix::libc::iovec;
#[cfg(not(feature = "disable-mimalloc"))]
#[global_allocator]
@@ -61,4 +61,4 @@ pub fn to_iovec<T>(data: &[T]) -> iovec {
iov_base: data.as_ptr() as *mut c_void,
iov_len: data.len() * std::mem::size_of::<T>(),
}
-}
\ No newline at end of file
+}
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 78b3a1e9..436b3829 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use std::rc::Rc;
use std::sync::Arc;
use std::thread::available_parallelism;
@@ -32,6 +33,7 @@ use server::bootstrap::{
create_shard_executor, load_config, resolve_persister,
};
use server::configs::config_provider::{self};
+use server::configs::server::ServerConfig;
#[cfg(not(feature = "tokio-console"))]
use server::log::logger::Logging;
#[cfg(feature = "tokio-console")]
@@ -77,54 +79,44 @@ fn main() -> Result<(), ServerError> {
// TODO: I think we could get rid of config provider, since we support only TOML
// as config provider.
let config_provider = config_provider::resolve(&args.config_provider)?;
- let config = std::thread::scope(|scope| {
- let config = scope
- .spawn(move || {
- let mut rt = create_default_executor::<monoio::IoUringDriver>();
- rt.block_on(load_config(&config_provider))
- })
- .join()
- .expect("Failed to load config");
- config
- })?;
+ // Load config and create directories.
+ // Remove `local_data` directory if run with `--fresh` flag.
+ let mut rt = create_default_executor::<monoio::IoUringDriver>();
+ let config = rt
+ .block_on(async {
+ let config = load_config(&config_provider)
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to load config during bootstrap")
+ })?;
+ if args.fresh {
+ let system_path = config.system.get_system_path();
+ if monoio::fs::metadata(&system_path).await.is_ok() {
+ println!(
+ "Removing system path at: {} because `--fresh` flag was set",
+ system_path
+ );
+ //TODO: Impl dir walk and remove the files
+ /*
+ if let Err(e) = tokio::fs::remove_dir_all(&system_path).await {
+ eprintln!("Failed to remove system path at {}: {}", system_path, e);
+ }
+ */
+ }
+ }
+
+ // Create directories.
+ create_directories(&config.system).await?;
+ Ok::<ServerConfig, ServerError>(config)
+ })
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to load config")
+ })?;
// Initialize logging
let mut logging = Logging::new(config.telemetry.clone());
logging.early_init();
- // Create directories.
- // Remove `local_data` directory if run with `--fresh` flag.
- std::thread::scope(|scope| {
- scope
- .spawn(|| {
- let mut rt = create_default_executor::<monoio::IoUringDriver>();
- rt.block_on(async {
- if args.fresh {
- let system_path = config.system.get_system_path();
- if monoio::fs::metadata(&system_path).await.is_ok() {
- println!(
- "Removing system path at: {} because `--fresh` flag was set",
- system_path
- );
- //TODO: Impl dir walk and remove the files
- /*
- if let Err(e) = tokio::fs::remove_dir_all(&system_path).await {
- eprintln!("Failed to remove system path at {}: {}", system_path, e);
- }
- */
- }
- }
-
- // Create directories.
- create_directories(&config.system).await?;
- Ok::<(), ServerError>(())
- })
- })
- .join()
- .expect("Failed join thread")
- })
- .with_error_context(|err| format!("Failed to init server: {err}"))?;
-
// TODO: Make this configurable from config as a range
// for example this instance of Iggy will use cores from 0..4
let available_cpus = available_parallelism().expect("Failed to get num of cores");
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index b5e48525..dbd94b68 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -426,7 +426,7 @@ impl IggyShard {
self.shards.len() as u32
}
- pub fn broadcast_event_to_all_shards(&self, client_id: u32, event: Rc<ShardEvent>) {
+ pub fn broadcast_event_to_all_shards(&self, client_id: u32, event: ShardEvent) {
self.shards
.iter()
.filter_map(|shard| {
@@ -437,8 +437,12 @@ impl IggyShard {
}
})
.map(|conn| {
- let message = ShardMessage::Event(event.clone());
+ //TODO: Fixme
+ /*
+ let message = ShardMessage::Event(event);
conn.send(ShardFrame::new(client_id, message, None));
+ */
+ ()
})
.collect::<Vec<_>>();
}
diff --git a/core/server/src/shard/system/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs
index b4de2992..eb726509 100644
--- a/core/server/src/shard/system/snapshot/mod.rs
+++ b/core/server/src/shard/system/snapshot/mod.rs
@@ -27,7 +27,7 @@ use iggy_common::{IggyDuration, IggyError, Snapshot, SnapshotCompression, System
use monoio::fs::{File, OpenOptions};
use std::io::Cursor;
use std::path::PathBuf;
-use std::rc::Rc;
+use std::sync::Arc;
use std::time::Instant;
use tempfile::NamedTempFile;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
@@ -191,7 +191,7 @@ async fn get_test_snapshot() -> Result<NamedTempFile, std::io::Error> {
write_command_output_to_temp_file(Command::new("echo").arg("test")).await
}
-async fn get_server_logs(config: Rc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
+async fn get_server_logs(config: Arc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
let base_directory = PathBuf::from(config.get_system_path());
let logs_subdirectory = PathBuf::from(&config.logging.path);
let logs_path = base_directory.join(logs_subdirectory);
@@ -204,7 +204,7 @@ async fn get_server_logs(config: Rc<SystemConfig>) -> Result<NamedTempFile, std:
write_command_output_to_temp_file(Command::new("sh").args(["-c", &list_and_cat])).await
}
-async fn get_server_config(config: Rc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
+async fn get_server_config(config: Arc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
let base_directory = PathBuf::from(config.get_system_path());
let config_path = base_directory.join("runtime").join("current_config.toml");
@@ -213,7 +213,7 @@ async fn get_server_config(config: Rc<SystemConfig>) -> Result<NamedTempFile, st
async fn get_command_result(
snapshot_type: &SystemSnapshotType,
- config: Rc<SystemConfig>,
+ config: Arc<SystemConfig>,
) -> Result<NamedTempFile, std::io::Error> {
match snapshot_type {
SystemSnapshotType::FilesystemOverview => get_filesystem_overview().await,
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index 8d1e66eb..cbf4fdd4 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -22,12 +22,12 @@ use crate::{binary::command::ServerCommand, streaming::session::Session};
#[derive(Debug)]
pub enum ShardMessage {
Command(ServerCommand),
- Event(Rc<ShardEvent>),
+ Event(ShardEvent),
}
#[derive(Debug)]
pub enum ShardEvent {
- NewSession(Rc<Session>),
+ NewSession(),
}
impl From<ServerCommand> for ShardMessage {
@@ -36,8 +36,8 @@ impl From<ServerCommand> for ShardMessage {
}
}
-impl From<Rc<ShardEvent>> for ShardMessage {
- fn from(event: Rc<ShardEvent>) -> Self {
+impl From<ShardEvent> for ShardMessage {
+ fn from(event: ShardEvent) -> Self {
ShardMessage::Event(event)
}
}
diff --git a/core/server/src/streaming/partitions/messages.rs b/core/server/src/streaming/partitions/messages.rs
index a2d71c8e..b136aeea 100644
--- a/core/server/src/streaming/partitions/messages.rs
+++ b/core/server/src/streaming/partitions/messages.rs
@@ -711,7 +711,7 @@ mod tests {
let partition_id = 3;
let with_segment = true;
let temp_dir = TempDir::new().unwrap();
- let config = Rc::new(SystemConfig {
+ let config = Arc::new(SystemConfig {
path: temp_dir.path().to_path_buf().to_str().unwrap().to_string(),
message_deduplication: MessageDeduplicationConfig {
enabled: deduplication_enabled,
diff --git a/core/server/src/streaming/partitions/partition.rs b/core/server/src/streaming/partitions/partition.rs
index 4a00ca85..1b8bacf8 100644
--- a/core/server/src/streaming/partitions/partition.rs
+++ b/core/server/src/streaming/partitions/partition.rs
@@ -59,7 +59,7 @@ pub struct Partition {
pub(crate) consumer_offsets: DashMap<u32, ConsumerOffset>,
pub(crate) consumer_group_offsets: DashMap<u32, ConsumerOffset>,
pub(crate) segments: Vec<Segment>,
- pub(crate) config: Rc<SystemConfig>,
+ pub(crate) config: Arc<SystemConfig>,
pub(crate) storage: Rc<SystemStorage>,
}
@@ -89,7 +89,7 @@ impl Partition {
topic_id: u32,
partition_id: u32,
with_segment: bool,
- config: Rc<SystemConfig>,
+ config: Arc<SystemConfig>,
storage: Rc<SystemStorage>,
message_expiry: IggyExpiry,
messages_count_of_parent_stream: Arc<AtomicU64>,
@@ -217,7 +217,7 @@ mod tests {
#[tokio::test]
async fn should_be_created_with_a_single_segment_given_valid_parameters() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Rc::new(SystemConfig {
+ let config = Arc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
@@ -266,7 +266,7 @@ mod tests {
#[tokio::test]
async fn should_not_initialize_segments_given_false_with_segment_parameter() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Rc::new(SystemConfig {
+ let config = Arc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
diff --git a/core/server/src/streaming/segments/segment.rs b/core/server/src/streaming/segments/segment.rs
index 252fe7f1..60886df6 100644
--- a/core/server/src/streaming/segments/segment.rs
+++ b/core/server/src/streaming/segments/segment.rs
@@ -61,7 +61,7 @@ pub struct Segment {
pub(super) index_reader: Option<IndexReader>,
pub(super) message_expiry: IggyExpiry,
pub(super) accumulator: MessagesAccumulator,
- pub(super) config: Rc<SystemConfig>,
+ pub(super) config: Arc<SystemConfig>,
pub(super) indexes: IggyIndexesMut,
pub(super) messages_size: Arc<AtomicU64>,
pub(super) indexes_size: Arc<AtomicU64>,
@@ -74,7 +74,7 @@ impl Segment {
topic_id: u32,
partition_id: u32,
start_offset: u64,
- config: Rc<SystemConfig>,
+ config: Arc<SystemConfig>,
message_expiry: IggyExpiry,
size_of_parent_stream: Arc<AtomicU64>,
size_of_parent_topic: Arc<AtomicU64>,
@@ -451,7 +451,7 @@ mod tests {
let topic_id = 2;
let partition_id = 3;
let start_offset = 0;
- let config = Rc::new(SystemConfig::default());
+ let config = Arc::new(SystemConfig::default());
let path = config.get_segment_path(stream_id, topic_id, partition_id, start_offset);
let messages_file_path = Segment::get_messages_file_path(&path);
let index_path = Segment::get_index_path(&path);
@@ -500,7 +500,7 @@ mod tests {
let topic_id = 2;
let partition_id = 3;
let start_offset = 0;
- let config = Rc::new(SystemConfig {
+ let config = Arc::new(SystemConfig {
segment: SegmentConfig {
cache_indexes: CacheIndexesConfig::None,
..Default::default()
diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs
index eaf96dc3..4b41a2b3 100644
--- a/core/server/src/streaming/storage.rs
+++ b/core/server/src/streaming/storage.rs
@@ -33,7 +33,6 @@ use iggy_common::IggyError;
use mockall::automock;
use std::fmt::Debug;
use std::future::Future;
-use std::rc::Rc;
use std::sync::Arc;
macro_rules! forward_async_methods {
@@ -137,7 +136,7 @@ pub trait PartitionStorage {
#[derive(Debug)]
pub struct SystemStorage {
- pub info: Rc<SystemInfoStorageKind>,
+ pub info: Arc<SystemInfoStorageKind>,
pub stream: Arc<StreamStorageKind>,
pub topic: Arc<TopicStorageKind>,
pub partition: Arc<PartitionStorageKind>,
@@ -145,9 +144,9 @@ pub struct SystemStorage {
}
impl SystemStorage {
- pub fn new(config: Rc<SystemConfig>, persister: Arc<PersisterKind>) -> Self {
+ pub fn new(config: Arc<SystemConfig>, persister: Arc<PersisterKind>) -> Self {
Self {
- info: Rc::new(SystemInfoStorageKind::File(FileSystemInfoStorage::new(
+ info: Arc::new(SystemInfoStorageKind::File(FileSystemInfoStorage::new(
config.get_state_info_path(),
persister.clone(),
))),
diff --git a/core/server/src/streaming/streams/stream.rs b/core/server/src/streaming/streams/stream.rs
index 03681b75..6a858fd5 100644
--- a/core/server/src/streaming/streams/stream.rs
+++ b/core/server/src/streaming/streams/stream.rs
@@ -40,7 +40,7 @@ pub struct Stream {
pub segments_count: Arc<AtomicU32>,
pub(crate) topics: AHashMap<u32, Topic>,
pub(crate) topics_ids: AHashMap<String, u32>,
- pub(crate) config: Rc<SystemConfig>,
+ pub(crate) config: Arc<SystemConfig>,
pub(crate) storage: Rc<SystemStorage>,
}
@@ -48,7 +48,7 @@ impl Stream {
pub fn empty(
id: u32,
name: &str,
- config: Rc<SystemConfig>,
+ config: Arc<SystemConfig>,
storage: Rc<SystemStorage>,
) -> Self {
Stream::create(id, name, config, storage)
@@ -57,7 +57,7 @@ impl Stream {
pub fn create(
id: u32,
name: &str,
- config: Rc<SystemConfig>,
+ config: Arc<SystemConfig>,
storage: Rc<SystemStorage>,
) -> Self {
let path = config.get_stream_path(id);
@@ -106,7 +106,7 @@ mod tests {
#[test]
fn should_be_created_given_valid_parameters() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Rc::new(SystemConfig {
+ let config = Arc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
@@ -117,7 +117,7 @@ mod tests {
MemoryPool::init_pool(config.clone());
let id = 1;
let name = "test";
- let config = Rc::new(SystemConfig::default());
+ let config = Arc::new(SystemConfig::default());
let path = config.get_stream_path(id);
let topics_path = config.get_topics_path(id);
diff --git a/core/server/src/streaming/streams/topics.rs b/core/server/src/streaming/streams/topics.rs
index ed350bee..1d47cc36 100644
--- a/core/server/src/streaming/streams/topics.rs
+++ b/core/server/src/streaming/streams/topics.rs
@@ -287,7 +287,7 @@ mod tests {
#[tokio::test]
async fn should_get_topic_by_id_and_name() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Rc::new(SystemConfig {
+ let config = Arc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index bf249d2e..28a8705c 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -440,7 +440,7 @@ mod tests {
async fn get_topic() -> Topic {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Rc::new(SystemConfig {
+ let config = Arc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
diff --git a/core/server/src/streaming/topics/messages.rs b/core/server/src/streaming/topics/messages.rs
index e8c52385..c01687a6 100644
--- a/core/server/src/streaming/topics/messages.rs
+++ b/core/server/src/streaming/topics/messages.rs
@@ -322,7 +322,7 @@ mod tests {
async fn init_topic(partitions_count: u32) -> Topic {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Rc::new(SystemConfig {
+ let config = Arc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs
index bfb7ead0..db74fc98 100644
--- a/core/server/src/streaming/topics/storage.rs
+++ b/core/server/src/streaming/topics/storage.rs
@@ -34,7 +34,7 @@ use std::path::Path;
use std::sync::Arc;
use tokio::fs;
use tokio::fs::create_dir_all;
-use tokio::sync::{Mutex};
+use tokio::sync::Mutex;
use tracing::{error, info, warn};
#[derive(Debug)]
diff --git a/core/server/src/streaming/topics/topic.rs b/core/server/src/streaming/topics/topic.rs
index 7e8ca4ac..2e9e38c9 100644
--- a/core/server/src/streaming/topics/topic.rs
+++ b/core/server/src/streaming/topics/topic.rs
@@ -49,7 +49,7 @@ pub struct Topic {
pub(crate) messages_count_of_parent_stream: Arc<AtomicU64>,
pub(crate) messages_count: Arc<AtomicU64>,
pub(crate) segments_count_of_parent_stream: Arc<AtomicU32>,
- pub(crate) config: Rc<SystemConfig>,
+ pub(crate) config: Arc<SystemConfig>,
pub(crate) partitions: AHashMap<u32, IggyRwLock<Partition>>,
pub(crate) storage: Rc<SystemStorage>,
pub(crate) consumer_groups: RefCell<AHashMap<u32, ConsumerGroup>>,
@@ -72,7 +72,7 @@ impl Topic {
size_of_parent_stream: Arc<AtomicU64>,
messages_count_of_parent_stream: Arc<AtomicU64>,
segments_count_of_parent_stream: Arc<AtomicU32>,
- config: Rc<SystemConfig>,
+ config: Arc<SystemConfig>,
storage: Rc<SystemStorage>,
) -> Topic {
Topic::create(
@@ -100,7 +100,7 @@ impl Topic {
topic_id: u32,
name: &str,
partitions_count: u32,
- config: Rc<SystemConfig>,
+ config: Arc<SystemConfig>,
storage: Rc<SystemStorage>,
size_of_parent_stream: Arc<AtomicU64>,
messages_count_of_parent_stream: Arc<AtomicU64>,
@@ -290,7 +290,7 @@ mod tests {
#[tokio::test]
async fn should_be_created_given_valid_parameters() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Rc::new(SystemConfig {
+ let config = Arc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
diff --git a/core/server/src/streaming/utils/memory_pool.rs b/core/server/src/streaming/utils/memory_pool.rs
index e05b7359..198e8626 100644
--- a/core/server/src/streaming/utils/memory_pool.rs
+++ b/core/server/src/streaming/utils/memory_pool.rs
@@ -154,7 +154,7 @@ impl MemoryPool {
}
/// Initialize the global pool from the given config.
- pub fn init_pool(config: Rc<SystemConfig>) {
+ pub fn init_pool(config: Arc<SystemConfig>) {
let is_enabled = config.memory_pool.enabled;
let memory_limit = config.memory_pool.size.as_bytes_usize();
let bucket_capacity = config.memory_pool.bucket_capacity as usize;
@@ -468,7 +468,7 @@ mod tests {
fn initialize_pool_for_tests() {
TEST_INIT.call_once(|| {
- let config = Rc::new(SystemConfig {
+ let config = Arc::new(SystemConfig {
memory_pool: MemoryPoolConfig {
enabled: true,
size: IggyByteSize::from_str("4GiB").unwrap(),
diff --git a/core/server/src/tcp/sender.rs b/core/server/src/tcp/sender.rs
index 22796812..2af9f494 100644
--- a/core/server/src/tcp/sender.rs
+++ b/core/server/src/tcp/sender.rs
@@ -28,10 +28,7 @@ use tracing::debug;
const STATUS_OK: &[u8] = &[0; 4];
-pub(crate) async fn read<T, B>(
- stream: &mut T,
- buffer: B,
-) -> (Result<usize, IggyError>, B)
+pub(crate) async fn read<T, B>(stream: &mut T, buffer: B) -> (Result<usize, IggyError>, B)
where
T: AsyncReadRent + AsyncWriteRent + Unpin,
B: IoBufMut,
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index ffa01eb1..ac29e2b9 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -54,8 +54,11 @@ pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) {
//TODO: Those can be shared with other shards.
shard.add_active_session(session.clone());
// Broadcast session to all shards.
+ //TODO: Fixme
+ /*
let event = Rc::new(ShardEvent::NewSession(session.clone()));
shard.broadcast_event_to_all_shards(session.client_id, event);
+ */
let _client_id = session.client_id;
info!("Created new session: {session}");
(iggy) 01/30: initial setup and migration of shards
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit af3dcac2ea870b6b4505cc569b287cf29bae68c8
Author: numinex <g....@gmail.com>
AuthorDate: Sun May 11 17:17:51 2025 +0200
initial setup and migration of shards
---
Cargo.lock | 91 +++++++++++++++-
core/server/Cargo.toml | 2 +
core/server/src/bootstrap.rs | 0
core/server/src/lib.rs | 2 +
core/server/src/shard/connector.rs | 140 +++++++++++++++++++++++++
core/server/src/shard/frame.rs | 68 ++++++++++++
core/server/src/shard/mod.rs | 59 +++++++++++
core/server/src/{lib.rs => shard/namespace.rs} | 48 +++------
8 files changed, 375 insertions(+), 35 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index ef4c02c9..c0fbfbea 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -142,7 +142,7 @@ dependencies = [
"actix-utils",
"futures-core",
"futures-util",
- "mio",
+ "mio 1.0.3",
"socket2",
"tokio",
"tracing",
@@ -631,6 +631,17 @@ dependencies = [
"webpki-roots 0.26.11",
]
+[[package]]
+name = "auto-const-array"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd73835ad7deb4bd2b389e6f10333b143f025d607c55ca04c66a0bcc6bb2fc6d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
[[package]]
name = "autocfg"
version = "1.5.0"
@@ -2662,6 +2673,15 @@ dependencies = [
"slab",
]
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder",
+]
+
[[package]]
name = "generator"
version = "0.8.5"
@@ -4114,6 +4134,16 @@ dependencies = [
"rustversion",
]
+[[package]]
+name = "io-uring"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "595a0399f411a508feb2ec1e970a4a30c249351e30208960d58298de8660b0e5"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+]
+
[[package]]
name = "ipnet"
version = "2.11.0"
@@ -4785,6 +4815,37 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "monoio"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3bd0f8bcde87b1949f95338b547543fcab187bc7e7a5024247e359a5e828ba6a"
+dependencies = [
+ "auto-const-array",
+ "bytes",
+ "fxhash",
+ "io-uring",
+ "libc",
+ "memchr",
+ "mio 0.8.11",
+ "monoio-macros",
+ "nix 0.26.4",
+ "pin-project-lite",
+ "socket2",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "monoio-macros"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "176a5f5e69613d9e88337cf2a65e11135332b4efbcc628404a7c555e4452084c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
[[package]]
name = "nanorand"
version = "0.7.0"
@@ -4800,6 +4861,19 @@ version = "6.6.666"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf5a574dadd7941adeaa71823ecba5e28331b8313fb2e1c6a5c7e5981ea53ad6"
+[[package]]
+name = "nix"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "libc",
+ "memoffset",
+ "pin-utils",
+]
+
[[package]]
name = "nix"
version = "0.30.1"
@@ -4864,7 +4938,7 @@ dependencies = [
"kqueue",
"libc",
"log",
- "mio",
+ "mio 1.0.3",
"notify-types",
"walkdir",
"windows-sys 0.59.0",
@@ -6859,7 +6933,8 @@ dependencies = [
"mimalloc",
"mockall",
"moka",
- "nix",
+ "monoio",
+ "nix 0.30.1",
"once_cell",
"opentelemetry",
"opentelemetry-appender-tracing",
@@ -6876,6 +6951,7 @@ dependencies = [
"serde",
"serde_with",
"serial_test",
+ "sharded_queue",
"static-toml",
"strum",
"sysinfo 0.35.2",
@@ -6927,6 +7003,15 @@ dependencies = [
"lazy_static",
]
+[[package]]
+name = "sharded_queue"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3499be28bd82560e75ad10457698607ba3cc389175ab47ac93279834ce1fab4"
+dependencies = [
+ "crossbeam-utils",
+]
+
[[package]]
name = "shlex"
version = "1.3.0"
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 3a0cf94e..21b4ddd9 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -63,6 +63,7 @@ jsonwebtoken = "9.3.1"
lending-iterator = "0.1.7"
mimalloc = { workspace = true, optional = true }
moka = { version = "0.12.10", features = ["future"] }
+monoio = "0.2.4"
nix = { version = "0.30", features = ["fs"] }
once_cell = "1.21.3"
opentelemetry = { version = "0.30.0", features = ["trace", "logs"] }
@@ -96,6 +97,7 @@ rustls-pemfile = "2.2.0"
serde = { workspace = true }
serde_with = { workspace = true }
static-toml = "1.3.0"
+sharded_queue = "2.0.1"
strum = { workspace = true }
sysinfo = { workspace = true }
tempfile = { workspace = true }
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
new file mode 100644
index 00000000..e69de29b
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index 91d17586..572301e0 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -26,12 +26,14 @@ static GLOBAL: MiMalloc = MiMalloc;
#[cfg(windows)]
compile_error!("iggy-server doesn't support windows.");
+mod bootstrap;
pub mod archiver;
pub mod args;
pub mod binary;
pub mod channels;
pub(crate) mod compat;
pub mod configs;
+pub mod shard;
pub mod http;
pub mod log;
pub mod quic;
diff --git a/core/server/src/shard/connector.rs b/core/server/src/shard/connector.rs
new file mode 100644
index 00000000..349218ba
--- /dev/null
+++ b/core/server/src/shard/connector.rs
@@ -0,0 +1,140 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use futures::{task::AtomicWaker, Stream};
+use sharded_queue::ShardedQueue;
+use std::{
+ sync::{atomic::AtomicUsize, Arc},
+ task::Poll,
+};
+
+pub type StopSender = flume::Sender<()>;
+pub type StopReceiver = flume::Receiver<()>;
+
+#[derive(Clone)]
+pub struct ShardConnector<T> {
+ pub id: u16,
+ pub sender: Sender<T>,
+ pub receiver: Receiver<T>,
+ pub stop_receiver: StopReceiver,
+ pub stop_sender: StopSender,
+}
+
+// TODO(numinex) - replace flume with async_channel
+impl<T: Clone> ShardConnector<T> {
+ pub fn new(id: u16, max_concurrent_thread_count: usize) -> Self {
+ let channel = Arc::new(ShardedChannel::new(max_concurrent_thread_count));
+ let (sender, receiver) = channel.unbounded();
+ let (stop_sender, stop_receiver) = flume::bounded(1);
+ Self {
+ id,
+ receiver,
+ sender,
+ stop_receiver,
+ stop_sender,
+ }
+ }
+
+ pub fn send(&self, data: T) {
+ self.sender.send(data);
+ }
+}
+
+#[derive(Clone)]
+pub struct Receiver<T> {
+ channel: Arc<ShardedChannel<T>>,
+}
+
+#[derive(Clone)]
+pub struct Sender<T> {
+ channel: Arc<ShardedChannel<T>>,
+}
+
+impl<T> Sender<T> {
+ pub fn send(&self, data: T) {
+ self.channel
+ .task_queue
+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ self.channel.queue.push_back(data);
+ self.channel.waker.wake();
+ }
+}
+
+pub struct ShardedChannel<T> {
+ queue: ShardedQueue<T>,
+ task_queue: AtomicUsize,
+ waker: AtomicWaker,
+}
+
+impl<T> ShardedChannel<T> {
+ pub fn new(max_concurrent_thread_count: usize) -> Self {
+ let waker = AtomicWaker::new();
+
+ Self {
+ queue: ShardedQueue::new(max_concurrent_thread_count),
+ task_queue: AtomicUsize::new(0),
+ waker,
+ }
+ }
+}
+
+pub trait ShardedChannelsSplit<T: Clone> {
+ fn unbounded(&self) -> (Sender<T>, Receiver<T>);
+
+ fn sender(&self) -> Sender<T>;
+}
+
+impl<T: Clone> ShardedChannelsSplit<T> for Arc<ShardedChannel<T>> {
+ fn unbounded(&self) -> (Sender<T>, Receiver<T>) {
+ let tx = self.sender();
+ let rx = Receiver {
+ channel: Arc::clone(self),
+ };
+
+ (tx, rx)
+ }
+
+ fn sender(&self) -> Sender<T> {
+ Sender {
+ channel: Arc::clone(self),
+ }
+ }
+}
+
+impl<T> Stream for Receiver<T> {
+ type Item = T;
+ fn poll_next(
+ self: std::pin::Pin<&mut Self>,
+ cx: &mut std::task::Context<'_>,
+ ) -> std::task::Poll<Option<Self::Item>> {
+ let old = self
+ .channel
+ .task_queue
+ .load(std::sync::atomic::Ordering::Relaxed);
+ if old == 0 {
+ self.channel.waker.register(cx.waker());
+ return Poll::Pending;
+ }
+
+ assert!(old > 0);
+ self.channel
+ .task_queue
+ .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
+ let item = self.channel.queue.pop_front_or_spin_wait_item();
+ Poll::Ready(Some(item))
+ }
+}
diff --git a/core/server/src/shard/frame.rs b/core/server/src/shard/frame.rs
new file mode 100644
index 00000000..b1524ad6
--- /dev/null
+++ b/core/server/src/shard/frame.rs
@@ -0,0 +1,68 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use bytes::Bytes;
+use flume::Sender;
+use iggy_common::IggyError;
+
+#[derive(Debug, Clone)]
+pub enum ShardMessage {
+ //TODO: Fixme
+ //Command(ServerCommand),
+ Event(ShardEvent),
+}
+
+#[derive(Debug, Clone)]
+pub enum ShardEvent {
+}
+
+#[derive(Debug)]
+pub enum ShardResponse {
+ BinaryResponse(Bytes),
+ ErrorResponse(IggyError),
+}
+
+#[derive(Debug, Clone)]
+pub struct ShardFrame {
+ pub client_id: u32,
+ pub message: ShardMessage,
+ pub response_sender: Option<Sender<ShardResponse>>,
+}
+
+impl ShardFrame {
+ pub fn new(
+ client_id: u32,
+ message: ShardMessage,
+ response_sender: Option<Sender<ShardResponse>>,
+ ) -> Self {
+ Self {
+ client_id,
+ message,
+ response_sender,
+ }
+ }
+}
+
+#[macro_export]
+macro_rules! handle_response {
+ ($sender:expr, $response:expr) => {
+ match $response {
+ ShardResponse::BinaryResponse(payload) => $sender.send_ok_response(&payload).await?,
+ ShardResponse::ErrorResponse(err) => $sender.send_error_response(err).await?,
+ }
+ };
+}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
new file mode 100644
index 00000000..6ccbcab3
--- /dev/null
+++ b/core/server/src/shard/mod.rs
@@ -0,0 +1,59 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+mod connector;
+mod namespace;
+mod frame;
+
+
+
+use std::cell::RefCell;
+use ahash::HashMap;
+use connector::ShardConnector;
+use frame::ShardFrame;
+use namespace::IggyNamespace;
+struct Shard {
+ id: u16,
+ connection: ShardConnector<ShardFrame>,
+}
+
+struct ShardInfo {
+ id: u16,
+}
+
+pub struct IggyShard {
+ pub id: u16,
+ shards: Vec<Shard>,
+ shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
+
+ pub(crate) permissioner: RefCell<Permissioner>,
+ pub(crate) storage: Rc<SystemStorage>,
+ pub(crate) streams: RwLock<HashMap<u32, Stream>>,
+ pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
+ pub(crate) users: RefCell<HashMap<UserId, User>>,
+
+ // TODO - get rid of this dynamic dispatch.
+ pub(crate) state: Rc<FileState>,
+ pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
+ pub(crate) config: ServerConfig,
+ pub(crate) client_manager: RefCell<ClientManager>,
+ pub(crate) active_sessions: RefCell<Vec<Session>>,
+ pub(crate) metrics: Metrics,
+ pub message_receiver: Cell<Option<Receiver<ShardFrame>>>,
+ stop_receiver: StopReceiver,
+ stop_sender: StopSender,
+}
\ No newline at end of file
diff --git a/core/server/src/lib.rs b/core/server/src/shard/namespace.rs
similarity index 51%
copy from core/server/src/lib.rs
copy to core/server/src/shard/namespace.rs
index 91d17586..4b972125 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/shard/namespace.rs
@@ -15,39 +15,23 @@
* specific language governing permissions and limitations
* under the License.
*/
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct IggyNamespace {
+ pub(crate) stream_id: u32,
+ pub(crate) topic_id: u32,
+ pub(crate) partition_id: u32,
+}
-#[cfg(not(feature = "disable-mimalloc"))]
-use mimalloc::MiMalloc;
-
-#[cfg(not(feature = "disable-mimalloc"))]
-#[global_allocator]
-static GLOBAL: MiMalloc = MiMalloc;
-
-#[cfg(windows)]
-compile_error!("iggy-server doesn't support windows.");
-
-pub mod archiver;
-pub mod args;
-pub mod binary;
-pub mod channels;
-pub(crate) mod compat;
-pub mod configs;
-pub mod http;
-pub mod log;
-pub mod quic;
-pub mod server_error;
-pub mod state;
-pub mod streaming;
-pub mod tcp;
-pub mod versioning;
-
-const VERSION: &str = env!("CARGO_PKG_VERSION");
-const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME";
-const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD";
+impl IggyNamespace {
+ pub fn new(stream_id: u32, topic_id: u32, partition_id: u32) -> Self {
+ Self {
+ stream_id,
+ topic_id,
+ partition_id,
+ }
+ }
-pub(crate) fn map_toggle_str<'a>(enabled: bool) -> &'a str {
- match enabled {
- true => "enabled",
- false => "disabled",
+ pub fn generate_hash(&self) -> u32 {
+ todo!();
}
}
(iggy) 25/30: feat(io_uring): fix tcp tls server (#1952)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit e1e31ea824212b57e780723209eaf0f80dfbf5f5
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Tue Jul 1 11:14:39 2025 +0200
feat(io_uring): fix tcp tls server (#1952)
Closes #1936
---
Cargo.lock | 56 ++----------
core/server/Cargo.toml | 2 +-
core/server/src/binary/sender.rs | 5 +-
core/server/src/tcp/tcp_listener.rs | 14 +--
core/server/src/tcp/tcp_server.rs | 16 +++-
core/server/src/tcp/tcp_tls_listener.rs | 153 +++++++++++++++++---------------
core/server/src/tcp/tcp_tls_sender.rs | 5 +-
7 files changed, 110 insertions(+), 141 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 6898bb77..ca282813 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1686,16 +1686,6 @@ dependencies = [
"version_check",
]
-[[package]]
-name = "core-foundation"
-version = "0.9.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
-dependencies = [
- "core-foundation-sys",
- "libc",
-]
-
[[package]]
name = "core-foundation"
version = "0.10.1"
@@ -4921,15 +4911,15 @@ dependencies = [
]
[[package]]
-name = "monoio-native-tls"
+name = "monoio-rustls"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9022f5aaa19f9688f97bfcfa0c4a4318d424851995badb356674ca742652cdb"
+checksum = "6e31f422825bd7fb19957af6eaf89d7234ba143fcc0e515f5a2f526e332d1875"
dependencies = [
"bytes",
"monoio",
"monoio-io-wrapper",
- "native-tls",
+ "rustls",
"thiserror 1.0.69",
]
@@ -4942,23 +4932,6 @@ dependencies = [
"getrandom 0.2.16",
]
-[[package]]
-name = "native-tls"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e"
-dependencies = [
- "libc",
- "log",
- "openssl",
- "openssl-probe",
- "openssl-sys",
- "schannel",
- "security-framework 2.11.1",
- "security-framework-sys",
- "tempfile",
-]
-
[[package]]
name = "never-say-never"
version = "6.6.666"
@@ -6631,7 +6604,7 @@ dependencies = [
"openssl-probe",
"rustls-pki-types",
"schannel",
- "security-framework 3.2.0",
+ "security-framework",
]
[[package]]
@@ -6659,7 +6632,7 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1"
dependencies = [
- "core-foundation 0.10.1",
+ "core-foundation",
"core-foundation-sys",
"jni",
"log",
@@ -6668,7 +6641,7 @@ dependencies = [
"rustls-native-certs",
"rustls-platform-verifier-android",
"rustls-webpki",
- "security-framework 3.2.0",
+ "security-framework",
"security-framework-sys",
"webpki-root-certs 0.26.11",
"windows-sys 0.59.0",
@@ -6816,19 +6789,6 @@ dependencies = [
"zeroize",
]
-[[package]]
-name = "security-framework"
-version = "2.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
-dependencies = [
- "bitflags 2.9.1",
- "core-foundation 0.9.4",
- "core-foundation-sys",
- "libc",
- "security-framework-sys",
-]
-
[[package]]
name = "security-framework"
version = "3.2.0"
@@ -6836,7 +6796,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
dependencies = [
"bitflags 2.9.1",
- "core-foundation 0.10.1",
+ "core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
@@ -7054,7 +7014,7 @@ dependencies = [
"mockall",
"moka",
"monoio",
- "monoio-native-tls",
+ "monoio-rustls",
"nix 0.30.1",
"once_cell",
"opentelemetry",
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 3e6c801e..4b330383 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -68,7 +68,7 @@ hash32 = "1.0.0"
mimalloc = { workspace = true, optional = true }
moka = { version = "0.12.10", features = ["future"] }
monoio = { version = "0.2.4", features = ["mkdirat", "unlinkat", "renameat", "sync"] }
-monoio-native-tls = "0.4.0"
+monoio-rustls = "0.4.0"
nix = { version = "0.30", features = ["fs"] }
once_cell = "1.21.3"
opentelemetry = { version = "0.30.0", features = ["trace", "logs"] }
diff --git a/core/server/src/binary/sender.rs b/core/server/src/binary/sender.rs
index adbf05ee..45066c75 100644
--- a/core/server/src/binary/sender.rs
+++ b/core/server/src/binary/sender.rs
@@ -25,10 +25,11 @@ use crate::{quic::quic_sender::QuicSender, server_error::ServerError};
use bytes::BytesMut;
use iggy_common::IggyError;
use monoio::buf::IoBufMut;
+use monoio::io::{AsyncReadRent, AsyncWriteRent};
use monoio::net::TcpStream;
-use monoio_native_tls::TlsStream;
use nix::libc;
use quinn::{RecvStream, SendStream};
+use monoio_rustls::{ServerTlsStream, TlsStream};
macro_rules! forward_async_methods {
(
@@ -84,7 +85,7 @@ impl SenderKind {
Self::Tcp(TcpSender { stream })
}
- pub fn get_tcp_tls_sender(stream: TlsStream<TcpStream>) -> Self {
+ pub fn get_tcp_tls_sender(stream: ServerTlsStream<TcpStream>) -> Self {
Self::TcpTls(TcpTlsSender { stream })
}
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index 90b1eabb..4f528016 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -21,25 +21,15 @@ use crate::shard::IggyShard;
use crate::shard::transmission::event::ShardEvent;
use crate::streaming::clients::client_manager::Transport;
use crate::tcp::connection_handler::{handle_connection, handle_error};
-use crate::tcp::tcp_socket;
use futures::FutureExt;
use iggy_common::IggyError;
+use socket2::Socket;
use std::net::SocketAddr;
use std::rc::Rc;
use std::time::Duration;
use tracing::{error, info};
-pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) -> Result<(), IggyError> {
- let ip_v6 = shard.config.tcp.ipv6;
- let socket_config = &shard.config.tcp.socket;
- let addr: SocketAddr = shard
- .config
- .tcp
- .address
- .parse()
- .expect("Failed to parse TCP address");
-
- let socket = tcp_socket::build(ip_v6, socket_config);
+pub async fn start(server_name: &'static str, addr: SocketAddr, socket: Socket, shard: Rc<IggyShard>) -> Result<(), IggyError> {
socket
.bind(&addr.into())
.expect("Failed to bind TCP listener");
diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs
index e558d08f..ac019bb0 100644
--- a/core/server/src/tcp/tcp_server.rs
+++ b/core/server/src/tcp/tcp_server.rs
@@ -17,8 +17,9 @@
*/
use crate::shard::IggyShard;
-use crate::tcp::tcp_listener;
+use crate::tcp::{tcp_listener, tcp_socket, tcp_tls_listener};
use iggy_common::IggyError;
+use std::net::SocketAddr;
use std::rc::Rc;
use tracing::info;
@@ -30,11 +31,20 @@ pub async fn spawn_tcp_server(shard: Rc<IggyShard>) -> Result<(), IggyError> {
} else {
"Iggy TCP"
};
+ let ip_v6 = shard.config.tcp.ipv6;
+ let socket_config = &shard.config.tcp.socket;
+ let addr: SocketAddr = shard
+ .config
+ .tcp
+ .address
+ .parse()
+ .expect("Failed to parse TCP address");
+ let socket = tcp_socket::build(ip_v6, socket_config);
info!("Initializing {server_name} server...");
// TODO: Fixme -- storing addr of the server inside of the config for integration tests...
match shard.config.tcp.tls.enabled {
- true => unimplemented!("TLS support is not implemented yet"),
- false => tcp_listener::start(server_name, shard.clone()).await?,
+ true => tcp_tls_listener::start(server_name, addr, socket, shard.clone()).await?,
+ false => tcp_listener::start(server_name, addr, socket, shard.clone()).await?,
};
Ok(())
diff --git a/core/server/src/tcp/tcp_tls_listener.rs b/core/server/src/tcp/tcp_tls_listener.rs
index 53df4b0b..2c81eb60 100644
--- a/core/server/src/tcp/tcp_tls_listener.rs
+++ b/core/server/src/tcp/tcp_tls_listener.rs
@@ -19,33 +19,31 @@
use crate::binary::sender::SenderKind;
use crate::configs::tcp::TcpTlsConfig;
use crate::shard::IggyShard;
+use crate::shard::transmission::event::ShardEvent;
use crate::streaming::clients::client_manager::Transport;
use crate::tcp::connection_handler::{handle_connection, handle_error};
-use monoio_native_tls::TlsAcceptor;
+use futures::FutureExt;
+use iggy_common::IggyError;
+use monoio_rustls::TlsAcceptor;
use rustls::ServerConfig;
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
use rustls_pemfile::{certs, private_key};
+use socket2::Socket;
use std::io::BufReader;
use std::net::SocketAddr;
use std::rc::Rc;
use std::sync::Arc;
-use tokio::net::TcpSocket;
-use tokio::sync::oneshot;
+use std::time::Duration;
use tracing::{error, info};
pub(crate) async fn start(
- address: &str,
- config: TcpTlsConfig,
- socket: TcpSocket,
- system: Rc<IggyShard>,
-) -> SocketAddr {
- //TODO: Fixme
- todo!();
- /*
- let address = address.to_string();
- let (tx, rx) = oneshot::channel();
- tokio::spawn(async move {
+ server_name: &'static str,
+ addr: SocketAddr,
+ socket: Socket,
+ shard: Rc<IggyShard>,
+) -> Result<(), IggyError> {
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+ let config = &shard.config.tcp.tls;
let (certs, key) =
if config.self_signed && !std::path::Path::new(&config.cert_file).exists() {
@@ -64,78 +62,87 @@ pub(crate) async fn start(
let acceptor = TlsAcceptor::from(Arc::new(server_config));
- let addr = address.parse();
- if addr.is_err() {
- panic!("Unable to parse address {address:?}");
- }
-
- let addr = addr.unwrap();
socket
- .bind(addr)
+ .bind(&addr.into())
.unwrap_or_else(|e| panic!("Unable to bind socket to address '{addr}': {e}",));
- let listener = socket
- .listen(1024)
- .unwrap_or_else(|e| panic!("Unable to start TCP TLS server on '{address}': {e}",));
-
- let local_addr = listener
- .local_addr()
- .unwrap_or_else(|e| panic!("Failed to get local address for TCP TLS listener: {e}",));
-
- tx.send(local_addr).unwrap_or_else(|_| {
- panic!("Failed to send the local address '{local_addr}' for TCP TLS listener")
- });
+ let listener: std::net::TcpListener = socket.into();
+ let listener = monoio::net::TcpListener::from_std(listener).unwrap();
+ info!("{server_name} server has started on: {:?}", addr);
loop {
- match listener.accept().await {
- Ok((stream, address)) => {
- info!("Accepted new TCP TLS connection: {}", address);
- let session = system
- .read()
- .await
- .add_client(&address, Transport::Tcp)
- .await;
-
- let client_id = session.client_id;
- let acceptor = acceptor.clone();
- let system_clone = system.clone();
- match acceptor.accept(stream).await {
- Ok(stream) => {
- let mut sender = SenderKind::get_tcp_tls_sender(stream);
- tokio::spawn(async move {
- if let Err(error) =
- handle_connection(session, &mut sender, system_clone.clone())
- .await
- {
- handle_error(error);
- system_clone.read().await.delete_client(client_id).await;
- if let Err(error) = sender.shutdown().await {
- error!(
- "Failed to shutdown TCP stream for client: {client_id}, address: {address}. {error}"
- );
- } else {
- info!(
- "Successfully closed TCP stream for client: {client_id}, address: {address}."
- );
+ let shutdown_check = async {
+ loop {
+ if shard.is_shutting_down() {
+ return;
+ }
+ monoio::time::sleep(Duration::from_millis(100)).await;
+ }
+ };
+
+ let accept_future = listener.accept();
+ futures::select! {
+ _ = shutdown_check.fuse() => {
+ info!("TCP TLS server detected shutdown flag, no longer accepting connections");
+ break;
+ }
+ result = accept_future.fuse() => {
+ match result {
+ Ok((stream, address)) => {
+ if shard.is_shutting_down() {
+ info!("Rejecting new TLS connection from {} during shutdown", address);
+ continue;
+ }
+ let shard_clone = shard.clone();
+ info!("Accepted new TCP TLS connection: {}", address);
+ let transport = Transport::Tcp;
+ let session = shard_clone.add_client(&address, transport);
+ //TODO: Those can be shared with other shards.
+ shard_clone.add_active_session(session.clone());
+ // Broadcast session to all shards.
+ let event = ShardEvent::NewSession { address, transport };
+ // TODO: Fixme look inside of broadcast_event_to_all_shards method.
+ let _responses = shard_clone.broadcast_event_to_all_shards(event.into());
+
+ let client_id = session.client_id;
+ info!("Created new session: {session}");
+ let acceptor = acceptor.clone();
+
+ let conn_stop_receiver = shard_clone.task_registry.add_connection(client_id);
+
+ let shard_for_conn = shard_clone.clone();
+ shard_clone.task_registry.spawn_tracked(async move {
+ match acceptor.accept(stream).await {
+ Ok(tls_stream) => {
+ let mut sender = SenderKind::get_tcp_tls_sender(tls_stream.into());
+ if let Err(error) = handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver).await {
+ handle_error(error);
+ }
+ shard_for_conn.task_registry.remove_connection(&client_id);
+
+ if let Err(error) = sender.shutdown().await {
+ error!(
+ "Failed to shutdown TCP TLS stream for client: {client_id}, address: {address}. {error}"
+ );
+ } else {
+ info!(
+ "Successfully closed TCP TLS stream for client: {client_id}, address: {address}."
+ );
+ }
+ }
+ Err(e) => {
+ error!("Failed to accept TLS connection from '{address}': {e}");
+ shard_for_conn.task_registry.remove_connection(&client_id);
}
}
});
}
- Err(e) => {
- error!("Failed to accept TLS connection from '{address}': {e}");
- system_clone.read().await.delete_client(client_id).await;
- }
+ Err(error) => error!("Unable to accept TCP TLS socket. {error}"),
}
}
- Err(error) => error!("Unable to accept TCP TLS socket. {error}"),
}
}
- });
- match rx.await {
- Ok(addr) => addr,
- Err(_) => panic!("Failed to get the local address for TCP TLS listener."),
- }
- */
+ Ok(())
}
fn generate_self_signed_cert()
diff --git a/core/server/src/tcp/tcp_tls_sender.rs b/core/server/src/tcp/tcp_tls_sender.rs
index 167427c1..b214e5a6 100644
--- a/core/server/src/tcp/tcp_tls_sender.rs
+++ b/core/server/src/tcp/tcp_tls_sender.rs
@@ -25,12 +25,13 @@ use iggy_common::IggyError;
use monoio::buf::IoBufMut;
use monoio::io::AsyncWriteRent;
use monoio::net::TcpStream;
-use monoio_native_tls::TlsStream;
+use monoio_rustls::ServerTlsStream;
+//use tokio_rustls::server::TlsStream;
use nix::libc;
#[derive(Debug)]
pub struct TcpTlsSender {
- pub(crate) stream: TlsStream<TcpStream>,
+ pub(crate) stream: ServerTlsStream<TcpStream>,
}
impl Sender for TcpTlsSender {
(iggy) 19/30: feat(io_uring): fix send/poll (#1930)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 235a7cc7263cc1d4bd2efed0d6a0847fd17ab344
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Sun Jun 29 22:49:43 2025 +0200
feat(io_uring): fix send/poll (#1930)
---
Cargo.lock | 12 ++
core/integration/tests/streaming/get_by_offset.rs | 3 +-
.../tests/streaming/get_by_timestamp.rs | 4 +-
core/integration/tests/streaming/messages.rs | 6 +-
core/integration/tests/streaming/partition.rs | 15 +--
core/integration/tests/streaming/stream.rs | 6 +-
core/integration/tests/streaming/topic.rs | 29 ++--
core/integration/tests/streaming/topic_messages.rs | 25 ++--
core/server/Cargo.toml | 2 +-
.../handlers/messages/poll_messages_handler.rs | 67 ++++-----
.../handlers/messages/send_messages_handler.rs | 64 ++++-----
.../handlers/streams/create_stream_handler.rs | 12 +-
.../binary/handlers/topics/create_topic_handler.rs | 18 ++-
.../binary/handlers/users/login_user_handler.rs | 18 ++-
core/server/src/bootstrap.rs | 11 +-
core/server/src/configs/displays.rs | 5 +-
core/server/src/io/file.rs | 9 +-
core/server/src/main.rs | 35 ++---
core/server/src/shard/builder.rs | 2 +-
core/server/src/shard/mod.rs | 149 ++++++++++++++++-----
core/server/src/shard/system/messages.rs | 4 +-
core/server/src/shard/system/streams.rs | 41 ++++--
core/server/src/shard/system/topics.rs | 98 ++++++++++++--
core/server/src/shard/system/users.rs | 15 +++
core/server/src/shard/tasks/messages.rs | 38 ++++++
core/server/src/shard/tasks/mod.rs | 1 +
core/server/src/shard/transmission/event.rs | 29 ++--
core/server/src/shard/transmission/frame.rs | 7 +-
core/server/src/shard/transmission/message.rs | 40 ++++--
core/server/src/shard/transmission/mod.rs | 2 +-
core/server/src/state/file.rs | 6 +-
.../src/streaming/segments/indexes/index_reader.rs | 20 ++-
.../src/streaming/segments/indexes/index_writer.rs | 18 ++-
.../src/streaming/segments/indexes/indexes_mut.rs | 7 +-
core/server/src/streaming/segments/messages/mod.rs | 64 ++-------
core/server/src/streaming/segments/segment.rs | 3 +-
.../streaming/segments/types/messages_batch_mut.rs | 4 +
.../src/streaming/segments/writing_messages.rs | 3 +-
core/server/src/streaming/streams/storage.rs | 17 +--
core/server/src/streaming/topics/messages.rs | 2 +-
core/server/src/streaming/topics/storage.rs | 15 ++-
core/server/src/streaming/utils/memory_pool.rs | 3 +-
core/server/src/streaming/utils/pooled_buffer.rs | 12 +-
core/server/src/tcp/connection_handler.rs | 10 +-
core/server/src/tcp/sender.rs | 8 +-
core/server/src/tcp/tcp_listener.rs | 23 ++--
core/server/src/tcp/tcp_server.rs | 9 +-
core/server/src/tcp/tcp_socket.rs | 15 ++-
48 files changed, 631 insertions(+), 375 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index f06db7bf..8a3b9e16 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4875,6 +4875,7 @@ checksum = "3bd0f8bcde87b1949f95338b547543fcab187bc7e7a5024247e359a5e828ba6a"
dependencies = [
"auto-const-array",
"bytes",
+ "flume",
"fxhash",
"io-uring",
"libc",
@@ -4882,8 +4883,10 @@ dependencies = [
"mio 0.8.11",
"monoio-macros",
"nix 0.26.4",
+ "once_cell",
"pin-project-lite",
"socket2",
+ "threadpool",
"windows-sys 0.48.0",
]
@@ -7554,6 +7557,15 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "threadpool"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
+dependencies = [
+ "num_cpus",
+]
+
[[package]]
name = "time"
version = "0.3.41"
diff --git a/core/integration/tests/streaming/get_by_offset.rs b/core/integration/tests/streaming/get_by_offset.rs
index df9f8f40..e5cc2a33 100644
--- a/core/integration/tests/streaming/get_by_offset.rs
+++ b/core/integration/tests/streaming/get_by_offset.rs
@@ -127,8 +127,7 @@ async fn test_get_messages_by_offset(
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
IggyTimestamp::now(),
- )
- .await;
+ );
setup.create_partitions_directory(stream_id, topic_id).await;
partition.persist().await.unwrap();
diff --git a/core/integration/tests/streaming/get_by_timestamp.rs b/core/integration/tests/streaming/get_by_timestamp.rs
index 93a178aa..7aff4a89 100644
--- a/core/integration/tests/streaming/get_by_timestamp.rs
+++ b/core/integration/tests/streaming/get_by_timestamp.rs
@@ -18,7 +18,6 @@
use crate::streaming::common::test_setup::TestSetup;
use bytes::BytesMut;
-use iggy::prelude::Confirmation;
use iggy::prelude::*;
use server::configs::cache_indexes::CacheIndexesConfig;
use server::configs::system::{PartitionConfig, SegmentConfig, SystemConfig};
@@ -129,8 +128,7 @@ async fn test_get_messages_by_timestamp(
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
IggyTimestamp::now(),
- )
- .await;
+ );
setup.create_partitions_directory(stream_id, topic_id).await;
partition.persist().await.unwrap();
diff --git a/core/integration/tests/streaming/messages.rs b/core/integration/tests/streaming/messages.rs
index 6ab6ca5b..c6363d9a 100644
--- a/core/integration/tests/streaming/messages.rs
+++ b/core/integration/tests/streaming/messages.rs
@@ -58,8 +58,7 @@ async fn should_persist_messages_and_then_load_them_from_disk() {
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
IggyTimestamp::now(),
- )
- .await;
+ );
let mut messages = Vec::with_capacity(messages_count as usize);
let mut appended_messages = Vec::with_capacity(messages_count as usize);
@@ -126,8 +125,7 @@ async fn should_persist_messages_and_then_load_them_from_disk() {
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
now,
- )
- .await;
+ );
let partition_state = PartitionState {
id: partition.partition_id,
created_at: now,
diff --git a/core/integration/tests/streaming/partition.rs b/core/integration/tests/streaming/partition.rs
index 4fff0d04..6d811e59 100644
--- a/core/integration/tests/streaming/partition.rs
+++ b/core/integration/tests/streaming/partition.rs
@@ -49,8 +49,7 @@ async fn should_persist_partition_with_segment() {
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
IggyTimestamp::now(),
- )
- .await;
+ );
partition.persist().await.unwrap();
@@ -81,8 +80,7 @@ async fn should_load_existing_partition_from_disk() {
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
IggyTimestamp::now(),
- )
- .await;
+ );
partition.persist().await.unwrap();
assert_persisted_partition(&partition.partition_path, with_segment).await;
@@ -101,8 +99,7 @@ async fn should_load_existing_partition_from_disk() {
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
now,
- )
- .await;
+ );
let partition_state = PartitionState {
id: partition.partition_id,
created_at: now,
@@ -151,8 +148,7 @@ async fn should_delete_existing_partition_from_disk() {
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
IggyTimestamp::now(),
- )
- .await;
+ );
partition.persist().await.unwrap();
assert_persisted_partition(&partition.partition_path, with_segment).await;
@@ -185,8 +181,7 @@ async fn should_purge_existing_partition_on_disk() {
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
IggyTimestamp::now(),
- )
- .await;
+ );
partition.persist().await.unwrap();
assert_persisted_partition(&partition.partition_path, with_segment).await;
let messages = create_messages();
diff --git a/core/integration/tests/streaming/stream.rs b/core/integration/tests/streaming/stream.rs
index 5b340750..677e2d32 100644
--- a/core/integration/tests/streaming/stream.rs
+++ b/core/integration/tests/streaming/stream.rs
@@ -145,10 +145,10 @@ async fn should_purge_existing_stream_on_disk() {
let topic = stream
.get_topic(&Identifier::numeric(topic_id).unwrap())
.unwrap();
- topic
- .append_messages(&Partitioning::partition_id(1), batch)
- .await
+ let partition_id = topic
+ .calculate_partition_id(&Partitioning::partition_id(1))
.unwrap();
+ topic.append_messages(partition_id, batch).await.unwrap();
let (_, loaded_messages) = topic
.get_messages(
PollingConsumer::Consumer(1, 1),
diff --git a/core/integration/tests/streaming/topic.rs b/core/integration/tests/streaming/topic.rs
index f75ffe2b..e3d17df2 100644
--- a/core/integration/tests/streaming/topic.rs
+++ b/core/integration/tests/streaming/topic.rs
@@ -38,7 +38,7 @@ async fn should_persist_topics_with_partitions_directories_and_info_file() {
let topic_ids = get_topic_ids();
for topic_id in topic_ids {
let name = format!("test-{topic_id}");
- let topic = Topic::create(
+ let created_topic_info = Topic::create(
stream_id,
topic_id,
&name,
@@ -53,8 +53,8 @@ async fn should_persist_topics_with_partitions_directories_and_info_file() {
MaxTopicSize::ServerDefault,
1,
)
- .await
.unwrap();
+ let topic = created_topic_info.topic;
topic.persist().await.unwrap();
@@ -76,7 +76,7 @@ async fn should_load_existing_topic_from_disk() {
let topic_ids = get_topic_ids();
for topic_id in topic_ids {
let name = format!("test-{topic_id}");
- let topic = Topic::create(
+ let created_topic_info = Topic::create(
stream_id,
topic_id,
&name,
@@ -91,8 +91,9 @@ async fn should_load_existing_topic_from_disk() {
MaxTopicSize::ServerDefault,
1,
)
- .await
.unwrap();
+ let topic = created_topic_info.topic;
+
topic.persist().await.unwrap();
assert_persisted_topic(
&topic.path,
@@ -102,7 +103,7 @@ async fn should_load_existing_topic_from_disk() {
.await;
let created_at = IggyTimestamp::now();
- let mut loaded_topic = Topic::empty(
+ let created_topic_info = Topic::empty(
stream_id,
topic_id,
&name,
@@ -111,8 +112,8 @@ async fn should_load_existing_topic_from_disk() {
Arc::new(AtomicU32::new(0)),
setup.config.clone(),
setup.storage.clone(),
- )
- .await;
+ );
+ let mut loaded_topic = created_topic_info.topic;
let topic_state = TopicState {
id: topic_id,
name,
@@ -153,7 +154,7 @@ async fn should_delete_existing_topic_from_disk() {
let topic_ids = get_topic_ids();
for topic_id in topic_ids {
let name = format!("test-{topic_id}");
- let topic = Topic::create(
+ let created_topic_info = Topic::create(
stream_id,
topic_id,
&name,
@@ -168,8 +169,8 @@ async fn should_delete_existing_topic_from_disk() {
MaxTopicSize::ServerDefault,
1,
)
- .await
.unwrap();
+ let topic = created_topic_info.topic;
topic.persist().await.unwrap();
assert_persisted_topic(
&topic.path,
@@ -193,7 +194,7 @@ async fn should_purge_existing_topic_on_disk() {
let topic_ids = get_topic_ids();
for topic_id in topic_ids {
let name = format!("test-{topic_id}");
- let topic = Topic::create(
+ let created_topic_info = Topic::create(
stream_id,
topic_id,
&name,
@@ -208,8 +209,8 @@ async fn should_purge_existing_topic_on_disk() {
MaxTopicSize::ServerDefault,
1,
)
- .await
.unwrap();
+ let topic = created_topic_info.topic;
topic.persist().await.unwrap();
assert_persisted_topic(
&topic.path,
@@ -225,10 +226,10 @@ async fn should_purge_existing_topic_on_disk() {
.map(|msg| msg.get_size_bytes().as_bytes_u32())
.sum::<u32>();
let batch = IggyMessagesBatchMut::from_messages(&messages, batch_size);
- topic
- .append_messages(&Partitioning::partition_id(1), batch)
- .await
+ let partition_id = topic
+ .calculate_partition_id(&Partitioning::partition_id(1))
.unwrap();
+ topic.append_messages(partition_id, batch).await.unwrap();
let (_, loaded_messages) = topic
.get_messages(
PollingConsumer::Consumer(1, 1),
diff --git a/core/integration/tests/streaming/topic_messages.rs b/core/integration/tests/streaming/topic_messages.rs
index 19de255f..3e5362ac 100644
--- a/core/integration/tests/streaming/topic_messages.rs
+++ b/core/integration/tests/streaming/topic_messages.rs
@@ -62,7 +62,8 @@ async fn assert_polling_messages() {
.map(|m| m.get_size_bytes())
.sum::<IggyByteSize>();
let batch = IggyMessagesBatchMut::from_messages(&messages, batch_size.as_bytes_u32());
- topic.append_messages(&partitioning, batch).await.unwrap();
+ let partition_id = topic.calculate_partition_id(&partitioning).unwrap();
+ topic.append_messages(partition_id, batch).await.unwrap();
let consumer = PollingConsumer::Consumer(1, partition_id);
let (_, polled_messages) = topic
@@ -106,10 +107,8 @@ async fn given_key_none_messages_should_be_appended_to_the_next_partition_using_
.expect("Failed to create message")],
batch_size.as_bytes_u32(),
);
- topic
- .append_messages(&partitioning, messages)
- .await
- .unwrap();
+ let partition_id = topic.calculate_partition_id(&partitioning).unwrap();
+ topic.append_messages(partition_id, messages).await.unwrap();
}
for i in 1..=partitions_count {
assert_messages(&topic, i, messages_per_partition_count).await;
@@ -135,10 +134,8 @@ async fn given_key_partition_id_messages_should_be_appended_to_the_chosen_partit
.expect("Failed to create message")],
batch_size.as_bytes_u32(),
);
- topic
- .append_messages(&partitioning, messages)
- .await
- .unwrap();
+ let partition_id = topic.calculate_partition_id(&partitioning).unwrap();
+ topic.append_messages(partition_id, messages).await.unwrap();
}
for i in 1..=partitions_count {
@@ -168,10 +165,8 @@ async fn given_key_messages_key_messages_should_be_appended_to_the_calculated_pa
.expect("Failed to create message")],
batch_size.as_bytes_u32(),
);
- topic
- .append_messages(&partitioning, messages)
- .await
- .unwrap();
+ let partition_id = topic.calculate_partition_id(&partitioning).unwrap();
+ topic.append_messages(partition_id, messages).await.unwrap();
}
let mut messages_count_per_partition = HashMap::new();
@@ -212,7 +207,7 @@ async fn init_topic(setup: &TestSetup, partitions_count: u32) -> Topic {
setup.create_topics_directory(stream_id).await;
let id = 2;
let name = "test";
- let topic = Topic::create(
+ let created_topic_info = Topic::create(
stream_id,
id,
name,
@@ -227,8 +222,8 @@ async fn init_topic(setup: &TestSetup, partitions_count: u32) -> Topic {
MaxTopicSize::ServerDefault,
1,
)
- .await
.unwrap();
+ let topic = created_topic_info.topic;
topic.persist().await.unwrap();
topic
}
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 3b9694f5..8da2f495 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -66,7 +66,7 @@ lending-iterator = "0.1.7"
hash32 = "1.0.0"
mimalloc = { workspace = true, optional = true }
moka = { version = "0.12.10", features = ["future"] }
-monoio = { version = "0.2.4", features = ["mkdirat", "unlinkat", "renameat"] }
+monoio = { version = "0.2.4", features = ["mkdirat", "unlinkat", "renameat", "sync"] }
monoio-native-tls = "0.4.0"
nix = { version = "0.30", features = ["fs"] }
once_cell = "1.21.3"
diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
index c590dcf2..3abfb6a1 100644
--- a/core/server/src/binary/handlers/messages/poll_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
@@ -21,10 +21,10 @@ use crate::binary::handlers::messages::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::sender::SenderKind;
use crate::shard::namespace::IggyNamespace;
+use crate::shard::system::messages::PollingArgs;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{ShardMessage, ShardRequest};
+use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
use crate::shard::{IggyShard, ShardRequestResult};
-use crate::shard::system::messages::PollingArgs;
use crate::streaming::segments::IggyMessagesBatchSet;
use crate::streaming::session::Session;
use crate::to_iovec;
@@ -112,53 +112,44 @@ impl ServerCommandHandler for PollMessages {
};
let namespace = IggyNamespace::new(stream.stream_id, topic.topic_id, partition_id);
- let request = ShardRequest::PollMessages {
- consumer,
- partition_id,
+ let payload = ShardRequestPayload::PollMessages {
+ consumer: consumer.clone(),
args,
count,
};
+ let request = ShardRequest::new(stream.stream_id, topic.topic_id, partition_id, payload);
let message = ShardMessage::Request(request);
let (metadata, batch) = match shard.send_request_to_shard(&namespace, message).await {
- ShardRequestResult::SameShard(message) => {
- match message {
- ShardMessage::Request(request) => {
- match request {
- ShardRequest::PollMessages {
- consumer,
- partition_id,
- args,
- count,
- } => {
- topic.get_messages(consumer, partition_id, args.strategy, count).await?
-
- }
- _ => unreachable!(
- "Expected a SendMessages request inside of SendMessages handler, impossible state"
- ),
- }
- }
- _ => unreachable!(
- "Expected a request message inside of an command handler, impossible state"
- ),
- }
- }
- ShardRequestResult::Result(result) => {
- match result? {
- ShardResponse::PollMessages(response) => {
- response
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
+ ShardRequestResult::SameShard(message) => match message {
+ ShardMessage::Request(request) => match request.payload {
+ ShardRequestPayload::PollMessages {
+ consumer,
+ args,
+ count,
+ } => {
+ topic
+ .get_messages(consumer, partition_id, args.strategy, count)
+ .await?
}
_ => unreachable!(
- "Expected a PollMessages response inside of PollMessages handler, impossible state"
+ "Expected a SendMessages request inside of SendMessages handler, impossible state"
),
+ },
+ _ => unreachable!(
+ "Expected a request message inside of an command handler, impossible state"
+ ),
+ },
+ ShardRequestResult::Result(result) => match result? {
+ ShardResponse::PollMessages(response) => response,
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
}
- }
+ _ => unreachable!(
+ "Expected a PollMessages response inside of PollMessages handler, impossible state"
+ ),
+ },
};
-
// Collect all chunks first into a Vec to extend their lifetimes.
// This ensures the Bytes (in reality Arc<[u8]>) references from each IggyMessagesBatch stay alive
// throughout the async vectored I/O operation, preventing "borrowed value does not live
diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs
index 61e2c39c..f4497bf6 100644
--- a/core/server/src/binary/handlers/messages/send_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs
@@ -21,7 +21,7 @@ use crate::binary::command::{BinaryServerCommand, ServerCommandHandler};
use crate::binary::sender::SenderKind;
use crate::shard::namespace::IggyNamespace;
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{ShardMessage, ShardRequest};
+use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
use crate::shard::{IggyShard, ShardRequestResult};
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
@@ -150,12 +150,8 @@ impl ServerCommandHandler for SendMessages {
let batch = shard.maybe_encrypt_messages(batch)?;
let namespace = IggyNamespace::new(stream.stream_id, topic.topic_id, partition_id);
- let request = ShardRequest::SendMessages {
- stream_id,
- topic_id,
- partition_id,
- batch,
- };
+ let payload = ShardRequestPayload::SendMessages { batch };
+ let request = ShardRequest::new(stream.stream_id, topic.topic_id, partition_id, payload);
let message = ShardMessage::Request(request);
// Egh... I don't like those nested match statements,
// Technically there is only two `request` types that will ever be dispatched
@@ -164,43 +160,31 @@ impl ServerCommandHandler for SendMessages {
// how to make this code reusable, in a sense that we will have exactly the same code inside of
// `PollMessages` handler, but with different request....
match shard.send_request_to_shard(&namespace, message).await {
- ShardRequestResult::SameShard(message) => {
- match message {
- ShardMessage::Request(request) => {
- match request {
- ShardRequest::SendMessages {
- stream_id,
- topic_id,
- partition_id,
- batch,
- } => {
- // Just shut up rust analyzer.
- let _stream_id = stream_id;
- let _topic_id = topic_id;
- topic.append_messages(partition_id, batch).await?
- }
- _ => unreachable!(
- "Expected a SendMessages request inside of SendMessages handler, impossible state"
- ),
+ ShardRequestResult::SameShard(message) => match message {
+ ShardMessage::Request(request) => {
+ let partition_id = request.partition_id;
+ match request.payload {
+ ShardRequestPayload::SendMessages { batch } => {
+ topic.append_messages(partition_id, batch).await?
}
+ _ => unreachable!(
+ "Expected a SendMessages request inside of SendMessages handler, impossible state"
+ ),
}
- _ => unreachable!(
- "Expected a request message inside of an command handler, impossible state"
- ),
}
- }
- ShardRequestResult::Result(result) => {
- match result? {
- ShardResponse::SendMessages => {
- ()
- }
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!("Expected a SendMessages response inside of SendMessages handler, impossible state"),
+ _ => unreachable!(
+ "Expected a request message inside of an command handler, impossible state"
+ ),
+ },
+ ShardRequestResult::Result(result) => match result? {
+ ShardResponse::SendMessages => (),
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
}
-
- }
+ _ => unreachable!(
+ "Expected a SendMessages response inside of SendMessages handler, impossible state"
+ ),
+ },
};
sender.send_empty_ok_response().await?;
Ok(())
diff --git a/core/server/src/binary/handlers/streams/create_stream_handler.rs b/core/server/src/binary/handlers/streams/create_stream_handler.rs
index a00afde7..0e746b76 100644
--- a/core/server/src/binary/handlers/streams/create_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/create_stream_handler.rs
@@ -21,6 +21,7 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::{handlers::streams::COMPONENT, sender::SenderKind};
use crate::shard::IggyShard;
+use crate::shard::transmission::event::ShardEvent;
use crate::state::command::EntryCommand;
use crate::state::models::CreateStreamWithId;
use crate::streaming::session::Session;
@@ -46,28 +47,31 @@ impl ServerCommandHandler for CreateStream {
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
let stream_id = self.stream_id;
-
+ let name = self.name.clone();
let created_stream_id = shard
- .create_stream(session, self.stream_id, &self.name)
+ .create_stream(session, stream_id, &name)
.await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to create stream with id: {stream_id:?}, session: {session}"
)
})?;
+ let event = ShardEvent::CreatedStream { stream_id, name };
+ // Broadcast the event to all shards.
+ let _responses = shard.broadcast_event_to_all_shards(event.into());
+
let stream = shard.find_stream(session, &created_stream_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to find created stream with id: {created_stream_id:?}, session: {session}"
)
})?;
- let stream_id = stream.stream_id;
let response = mapper::map_stream(&stream);
shard
.state
.apply(session.get_user_id(), &EntryCommand::CreateStream(CreateStreamWithId {
- stream_id,
+ stream_id: stream.stream_id,
command: self
})) .await
.with_error_context(|error| {
diff --git a/core/server/src/binary/handlers/topics/create_topic_handler.rs b/core/server/src/binary/handlers/topics/create_topic_handler.rs
index db09863e..6785e4f0 100644
--- a/core/server/src/binary/handlers/topics/create_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/create_topic_handler.rs
@@ -21,6 +21,7 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::{handlers::topics::COMPONENT, sender::SenderKind};
use crate::shard::IggyShard;
+use crate::shard::transmission::event::ShardEvent;
use crate::state::command::EntryCommand;
use crate::state::models::CreateTopicWithId;
use crate::streaming::session::Session;
@@ -62,14 +63,27 @@ impl ServerCommandHandler for CreateTopic {
.await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to create topic for stream_id: {stream_id}, topic_id: {topic_id:?}"
))?;
+ let event = ShardEvent::CreatedTopic {
+ stream_id: stream_id.clone(),
+ topic_id,
+ name: self.name.clone(),
+ partitions_count: self.partitions_count,
+ message_expiry: self.message_expiry,
+ compression_algorithm: self.compression_algorithm,
+ max_topic_size: self.max_topic_size,
+ replication_factor: self.replication_factor,
+ };
+ // Broadcast the event to all shards.
+ let _responses = shard.broadcast_event_to_all_shards(event.into());
+
let stream = shard
- .find_stream(session, &self.stream_id)
+ .get_stream(&self.stream_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get stream for stream_id: {stream_id}"
)
})?;
- let topic = shard.find_topic(session, &stream, &created_topic_id)
+ let topic = stream.get_topic(&created_topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get topic with ID: {created_topic_id} in stream with ID: {stream_id}"
diff --git a/core/server/src/binary/handlers/users/login_user_handler.rs b/core/server/src/binary/handlers/users/login_user_handler.rs
index 21cbba49..b6b163b6 100644
--- a/core/server/src/binary/handlers/users/login_user_handler.rs
+++ b/core/server/src/binary/handlers/users/login_user_handler.rs
@@ -23,6 +23,7 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::{handlers::users::COMPONENT, sender::SenderKind};
use crate::shard::IggyShard;
+use crate::shard::transmission::event::ShardEvent;
use crate::streaming::session::Session;
use anyhow::Result;
use error_set::ErrContext;
@@ -39,19 +40,30 @@ impl ServerCommandHandler for LoginUser {
async fn handle(
self,
sender: &mut SenderKind,
- length: u32,
+ _length: u32,
session: &Rc<Session>,
shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
+ let LoginUser {
+ username, password, ..
+ } = self;
let user = shard
- .login_user(&self.username, &self.password, Some(session))
+ .login_user(&username, &password, Some(session))
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to login user with name: {}, session: {session}",
- self.username
+ username
)
})?;
+ let event = ShardEvent::LoginUser {
+ client_id: session.client_id,
+ username,
+ password,
+ };
+ // Broadcast the event to all shards.
+ let _responses = shard.broadcast_event_to_all_shards(event.into());
+
let identity_info = mapper::map_identity_info(user.id);
sender.send_ok_response(&identity_info).await?;
Ok(())
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index ebf8946e..2d9f7eaa 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -16,6 +16,7 @@ use crate::{
streaming::{
persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind},
users::user::User,
+ utils::file::overwrite,
},
};
use std::{env, fs::remove_dir_all, ops::Range, path::Path, sync::Arc};
@@ -47,6 +48,12 @@ pub async fn create_directories(config: &SystemConfig) -> Result<(), IggyError>
if !Path::new(&state_path).exists() && create_dir_all(&state_path).await.is_err() {
return Err(IggyError::CannotCreateStateDirectory(state_path));
}
+ let state_log = config.get_state_messages_file_path();
+ if !Path::new(&state_log).exists() {
+ if let Err(_) = overwrite(&state_log).await {
+ return Err(IggyError::CannotCreateStateDirectory(state_log));
+ }
+ }
let streams_path = config.get_streams_path();
if !Path::new(&streams_path).exists() && create_dir_all(&streams_path).await.is_err() {
@@ -68,10 +75,6 @@ pub async fn create_directories(config: &SystemConfig) -> Result<(), IggyError>
config.get_system_path()
);
Ok(())
-
- // TODO: Move this to individual shard level
- /*
- */
}
pub fn create_root_user() -> User {
diff --git a/core/server/src/configs/displays.rs b/core/server/src/configs/displays.rs
index 51873421..b65804b5 100644
--- a/core/server/src/configs/displays.rs
+++ b/core/server/src/configs/displays.rs
@@ -290,10 +290,7 @@ impl Display for SegmentConfig {
write!(
f,
"{{ size_bytes: {}, cache_indexes: {}, message_expiry: {}, archive_expired: {} }}",
- self.size,
- self.cache_indexes,
- self.message_expiry,
- self.archive_expired,
+ self.size, self.cache_indexes, self.message_expiry, self.archive_expired,
)
}
}
diff --git a/core/server/src/io/file.rs b/core/server/src/io/file.rs
index 61e51023..95129aa8 100644
--- a/core/server/src/io/file.rs
+++ b/core/server/src/io/file.rs
@@ -94,14 +94,9 @@ impl AsyncWriteRent for IggyFile {
(Ok(n), buf)
}
+ // This is bait!!!!
async fn writev<T: IoVecBuf>(&mut self, buf_vec: T) -> BufResult<usize, T> {
- let slice = match IoVecWrapper::new(buf_vec) {
- Ok(slice) => slice,
- Err(buf) => return (Ok(0), buf),
- };
-
- let (result, slice) = self.write(slice).await;
- (result, slice.into_inner())
+ unimplemented!("writev is not implemented for IggyFile");
}
async fn flush(&mut self) -> std::io::Result<()> {
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 436b3829..5875e6bd 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -26,6 +26,7 @@ use dotenvy::dotenv;
use error_set::ErrContext;
use figlet_rs::FIGfont;
use iggy_common::create_user::CreateUser;
+use iggy_common::defaults::DEFAULT_ROOT_USER_ID;
use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError};
use server::args::Args;
use server::bootstrap::{
@@ -46,6 +47,7 @@ use server::state::command::EntryCommand;
use server::state::file::FileState;
use server::state::models::CreateUserWithId;
use server::state::system::SystemState;
+use server::streaming::utils::{MemoryPool, crypto};
use server::versioning::SemanticVersion;
use server::{IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV, map_toggle_str};
use tokio::time::Instant;
@@ -117,21 +119,27 @@ fn main() -> Result<(), ServerError> {
let mut logging = Logging::new(config.telemetry.clone());
logging.early_init();
+ // From this point on, we can use tracing macros to log messages.
+ logging.late_init(config.system.get_system_path(), &config.system.logging)?;
+
// TODO: Make this configurable from config as a range
// for example this instance of Iggy will use cores from 0..4
let available_cpus = available_parallelism().expect("Failed to get num of cores");
let shards_count = available_cpus.into();
let shards_set = 0..shards_count;
let connections = create_shard_connections(shards_set.clone());
+ let gate = Arc::new(Gate::new());
+ let mut handles = Vec::with_capacity(shards_set.len());
for shard_id in shards_set {
- let gate: Arc<Gate<()>> = Arc::new(Gate::new());
let id = shard_id as u16;
+ let gate = gate.clone();
let connections = connections.clone();
let config = config.clone();
let state_persister = resolve_persister(config.system.state.enforce_fsync);
- std::thread::Builder::new()
+ let handle = std::thread::Builder::new()
.name(format!("shard-{id}"))
.spawn(move || {
+ MemoryPool::init_pool(config.system.clone());
monoio::utils::bind_to_cpu_set(Some(shard_id))
.expect(format!("Failed to set CPU affinity for shard-{id}").as_str());
@@ -159,7 +167,6 @@ fn main() -> Result<(), ServerError> {
// We can't use std::sync::Once because it doesn't support async.
// Trait bound on the closure is FnOnce.
- let gate = gate.clone();
// Peak into the state to check if the root user exists.
// If it does not exist, create it.
gate.with_async::<Result<(), IggyError>>(async |gate_state| {
@@ -180,9 +187,7 @@ fn main() -> Result<(), ServerError> {
entry
.command()
.and_then(|command| match command {
- EntryCommand::CreateUser(payload)
- if payload.command.username
- == IGGY_ROOT_USERNAME_ENV && payload.command.password == IGGY_ROOT_PASSWORD_ENV =>
+ EntryCommand::CreateUser(payload) if payload.user_id == DEFAULT_ROOT_USER_ID =>
{
Ok(true)
}
@@ -238,19 +243,20 @@ fn main() -> Result<(), ServerError> {
.into();
//TODO: If one of the shards fails to initialize, we should crash the whole program;
- shard.run().await.expect("Failed to run shard");
+ if let Err(e) = shard.run().await {
+ error!("Failed to run shard-{id}: {e}");
+ }
//TODO: If one of the shards fails to initialize, we should crash the whole program;
- shard.assert_init();
- let shard = Rc::new(shard);
+ //shard.assert_init();
})
})
- .expect(format!("Failed to spawn thread for shard-{id}").as_str())
- .join()
- .expect(format!("Failed to join thread for shard-{id}").as_str());
+ .expect(format!("Failed to spawn thread for shard-{id}").as_str());
+ handles.push(handle);
}
- // From this point on, we can use tracing macros to log messages.
- logging.late_init(config.system.get_system_path(), &config.system.logging)?;
+ handles.into_iter().for_each(|handle| {
+ handle.join().expect("Failed to join shard thread");
+ });
/*
#[cfg(feature = "disable-mimalloc")]
@@ -260,7 +266,6 @@ fn main() -> Result<(), ServerError> {
#[cfg(not(feature = "disable-mimalloc"))]
info!("Using mimalloc allocator");
- MemoryPool::init_pool(config.system.clone());
let system = SharedSystem::new(System::new(
config.system.clone(),
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index a2a3fdfb..beda9ae2 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -113,7 +113,7 @@ impl IggyShardBuilder {
version: version,
stop_receiver: stop_receiver,
stop_sender: stop_sender,
- frame_receiver: Cell::new(Some(frame_receiver)),
+ messages_receiver: Cell::new(Some(frame_receiver)),
metrics: Metrics::init(),
users: Default::default(),
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 7976cca3..cfbd6eac 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -20,13 +20,14 @@ pub mod builder;
pub mod gate;
pub mod namespace;
pub mod system;
+pub mod tasks;
pub mod transmission;
use ahash::{AHashMap, AHashSet, HashMap};
use builder::IggyShardBuilder;
use error_set::ErrContext;
use futures::future::try_join_all;
-use iggy_common::{EncryptorKind, IggyError, UserId};
+use iggy_common::{EncryptorKind, Identifier, IggyError, UserId};
use namespace::IggyNamespace;
use std::{
cell::{Cell, RefCell},
@@ -46,23 +47,29 @@ use crate::{
configs::server::ServerConfig,
shard::{
system::info::SystemInfo,
+ tasks::messages::spawn_shard_message_task,
transmission::{
+ event::ShardEvent,
frame::{ShardFrame, ShardResponse},
- message::{ShardEvent, ShardMessage, ShardRequest},
+ message::{ShardMessage, ShardRequest, ShardRequestPayload},
},
},
state::{
- file::FileState, system::{StreamState, SystemState, UserState}, StateKind
+ StateKind,
+ file::FileState,
+ system::{StreamState, SystemState, UserState},
},
streaming::{
clients::client_manager::ClientManager,
diagnostics::metrics::Metrics,
+ partitions::partition,
personal_access_tokens::personal_access_token::PersonalAccessToken,
session::Session,
storage::SystemStorage,
streams::stream::Stream,
users::{permissioner::Permissioner, user::User},
},
+ tcp::tcp_server::spawn_tcp_server,
versioning::SemanticVersion,
};
@@ -90,16 +97,15 @@ impl Shard {
.sender
.send(ShardFrame::new(message, Some(sender.clone()))); // Apparently sender needs to be cloned, otherwise channel will close...
//TODO: Fixme
- let response = receiver.recv().await
- .map_err(|err| {
- error!("Failed to receive response from shard: {err}");
- IggyError::ShardCommunicationError(self.id)
- })?;
+ let response = receiver.recv().await.map_err(|err| {
+ error!("Failed to receive response from shard: {err}");
+ IggyError::ShardCommunicationError(self.id)
+ })?;
Ok(response)
}
}
-struct ShardInfo {
+pub struct ShardInfo {
id: u16,
}
@@ -135,7 +141,7 @@ pub struct IggyShard {
pub(crate) users: RefCell<HashMap<UserId, User>>,
pub(crate) metrics: Metrics,
- pub frame_receiver: Cell<Option<Receiver<ShardFrame>>>,
+ pub messages_receiver: Cell<Option<Receiver<ShardFrame>>>,
stop_receiver: StopReceiver,
stop_sender: StopSender,
}
@@ -172,17 +178,16 @@ impl IggyShard {
// have the correct statistics when the server starts.
//self.get_stats().await?;
self.init().await?;
- self.assert_init();
+ // TODO: Fixme
+ //self.assert_init();
info!("Initiated shard with ID: {}", self.id);
// Create all tasks (tcp listener, http listener, command processor, in the future also the background jobs).
- /*
- let mut tasks: Vec<Task> = vec![Box::pin(spawn_shard_message_task(shard.clone()))];
+ let mut tasks: Vec<Task> = vec![Box::pin(spawn_shard_message_task(self.clone()))];
if self.config.tcp.enabled {
tasks.push(Box::pin(spawn_tcp_server(self.clone())));
}
let result = try_join_all(tasks).await;
result?;
- */
Ok(())
}
@@ -427,6 +432,7 @@ impl IggyShard {
#[instrument(skip_all, name = "trace_shutdown")]
pub async fn shutdown(&mut self) -> Result<(), IggyError> {
+ //TODO: Fixme, impl cooperative shutdown.
self.persist_messages().await?;
Ok(())
}
@@ -449,6 +455,81 @@ impl IggyShard {
self.shards.len() as u32
}
+ pub async fn handle_shard_message(&self, message: ShardMessage) -> Option<ShardResponse> {
+ match message {
+ ShardMessage::Request(request) => match self.handle_request(request).await {
+ Ok(response) => Some(response),
+ Err(err) => Some(ShardResponse::ErrorResponse(err)),
+ },
+ ShardMessage::Event(event) => match self.handle_event(event).await {
+ Ok(_) => Some(ShardResponse::Event),
+ Err(err) => Some(ShardResponse::ErrorResponse(err)),
+ },
+ }
+ }
+
+ async fn handle_request(&self, request: ShardRequest) -> Result<ShardResponse, IggyError> {
+ let stream = self.get_stream(&Identifier::numeric(request.stream_id)?)?;
+ let topic = stream.get_topic(&Identifier::numeric(request.topic_id)?)?;
+ let partition_id = request.partition_id;
+ match request.payload {
+ ShardRequestPayload::SendMessages { batch } => {
+ topic.append_messages(partition_id, batch).await?;
+ Ok(ShardResponse::SendMessages)
+ }
+ ShardRequestPayload::PollMessages {
+ args,
+ consumer,
+ count,
+ } => {
+ let (metadata, batch) = topic
+ .get_messages(consumer, partition_id, args.strategy, count)
+ .await?;
+ Ok(ShardResponse::PollMessages((metadata, batch)))
+ }
+ }
+ }
+
+ async fn handle_event(&self, event: Arc<ShardEvent>) -> Result<(), IggyError> {
+ match &*event {
+ ShardEvent::CreatedStream { stream_id, name } => {
+ self.create_stream_bypass_auth(*stream_id, name).await
+ }
+ ShardEvent::CreatedTopic {
+ stream_id,
+ topic_id,
+ name,
+ partitions_count,
+ message_expiry,
+ compression_algorithm,
+ max_topic_size,
+ replication_factor,
+ } => {
+ self.create_topic_bypass_auth(
+ stream_id,
+ *topic_id,
+ name,
+ *partitions_count,
+ *message_expiry,
+ *compression_algorithm,
+ *max_topic_size,
+ *replication_factor,
+ )
+ .await
+ }
+ ShardEvent::LoginUser {
+ client_id,
+ username,
+ password,
+ } => self.login_user_event(*client_id, username, password),
+ ShardEvent::NewSession { address, transport } => {
+ let session = self.add_client(address, *transport);
+ self.add_active_session(session);
+ Ok(())
+ }
+ }
+ }
+
pub async fn send_request_to_shard(
&self,
namespace: &IggyNamespace,
@@ -479,6 +560,25 @@ impl IggyShard {
}
}
+ pub fn broadcast_event_to_all_shards(&self, event: Arc<ShardEvent>) -> Vec<ShardResponse> {
+ self.shards
+ .iter()
+ .filter_map(|shard| {
+ if shard.id != self.id {
+ Some(shard.connection.clone())
+ } else {
+ None
+ }
+ })
+ .map(|conn| {
+ // TODO: Fixme, maybe we should send response_sender
+ // and propagate errors back.
+ conn.send(ShardFrame::new(event.clone().into(), None));
+ ShardResponse::Event
+ })
+ .collect()
+ }
+
fn find_shard(&self, namespace: &IggyNamespace) -> Option<&Shard> {
let shards_table = self.shards_table.borrow();
shards_table.get(namespace).map(|shard_info| {
@@ -496,27 +596,6 @@ impl IggyShard {
self.shards_table.borrow_mut().extend(records);
}
- pub fn broadcast_event_to_all_shards(&self, client_id: u32, event: ShardEvent) {
- self.shards
- .iter()
- .filter_map(|shard| {
- if shard.id != self.id {
- Some(shard.connection.clone())
- } else {
- None
- }
- })
- .map(|conn| {
- //TODO: Fixme
- /*
- let message = ShardMessage::Event(event);
- conn.send(ShardFrame::new(client_id, message, None));
- */
- ()
- })
- .collect::<Vec<_>>();
- }
-
pub fn add_active_session(&self, session: Rc<Session>) {
self.active_sessions.borrow_mut().push(session);
}
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index 76337a1a..36ae234b 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -25,8 +25,8 @@ use crate::streaming::utils::PooledBuffer;
use async_zip::tokio::read::stream;
use error_set::ErrContext;
use iggy_common::{
- BytesSerializable, Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier,
- IggyError, Partitioning, PollingStrategy,
+ BytesSerializable, Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier, IggyError,
+ Partitioning, PollingStrategy,
};
use tracing::{error, trace};
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index fecec536..47068f11 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -186,6 +186,19 @@ impl IggyShard {
}))
}
+ pub async fn create_stream_bypass_auth(
+ &self,
+ stream_id: Option<u32>,
+ name: &str,
+ ) -> Result<(), IggyError> {
+ let stream = self.create_stream_base(stream_id, name)?;
+ let id = stream.stream_id;
+ self.streams_ids.borrow_mut().insert(name.to_owned(), id);
+ self.streams.borrow_mut().insert(id, stream);
+ self.metrics.increment_streams(1);
+ Ok(())
+ }
+
pub async fn create_stream(
&self,
session: &Session,
@@ -199,6 +212,25 @@ impl IggyShard {
if self.streams_ids.borrow().contains_key(name) {
return Err(IggyError::StreamNameAlreadyExists(name.to_owned()));
}
+ let stream = self
+ .create_stream_base(stream_id, name)
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to create stream with name: {name}")
+ })?;
+ let id = stream.stream_id;
+
+ stream.persist().await?;
+ info!("Created stream with ID: {id}, name: '{name}'.");
+ self.streams_ids.borrow_mut().insert(name.to_owned(), id);
+ self.streams.borrow_mut().insert(id, stream);
+ self.metrics.increment_streams(1);
+ Ok(Identifier::numeric(id)?)
+ }
+
+ fn create_stream_base(&self, stream_id: Option<u32>, name: &str) -> Result<Stream, IggyError> {
+ if self.streams_ids.borrow().contains_key(name) {
+ return Err(IggyError::StreamNameAlreadyExists(name.to_owned()));
+ }
let mut id;
if stream_id.is_none() {
@@ -222,14 +254,7 @@ impl IggyShard {
}
let stream = Stream::create(id, name, self.config.system.clone(), self.storage.clone());
- stream.persist().await?;
- info!("Created stream with ID: {id}, name: '{name}'.");
- self.streams_ids
- .borrow_mut()
- .insert(name.to_owned(), stream.stream_id);
- self.streams.borrow_mut().insert(stream.stream_id, stream);
- self.metrics.increment_streams(1);
- Ok(Identifier::numeric(id)?)
+ Ok(stream)
}
pub async fn update_stream(
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index 252dae30..c247cf74 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -101,6 +101,52 @@ impl IggyShard {
Ok(Some(topic))
}
+ pub async fn create_topic_bypass_auth(
+ &self,
+ stream_id: &Identifier,
+ topic_id: Option<u32>,
+ name: &str,
+ partitions_count: u32,
+ message_expiry: IggyExpiry,
+ compression_algorithm: CompressionAlgorithm,
+ max_topic_size: MaxTopicSize,
+ replication_factor: Option<u8>,
+ ) -> Result<(), IggyError> {
+ let (stream_id, topic_id, partition_ids) = self
+ .create_topic_base(
+ stream_id,
+ topic_id,
+ name,
+ partitions_count,
+ message_expiry,
+ compression_algorithm,
+ max_topic_size,
+ replication_factor,
+ )
+ .await?;
+ // TODO: Figure out a way how to distribute the shards table among different shards,
+ // without the need to do code from below, everytime we handle a `ShardEvent`.
+ // I think we shouldn't be sharing it, maintain a single shard table per shard,
+ // but figure out a way how to distribute it smarter (maybe move the broadcast inside of the `insert_shard_table_records`) method.
+ let records = partition_ids.into_iter().map(|partition_id| {
+ let namespace = IggyNamespace::new(stream_id, topic_id, partition_id);
+ // TODO: This setup isn't deterministic.
+ // Imagine a scenario where client creates partition using `String` identifiers,
+ // but then for poll_messages requests uses numeric ones.
+ // the namespace wouldn't match, therefore we would get miss in the shard table.
+ let hash = namespace.generate_hash();
+ let shard_id = hash % self.get_available_shards_count();
+ let shard_info = ShardInfo::new(shard_id as u16);
+ (namespace, shard_info)
+ });
+ self.insert_shard_table_records(records);
+
+ self.metrics.increment_topics(1);
+ self.metrics.increment_partitions(partitions_count);
+ self.metrics.increment_segments(partitions_count);
+ Ok(())
+ }
+
#[allow(clippy::too_many_arguments)]
pub async fn create_topic(
&self,
@@ -130,26 +176,18 @@ impl IggyShard {
})?;
}
- // TODO: Make create topic sync, and extract the storage persister out of it
- // perform disk i/o outside of the borrow_mut of the stream.
- let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}")
- })?;
- let stream_id = stream.stream_id;
- let (topic_id, partition_ids) = stream
- .create_topic(
+ let (stream_id, topic_id, partition_ids) = self
+ .create_topic_base(
+ stream_id,
topic_id,
name,
partitions_count,
message_expiry,
compression_algorithm,
max_topic_size,
- replication_factor.unwrap_or(1),
+ replication_factor,
)
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to create topic with name: {name} in stream ID: {stream_id}")
- })?;
+ .await?;
let records = partition_ids.into_iter().map(|partition_id| {
let namespace = IggyNamespace::new(stream_id, topic_id, partition_id);
// TODO: This setup isn't deterministic.
@@ -170,6 +208,40 @@ impl IggyShard {
Ok(Identifier::numeric(topic_id)?)
}
+ async fn create_topic_base(
+ &self,
+ stream_id: &Identifier,
+ topic_id: Option<u32>,
+ name: &str,
+ partitions_count: u32,
+ message_expiry: IggyExpiry,
+ compression_algorithm: CompressionAlgorithm,
+ max_topic_size: MaxTopicSize,
+ replication_factor: Option<u8>,
+ ) -> Result<(u32, u32, Vec<u32>), IggyError> {
+ // TODO: Make create topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let (topic_id, partition_ids) = stream
+ .create_topic(
+ topic_id,
+ name,
+ partitions_count,
+ message_expiry,
+ compression_algorithm,
+ max_topic_size,
+ replication_factor.unwrap_or(1),
+ )
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to create topic with name: {name} in stream ID: {stream_id}")
+ })?;
+ Ok((stream_id, topic_id, partition_ids))
+ }
+
#[allow(clippy::too_many_arguments)]
pub async fn update_topic(
&self,
diff --git a/core/server/src/shard/system/users.rs b/core/server/src/shard/system/users.rs
index f3b271a7..f0e87fc0 100644
--- a/core/server/src/shard/system/users.rs
+++ b/core/server/src/shard/system/users.rs
@@ -372,6 +372,21 @@ impl IggyShard {
Ok(())
}
+ pub fn login_user_event(
+ &self,
+ client_id: u32,
+ username: &str,
+ password: &str,
+ ) -> Result<(), IggyError> {
+ let active_sessions = self.active_sessions.borrow();
+ let session = active_sessions
+ .iter()
+ .find(|s| s.client_id == client_id)
+ .expect(format!("At this point session for {}, should exist.", client_id).as_str());
+ self.login_user_with_credentials(username, Some(password), Some(session))?;
+ Ok(())
+ }
+
pub fn login_user(
&self,
username: &str,
diff --git a/core/server/src/shard/tasks/messages.rs b/core/server/src/shard/tasks/messages.rs
new file mode 100644
index 00000000..87660290
--- /dev/null
+++ b/core/server/src/shard/tasks/messages.rs
@@ -0,0 +1,38 @@
+use futures::StreamExt;
+use iggy_common::IggyError;
+use std::rc::Rc;
+use tracing::error;
+
+use crate::shard::{IggyShard, transmission::frame::ShardFrame};
+
+async fn run_shard_messages_receiver(shard: Rc<IggyShard>) -> Result<(), IggyError> {
+ let mut messages_receiver = shard.messages_receiver.take().unwrap();
+ loop {
+ if let Some(frame) = messages_receiver.next().await {
+ let ShardFrame {
+ message,
+ response_sender,
+ } = frame;
+ match (shard.handle_shard_message(message).await, response_sender) {
+ (Some(response), Some(response_sender)) => {
+ response_sender
+ .send(response)
+ .await
+ .expect("Failed to send response back to origin shard.");
+ }
+ _ => {}
+ };
+ }
+ }
+}
+
+pub async fn spawn_shard_message_task(shard: Rc<IggyShard>) -> Result<(), IggyError> {
+ monoio::spawn(async move {
+ let result = run_shard_messages_receiver(shard).await;
+ if let Err(err) = &result {
+ error!("Error running shard: {err}");
+ }
+ result
+ })
+ .await
+}
diff --git a/core/server/src/shard/tasks/mod.rs b/core/server/src/shard/tasks/mod.rs
new file mode 100644
index 00000000..ba63992f
--- /dev/null
+++ b/core/server/src/shard/tasks/mod.rs
@@ -0,0 +1 @@
+pub mod messages;
diff --git a/core/server/src/shard/transmission/event.rs b/core/server/src/shard/transmission/event.rs
index 45815a50..49794c3f 100644
--- a/core/server/src/shard/transmission/event.rs
+++ b/core/server/src/shard/transmission/event.rs
@@ -4,26 +4,27 @@ use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize};
use crate::streaming::clients::client_manager::Transport;
+#[derive(Debug)]
pub enum ShardEvent {
CreatedStream {
stream_id: Option<u32>,
- topic_id: Identifier
+ name: String,
},
//DeletedStream(Identifier),
//UpdatedStream(Identifier, String),
//PurgedStream(Identifier),
//CreatedPartitions(Identifier, Identifier, u32),
//DeletedPartitions(Identifier, Identifier, u32),
- CreatedTopic(
- Identifier,
- Option<u32>,
- String,
- u32,
- IggyExpiry,
- CompressionAlgorithm,
- MaxTopicSize,
- Option<u8>,
- ),
+ CreatedTopic {
+ stream_id: Identifier,
+ topic_id: Option<u32>,
+ name: String,
+ partitions_count: u32,
+ message_expiry: IggyExpiry,
+ compression_algorithm: CompressionAlgorithm,
+ max_topic_size: MaxTopicSize,
+ replication_factor: Option<u8>,
+ },
//CreatedConsumerGroup(Identifier, Identifier, Option<u32>, String),
//DeletedConsumerGroup(Identifier, Identifier, Identifier),
/*
@@ -42,6 +43,7 @@ pub enum ShardEvent {
//CreatedUser(String, String, UserStatus, Option<Permissions>),
//DeletedUser(Identifier),
LoginUser {
+ client_id: u32,
username: String,
password: String,
},
@@ -53,8 +55,7 @@ pub enum ShardEvent {
//LoginWithPersonalAccessToken(String),
//StoredConsumerOffset(Identifier, Identifier, PollingConsumer, u64),
NewSession {
- user_id: u32,
- socket_addr: SocketAddr,
+ address: SocketAddr,
transport: Transport,
},
-}
\ No newline at end of file
+}
diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs
index ff519ab4..6539ec4c 100644
--- a/core/server/src/shard/transmission/frame.rs
+++ b/core/server/src/shard/transmission/frame.rs
@@ -18,13 +18,16 @@
use async_channel::Sender;
use iggy_common::IggyError;
-use crate::{binary::handlers::messages::poll_messages_handler::IggyPollMetadata, shard::transmission::message::ShardMessage, streaming::segments::IggyMessagesBatchSet};
+use crate::{
+ binary::handlers::messages::poll_messages_handler::IggyPollMetadata,
+ shard::transmission::message::ShardMessage, streaming::segments::IggyMessagesBatchSet,
+};
#[derive(Debug)]
pub enum ShardResponse {
PollMessages((IggyPollMetadata, IggyMessagesBatchSet)),
SendMessages,
- ShardEvent,
+ Event,
ErrorResponse(IggyError),
}
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index 360e7a3c..aa196fde 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -1,4 +1,4 @@
-use std::rc::Rc;
+use std::{rc::Rc, sync::Arc};
use iggy_common::PollingStrategy;
@@ -19,29 +19,47 @@ use iggy_common::PollingStrategy;
* specific language governing permissions and limitations
* under the License.
*/
-use crate::{shard::system::messages::PollingArgs, streaming::{polling_consumer::PollingConsumer, segments::IggyMessagesBatchMut, session::Session}};
+use crate::{
+ shard::{system::messages::PollingArgs, transmission::event::ShardEvent},
+ streaming::{polling_consumer::PollingConsumer, segments::IggyMessagesBatchMut},
+};
#[derive(Debug)]
pub enum ShardMessage {
Request(ShardRequest),
- Event(ShardEvent),
+ Event(Arc<ShardEvent>),
}
#[derive(Debug)]
-pub enum ShardEvent {
- NewSession(),
+pub struct ShardRequest {
+ pub stream_id: u32,
+ pub topic_id: u32,
+ pub partition_id: u32,
+ pub payload: ShardRequestPayload,
}
-#[derive(Debug)]
-pub enum ShardRequest {
- SendMessages {
+impl ShardRequest {
+ pub fn new(
stream_id: u32,
topic_id: u32,
partition_id: u32,
+ payload: ShardRequestPayload,
+ ) -> Self {
+ Self {
+ stream_id,
+ topic_id,
+ partition_id,
+ payload,
+ }
+ }
+}
+
+#[derive(Debug)]
+pub enum ShardRequestPayload {
+ SendMessages {
batch: IggyMessagesBatchMut,
},
PollMessages {
- partition_id: u32,
args: PollingArgs,
consumer: PollingConsumer,
count: u32,
@@ -54,8 +72,8 @@ impl From<ShardRequest> for ShardMessage {
}
}
-impl From<ShardEvent> for ShardMessage {
- fn from(event: ShardEvent) -> Self {
+impl From<Arc<ShardEvent>> for ShardMessage {
+ fn from(event: Arc<ShardEvent>) -> Self {
ShardMessage::Event(event)
}
}
diff --git a/core/server/src/shard/transmission/mod.rs b/core/server/src/shard/transmission/mod.rs
index 4ed9bd1e..107bdcbd 100644
--- a/core/server/src/shard/transmission/mod.rs
+++ b/core/server/src/shard/transmission/mod.rs
@@ -16,7 +16,7 @@
* under the License.
*/
-pub mod event;
pub mod connector;
+pub mod event;
pub mod frame;
pub mod message;
diff --git a/core/server/src/state/file.rs b/core/server/src/state/file.rs
index b845d877..e8eb6ee6 100644
--- a/core/server/src/state/file.rs
+++ b/core/server/src/state/file.rs
@@ -117,7 +117,6 @@ impl State for FileState {
)
})
.map_err(|_| IggyError::CannotReadFile)?;
- let file = IggyFile::new(file);
let file_size = file
.metadata()
.await
@@ -129,6 +128,8 @@ impl State for FileState {
})
.map_err(|_| IggyError::CannotReadFileMetadata)?
.len();
+
+ let file = IggyFile::new(file);
if file_size == 0 {
info!("State file is empty");
return Ok(Vec::new());
@@ -150,7 +151,8 @@ impl State for FileState {
.with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} index. {error}"))
.map_err(|_| IggyError::InvalidNumberEncoding)?;
total_size += 8;
- if entries_count > 0 && index != current_index + 1 {
+ // Greater than one, because one of the entries after a fresh reboot is the default root user.
+ if entries_count > 1 && index != current_index + 1 {
error!(
"State file is corrupted, expected index: {}, got: {}",
current_index + 1,
diff --git a/core/server/src/streaming/segments/indexes/index_reader.rs b/core/server/src/streaming/segments/indexes/index_reader.rs
index c373ce0f..240e62a1 100644
--- a/core/server/src/streaming/segments/indexes/index_reader.rs
+++ b/core/server/src/streaming/segments/indexes/index_reader.rs
@@ -17,7 +17,7 @@
*/
use super::IggyIndexesMut;
-use crate::streaming::utils::PooledBuffer;
+use crate::{io::file::IggyFile, streaming::utils::PooledBuffer};
use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::{INDEX_SIZE, IggyError, IggyIndex, IggyIndexView};
@@ -30,15 +30,14 @@ use std::{
atomic::{AtomicU64, Ordering},
},
};
-use tokio::fs::OpenOptions;
-use tokio::task::spawn_blocking;
+use monoio::fs::OpenOptions;
use tracing::{error, trace};
/// A dedicated struct for reading from the index file.
#[derive(Debug)]
pub struct IndexReader {
file_path: String,
- file: Arc<StdFile>,
+ file: IggyFile,
index_size_bytes: Arc<AtomicU64>,
}
@@ -52,13 +51,14 @@ impl IndexReader {
.with_error_context(|error| format!("Failed to open index file: {file_path}. {error}"))
.map_err(|_| IggyError::CannotReadFile)?;
+ let file = IggyFile::new(file);
trace!(
"Opened index file for reading: {file_path}, size: {}",
index_size_bytes.load(Ordering::Acquire)
);
Ok(Self {
file_path: file_path.to_string(),
- file: Arc::new(file.into_std().await),
+ file,
index_size_bytes,
})
}
@@ -334,21 +334,19 @@ impl IndexReader {
len: u32,
use_pool: bool,
) -> Result<PooledBuffer, std::io::Error> {
- let file = self.file.clone();
- spawn_blocking(move || {
if use_pool {
let mut buf = PooledBuffer::with_capacity(len as usize);
unsafe { buf.set_len(len as usize) };
- file.read_exact_at(&mut buf, offset as u64)?;
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ result?;
Ok(buf)
} else {
let mut buf = BytesMut::with_capacity(len as usize);
unsafe { buf.set_len(len as usize) };
- file.read_exact_at(&mut buf, offset as u64)?;
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ result?;
Ok(PooledBuffer::from_existing(buf))
}
- })
- .await?
}
/// Gets the nth index from the index file.
diff --git a/core/server/src/streaming/segments/indexes/index_writer.rs b/core/server/src/streaming/segments/indexes/index_writer.rs
index 71b18b6d..120bfe4e 100644
--- a/core/server/src/streaming/segments/indexes/index_writer.rs
+++ b/core/server/src/streaming/segments/indexes/index_writer.rs
@@ -19,21 +19,22 @@
use error_set::ErrContext;
use iggy_common::INDEX_SIZE;
use iggy_common::IggyError;
+use monoio::fs::OpenOptions;
+use monoio::io::AsyncWriteRentExt;
use std::sync::{
Arc,
atomic::{AtomicU64, Ordering},
};
-use tokio::{
- fs::{File, OpenOptions},
- io::AsyncWriteExt,
-};
use tracing::trace;
+use crate::io::file::IggyFile;
+use crate::streaming::utils::PooledBuffer;
+
/// A dedicated struct for writing to the index file.
#[derive(Debug)]
pub struct IndexWriter {
file_path: String,
- file: File,
+ file: IggyFile,
index_size_bytes: Arc<AtomicU64>,
fsync: bool,
}
@@ -55,6 +56,7 @@ impl IndexWriter {
.with_error_context(|error| format!("Failed to open index file: {file_path}. {error}"))
.map_err(|_| IggyError::CannotReadFile)?;
+ let file = IggyFile::from(file);
if file_exists {
let _ = file.sync_all().await.with_error_context(|error| {
format!("Failed to fsync index file after creation: {file_path}. {error}",)
@@ -86,16 +88,18 @@ impl IndexWriter {
}
/// Appends multiple index buffer to the index file in a single operation.
- pub async fn save_indexes(&mut self, indexes: &[u8]) -> Result<(), IggyError> {
+ pub async fn save_indexes(&mut self, indexes: PooledBuffer) -> Result<(), IggyError> {
if indexes.is_empty() {
return Ok(());
}
let count = indexes.len() / INDEX_SIZE;
+ let len = indexes.len();
self.file
.write_all(indexes)
.await
+ .0
.with_error_context(|error| {
format!(
"Failed to write {} indexes to file: {}. {error}",
@@ -105,7 +109,7 @@ impl IndexWriter {
.map_err(|_| IggyError::CannotSaveIndexToSegment)?;
self.index_size_bytes
- .fetch_add(indexes.len() as u64, Ordering::Release);
+ .fetch_add(len as u64, Ordering::Release);
if self.fsync {
let _ = self.fsync().await;
diff --git a/core/server/src/streaming/segments/indexes/indexes_mut.rs b/core/server/src/streaming/segments/indexes/indexes_mut.rs
index 0dacb907..1b5e8e1e 100644
--- a/core/server/src/streaming/segments/indexes/indexes_mut.rs
+++ b/core/server/src/streaming/segments/indexes/indexes_mut.rs
@@ -221,9 +221,12 @@ impl IggyIndexesMut {
}
/// Gets the unsaved part of the index buffer
- pub fn unsaved_slice(&self) -> &[u8] {
+ pub fn unsaved_slice(&self) -> PooledBuffer {
let start_pos = self.saved_count as usize * INDEX_SIZE;
- &self.buffer[start_pos..]
+ // TODO: Dunno how to handle this better, maybe we should have a `split` method,
+ // That splits the underlying Indexes buffer into two parts
+ // saved on disk and not saved yet.
+ PooledBuffer::from(&self.buffer[start_pos..])
}
/// Mark all indexes as saved to disk
diff --git a/core/server/src/streaming/segments/messages/mod.rs b/core/server/src/streaming/segments/messages/mod.rs
index 512c3894..b79b1af0 100644
--- a/core/server/src/streaming/segments/messages/mod.rs
+++ b/core/server/src/streaming/segments/messages/mod.rs
@@ -19,71 +19,35 @@
mod messages_reader;
mod messages_writer;
-use crate::{io::file::IggyFile, to_iovec};
+use crate::{io::file::IggyFile};
use super::IggyMessagesBatchSet;
use error_set::ErrContext;
use iggy_common::IggyError;
-use monoio::io::AsyncWriteRent;
-use std::{io::IoSlice, mem::take};
+use monoio::io::AsyncWriteRentExt;
pub use messages_reader::MessagesReader;
pub use messages_writer::MessagesWriter;
-use nix::libc::iovec;
/// Vectored write a batches of messages to file
async fn write_batch(
file: &mut IggyFile,
file_path: &str,
- batches: IggyMessagesBatchSet,
+ mut batches: IggyMessagesBatchSet,
) -> Result<usize, IggyError> {
- let mut slices = batches.iter().map(|b| to_iovec(&b)).collect::<Vec<iovec>>();
+ //let mut slices = batches.iter().map(|b| to_iovec(&b)).collect::<Vec<iovec>>();
let mut total_written = 0;
-
- loop {
- let (result, vomited) = file.writev(slices).await;
- slices = vomited;
- let bytes_written = result
- .with_error_context(|error| {
- format!("Failed to write messages to file: {file_path}, error: {error}",)
- })
- .map_err(|_| IggyError::CannotWriteToFile)?;
-
- total_written += bytes_written;
- advance_slices(&mut slices.as_mut_slice(), bytes_written);
- if slices.is_empty() {
- break;
- }
+ // TODO: Fork monoio, piece of shit runtime.
+ for batch in batches.iter_mut() {
+ let messages = batch.take_messages();
+ let writen = file.write_all(messages).await.0.with_error_context(|error| {
+ format!(
+ "Failed to write messages to file: {file_path}, error: {error}",
+ )
+ // TODO: Better error variant.
+ }).map_err(|_| IggyError::CannotAppendMessage)?;
+ total_written += writen;
}
-
Ok(total_written)
}
-
-fn advance_slices(mut bufs: &mut [iovec], n: usize) {
- // Number of buffers to remove.
- let mut remove = 0;
- // Remaining length before reaching n. This prevents overflow
- // that could happen if the length of slices in `bufs` were instead
- // accumulated. Those slice may be aliased and, if they are large
- // enough, their added length may overflow a `usize`.
- let mut left = n;
- for buf in bufs.iter() {
- if let Some(remainder) = left.checked_sub(buf.iov_len as _) {
- left = remainder;
- remove += 1;
- } else {
- break;
- }
- }
-
- bufs = &mut bufs[remove..];
- if bufs.is_empty() {
- assert!(left == 0, "advancing io slices beyond their length");
- } else {
- unsafe {
- bufs[0].iov_len -= n;
- bufs[0].iov_base = bufs[0].iov_base.add(n);
- }
- }
-}
diff --git a/core/server/src/streaming/segments/segment.rs b/core/server/src/streaming/segments/segment.rs
index 92dd254f..9887df90 100644
--- a/core/server/src/streaming/segments/segment.rs
+++ b/core/server/src/streaming/segments/segment.rs
@@ -313,7 +313,8 @@ impl Segment {
}
if let Some(index_writer) = self.index_writer.take() {
- tokio::spawn(async move {
+ //TODO: Fixme not sure whether we should spawn a task here.
+ monoio::spawn(async move {
let _ = index_writer.fsync().await;
drop(index_writer)
});
diff --git a/core/server/src/streaming/segments/types/messages_batch_mut.rs b/core/server/src/streaming/segments/types/messages_batch_mut.rs
index ea549291..6a76b582 100644
--- a/core/server/src/streaming/segments/types/messages_batch_mut.rs
+++ b/core/server/src/streaming/segments/types/messages_batch_mut.rs
@@ -264,6 +264,10 @@ impl IggyMessagesBatchMut {
(indexes, messages)
}
+ pub fn take_messages(&mut self) -> PooledBuffer {
+ std::mem::take(&mut self.messages)
+ }
+
/// Take the indexes from the batch
pub fn take_indexes(&mut self) -> IggyIndexesMut {
std::mem::take(&mut self.indexes)
diff --git a/core/server/src/streaming/segments/writing_messages.rs b/core/server/src/streaming/segments/writing_messages.rs
index 9c385b02..8313f589 100644
--- a/core/server/src/streaming/segments/writing_messages.rs
+++ b/core/server/src/streaming/segments/writing_messages.rs
@@ -99,6 +99,7 @@ impl Segment {
self.last_index_position += saved_bytes.as_bytes_u64() as u32;
let unsaved_indexes_slice = self.indexes.unsaved_slice();
+ let len = unsaved_indexes_slice.len();
self.index_writer
.as_mut()
.expect("Index writer not initialized")
@@ -107,7 +108,7 @@ impl Segment {
.with_error_context(|error| {
format!(
"Failed to save index of {} indexes to {self}. {error}",
- unsaved_indexes_slice.len()
+ len
)
})?;
diff --git a/core/server/src/streaming/streams/storage.rs b/core/server/src/streaming/streams/storage.rs
index ebd4946a..5bf4c554 100644
--- a/core/server/src/streaming/streams/storage.rs
+++ b/core/server/src/streaming/streams/storage.rs
@@ -20,7 +20,6 @@ use crate::state::system::StreamState;
use crate::streaming::storage::StreamStorage;
use crate::streaming::streams::COMPONENT;
use crate::streaming::streams::stream::Stream;
-use crate::streaming::topics::topic::CreatedTopicInfo;
use crate::streaming::topics::topic::Topic;
use ahash::AHashSet;
use error_set::ErrContext;
@@ -28,11 +27,11 @@ use futures::future::join_all;
use iggy_common::IggyError;
use iggy_common::IggyTimestamp;
use serde::{Deserialize, Serialize};
+use tokio::sync::Mutex;
use std::path::Path;
use std::sync::Arc;
-use tokio::fs;
-use tokio::fs::create_dir_all;
-use tokio::sync::Mutex;
+use monoio::fs;
+use monoio::fs::create_dir_all;
use tracing::{error, info, warn};
#[derive(Debug)]
@@ -52,13 +51,13 @@ impl StreamStorage for FileStreamStorage {
}
let mut unloaded_topics = Vec::new();
- let dir_entries = fs::read_dir(&stream.topics_path).await;
+ let dir_entries = std::fs::read_dir(&stream.topics_path);
if dir_entries.is_err() {
return Err(IggyError::CannotReadTopics(stream.stream_id));
}
let mut dir_entries = dir_entries.unwrap();
- while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) {
+ while let Some(dir_entry) = dir_entries.next().transpose().unwrap_or(None) {
let name = dir_entry.file_name().into_string().unwrap();
let topic_id = name.parse::<u32>();
if topic_id.is_err() {
@@ -73,7 +72,8 @@ impl StreamStorage for FileStreamStorage {
error!(
"Topic with ID: '{topic_id}' for stream with ID: '{stream_id}' was not found in state, but exists on disk and will be removed."
);
- if let Err(error) = fs::remove_dir_all(&dir_entry.path()).await {
+ // TODO: Replace this with the dir walk impl that is mentioed in main function.
+ if let Err(error) = std::fs::remove_dir_all(&dir_entry.path()) {
error!("Cannot remove topic directory: {error}");
} else {
warn!(
@@ -151,6 +151,7 @@ impl StreamStorage for FileStreamStorage {
}
}
+ // TODO: Refactor....
let loaded_topics = Arc::new(Mutex::new(Vec::new()));
let mut load_topics = Vec::new();
for mut topic in unloaded_topics {
@@ -222,7 +223,7 @@ impl StreamStorage for FileStreamStorage {
async fn delete(&self, stream: &Stream) -> Result<(), IggyError> {
info!("Deleting stream with ID: {}...", stream.stream_id);
- if fs::remove_dir_all(&stream.path).await.is_err() {
+ if std::fs::remove_dir_all(&stream.path).is_err() {
return Err(IggyError::CannotDeleteStreamDirectory(stream.stream_id));
}
info!("Deleted stream with ID: {}.", stream.stream_id);
diff --git a/core/server/src/streaming/topics/messages.rs b/core/server/src/streaming/topics/messages.rs
index 2f8f0557..f56ed560 100644
--- a/core/server/src/streaming/topics/messages.rs
+++ b/core/server/src/streaming/topics/messages.rs
@@ -25,8 +25,8 @@ use crate::streaming::utils::hash;
use ahash::AHashMap;
use error_set::ErrContext;
use iggy_common::locking::IggySharedMutFn;
-use iggy_common::{IggyTimestamp, PollingStrategy};
use iggy_common::{IggyError, IggyExpiry, Partitioning, PartitioningKind, PollingKind};
+use iggy_common::{IggyTimestamp, PollingStrategy};
use std::sync::atomic::Ordering;
use tracing::trace;
diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs
index 827ba6c9..1025dbd3 100644
--- a/core/server/src/streaming/topics/storage.rs
+++ b/core/server/src/streaming/topics/storage.rs
@@ -29,11 +29,11 @@ use futures::future::join_all;
use iggy_common::IggyError;
use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
+use monoio::fs::create_dir_all;
use serde::{Deserialize, Serialize};
use std::path::Path;
use std::sync::Arc;
-use tokio::fs;
-use tokio::fs::create_dir_all;
+use monoio::fs;
use tokio::sync::Mutex;
use tracing::{error, info, warn};
@@ -61,14 +61,14 @@ impl TopicStorage for FileTopicStorage {
topic.compression_algorithm = state.compression_algorithm;
topic.replication_factor = state.replication_factor.unwrap_or(1);
- let mut dir_entries = fs::read_dir(&topic.partitions_path).await
+ let mut dir_entries = std::fs::read_dir(&topic.partitions_path)
.with_context(|| format!("Failed to read partition with ID: {} for stream with ID: {} for topic with ID: {} and path: {}",
topic.topic_id, topic.stream_id, topic.topic_id, &topic.partitions_path))
.map_err(|_| IggyError::CannotReadPartitions)?;
let mut unloaded_partitions = Vec::new();
- while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) {
- let metadata = dir_entry.metadata().await;
+ while let Some(dir_entry) = dir_entries.next().transpose().unwrap_or(None) {
+ let metadata = dir_entry.metadata();
if metadata.is_err() || metadata.unwrap().is_file() {
continue;
}
@@ -88,7 +88,8 @@ impl TopicStorage for FileTopicStorage {
error!(
"Partition with ID: '{partition_id}' for stream with ID: '{stream_id}' and topic with ID: '{topic_id}' was not found in state, but exists on disk and will be removed."
);
- if let Err(error) = fs::remove_dir_all(&dir_entry.path()).await {
+ // TODO: Replace this with the dir walk impl that is mentioed in main function.
+ if let Err(error) = std::fs::remove_dir_all(&dir_entry.path()) {
error!("Cannot remove partition directory: {error}");
} else {
warn!(
@@ -269,7 +270,7 @@ impl TopicStorage for FileTopicStorage {
async fn delete(&self, topic: &Topic) -> Result<(), IggyError> {
info!("Deleting topic {topic}...");
- if fs::remove_dir_all(&topic.path).await.is_err() {
+ if std::fs::remove_dir_all(&topic.path).is_err() {
return Err(IggyError::CannotDeleteTopicDirectory(
topic.topic_id,
topic.stream_id,
diff --git a/core/server/src/streaming/utils/memory_pool.rs b/core/server/src/streaming/utils/memory_pool.rs
index 198e8626..16ad2087 100644
--- a/core/server/src/streaming/utils/memory_pool.rs
+++ b/core/server/src/streaming/utils/memory_pool.rs
@@ -155,7 +155,8 @@ impl MemoryPool {
/// Initialize the global pool from the given config.
pub fn init_pool(config: Arc<SystemConfig>) {
- let is_enabled = config.memory_pool.enabled;
+ // TODO: Fixme (stary napraw).
+ let is_enabled = false;
let memory_limit = config.memory_pool.size.as_bytes_usize();
let bucket_capacity = config.memory_pool.bucket_capacity as usize;
diff --git a/core/server/src/streaming/utils/pooled_buffer.rs b/core/server/src/streaming/utils/pooled_buffer.rs
index 0aa1dd3a..a8664325 100644
--- a/core/server/src/streaming/utils/pooled_buffer.rs
+++ b/core/server/src/streaming/utils/pooled_buffer.rs
@@ -18,7 +18,7 @@
use super::memory_pool::{BytesMutExt, memory_pool};
use bytes::{Buf, BufMut, BytesMut};
-use monoio::buf::IoBufMut;
+use monoio::buf::{IoBuf, IoBufMut};
use std::ops::{Deref, DerefMut};
#[derive(Debug)]
@@ -230,3 +230,13 @@ unsafe impl IoBufMut for PooledBuffer {
unsafe { self.inner.set_init(pos) }
}
}
+
+unsafe impl IoBuf for PooledBuffer {
+ fn read_ptr(&self) -> *const u8 {
+ self.inner.read_ptr()
+ }
+
+ fn bytes_init(&self) -> usize {
+ self.inner.bytes_init()
+ }
+}
diff --git a/core/server/src/tcp/connection_handler.rs b/core/server/src/tcp/connection_handler.rs
index 9c92d27d..5622d2a8 100644
--- a/core/server/src/tcp/connection_handler.rs
+++ b/core/server/src/tcp/connection_handler.rs
@@ -35,8 +35,14 @@ pub(crate) async fn handle_connection(
sender: &mut SenderKind,
shard: &Rc<IggyShard>,
) -> Result<(), ConnectionError> {
- let length_buffer = BytesMut::with_capacity(INITIAL_BYTES_LENGTH);
- let code_buffer = BytesMut::with_capacity(INITIAL_BYTES_LENGTH);
+ let mut length_buffer = BytesMut::with_capacity(INITIAL_BYTES_LENGTH);
+ unsafe {
+ length_buffer.set_len(INITIAL_BYTES_LENGTH);
+ }
+ let mut code_buffer = BytesMut::with_capacity(INITIAL_BYTES_LENGTH);
+ unsafe {
+ code_buffer.set_len(INITIAL_BYTES_LENGTH);
+ }
loop {
let (read_length, initial_buffer) = match sender.read(length_buffer.clone()).await {
(Ok(read_length), initial_buffer) => (read_length, initial_buffer),
diff --git a/core/server/src/tcp/sender.rs b/core/server/src/tcp/sender.rs
index 2af9f494..2f445d4f 100644
--- a/core/server/src/tcp/sender.rs
+++ b/core/server/src/tcp/sender.rs
@@ -119,14 +119,14 @@ where
status
);
let prefix = [
- libc::iovec {
- iov_base: length.as_ptr() as _,
- iov_len: length.len(),
- },
libc::iovec {
iov_base: status.as_ptr() as _,
iov_len: status.len(),
},
+ libc::iovec {
+ iov_base: length.as_ptr() as _,
+ iov_len: length.len(),
+ },
];
slices.splice(0..0, prefix);
stream
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index ac29e2b9..c0080edf 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -18,15 +18,16 @@
use crate::binary::sender::SenderKind;
use crate::shard::IggyShard;
-use crate::shard::transmission::message::ShardEvent;
+use crate::shard::transmission::event::ShardEvent;
use crate::streaming::clients::client_manager::Transport;
use crate::tcp::connection_handler::{handle_connection, handle_error};
use crate::tcp::tcp_socket;
+use iggy_common::IggyError;
use std::net::SocketAddr;
use std::rc::Rc;
use tracing::{error, info};
-pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) {
+pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) -> Result<(), IggyError> {
let ip_v6 = shard.config.tcp.ipv6;
let socket_config = &shard.config.tcp.socket;
let addr: SocketAddr = shard
@@ -40,8 +41,8 @@ pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) {
monoio::spawn(async move {
socket
.bind(&addr.into())
- .expect("Failed to bind eTCP listener");
- socket.listen(1024);
+ .expect("Failed to bind TCP listener");
+ socket.listen(1024).unwrap();
let listener: std::net::TcpListener = socket.into();
let listener = monoio::net::TcpListener::from_std(listener).unwrap();
info!("{server_name} server has started on: {:?}", addr);
@@ -50,15 +51,14 @@ pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) {
Ok((stream, address)) => {
let shard = shard.clone();
info!("Accepted new TCP connection: {address}");
- let session = shard.add_client(&address, Transport::Tcp);
+ let transport = Transport::Tcp;
+ let session = shard.add_client(&address, transport);
//TODO: Those can be shared with other shards.
shard.add_active_session(session.clone());
// Broadcast session to all shards.
- //TODO: Fixme
- /*
- let event = Rc::new(ShardEvent::NewSession(session.clone()));
- shard.broadcast_event_to_all_shards(session.client_id, event);
- */
+ let event = ShardEvent::NewSession { address, transport };
+ // TODO: Fixme look inside of broadcast_event_to_all_shards method.
+ let _responses = shard.broadcast_event_to_all_shards(event.into());
let _client_id = session.client_id;
info!("Created new session: {session}");
@@ -85,5 +85,6 @@ pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) {
Err(error) => error!("Unable to accept TCP socket. {error}"),
}
}
- });
+ })
+ .await
}
diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs
index 9b8e1c04..0430f773 100644
--- a/core/server/src/tcp/tcp_server.rs
+++ b/core/server/src/tcp/tcp_server.rs
@@ -24,17 +24,18 @@ use tracing::info;
/// Starts the TCP server.
/// Returns the address the server is listening on.
-pub async fn start(shard: Rc<IggyShard>) -> Result<(), IggyError> {
+pub async fn spawn_tcp_server(shard: Rc<IggyShard>) -> Result<(), IggyError> {
let server_name = if shard.config.tcp.tls.enabled {
"Iggy TCP TLS"
} else {
"Iggy TCP"
};
info!("Initializing {server_name} server...");
- let addr = match shard.config.tcp.tls.enabled {
+ // TODO: Fixme -- storing addr of the server inside of the config for integration tests...
+ let result = match shard.config.tcp.tls.enabled {
true => unimplemented!("TLS support is not implemented yet"),
false => tcp_listener::start(server_name, shard).await,
};
- info!("{server_name} server has started on: {:?}", addr);
- Ok(())
+ //info!("{server_name} server has started on: {:?}", addr);
+ result
}
diff --git a/core/server/src/tcp/tcp_socket.rs b/core/server/src/tcp/tcp_socket.rs
index 747540fa..7ea3acf0 100644
--- a/core/server/src/tcp/tcp_socket.rs
+++ b/core/server/src/tcp/tcp_socket.rs
@@ -30,6 +30,15 @@ pub fn build(ipv6: bool, config: &TcpSocketConfig) -> Socket {
.expect("Unable to create an ipv4 socket")
};
+ // Required by the thread-per-core model...
+ // We create bunch of sockets on different threads, that bind to exactly the same address and port.
+ socket
+ .set_reuse_address(true)
+ .expect("Unable to set SO_REUSEADDR on socket");
+ socket
+ .set_reuse_port(true)
+ .expect("Unable to set SO_REUSEPORT on socket");
+
if config.override_defaults {
config
.recv_buffer_size
@@ -54,12 +63,6 @@ pub fn build(ipv6: bool, config: &TcpSocketConfig) -> Socket {
socket
.set_linger(Some(config.linger.get_duration()))
.expect("Unable to set SO_LINGER on socket");
- socket
- .set_reuse_address(true)
- .expect("Unable to set SO_REUSEADDR on socket");
- socket
- .set_reuse_port(true)
- .expect("Unable to set SO_REUSEPORT on socket");
}
socket
(iggy) 17/30: fix segment writing/reading
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 492ea25d0be73d9216ef1ed678007ebe1b77869a
Author: numinex <g....@gmail.com>
AuthorDate: Sat Jun 28 14:05:05 2025 +0200
fix segment writing/reading
---
core/common/src/types/confirmation/mod.rs | 1 -
core/integration/tests/archiver/disk.rs | 6 +
core/integration/tests/state/mod.rs | 7 +-
.../tests/streaming/common/test_setup.rs | 5 +-
core/integration/tests/streaming/get_by_offset.rs | 2 +-
.../tests/streaming/get_by_timestamp.rs | 2 +-
core/integration/tests/streaming/messages.rs | 2 +-
core/integration/tests/streaming/mod.rs | 2 +-
core/integration/tests/streaming/partition.rs | 2 +-
core/integration/tests/streaming/segment.rs | 160 +--------------
.../tests/streaming/{system.rs => shard.rs} | 5 +-
core/integration/tests/streaming/snapshot.rs | 5 +-
core/integration/tests/streaming/stream.rs | 2 +-
core/integration/tests/streaming/topic.rs | 2 +-
core/integration/tests/streaming/topic_messages.rs | 8 +-
core/server/Cargo.toml | 2 +-
core/server/src/archiver/disk.rs | 5 +-
core/server/src/archiver/mod.rs | 8 +-
core/server/src/archiver/s3.rs | 4 +
.../handlers/messages/send_messages_handler.rs | 1 -
core/server/src/bootstrap.rs | 1 -
.../src/compat/index_rebuilding/index_rebuilder.rs | 43 +++--
core/server/src/io/file.rs | 122 ++++++++++++
core/server/src/io/mod.rs | 3 +
core/server/src/io/reader.rs | 44 +++++
core/server/src/io/writer.rs | 57 ++++++
core/server/src/lib.rs | 1 +
core/server/src/shard/system/messages.rs | 3 +-
core/server/src/shard/system/storage.rs | 14 +-
core/server/src/state/file.rs | 25 ++-
core/server/src/state/mod.rs | 8 +-
core/server/src/streaming/partitions/messages.rs | 29 ++-
core/server/src/streaming/partitions/storage.rs | 44 +++--
core/server/src/streaming/persistence/persister.rs | 58 +++---
core/server/src/streaming/persistence/task.rs | 24 +--
.../streaming/segments/messages/messages_reader.rs | 30 ++-
.../streaming/segments/messages/messages_writer.rs | 94 +++------
core/server/src/streaming/segments/messages/mod.rs | 58 ++++--
.../streaming/segments/messages/persister_task.rs | 215 ---------------------
core/server/src/streaming/segments/segment.rs | 7 +-
.../src/streaming/segments/writing_messages.rs | 13 +-
core/server/src/streaming/topics/messages.rs | 10 +-
core/server/src/streaming/topics/persistence.rs | 2 +-
core/server/src/streaming/utils/file.rs | 12 +-
44 files changed, 516 insertions(+), 632 deletions(-)
diff --git a/core/common/src/types/confirmation/mod.rs b/core/common/src/types/confirmation/mod.rs
index 5289a627..bc26cffa 100644
--- a/core/common/src/types/confirmation/mod.rs
+++ b/core/common/src/types/confirmation/mod.rs
@@ -24,7 +24,6 @@ use strum::{Display, EnumString};
pub enum Confirmation {
#[default]
Wait,
- NoWait,
}
#[cfg(test)]
diff --git a/core/integration/tests/archiver/disk.rs b/core/integration/tests/archiver/disk.rs
index f676d23f..00901190 100644
--- a/core/integration/tests/archiver/disk.rs
+++ b/core/integration/tests/archiver/disk.rs
@@ -110,11 +110,16 @@ async fn should_fail_when_file_to_archive_does_not_exist() {
}
async fn create_file(path: &str, content: &str) {
+ // TODO: Fixme
+ /*
let mut file = file::overwrite(path).await.unwrap();
file.write_all(content.as_bytes()).await.unwrap();
+ */
}
async fn assert_archived_file(file_to_archive_path: &str, archived_file_path: &str, content: &str) {
+ // TODO: Fixme
+ /*
assert!(Path::new(&file_to_archive_path).exists());
assert!(Path::new(&archived_file_path).exists());
let archived_file = file::open(archived_file_path).await;
@@ -126,4 +131,5 @@ async fn assert_archived_file(file_to_archive_path: &str, archived_file_path: &s
.await
.unwrap();
assert_eq!(content, archived_file_content);
+ */
}
diff --git a/core/integration/tests/state/mod.rs b/core/integration/tests/state/mod.rs
index d6729d2e..e125a395 100644
--- a/core/integration/tests/state/mod.rs
+++ b/core/integration/tests/state/mod.rs
@@ -50,11 +50,8 @@ impl StateSetup {
let version = SemanticVersion::from_str("1.2.3").unwrap();
let persister = PersisterKind::FileWithSync(FileWithSyncPersister {});
- let encryptor = encryption_key.map(|key| {
- Arc::new(EncryptorKind::Aes256Gcm(
- Aes256GcmEncryptor::new(key).unwrap(),
- ))
- });
+ let encryptor = encryption_key
+ .map(|key| EncryptorKind::Aes256Gcm(Aes256GcmEncryptor::new(key).unwrap()));
let state = FileState::new(
&messages_file_path,
&version,
diff --git a/core/integration/tests/streaming/common/test_setup.rs b/core/integration/tests/streaming/common/test_setup.rs
index c5971762..19363249 100644
--- a/core/integration/tests/streaming/common/test_setup.rs
+++ b/core/integration/tests/streaming/common/test_setup.rs
@@ -20,13 +20,14 @@ use server::configs::system::SystemConfig;
use server::streaming::persistence::persister::{FileWithSyncPersister, PersisterKind};
use server::streaming::storage::SystemStorage;
use server::streaming::utils::MemoryPool;
+use std::rc::Rc;
use std::sync::Arc;
use tokio::fs;
use uuid::Uuid;
pub struct TestSetup {
pub config: Arc<SystemConfig>,
- pub storage: Arc<SystemStorage>,
+ pub storage: Rc<SystemStorage>,
}
impl TestSetup {
@@ -42,7 +43,7 @@ impl TestSetup {
let config = Arc::new(config);
fs::create_dir(config.get_system_path()).await.unwrap();
let persister = PersisterKind::FileWithSync(FileWithSyncPersister {});
- let storage = Arc::new(SystemStorage::new(config.clone(), Arc::new(persister)));
+ let storage = Rc::new(SystemStorage::new(config.clone(), Arc::new(persister)));
MemoryPool::init_pool(config.clone());
TestSetup { config, storage }
}
diff --git a/core/integration/tests/streaming/get_by_offset.rs b/core/integration/tests/streaming/get_by_offset.rs
index 68690c4f..df9f8f40 100644
--- a/core/integration/tests/streaming/get_by_offset.rs
+++ b/core/integration/tests/streaming/get_by_offset.rs
@@ -195,7 +195,7 @@ async fn test_get_messages_by_offset(
let batch = IggyMessagesBatchMut::from_messages(messages_slice_to_append, messages_size);
assert_eq!(batch.count(), batch_len);
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
batch_offsets.push(partition.current_offset);
current_pos += batch_len as usize;
diff --git a/core/integration/tests/streaming/get_by_timestamp.rs b/core/integration/tests/streaming/get_by_timestamp.rs
index ae0ab88b..d0cc2e5b 100644
--- a/core/integration/tests/streaming/get_by_timestamp.rs
+++ b/core/integration/tests/streaming/get_by_timestamp.rs
@@ -203,7 +203,7 @@ async fn test_get_messages_by_timestamp(
let batch = IggyMessagesBatchMut::from_messages(messages_slice_to_append, messages_size);
assert_eq!(batch.count(), batch_len);
partition
- .append_messages(batch, Some(Confirmation::Wait))
+ .append_messages(batch)
.await
.unwrap();
diff --git a/core/integration/tests/streaming/messages.rs b/core/integration/tests/streaming/messages.rs
index e3529ab7..6ab6ca5b 100644
--- a/core/integration/tests/streaming/messages.rs
+++ b/core/integration/tests/streaming/messages.rs
@@ -104,7 +104,7 @@ async fn should_persist_messages_and_then_load_them_from_disk() {
.map(|msg| msg.get_size_bytes().as_bytes_u32())
.sum::<u32>();
let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
assert_eq!(
partition.unsaved_messages_count, 0,
"Expected unsaved messages count to be 0, but got {}",
diff --git a/core/integration/tests/streaming/mod.rs b/core/integration/tests/streaming/mod.rs
index b5df4c0a..f51b875b 100644
--- a/core/integration/tests/streaming/mod.rs
+++ b/core/integration/tests/streaming/mod.rs
@@ -26,9 +26,9 @@ mod get_by_timestamp;
mod messages;
mod partition;
mod segment;
+mod shard;
mod snapshot;
mod stream;
-mod system;
mod topic;
mod topic_messages;
diff --git a/core/integration/tests/streaming/partition.rs b/core/integration/tests/streaming/partition.rs
index 5a7e5866..4fff0d04 100644
--- a/core/integration/tests/streaming/partition.rs
+++ b/core/integration/tests/streaming/partition.rs
@@ -196,7 +196,7 @@ async fn should_purge_existing_partition_on_disk() {
.map(|msg| msg.get_size_bytes().as_bytes_u32())
.sum();
let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
let loaded_messages = partition.get_messages_by_offset(0, 100).await.unwrap();
assert_eq!(loaded_messages.count(), messages_count);
partition.purge().await.unwrap();
diff --git a/core/integration/tests/streaming/segment.rs b/core/integration/tests/streaming/segment.rs
index 403418d8..c5ec065e 100644
--- a/core/integration/tests/streaming/segment.rs
+++ b/core/integration/tests/streaming/segment.rs
@@ -25,7 +25,6 @@ use server::configs::server::{DataMaintenanceConfig, PersonalAccessTokenConfig};
use server::configs::system::{PartitionConfig, SegmentConfig, SystemConfig};
use server::streaming::segments::*;
use server::streaming::session::Session;
-use server::streaming::systems::system::System;
use std::fs::DirEntry;
use std::net::{Ipv4Addr, SocketAddr};
use std::str::FromStr;
@@ -192,7 +191,7 @@ async fn should_persist_and_load_segment_with_messages() {
let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
segment.append_batch(0, batch, None).await.unwrap();
- segment.persist_messages(None).await.unwrap();
+ segment.persist_messages().await.unwrap();
let mut loaded_segment = Segment::create(
stream_id,
topic_id,
@@ -216,158 +215,6 @@ async fn should_persist_and_load_segment_with_messages() {
assert_eq!(messages.count(), messages_count);
}
-#[tokio::test]
-async fn should_persist_and_load_segment_with_messages_with_nowait_confirmation() {
- let setup = TestSetup::init_with_config(SystemConfig {
- segment: SegmentConfig {
- server_confirmation: Confirmation::NoWait,
- ..Default::default()
- },
- ..Default::default()
- })
- .await;
- let stream_id = 1;
- let topic_id = 2;
- let partition_id = 3;
- let start_offset = 0;
- let mut segment = Segment::create(
- stream_id,
- topic_id,
- partition_id,
- start_offset,
- setup.config.clone(),
- IggyExpiry::NeverExpire,
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- true,
- );
-
- setup
- .create_partition_directory(stream_id, topic_id, partition_id)
- .await;
- segment.persist().await.unwrap();
- assert_persisted_segment(
- &setup
- .config
- .get_partition_path(stream_id, topic_id, partition_id),
- start_offset,
- )
- .await;
- let messages_count = 10;
- let mut messages = Vec::new();
- let mut messages_size = 0;
- for i in 0..messages_count {
- let message = IggyMessage::builder()
- .id(i as u128)
- .payload(Bytes::from("test"))
- .build()
- .expect("Failed to create message");
- messages_size += message.get_size_bytes().as_bytes_u32();
- messages.push(message);
- }
- let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- segment.append_batch(0, batch, None).await.unwrap();
- segment
- .persist_messages(Some(Confirmation::NoWait))
- .await
- .unwrap();
- sleep(Duration::from_millis(200)).await;
- let mut loaded_segment = Segment::create(
- stream_id,
- topic_id,
- partition_id,
- start_offset,
- setup.config.clone(),
- IggyExpiry::NeverExpire,
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- false,
- );
- loaded_segment.load_from_disk().await.unwrap();
- let messages = loaded_segment
- .get_messages_by_offset(0, messages_count)
- .await
- .unwrap();
- assert_eq!(messages.count(), messages_count);
-}
-
-#[tokio::test]
-async fn given_all_expired_messages_segment_should_be_expired() {
- let config = SystemConfig {
- partition: PartitionConfig {
- enforce_fsync: true,
- ..Default::default()
- },
- segment: SegmentConfig {
- size: IggyByteSize::from_str("10B").unwrap(), // small size to force expiration
- ..Default::default()
- },
- ..Default::default()
- };
- let setup = TestSetup::init_with_config(config).await;
- let stream_id = 1;
- let topic_id = 2;
- let partition_id = 3;
- let start_offset = 0;
- let message_expiry_us = 100000;
- let message_expiry = message_expiry_us.into();
- let mut segment = Segment::create(
- stream_id,
- topic_id,
- partition_id,
- start_offset,
- setup.config.clone(),
- message_expiry,
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- Arc::new(AtomicU64::new(0)),
- true,
- );
-
- setup
- .create_partition_directory(stream_id, topic_id, partition_id)
- .await;
- segment.persist().await.unwrap();
- assert_persisted_segment(
- &setup
- .config
- .get_partition_path(stream_id, topic_id, partition_id),
- start_offset,
- )
- .await;
- let messages_count = 10;
- let mut messages = Vec::new();
- let mut messages_size = 0;
- for i in 0..messages_count {
- let message = IggyMessage::builder()
- .id(i as u128)
- .payload(Bytes::from("test"))
- .build()
- .expect("Failed to create message");
- messages_size += message.get_size_bytes().as_bytes_u32();
- messages.push(message);
- }
- let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- segment.append_batch(0, batch, None).await.unwrap();
- segment.persist_messages(None).await.unwrap();
- let not_expired_ts = IggyTimestamp::now();
- let expired_ts = not_expired_ts + IggyDuration::from(message_expiry_us + 1);
-
- assert!(segment.is_expired(expired_ts).await);
- assert!(!segment.is_expired(not_expired_ts).await);
-}
-
#[tokio::test]
async fn given_at_least_one_not_expired_message_segment_should_not_be_expired() {
let config = SystemConfig {
@@ -444,7 +291,7 @@ async fn given_at_least_one_not_expired_message_segment_should_not_be_expired()
let second_message_expired_ts =
IggyTimestamp::now() + IggyDuration::from(message_expiry_us * 2);
- segment.persist_messages(None).await.unwrap();
+ segment.persist_messages().await.unwrap();
assert!(
!segment.is_expired(nothing_expired_ts).await,
@@ -460,6 +307,8 @@ async fn given_at_least_one_not_expired_message_segment_should_not_be_expired()
);
}
+/*
+//TODO: Fixme use shard instead of system
#[tokio::test]
async fn should_delete_persisted_segments() -> Result<(), Box<dyn std::error::Error>> {
let config = SystemConfig {
@@ -648,6 +497,7 @@ async fn should_delete_persisted_segments() -> Result<(), Box<dyn std::error::Er
Ok(())
}
+*/
// Helper function to extract segment offsets from DirEntry list
fn get_segment_offsets(segments: &[DirEntry]) -> Vec<u64> {
diff --git a/core/integration/tests/streaming/system.rs b/core/integration/tests/streaming/shard.rs
similarity index 98%
rename from core/integration/tests/streaming/system.rs
rename to core/integration/tests/streaming/shard.rs
index ce950e84..454a5841 100644
--- a/core/integration/tests/streaming/system.rs
+++ b/core/integration/tests/streaming/shard.rs
@@ -20,10 +20,11 @@ use crate::streaming::common::test_setup::TestSetup;
use iggy::prelude::Identifier;
use server::configs::server::{DataMaintenanceConfig, PersonalAccessTokenConfig};
use server::streaming::session::Session;
-use server::streaming::systems::system::System;
use std::net::{Ipv4Addr, SocketAddr};
use tokio::fs;
+//TODO: Fix me use shard instead of system
+/*
#[tokio::test]
async fn should_initialize_system_and_base_directories() {
let setup = TestSetup::init().await;
@@ -127,3 +128,5 @@ async fn assert_persisted_stream(streams_path: &str, stream_id: u32) {
let stream_metadata = fs::metadata(stream_path).await.unwrap();
assert!(stream_metadata.is_dir());
}
+
+*/
diff --git a/core/integration/tests/streaming/snapshot.rs b/core/integration/tests/streaming/snapshot.rs
index 52b9419a..ae576c6a 100644
--- a/core/integration/tests/streaming/snapshot.rs
+++ b/core/integration/tests/streaming/snapshot.rs
@@ -20,11 +20,12 @@ use crate::streaming::common::test_setup::TestSetup;
use iggy::prelude::{SnapshotCompression, SystemSnapshotType};
use server::configs::server::{DataMaintenanceConfig, PersonalAccessTokenConfig};
use server::streaming::session::Session;
-use server::streaming::systems::system::System;
use std::io::{Cursor, Read};
use std::net::{Ipv4Addr, SocketAddr};
use zip::ZipArchive;
+//TODO: Fix me use shard instead of system
+/*
#[tokio::test]
async fn should_create_snapshot_file() {
let setup = TestSetup::init().await;
@@ -55,3 +56,5 @@ async fn should_create_snapshot_file() {
test_file.read_to_string(&mut test_content).unwrap();
assert_eq!(test_content, "test\n");
}
+
+*/
diff --git a/core/integration/tests/streaming/stream.rs b/core/integration/tests/streaming/stream.rs
index 417c00d4..5b340750 100644
--- a/core/integration/tests/streaming/stream.rs
+++ b/core/integration/tests/streaming/stream.rs
@@ -146,7 +146,7 @@ async fn should_purge_existing_stream_on_disk() {
.get_topic(&Identifier::numeric(topic_id).unwrap())
.unwrap();
topic
- .append_messages(&Partitioning::partition_id(1), batch, None)
+ .append_messages(&Partitioning::partition_id(1), batch)
.await
.unwrap();
let (_, loaded_messages) = topic
diff --git a/core/integration/tests/streaming/topic.rs b/core/integration/tests/streaming/topic.rs
index 3ae2c5f0..f75ffe2b 100644
--- a/core/integration/tests/streaming/topic.rs
+++ b/core/integration/tests/streaming/topic.rs
@@ -226,7 +226,7 @@ async fn should_purge_existing_topic_on_disk() {
.sum::<u32>();
let batch = IggyMessagesBatchMut::from_messages(&messages, batch_size);
topic
- .append_messages(&Partitioning::partition_id(1), batch, None)
+ .append_messages(&Partitioning::partition_id(1), batch)
.await
.unwrap();
let (_, loaded_messages) = topic
diff --git a/core/integration/tests/streaming/topic_messages.rs b/core/integration/tests/streaming/topic_messages.rs
index f716ef9d..f21263c4 100644
--- a/core/integration/tests/streaming/topic_messages.rs
+++ b/core/integration/tests/streaming/topic_messages.rs
@@ -63,7 +63,7 @@ async fn assert_polling_messages() {
.sum::<IggyByteSize>();
let batch = IggyMessagesBatchMut::from_messages(&messages, batch_size.as_bytes_u32());
topic
- .append_messages(&partitioning, batch, None)
+ .append_messages(&partitioning, batch)
.await
.unwrap();
@@ -110,7 +110,7 @@ async fn given_key_none_messages_should_be_appended_to_the_next_partition_using_
batch_size.as_bytes_u32(),
);
topic
- .append_messages(&partitioning, messages, None)
+ .append_messages(&partitioning, messages)
.await
.unwrap();
}
@@ -139,7 +139,7 @@ async fn given_key_partition_id_messages_should_be_appended_to_the_chosen_partit
batch_size.as_bytes_u32(),
);
topic
- .append_messages(&partitioning, messages, None)
+ .append_messages(&partitioning, messages)
.await
.unwrap();
}
@@ -172,7 +172,7 @@ async fn given_key_messages_key_messages_should_be_appended_to_the_calculated_pa
batch_size.as_bytes_u32(),
);
topic
- .append_messages(&partitioning, messages, None)
+ .append_messages(&partitioning, messages)
.await
.unwrap();
}
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 70f6a7fd..3b9694f5 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -66,7 +66,7 @@ lending-iterator = "0.1.7"
hash32 = "1.0.0"
mimalloc = { workspace = true, optional = true }
moka = { version = "0.12.10", features = ["future"] }
-monoio = { version = "0.2.4", features = ["mkdirat", "unlinkat"] }
+monoio = { version = "0.2.4", features = ["mkdirat", "unlinkat", "renameat"] }
monoio-native-tls = "0.4.0"
nix = { version = "0.30", features = ["fs"] }
once_cell = "1.21.3"
diff --git a/core/server/src/archiver/disk.rs b/core/server/src/archiver/disk.rs
index c0affe7f..0eafa20d 100644
--- a/core/server/src/archiver/disk.rs
+++ b/core/server/src/archiver/disk.rs
@@ -70,7 +70,8 @@ impl Archiver for DiskArchiver {
files: &[&str],
base_directory: Option<String>,
) -> Result<(), ArchiverError> {
- debug!("Archiving files on disk: {:?}", files);
+ //TODO: Fixme figure this out, we can't use tokio methods there.
+ /* debug!("Archiving files on disk: {:?}", files);
for file in files {
debug!("Archiving file: {file}");
let source = Path::new(file);
@@ -93,7 +94,7 @@ impl Archiver for DiskArchiver {
})?;
debug!("Archived file: {file} at: {destination_path}");
}
-
+ */
Ok(())
}
}
diff --git a/core/server/src/archiver/mod.rs b/core/server/src/archiver/mod.rs
index 1ecf48b7..6d61dcad 100644
--- a/core/server/src/archiver/mod.rs
+++ b/core/server/src/archiver/mod.rs
@@ -53,18 +53,18 @@ impl FromStr for ArchiverKindType {
}
}
-pub trait Archiver: Send {
- fn init(&self) -> impl Future<Output = Result<(), ArchiverError>> + Send;
+pub trait Archiver {
+ fn init(&self) -> impl Future<Output = Result<(), ArchiverError>>;
fn is_archived(
&self,
file: &str,
base_directory: Option<String>,
- ) -> impl Future<Output = Result<bool, ArchiverError>> + Send;
+ ) -> impl Future<Output = Result<bool, ArchiverError>>;
fn archive(
&self,
files: &[&str],
base_directory: Option<String>,
- ) -> impl Future<Output = Result<(), ArchiverError>> + Send;
+ ) -> impl Future<Output = Result<(), ArchiverError>>;
}
#[derive(Debug)]
diff --git a/core/server/src/archiver/s3.rs b/core/server/src/archiver/s3.rs
index 036945d9..bbc9cd3b 100644
--- a/core/server/src/archiver/s3.rs
+++ b/core/server/src/archiver/s3.rs
@@ -157,6 +157,9 @@ impl Archiver for S3Archiver {
let base_directory = base_directory.as_deref().unwrap_or_default();
let destination = Path::new(&base_directory).join(path);
let destination_path = destination.to_str().unwrap_or_default().to_owned();
+ // TODO: Fixme figure this out.
+ // The `put_object_stream` method requires `AsyncRead` trait from tokio as its reader.
+ /*
let response = self
.bucket
.put_object_stream(&mut file, destination_path)
@@ -188,6 +191,7 @@ impl Archiver for S3Archiver {
return Err(ArchiverError::CannotArchiveFile {
file_path: (*path).to_string(),
});
+ */
}
Ok(())
}
diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs
index 5603dfe9..d9a4770f 100644
--- a/core/server/src/binary/handlers/messages/send_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs
@@ -114,7 +114,6 @@ impl ServerCommandHandler for SendMessages {
&self.topic_id,
&self.partitioning,
batch,
- None,
)
.await?;
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index 4f356a9b..ebf8946e 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -122,7 +122,6 @@ where
pub fn create_shard_executor() -> Runtime<TimeDriver<monoio::IoUringDriver>> {
// TODO: Figure out what else we could tweak there
// We for sure want to disable the userspace interrupts on new cq entry (set_coop_taskrun)
- // let urb = io_uring::IoUring::builder();
// TODO: Shall we make the size of ring be configureable ?
let builder = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new()
//.uring_builder(urb.setup_coop_taskrun()) // broken shit.
diff --git a/core/server/src/compat/index_rebuilding/index_rebuilder.rs b/core/server/src/compat/index_rebuilding/index_rebuilder.rs
index 542620ee..1eebdacd 100644
--- a/core/server/src/compat/index_rebuilding/index_rebuilder.rs
+++ b/core/server/src/compat/index_rebuilding/index_rebuilder.rs
@@ -16,11 +16,13 @@
* under the License.
*/
-use crate::server_error::CompatError;
+use crate::io::file::IggyFile;
+use crate::io::writer::IggyWriter;
use crate::streaming::utils::file;
+use crate::{io::reader::IggyReader, server_error::CompatError};
use iggy_common::{IGGY_MESSAGE_HEADER_SIZE, IggyMessageHeader};
-use std::io::SeekFrom;
-use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, BufReader, BufWriter};
+use monoio::io::{AsyncReadRent, AsyncReadRentExt, AsyncWriteRent, AsyncWriteRentExt};
+use std::io::{Seek, SeekFrom};
pub struct IndexRebuilder {
pub messages_file_path: String,
@@ -38,16 +40,17 @@ impl IndexRebuilder {
}
async fn read_message_header(
- reader: &mut BufReader<tokio::fs::File>,
+ reader: &mut IggyReader<IggyFile>,
) -> Result<IggyMessageHeader, std::io::Error> {
- let mut buf = [0u8; IGGY_MESSAGE_HEADER_SIZE];
- reader.read_exact(&mut buf).await?;
- IggyMessageHeader::from_raw_bytes(&buf)
+ let buf = [0u8; IGGY_MESSAGE_HEADER_SIZE];
+ let (result, buf) = reader.read_exact(Box::new(buf)).await;
+ result?;
+ IggyMessageHeader::from_raw_bytes(&*buf)
.map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidData))
}
async fn write_index_entry(
- writer: &mut BufWriter<tokio::fs::File>,
+ writer: &mut IggyWriter<IggyFile>,
header: &IggyMessageHeader,
position: usize,
start_offset: u64,
@@ -55,20 +58,26 @@ impl IndexRebuilder {
// Write offset (4 bytes) - base_offset + last_offset_delta - start_offset
let offset = start_offset - header.offset;
debug_assert!(offset <= u32::MAX as u64);
- writer.write_u32_le(offset as u32).await?;
+ let (result, _) = writer.write_all(Box::new(offset.to_le_bytes())).await;
+ result?;
// Write position (4 bytes)
- writer.write_u32_le(position as u32).await?;
+ let (result, _) = writer.write_all(Box::new(position.to_le_bytes())).await;
+ result?;
// Write timestamp (8 bytes)
- writer.write_u64_le(header.timestamp).await?;
+ let (result, _) = writer
+ .write_all(Box::new(header.timestamp.to_le_bytes()))
+ .await;
+ result?;
Ok(())
}
pub async fn rebuild(&self) -> Result<(), CompatError> {
- let mut reader = BufReader::new(file::open(&self.messages_file_path).await?);
- let mut writer = BufWriter::new(file::overwrite(&self.index_path).await?);
+ let mut reader =
+ IggyReader::new(IggyFile::new(file::open(&self.messages_file_path).await?));
+ let mut writer = IggyWriter::new(IggyFile::new(file::overwrite(&self.index_path).await?));
let mut position = 0;
let mut next_position;
@@ -84,11 +93,9 @@ impl IndexRebuilder {
.await?;
// Skip message payload and headers
- reader
- .seek(SeekFrom::Current(
- header.payload_length as i64 + header.user_headers_length as i64,
- ))
- .await?;
+ reader.seek(SeekFrom::Current(
+ header.payload_length as i64 + header.user_headers_length as i64,
+ ))?;
// Update position for next iteration
position = next_position;
diff --git a/core/server/src/io/file.rs b/core/server/src/io/file.rs
new file mode 100644
index 00000000..61e51023
--- /dev/null
+++ b/core/server/src/io/file.rs
@@ -0,0 +1,122 @@
+use std::io::SeekFrom;
+
+use monoio::{
+ BufResult,
+ buf::{IoBuf, IoBufMut, IoVecBuf, IoVecBufMut, IoVecWrapper},
+ fs::{File, Metadata},
+ io::{AsyncReadRent, AsyncWriteRent},
+};
+
+/// Wrapper around `monoio::fs::File` to provide a consistent API for reading and writing files
+/// in an asynchronous context. This struct maintains the current position in the file and provides
+/// methods to read and write data at specific positions.
+#[derive(Debug)]
+pub struct IggyFile {
+ file: File,
+ position: u64,
+}
+
+impl From<File> for IggyFile {
+ fn from(file: File) -> Self {
+ Self { file, position: 0 }
+ }
+}
+
+impl std::io::Seek for IggyFile {
+ /// This method doesn't do bound checking aswell as, the `End` variant is not supported.
+ fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+ self.position = match pos {
+ SeekFrom::Start(n) => n as i64,
+ SeekFrom::End(_) => unreachable!("End variant is not supported in IggyFile"),
+ SeekFrom::Current(n) => self.position as i64 + n,
+ } as u64;
+ Ok(self.position)
+ }
+}
+
+impl IggyFile {
+ pub fn new(file: File) -> Self {
+ Self { file, position: 0 }
+ }
+
+ pub async fn metadata(&self) -> std::io::Result<Metadata> {
+ self.file.metadata().await
+ }
+
+ pub async fn write_at<T: IoBuf>(&self, buf: T, pos: usize) -> BufResult<usize, T> {
+ self.file.write_at(buf, pos as u64).await
+ }
+
+ pub async fn write_all_at<T: IoBuf>(&self, buf: T, pos: u64) -> BufResult<(), T> {
+ self.file.write_all_at(buf, pos).await
+ }
+
+ pub async fn read_exact_at<T: IoBufMut>(&self, buf: T, pos: u64) -> BufResult<(), T> {
+ self.file.read_exact_at(buf, pos).await
+ }
+
+ pub async fn read_at<T: IoBufMut>(&self, buf: T, pos: usize) -> BufResult<usize, T> {
+ self.file.read_at(buf, pos as u64).await
+ }
+
+ pub async fn sync_all(&self) -> std::io::Result<()> {
+ self.file.sync_all().await
+ }
+}
+
+impl AsyncReadRent for IggyFile {
+ /// Reads `exactly` `buf.len()` bytes into the buffer.
+ async fn read<T: IoBufMut>(&mut self, buf: T) -> BufResult<usize, T> {
+ let (res, buf) = self.file.read_at(buf, self.position).await;
+ let n = match res {
+ Ok(n) => n,
+ Err(e) => return (Err(e), buf),
+ };
+ self.position += n as u64;
+ (Ok(n), buf)
+ }
+
+ //TODO(numinex) - maybe implement this ?
+ fn readv<T: IoVecBufMut>(&mut self, buf: T) -> impl Future<Output = BufResult<usize, T>> {
+ async move { (Ok(0), buf) }
+ }
+}
+
+impl AsyncWriteRent for IggyFile {
+ /// Writes entire buffer to the file at the current position.
+ async fn write<B: IoBuf>(&mut self, buf: B) -> BufResult<usize, B> {
+ let (res, buf) = self.file.write_at(buf, self.position).await;
+ let n = match res {
+ Ok(n) => n,
+ Err(e) => return (Err(e), buf),
+ };
+ self.position += n as u64;
+ (Ok(n), buf)
+ }
+
+ async fn writev<T: IoVecBuf>(&mut self, buf_vec: T) -> BufResult<usize, T> {
+ let slice = match IoVecWrapper::new(buf_vec) {
+ Ok(slice) => slice,
+ Err(buf) => return (Ok(0), buf),
+ };
+
+ let (result, slice) = self.write(slice).await;
+ (result, slice.into_inner())
+ }
+
+ async fn flush(&mut self) -> std::io::Result<()> {
+ self.file.sync_all().await
+ }
+
+ //TODO(numinex) - How to implement this ?
+ async fn shutdown(&mut self) -> std::io::Result<()> {
+ panic!("shutdown is not supported for IggyFile");
+ /*
+ self.file.sync_all().await;
+ unsafe {
+ let file = *self.file;
+ file.close().await
+ }
+ */
+ }
+}
diff --git a/core/server/src/io/mod.rs b/core/server/src/io/mod.rs
new file mode 100644
index 00000000..ea2bfab5
--- /dev/null
+++ b/core/server/src/io/mod.rs
@@ -0,0 +1,3 @@
+pub mod file;
+pub mod reader;
+pub mod writer;
diff --git a/core/server/src/io/reader.rs b/core/server/src/io/reader.rs
new file mode 100644
index 00000000..63443214
--- /dev/null
+++ b/core/server/src/io/reader.rs
@@ -0,0 +1,44 @@
+use std::io::{Seek, SeekFrom};
+
+use monoio::{
+ BufResult,
+ buf::{IoBufMut, IoVecBufMut},
+ io::{AsyncReadRent, BufReader},
+};
+
+pub struct IggyReader<R: AsyncReadRent + Seek> {
+ inner: BufReader<R>,
+}
+
+impl<R: AsyncReadRent + Seek> IggyReader<R> {
+ pub fn new(reader: R) -> Self {
+ Self {
+ inner: BufReader::new(reader),
+ }
+ }
+
+ pub fn with_capacity(capacity: usize, reader: R) -> Self {
+ Self {
+ inner: BufReader::with_capacity(capacity, reader),
+ }
+ }
+}
+
+impl<R: AsyncReadRent + Seek> Seek for IggyReader<R> {
+ fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+ self.inner.get_mut().seek(pos)
+ }
+}
+
+impl<R> AsyncReadRent for IggyReader<R>
+where
+ R: AsyncReadRent + Seek,
+{
+ fn read<T: IoBufMut>(&mut self, buf: T) -> impl Future<Output = BufResult<usize, T>> {
+ self.inner.read(buf)
+ }
+
+ fn readv<T: IoVecBufMut>(&mut self, buf: T) -> impl Future<Output = BufResult<usize, T>> {
+ self.inner.readv(buf)
+ }
+}
diff --git a/core/server/src/io/writer.rs b/core/server/src/io/writer.rs
new file mode 100644
index 00000000..e249e405
--- /dev/null
+++ b/core/server/src/io/writer.rs
@@ -0,0 +1,57 @@
+use std::io::{Seek, SeekFrom};
+
+use monoio::{
+ BufResult,
+ io::{AsyncWriteRent, BufWriter},
+};
+
+pub struct IggyWriter<W: AsyncWriteRent + Seek> {
+ inner: BufWriter<W>,
+}
+
+impl<W: AsyncWriteRent + Seek> IggyWriter<W> {
+ pub fn new(writer: W) -> Self {
+ Self {
+ inner: BufWriter::new(writer),
+ }
+ }
+
+ pub fn with_capacity(capacity: usize, writer: W) -> Self {
+ Self {
+ inner: BufWriter::with_capacity(capacity, writer),
+ }
+ }
+}
+
+impl<W: AsyncWriteRent + Seek> Seek for IggyWriter<W> {
+ fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+ self.inner.get_mut().seek(pos)
+ }
+}
+
+impl<W> AsyncWriteRent for IggyWriter<W>
+where
+ W: AsyncWriteRent + Seek,
+{
+ fn write<T: monoio::buf::IoBuf>(
+ &mut self,
+ buf: T,
+ ) -> impl Future<Output = BufResult<usize, T>> {
+ self.inner.write(buf)
+ }
+
+ fn writev<T: monoio::buf::IoVecBuf>(
+ &mut self,
+ buf_vec: T,
+ ) -> impl Future<Output = BufResult<usize, T>> {
+ self.inner.writev(buf_vec)
+ }
+
+ fn flush(&mut self) -> impl Future<Output = std::io::Result<()>> {
+ self.inner.flush()
+ }
+
+ fn shutdown(&mut self) -> impl Future<Output = std::io::Result<()>> {
+ self.inner.shutdown()
+ }
+}
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index f618d3e0..68be8afd 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -36,6 +36,7 @@ pub mod channels;
pub(crate) mod compat;
pub mod configs;
pub mod http;
+pub mod io;
pub mod log;
pub mod quic;
pub mod server_error;
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index 7668566a..9a23a9bb 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -105,7 +105,6 @@ impl IggyShard {
topic_id: &Identifier,
partitioning: &Partitioning,
messages: IggyMessagesBatchMut,
- confirmation: Option<Confirmation>,
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
let stream = self.get_stream(stream_id).with_error_context(|error| {
@@ -133,7 +132,7 @@ impl IggyShard {
};
topic
- .append_messages(partitioning, messages, confirmation)
+ .append_messages(partitioning, messages)
.await?;
self.metrics.increment_messages(messages_count as u64);
diff --git a/core/server/src/shard/system/storage.rs b/core/server/src/shard/system/storage.rs
index cc413583..3d895d4c 100644
--- a/core/server/src/shard/system/storage.rs
+++ b/core/server/src/shard/system/storage.rs
@@ -17,6 +17,7 @@
*/
use super::COMPONENT;
+use crate::io::file::IggyFile;
use crate::shard::system::info::SystemInfo;
use crate::streaming::persistence::persister::PersisterKind;
use crate::streaming::storage::SystemInfoStorage;
@@ -25,6 +26,7 @@ use crate::streaming::utils::file;
use anyhow::Context;
use error_set::ErrContext;
use iggy_common::IggyError;
+use monoio::io::AsyncReadRentExt;
use std::sync::Arc;
use tokio::io::AsyncReadExt;
use tracing::info;
@@ -48,7 +50,7 @@ impl SystemInfoStorage for FileSystemInfoStorage {
return Err(IggyError::ResourceNotFound(self.path.to_owned()));
}
- let mut file = file.unwrap();
+ let file = file.unwrap();
let file_size = file
.metadata()
.await
@@ -60,13 +62,15 @@ impl SystemInfoStorage for FileSystemInfoStorage {
})
.map_err(|_| IggyError::CannotReadFileMetadata)?
.len() as usize;
+
+ let mut file = IggyFile::new(file);
let mut buffer = PooledBuffer::with_capacity(file_size);
buffer.put_bytes(0, file_size);
- file.read_exact(&mut buffer)
- .await
+ let (result, buffer) = file.read_exact(buffer).await;
+ result
.with_error_context(|error| {
format!(
- "{COMPONENT} (error: {error}) - failed to read file content from path: {}",
+ "{COMPONENT} Failed to read system info from file at path: {} (error: {error})",
self.path
)
})
@@ -83,7 +87,7 @@ impl SystemInfoStorage for FileSystemInfoStorage {
.with_context(|| "Failed to serialize system info")
.map_err(|_| IggyError::CannotSerializeResource)?;
self.persister
- .overwrite(&self.path, &data)
+ .overwrite(&self.path, data)
.await
.with_error_context(|error| {
format!(
diff --git a/core/server/src/state/file.rs b/core/server/src/state/file.rs
index 41117c3a..b845d877 100644
--- a/core/server/src/state/file.rs
+++ b/core/server/src/state/file.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+use crate::io::file::IggyFile;
+use crate::io::reader::IggyReader;
use crate::state::command::EntryCommand;
use crate::state::{COMPONENT, State, StateEntry};
use crate::streaming::persistence::persister::PersisterKind;
@@ -28,11 +30,11 @@ use iggy_common::EncryptorKind;
use iggy_common::IggyByteSize;
use iggy_common::IggyError;
use iggy_common::IggyTimestamp;
+use monoio::io::AsyncReadRentExt;
use std::fmt::Debug;
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
-use tokio::io::{AsyncReadExt, BufReader};
use tracing::{debug, error, info};
pub const BUF_READER_CAPACITY_BYTES: usize = 512 * 1000;
@@ -115,6 +117,7 @@ impl State for FileState {
)
})
.map_err(|_| IggyError::CannotReadFile)?;
+ let file = IggyFile::new(file);
let file_size = file
.metadata()
.await
@@ -137,7 +140,7 @@ impl State for FileState {
);
let mut entries = Vec::new();
let mut total_size: u64 = 0;
- let mut reader = BufReader::with_capacity(BUF_READER_CAPACITY_BYTES, file);
+ let mut reader = IggyReader::with_capacity(BUF_READER_CAPACITY_BYTES, file);
let mut current_index = 0;
let mut entries_count = 0;
loop {
@@ -215,9 +218,10 @@ impl State for FileState {
total_size += 4;
let mut context = BytesMut::with_capacity(context_length);
context.put_bytes(0, context_length);
- reader
- .read_exact(&mut context)
- .await
+ let (result, context) = reader.read_exact(context).await;
+
+ result
+ .with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} code. {error}"))
.map_err(|_| IggyError::CannotReadFile)?;
let context = context.freeze();
total_size += context_length as u64;
@@ -238,9 +242,9 @@ impl State for FileState {
total_size += 4;
let mut command = BytesMut::with_capacity(command_length);
command.put_bytes(0, command_length);
- reader
- .read_exact(&mut command)
- .await
+ let (result, command) = reader.read_exact(command).await;
+ result
+ .with_error_context(|error| format!("{FILE_STATE_PARSE_ERROR} command. {error}"))
.map_err(|_| IggyError::CannotReadFile)?;
total_size += command_length as u64;
let command_payload;
@@ -353,15 +357,16 @@ impl State for FileState {
command,
);
let bytes = entry.to_bytes();
+ let len = bytes.len();
self.entries_count.fetch_add(1, Ordering::SeqCst);
self.persister
- .append(&self.path, &bytes)
+ .append(&self.path, bytes)
.await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to append state entry data to file, path: {}, data size: {}",
self.path,
- bytes.len()
+ len
)
})?;
debug!("Applied state entry: {entry}");
diff --git a/core/server/src/state/mod.rs b/core/server/src/state/mod.rs
index 656fc852..6ea022a6 100644
--- a/core/server/src/state/mod.rs
+++ b/core/server/src/state/mod.rs
@@ -40,14 +40,14 @@ pub enum StateKind {
}
#[cfg_attr(test, automock)]
-pub trait State: Send {
- fn init(&self) -> impl Future<Output = Result<Vec<StateEntry>, IggyError>> + Send;
- fn load_entries(&self) -> impl Future<Output = Result<Vec<StateEntry>, IggyError>> + Send;
+pub trait State {
+ fn init(&self) -> impl Future<Output = Result<Vec<StateEntry>, IggyError>>;
+ fn load_entries(&self) -> impl Future<Output = Result<Vec<StateEntry>, IggyError>>;
fn apply(
&self,
user_id: u32,
command: &EntryCommand,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
}
impl StateKind {
diff --git a/core/server/src/streaming/partitions/messages.rs b/core/server/src/streaming/partitions/messages.rs
index b136aeea..cc961370 100644
--- a/core/server/src/streaming/partitions/messages.rs
+++ b/core/server/src/streaming/partitions/messages.rs
@@ -221,7 +221,6 @@ impl Partition {
pub async fn append_messages(
&mut self,
batch: IggyMessagesBatchMut,
- confirmation: Option<Confirmation>,
) -> Result<(), IggyError> {
if batch.count() == 0 {
return Ok(());
@@ -316,7 +315,7 @@ impl Partition {
}
);
- last_segment.persist_messages(confirmation).await.with_error_context(|error| {
+ last_segment.persist_messages().await.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to persist messages, partition id: {}, start offset: {}",
self.partition_id, last_segment.start_offset()
@@ -346,7 +345,7 @@ impl Partition {
self.partition_id
);
- last_segment.persist_messages(None).await.with_error_context(|error| {
+ last_segment.persist_messages().await.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to persist messages, partition id: {}, start offset: {}",
self.partition_id, last_segment.start_offset()
@@ -384,7 +383,7 @@ mod tests {
.sum();
let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
let loaded_messages = partition
.get_messages_by_offset(0, messages_count)
@@ -406,7 +405,7 @@ mod tests {
assert_eq!(batch.count(), messages_count);
let unique_messages_count = 3;
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
let loaded_messages = partition
.get_messages_by_offset(0, messages_count)
@@ -433,7 +432,7 @@ mod tests {
.sum();
let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
let loaded_messages = partition.get_messages_by_offset(0, 10).await.unwrap();
@@ -464,7 +463,7 @@ mod tests {
.sum();
let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
let loaded_messages = partition.get_messages_by_offset(0, 10).await.unwrap();
@@ -495,7 +494,7 @@ mod tests {
.sum();
let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
let loaded_messages = partition.get_messages_by_offset(0, 10).await.unwrap();
@@ -528,7 +527,7 @@ mod tests {
.sum();
let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
let loaded_messages = partition.get_messages_by_offset(0, 10).await.unwrap();
@@ -566,7 +565,7 @@ mod tests {
.sum();
let initial_batch = IggyMessagesBatchMut::from_messages(&initial_messages, initial_size);
partition
- .append_messages(initial_batch, None)
+ .append_messages(initial_batch)
.await
.unwrap();
@@ -584,7 +583,7 @@ mod tests {
let duplicate_batch =
IggyMessagesBatchMut::from_messages(&duplicate_messages, duplicate_size);
partition
- .append_messages(duplicate_batch, None)
+ .append_messages(duplicate_batch)
.await
.unwrap();
@@ -614,7 +613,7 @@ mod tests {
.sum();
let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
let loaded_messages = partition.get_messages_by_offset(0, 10).await.unwrap();
@@ -642,7 +641,7 @@ mod tests {
.map(|m| m.get_size_bytes().as_bytes_u32())
.sum();
let batch1 = IggyMessagesBatchMut::from_messages(&batch1, batch1_size);
- partition.append_messages(batch1, None).await.unwrap();
+ partition.append_messages(batch1).await.unwrap();
// Second batch with mix of new and duplicate messages
let batch2 = vec![
@@ -656,7 +655,7 @@ mod tests {
.map(|m| m.get_size_bytes().as_bytes_u32())
.sum();
let batch2 = IggyMessagesBatchMut::from_messages(&batch2, batch2_size);
- partition.append_messages(batch2, None).await.unwrap();
+ partition.append_messages(batch2).await.unwrap();
let loaded_messages = partition.get_messages_by_offset(0, 10).await.unwrap();
@@ -695,7 +694,7 @@ mod tests {
.sum();
let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- partition.append_messages(batch, None).await.unwrap();
+ partition.append_messages(batch).await.unwrap();
let loaded_messages = partition.get_messages_by_offset(0, 10).await.unwrap();
diff --git a/core/server/src/streaming/partitions/storage.rs b/core/server/src/streaming/partitions/storage.rs
index ba3c2981..954025d1 100644
--- a/core/server/src/streaming/partitions/storage.rs
+++ b/core/server/src/streaming/partitions/storage.rs
@@ -18,6 +18,7 @@
use crate::compat::index_rebuilding::index_rebuilder::IndexRebuilder;
use crate::configs::cache_indexes::CacheIndexesConfig;
+use crate::io::file::IggyFile;
use crate::state::system::PartitionState;
use crate::streaming::partitions::COMPONENT;
use crate::streaming::partitions::partition::{ConsumerOffset, Partition};
@@ -28,12 +29,12 @@ use crate::streaming::utils::file;
use error_set::ErrContext;
use iggy_common::ConsumerKind;
use iggy_common::IggyError;
+use monoio::fs;
+use monoio::fs::create_dir_all;
+use monoio::io::AsyncReadRentExt;
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::Ordering;
-use tokio::fs;
-use tokio::fs::create_dir_all;
-use tokio::io::AsyncReadExt;
use tracing::{error, info, trace, warn};
#[derive(Debug)]
@@ -61,27 +62,24 @@ impl PartitionStorage for FilePartitionStorage {
partition.partition_path
);
partition.created_at = state.created_at;
- let dir_entries = fs::read_dir(&partition.partition_path).await;
- if fs::read_dir(&partition.partition_path)
- .await
+ // TODO: Replace this with the dir walk impl, that is mentined
+ // in the main function.
+ let mut dir_entries = std::fs::read_dir(&partition.partition_path)
.with_error_context(|error| format!(
"{COMPONENT} (error: {error}) - failed to read partition with ID: {} for stream with ID: {} and topic with ID: {} and path: {}.",
partition.partition_id, partition.stream_id, partition.topic_id, partition.partition_path,
- )).is_err()
- {
- return Err(IggyError::CannotReadPartitions);
- }
-
- let mut dir_entries = dir_entries.unwrap();
+ ))
+ .map_err(|_| IggyError::CannotReadPartitions)?;
let mut log_files = Vec::new();
- while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) {
+ while let Some(dir_entry) = dir_entries.next() {
+ let dir_entry = dir_entry.unwrap();
let path = dir_entry.path();
let extension = path.extension();
if extension.is_none() || extension.unwrap() != LOG_EXTENSION {
continue;
}
- let metadata = dir_entry.metadata().await.unwrap();
+ let metadata = dir_entry.metadata().unwrap();
if metadata.is_dir() {
continue;
}
@@ -368,7 +366,7 @@ impl PartitionStorage for FilePartitionStorage {
));
}
- if fs::remove_dir_all(&partition.partition_path).await.is_err() {
+ if std::fs::remove_dir_all(&partition.partition_path).is_err() {
error!(
"Cannot delete partition directory: {} for partition with ID: {} for topic with ID: {} for stream with ID: {}.",
partition.partition_path,
@@ -391,7 +389,7 @@ impl PartitionStorage for FilePartitionStorage {
async fn save_consumer_offset(&self, offset: u64, path: &str) -> Result<(), IggyError> {
self.persister
- .overwrite(path, &offset.to_le_bytes())
+ .overwrite(path, Box::new(offset.to_le_bytes()))
.await
.with_error_context(|error| format!(
"{COMPONENT} (error: {error}) - failed to overwrite consumer offset with value: {offset}, path: {path}",
@@ -406,15 +404,17 @@ impl PartitionStorage for FilePartitionStorage {
path: &str,
) -> Result<Vec<ConsumerOffset>, IggyError> {
trace!("Loading consumer offsets from path: {path}...");
- let dir_entries = fs::read_dir(&path).await;
+ //TODO: replace with async once its there
+ let dir_entries = std::fs::read_dir(&path);
if dir_entries.is_err() {
return Err(IggyError::CannotReadConsumerOffsets(path.to_owned()));
}
let mut consumer_offsets = Vec::new();
let mut dir_entries = dir_entries.unwrap();
- while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) {
- let metadata = dir_entry.metadata().await;
+ while let Some(dir_entry) = dir_entries.next() {
+ let dir_entry = dir_entry.unwrap();
+ let metadata = dir_entry.metadata();
if metadata.is_err() {
break;
}
@@ -439,7 +439,7 @@ impl PartitionStorage for FilePartitionStorage {
let path = Arc::new(path.unwrap().to_string());
let consumer_id = consumer_id.unwrap();
- let mut file = file::open(&path)
+ let file = file::open(&path)
.await
.with_error_context(|error| {
format!(
@@ -447,6 +447,7 @@ impl PartitionStorage for FilePartitionStorage {
)
})
.map_err(|_| IggyError::CannotReadFile)?;
+ let mut file = IggyFile::new(file);
let offset = file
.read_u64_le()
.await
@@ -473,7 +474,8 @@ impl PartitionStorage for FilePartitionStorage {
return Ok(());
}
- if fs::remove_dir_all(path).await.is_err() {
+ //TODO: replace with async once its there
+ if std::fs::remove_dir_all(path).is_err() {
error!("Cannot delete consumer offsets directory: {}.", path);
return Err(IggyError::CannotDeleteConsumerOffsetsDirectory(
path.to_owned(),
diff --git a/core/server/src/streaming/persistence/persister.rs b/core/server/src/streaming/persistence/persister.rs
index 75282e04..6ba5c87b 100644
--- a/core/server/src/streaming/persistence/persister.rs
+++ b/core/server/src/streaming/persistence/persister.rs
@@ -20,10 +20,10 @@ use crate::streaming::persistence::COMPONENT;
use crate::streaming::utils::file;
use error_set::ErrContext;
use iggy_common::IggyError;
+use monoio::buf::IoBuf;
use std::fmt::Debug;
use std::future::Future;
use tokio::fs;
-use tokio::io::AsyncWriteExt;
#[cfg(test)]
use mockall::automock;
@@ -37,7 +37,7 @@ pub enum PersisterKind {
}
impl PersisterKind {
- pub async fn append(&self, path: &str, bytes: &[u8]) -> Result<(), IggyError> {
+ pub async fn append<B: IoBuf>(&self, path: &str, bytes: B) -> Result<(), IggyError> {
match self {
PersisterKind::File(p) => p.append(path, bytes).await,
PersisterKind::FileWithSync(p) => p.append(path, bytes).await,
@@ -46,7 +46,7 @@ impl PersisterKind {
}
}
- pub async fn overwrite(&self, path: &str, bytes: &[u8]) -> Result<(), IggyError> {
+ pub async fn overwrite<B: IoBuf>(&self, path: &str, bytes: B) -> Result<(), IggyError> {
match self {
PersisterKind::File(p) => p.overwrite(path, bytes).await,
PersisterKind::FileWithSync(p) => p.overwrite(path, bytes).await,
@@ -66,18 +66,15 @@ impl PersisterKind {
}
#[cfg_attr(test, automock)]
-pub trait Persister: Send {
- fn append(
+pub trait Persister {
+ fn append<B: IoBuf>(&self, path: &str, bytes: B)
+ -> impl Future<Output = Result<(), IggyError>>;
+ fn overwrite<B: IoBuf>(
&self,
path: &str,
- bytes: &[u8],
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn overwrite(
- &self,
- path: &str,
- bytes: &[u8],
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn delete(&self, path: &str) -> impl Future<Output = Result<(), IggyError>> + Send;
+ bytes: B,
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, path: &str) -> impl Future<Output = Result<(), IggyError>>;
}
#[derive(Debug)]
@@ -87,15 +84,16 @@ pub struct FilePersister;
pub struct FileWithSyncPersister;
impl Persister for FilePersister {
- async fn append(&self, path: &str, bytes: &[u8]) -> Result<(), IggyError> {
- let mut file = file::append(path)
+ async fn append<B: IoBuf>(&self, path: &str, bytes: B) -> Result<(), IggyError> {
+ let file = file::append(path)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to append to file: {path}")
})
.map_err(|_| IggyError::CannotAppendToFile)?;
- file.write_all(bytes)
+ file.write_all_at(bytes, 0)
.await
+ .0
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to write data to file: {path}")
})
@@ -103,15 +101,17 @@ impl Persister for FilePersister {
Ok(())
}
- async fn overwrite(&self, path: &str, bytes: &[u8]) -> Result<(), IggyError> {
- let mut file = file::overwrite(path)
+ async fn overwrite<B: IoBuf>(&self, path: &str, bytes: B) -> Result<(), IggyError> {
+ let file = file::overwrite(path)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to overwrite file: {path}")
})
.map_err(|_| IggyError::CannotOverwriteFile)?;
- file.write_all(bytes)
+ let position = 0;
+ file.write_all_at(bytes, position)
.await
+ .0
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to write data to file: {path}")
})
@@ -131,15 +131,21 @@ impl Persister for FilePersister {
}
impl Persister for FileWithSyncPersister {
- async fn append(&self, path: &str, bytes: &[u8]) -> Result<(), IggyError> {
- let mut file = file::append(path)
+ async fn append<B: IoBuf>(&self, path: &str, bytes: B) -> Result<(), IggyError> {
+ let file = file::append(path)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to append to file: {path}")
})
.map_err(|_| IggyError::CannotAppendToFile)?;
- file.write_all(bytes)
+ let position = file
+ .metadata()
+ .await
+ .map_err(|_| IggyError::CannotReadFileMetadata)?
+ .len();
+ file.write_all_at(bytes, position)
.await
+ .0
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to write data to file: {path}")
})
@@ -155,15 +161,17 @@ impl Persister for FileWithSyncPersister {
Ok(())
}
- async fn overwrite(&self, path: &str, bytes: &[u8]) -> Result<(), IggyError> {
- let mut file = file::overwrite(path)
+ async fn overwrite<B: IoBuf>(&self, path: &str, bytes: B) -> Result<(), IggyError> {
+ let file = file::overwrite(path)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to overwrite file: {path}")
})
.map_err(|_| IggyError::CannotOverwriteFile)?;
- file.write_all(bytes)
+ let position = 0;
+ file.write_all_at(bytes, position)
.await
+ .0
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to write data to file: {path}")
})
diff --git a/core/server/src/streaming/persistence/task.rs b/core/server/src/streaming/persistence/task.rs
index 70510ed5..d1c716aa 100644
--- a/core/server/src/streaming/persistence/task.rs
+++ b/core/server/src/streaming/persistence/task.rs
@@ -21,18 +21,26 @@ use bytes::Bytes;
use error_set::ErrContext;
use flume::{Receiver, Sender, unbounded};
use iggy_common::IggyError;
+use monoio::task;
use std::{sync::Arc, time::Duration};
-use tokio::task;
use tracing::error;
use super::persister::PersisterKind;
-#[derive(Debug)]
pub struct LogPersisterTask {
_sender: Option<Sender<Bytes>>,
_task_handle: Option<task::JoinHandle<()>>,
}
+impl std::fmt::Debug for LogPersisterTask {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("LogPersisterTask")
+ .field("_sender", &self._sender.is_some())
+ .field("_task_handle", &self._task_handle.is_some())
+ .finish()
+ }
+}
+
impl LogPersisterTask {
pub fn new(
path: String,
@@ -42,7 +50,7 @@ impl LogPersisterTask {
) -> Self {
let (sender, receiver): (Sender<Bytes>, Receiver<Bytes>) = unbounded();
- let task_handle = task::spawn(async move {
+ let task_handle = monoio::spawn(async move {
loop {
match receiver.recv_async().await {
Ok(data) => {
@@ -82,7 +90,7 @@ impl LogPersisterTask {
let mut retries = 0;
while retries < max_retries {
- match persister.append(path, &data).await {
+ match persister.append(path, data.clone()).await {
Ok(_) => return Ok(()),
Err(e) => {
error!(
@@ -121,13 +129,7 @@ impl Drop for LogPersisterTask {
self._sender.take();
if let Some(handle) = self._task_handle.take() {
- tokio::spawn(async move {
- if let Err(error) = handle.await {
- error!(
- "{COMPONENT} (error: {error}) - error while shutting down task in Drop.",
- );
- }
- });
+ monoio::spawn(async move { handle.await });
}
}
}
diff --git a/core/server/src/streaming/segments/messages/messages_reader.rs b/core/server/src/streaming/segments/messages/messages_reader.rs
index b7e4f434..0f413650 100644
--- a/core/server/src/streaming/segments/messages/messages_reader.rs
+++ b/core/server/src/streaming/segments/messages/messages_reader.rs
@@ -16,12 +16,13 @@
* under the License.
*/
+use crate::io::file::{IggyFile};
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
-use crate::streaming::utils::PooledBuffer;
+use crate::streaming::utils::{file, PooledBuffer};
use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::IggyError;
-use std::{fs::File as StdFile, os::unix::prelude::FileExt};
+use monoio::fs::File;
use std::{
io::ErrorKind,
sync::{
@@ -29,15 +30,13 @@ use std::{
atomic::{AtomicU64, Ordering},
},
};
-use tokio::fs::OpenOptions;
-use tokio::task::spawn_blocking;
use tracing::{error, trace};
/// A dedicated struct for reading from the messages file.
#[derive(Debug)]
pub struct MessagesReader {
file_path: String,
- file: Arc<StdFile>,
+ file: IggyFile,
messages_size_bytes: Arc<AtomicU64>,
}
@@ -47,10 +46,7 @@ impl MessagesReader {
file_path: &str,
messages_size_bytes: Arc<AtomicU64>,
) -> Result<Self, IggyError> {
- let file = OpenOptions::new()
- .read(true)
- .open(file_path)
- .await
+ let file = file::open_std(file_path)
.with_error_context(|error| {
format!("Failed to open messages file: {file_path}, error: {error}")
})
@@ -71,6 +67,7 @@ impl MessagesReader {
)
});
}
+ let file = File::from_std(file).unwrap();
trace!(
"Opened messages file for reading: {file_path}, size: {}",
@@ -79,7 +76,7 @@ impl MessagesReader {
Ok(Self {
file_path: file_path.to_string(),
- file: Arc::new(file.into_std().await),
+ file: file.into(),
messages_size_bytes,
})
}
@@ -177,20 +174,19 @@ impl MessagesReader {
len: u32,
use_pool: bool,
) -> Result<PooledBuffer, std::io::Error> {
- let file = self.file.clone();
- spawn_blocking(move || {
- if use_pool {
+ if use_pool {
let mut buf = PooledBuffer::with_capacity(len as usize);
unsafe { buf.set_len(len as usize) };
- file.read_exact_at(&mut buf, offset as u64)?;
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ result?;
Ok(buf)
+
} else {
let mut buf = BytesMut::with_capacity(len as usize);
unsafe { buf.set_len(len as usize) };
- file.read_exact_at(&mut buf, offset as u64)?;
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ result?;
Ok(PooledBuffer::from_existing(buf))
}
- })
- .await?
}
}
diff --git a/core/server/src/streaming/segments/messages/messages_writer.rs b/core/server/src/streaming/segments/messages/messages_writer.rs
index 01a661ac..0429817f 100644
--- a/core/server/src/streaming/segments/messages/messages_writer.rs
+++ b/core/server/src/streaming/segments/messages/messages_writer.rs
@@ -16,15 +16,17 @@
* under the License.
*/
-use super::PersisterTask;
-use crate::streaming::segments::{IggyMessagesBatchSet, messages::write_batch};
+use crate::{
+ io::file::IggyFile,
+ streaming::segments::{IggyMessagesBatchSet, messages::write_batch},
+};
use error_set::ErrContext;
use iggy_common::{Confirmation, IggyByteSize, IggyError};
+use monoio::fs::{File, OpenOptions};
use std::sync::{
Arc,
atomic::{AtomicU64, Ordering},
};
-use tokio::fs::{File, OpenOptions};
use tracing::{error, trace};
/// A dedicated struct for writing to the messages file.
@@ -32,9 +34,8 @@ use tracing::{error, trace};
pub struct MessagesWriter {
file_path: String,
/// Holds the file for synchronous writes; when asynchronous persistence is enabled, this will be None.
- file: Option<File>,
+ file: Option<IggyFile>,
/// When set, asynchronous writes are handled by this persister task.
- persister_task: Option<PersisterTask>,
messages_size_bytes: Arc<AtomicU64>,
fsync: bool,
}
@@ -49,7 +50,6 @@ impl MessagesWriter {
file_path: &str,
messages_size_bytes: Arc<AtomicU64>,
fsync: bool,
- server_confirmation: Confirmation,
file_exists: bool,
) -> Result<Self, IggyError> {
let file = OpenOptions::new()
@@ -82,23 +82,10 @@ impl MessagesWriter {
messages_size_bytes.load(Ordering::Acquire)
);
- let (file, persister_task) = match server_confirmation {
- Confirmation::NoWait => {
- let persister = PersisterTask::new(
- file,
- file_path.to_string(),
- fsync,
- messages_size_bytes.clone(),
- );
- (None, Some(persister))
- }
- Confirmation::Wait => (Some(file), None),
- };
-
+ let file = Some(IggyFile::new(file));
Ok(Self {
file_path: file_path.to_string(),
file,
- persister_task,
messages_size_bytes,
fsync,
})
@@ -108,7 +95,6 @@ impl MessagesWriter {
pub async fn save_batch_set(
&mut self,
batch_set: IggyMessagesBatchSet,
- confirmation: Confirmation,
) -> Result<IggyByteSize, IggyError> {
let messages_size = batch_set.size();
let messages_count = batch_set.count();
@@ -117,46 +103,32 @@ impl MessagesWriter {
"Saving batch set of size {messages_size} bytes ({containers_count} containers, {messages_count} messages) to messages file: {}",
self.file_path
);
- match confirmation {
- Confirmation::Wait => {
- if let Some(ref mut file) = self.file {
- write_batch(file, &self.file_path, batch_set)
- .await
- .with_error_context(|error| {
- format!(
- "Failed to write batch to messages file: {}. {error}",
- self.file_path
- )
- })
- .map_err(|_| IggyError::CannotWriteToFile)?;
- } else {
- error!("File handle is not available for synchronous write.");
- return Err(IggyError::CannotWriteToFile);
- }
-
- if self.fsync {
- let _ = self.fsync().await;
- }
-
- self.messages_size_bytes
- .fetch_add(messages_size as u64, Ordering::Release);
- trace!(
- "Written batch set of size {messages_size} bytes ({containers_count} containers, {messages_count} messages) to disk messages file: {}",
- self.file_path
- );
- }
- Confirmation::NoWait => {
- if let Some(task) = &self.persister_task {
- task.persist(batch_set).await;
- } else {
- panic!(
- "Confirmation::NoWait is used, but MessagesPersisterTask is not set for messages file: {}",
+ if let Some(ref mut file) = self.file {
+ write_batch(file, &self.file_path, batch_set)
+ .await
+ .with_error_context(|error| {
+ format!(
+ "Failed to write batch to messages file: {}. {error}",
self.file_path
- );
- }
- }
+ )
+ })
+ .map_err(|_| IggyError::CannotWriteToFile)?;
+ } else {
+ error!("File handle is not available for synchronous write.");
+ return Err(IggyError::CannotWriteToFile);
}
+ if self.fsync {
+ let _ = self.fsync().await;
+ }
+
+ self.messages_size_bytes
+ .fetch_add(messages_size as u64, Ordering::Release);
+ trace!(
+ "Written batch set of size {messages_size} bytes ({containers_count} containers, {messages_count} messages) to disk messages file: {}",
+ self.file_path
+ );
+
Ok(IggyByteSize::from(messages_size as u64))
}
@@ -172,10 +144,4 @@ impl MessagesWriter {
Ok(())
}
-
- pub async fn shutdown_persister_task(self) {
- if let Some(task) = self.persister_task {
- task.shutdown().await;
- }
- }
}
diff --git a/core/server/src/streaming/segments/messages/mod.rs b/core/server/src/streaming/segments/messages/mod.rs
index 7f9ef61e..512c3894 100644
--- a/core/server/src/streaming/segments/messages/mod.rs
+++ b/core/server/src/streaming/segments/messages/mod.rs
@@ -18,42 +18,72 @@
mod messages_reader;
mod messages_writer;
-mod persister_task;
+
+use crate::{io::file::IggyFile, to_iovec};
use super::IggyMessagesBatchSet;
use error_set::ErrContext;
use iggy_common::IggyError;
-use std::io::IoSlice;
-use tokio::{fs::File, io::AsyncWriteExt};
+use monoio::io::AsyncWriteRent;
+use std::{io::IoSlice, mem::take};
pub use messages_reader::MessagesReader;
pub use messages_writer::MessagesWriter;
-pub use persister_task::PersisterTask;
+
+use nix::libc::iovec;
/// Vectored write a batches of messages to file
async fn write_batch(
- file: &mut File,
+ file: &mut IggyFile,
file_path: &str,
batches: IggyMessagesBatchSet,
) -> Result<usize, IggyError> {
- let mut slices: Vec<IoSlice> = batches.iter().map(|b| IoSlice::new(b)).collect();
-
- let slices = &mut slices.as_mut_slice();
+ let mut slices = batches.iter().map(|b| to_iovec(&b)).collect::<Vec<iovec>>();
let mut total_written = 0;
- while !slices.is_empty() {
- let bytes_written = file
- .write_vectored(slices)
- .await
+ loop {
+ let (result, vomited) = file.writev(slices).await;
+ slices = vomited;
+ let bytes_written = result
.with_error_context(|error| {
format!("Failed to write messages to file: {file_path}, error: {error}",)
})
.map_err(|_| IggyError::CannotWriteToFile)?;
total_written += bytes_written;
-
- IoSlice::advance_slices(slices, bytes_written);
+ advance_slices(&mut slices.as_mut_slice(), bytes_written);
+ if slices.is_empty() {
+ break;
+ }
}
Ok(total_written)
}
+
+fn advance_slices(mut bufs: &mut [iovec], n: usize) {
+ // Number of buffers to remove.
+ let mut remove = 0;
+ // Remaining length before reaching n. This prevents overflow
+ // that could happen if the length of slices in `bufs` were instead
+ // accumulated. Those slice may be aliased and, if they are large
+ // enough, their added length may overflow a `usize`.
+ let mut left = n;
+ for buf in bufs.iter() {
+ if let Some(remainder) = left.checked_sub(buf.iov_len as _) {
+ left = remainder;
+ remove += 1;
+ } else {
+ break;
+ }
+ }
+
+ bufs = &mut bufs[remove..];
+ if bufs.is_empty() {
+ assert!(left == 0, "advancing io slices beyond their length");
+ } else {
+ unsafe {
+ bufs[0].iov_len -= n;
+ bufs[0].iov_base = bufs[0].iov_base.add(n);
+ }
+ }
+}
diff --git a/core/server/src/streaming/segments/messages/persister_task.rs b/core/server/src/streaming/segments/messages/persister_task.rs
deleted file mode 100644
index 74056293..00000000
--- a/core/server/src/streaming/segments/messages/persister_task.rs
+++ /dev/null
@@ -1,215 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-use crate::streaming::segments::IggyMessagesBatchSet;
-use error_set::ErrContext;
-use flume::{Receiver, unbounded};
-use std::{
- sync::{
- Arc,
- atomic::{AtomicU64, Ordering},
- },
- time::Duration,
-};
-use tokio::{fs::File, select, time::sleep};
-use tracing::{error, trace, warn};
-
-use super::write_batch;
-
-#[derive(Debug)]
-/// A command to the persister task.
-enum PersisterTaskCommand {
- WriteRequest(IggyMessagesBatchSet),
- Shutdown,
-}
-
-/// A background task that writes data asynchronously.
-#[derive(Debug)]
-pub struct PersisterTask {
- sender: flume::Sender<PersisterTaskCommand>,
- file_path: String, // used only for logging
- _handle: tokio::task::JoinHandle<()>,
-}
-
-impl PersisterTask {
- /// Creates a new persister task that takes ownership of `file`.
- pub fn new(file: File, file_path: String, fsync: bool, log_file_size: Arc<AtomicU64>) -> Self {
- let (sender, receiver) = unbounded();
- let log_file_size = log_file_size.clone();
- let file_path_clone = file_path.clone();
- let handle = tokio::spawn(async move {
- Self::run(file, file_path_clone, receiver, fsync, log_file_size).await;
- });
- Self {
- sender,
- file_path,
- _handle: handle,
- }
- }
-
- /// Sends the batch bytes to the persister task (fire-and-forget).
- pub async fn persist(&self, messages: IggyMessagesBatchSet) {
- if let Err(e) = self
- .sender
- .send_async(PersisterTaskCommand::WriteRequest(messages))
- .await
- {
- error!(
- "Failed to send write request to LogPersisterTask for file {}: {:?}",
- self.file_path, e
- );
- }
- }
-
- /// Sends the shutdown command to the persister task and waits for a response.
- pub async fn shutdown(self) {
- let start_time = tokio::time::Instant::now();
-
- if let Err(e) = self.sender.send_async(PersisterTaskCommand::Shutdown).await {
- error!(
- "Failed to send shutdown command to LogPersisterTask for file {}: {:?}",
- self.file_path, e
- );
- return;
- }
-
- let mut handle_future = self._handle;
-
- select! {
- result = &mut handle_future => {
- match result {
- Ok(_) => {
- let elapsed = start_time.elapsed();
- trace!(
- "PersisterTask shutdown complete for file {} in {:.2}s",
- self.file_path,
- elapsed.as_secs_f64()
- );
- }
- Err(e) => {
- error!(
- "Error during joining PersisterTask for file {}: {:?}",
- self.file_path, e
- );
- }
- }
- return;
- }
- _ = sleep(Duration::from_secs(1)) => {
- warn!(
- "PersisterTask for file {} is still shutting down after 1s",
- self.file_path
- );
- }
- }
-
- select! {
- result = &mut handle_future => {
- match result {
- Ok(_) => {
- let elapsed = start_time.elapsed();
- trace!(
- "PersisterTask shutdown complete for file {} in {:.2}s",
- self.file_path,
- elapsed.as_secs_f64()
- );
- }
- Err(e) => {
- error!(
- "Error during joining PersisterTask for file {}: {:?}",
- self.file_path, e
- );
- }
- }
- return;
- }
- _ = sleep(Duration::from_secs(4)) => {
- warn!(
- "PersisterTask for file {} is still shutting down after 5s",
- self.file_path
- );
- }
- }
-
- match handle_future.await {
- Ok(_) => {
- let elapsed = start_time.elapsed();
- warn!(
- "PersisterTask shutdown complete for file {} in {:.2}s",
- self.file_path,
- elapsed.as_secs_f64()
- );
- }
- Err(e) => {
- error!(
- "Error during joining PersisterTask for file {}: {:?}",
- self.file_path, e
- );
- }
- }
- }
-
- /// The background task loop. Processes write requests until the channel is closed.
- async fn run(
- mut file: File,
- file_path: String,
- receiver: Receiver<PersisterTaskCommand>,
- fsync: bool,
- log_file_size: Arc<AtomicU64>,
- ) {
- while let Ok(request) = receiver.recv_async().await {
- match request {
- PersisterTaskCommand::WriteRequest(messages) => {
- match write_batch(&mut file, &file_path, messages).await {
- Ok(bytes_written) => {
- if fsync {
- file.sync_all()
- .await
- .with_error_context(|error| {
- format!(
- "Failed to fsync messages file: {file_path}. {error}"
- )
- })
- .expect("Failed to fsync messages file");
- }
-
- log_file_size.fetch_add(bytes_written as u64, Ordering::Acquire);
- }
- Err(e) => {
- error!(
- "Failed to persist data in LogPersisterTask for file {file_path}: {:?}",
- e
- )
- }
- }
- }
- PersisterTaskCommand::Shutdown => {
- trace!("LogPersisterTask for file {file_path} received shutdown command");
- if let Err(e) = file.sync_all().await {
- error!(
- "Failed to sync_all() in LogPersisterTask for file {file_path}: {:?}",
- e
- );
- }
- break;
- }
- }
- }
- trace!("PersisterTask for file {file_path} has finished processing requests");
- }
-}
diff --git a/core/server/src/streaming/segments/segment.rs b/core/server/src/streaming/segments/segment.rs
index 60886df6..92dd254f 100644
--- a/core/server/src/streaming/segments/segment.rs
+++ b/core/server/src/streaming/segments/segment.rs
@@ -224,13 +224,10 @@ impl Segment {
let log_fsync = self.config.partition.enforce_fsync;
let index_fsync = self.config.partition.enforce_fsync;
- let server_confirmation = self.config.segment.server_confirmation;
-
let messages_writer = MessagesWriter::new(
&self.messages_path,
self.messages_size.clone(),
log_fsync,
- server_confirmation,
file_exists,
)
.await?;
@@ -304,9 +301,9 @@ impl Segment {
pub async fn shutdown_writing(&mut self) {
if let Some(log_writer) = self.messages_writer.take() {
- tokio::spawn(async move {
+ //TODO: Fixme not sure whether we should spawn a task here.
+ monoio::spawn(async move {
let _ = log_writer.fsync().await;
- log_writer.shutdown_persister_task().await;
});
} else {
warn!(
diff --git a/core/server/src/streaming/segments/writing_messages.rs b/core/server/src/streaming/segments/writing_messages.rs
index 607fd261..9c385b02 100644
--- a/core/server/src/streaming/segments/writing_messages.rs
+++ b/core/server/src/streaming/segments/writing_messages.rs
@@ -23,7 +23,6 @@ use crate::{
streaming::deduplication::message_deduplicator::MessageDeduplicator,
};
use error_set::ErrContext;
-use iggy_common::Confirmation;
use iggy_common::IggyError;
use std::sync::atomic::Ordering;
use tracing::{info, trace};
@@ -66,10 +65,7 @@ impl Segment {
Ok(())
}
- pub async fn persist_messages(
- &mut self,
- confirmation: Option<Confirmation>,
- ) -> Result<usize, IggyError> {
+ pub async fn persist_messages(&mut self) -> Result<usize, IggyError> {
if self.accumulator.is_empty() {
return Ok(0);
}
@@ -83,11 +79,6 @@ impl Segment {
let accumulator = std::mem::take(&mut self.accumulator);
let batches = accumulator.into_batch_set();
- let confirmation = match confirmation {
- Some(val) => val,
- None => self.config.segment.server_confirmation,
- };
-
let batch_size = batches.size();
let batch_count = batches.count();
@@ -97,7 +88,7 @@ impl Segment {
.messages_writer
.as_mut()
.expect("Messages writer not initialized")
- .save_batch_set(batches, confirmation)
+ .save_batch_set(batches)
.await
.with_error_context(|error| {
format!(
diff --git a/core/server/src/streaming/topics/messages.rs b/core/server/src/streaming/topics/messages.rs
index c01687a6..acbe8c28 100644
--- a/core/server/src/streaming/topics/messages.rs
+++ b/core/server/src/streaming/topics/messages.rs
@@ -80,7 +80,6 @@ impl Topic {
&self,
partitioning: &Partitioning,
messages: IggyMessagesBatchMut,
- confirmation: Option<Confirmation>,
) -> Result<(), IggyError> {
if !self.has_partitions() {
return Err(IggyError::NoPartitions(self.topic_id, self.stream_id));
@@ -108,7 +107,7 @@ impl Topic {
}
};
- self.append_messages_to_partition(messages, partition_id, confirmation)
+ self.append_messages_to_partition(messages, partition_id)
.await
}
@@ -134,7 +133,6 @@ impl Topic {
&self,
messages: IggyMessagesBatchMut,
partition_id: u32,
- confirmation: Option<Confirmation>,
) -> Result<(), IggyError> {
let partition = self.partitions.get(&partition_id);
partition
@@ -145,7 +143,7 @@ impl Topic {
))?
.write()
.await
- .append_messages(messages, confirmation)
+ .append_messages(messages)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to append messages")
@@ -230,7 +228,7 @@ mod tests {
.expect("Failed to create message with valid payload and headers");
let messages = IggyMessagesBatchMut::from_messages(&[message], 1);
topic
- .append_messages(&partitioning, messages, None)
+ .append_messages(&partitioning, messages)
.await
.unwrap();
}
@@ -263,7 +261,7 @@ mod tests {
.expect("Failed to create message with valid payload and headers");
let messages = IggyMessagesBatchMut::from_messages(&[message], 1);
topic
- .append_messages(&partitioning, messages, None)
+ .append_messages(&partitioning, messages)
.await
.unwrap();
}
diff --git a/core/server/src/streaming/topics/persistence.rs b/core/server/src/streaming/topics/persistence.rs
index a423c67d..f75e36ec 100644
--- a/core/server/src/streaming/topics/persistence.rs
+++ b/core/server/src/streaming/topics/persistence.rs
@@ -54,7 +54,7 @@ impl Topic {
let mut partition = partition.write().await;
let partition_id = partition.partition_id;
for segment in partition.get_segments_mut() {
- saved_messages_number += segment.persist_messages(None).await.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to persist messages in segment, partition ID: {partition_id}"))?;
+ saved_messages_number += segment.persist_messages().await.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to persist messages in segment, partition ID: {partition_id}"))?;
}
}
diff --git a/core/server/src/streaming/utils/file.rs b/core/server/src/streaming/utils/file.rs
index af26454d..7a5bb524 100644
--- a/core/server/src/streaming/utils/file.rs
+++ b/core/server/src/streaming/utils/file.rs
@@ -16,8 +16,12 @@
* under the License.
*/
+use monoio::fs::{File, OpenOptions, remove_file};
use std::path::Path;
-use tokio::fs::{File, OpenOptions, remove_file};
+
+pub fn open_std(path: &str) -> Result<std::fs::File, std::io::Error> {
+ std::fs::OpenOptions::new().read(true).open(path)
+}
pub async fn open(path: &str) -> Result<File, std::io::Error> {
OpenOptions::new().read(true).open(path).await
@@ -35,14 +39,16 @@ pub async fn overwrite(path: &str) -> Result<File, std::io::Error> {
.open(path)
.await
}
+
pub async fn remove(path: &str) -> Result<(), std::io::Error> {
remove_file(path).await
}
pub async fn rename(old_path: &str, new_path: &str) -> Result<(), std::io::Error> {
- tokio::fs::rename(Path::new(old_path), Path::new(new_path)).await
+ monoio::fs::rename(Path::new(old_path), Path::new(new_path)).await
}
pub async fn exists(path: &str) -> Result<bool, std::io::Error> {
- tokio::fs::try_exists(path).await
+ //TODO: Does monoio support that ?
+ std::fs::exists(path)
}
(iggy) 23/30: refactor(io_uring): refactor the send/poll method flow (#1944)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 63c0b2f4cdf78b2ea7470a7bece901f66683008e
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Mon Jun 30 20:26:35 2025 +0200
refactor(io_uring): refactor the send/poll method flow (#1944)
---
.../handlers/messages/poll_messages_handler.rs | 95 ++-------
.../handlers/messages/send_messages_handler.rs | 89 ++------
core/server/src/shard/mod.rs | 29 +--
core/server/src/shard/system/messages.rs | 236 +++++++++++++--------
core/server/src/shard/transmission/message.rs | 14 +-
core/server/src/streaming/streams/messages.rs | 90 +++++++-
6 files changed, 291 insertions(+), 262 deletions(-)
diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
index 3abfb6a1..dfc4a89f 100644
--- a/core/server/src/binary/handlers/messages/poll_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
@@ -20,11 +20,11 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::messages::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::shard::namespace::IggyNamespace;
use crate::shard::system::messages::PollingArgs;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
-use crate::shard::{IggyShard, ShardRequestResult};
use crate::streaming::segments::IggyMessagesBatchSet;
use crate::streaming::session::Session;
use crate::to_iovec;
@@ -63,93 +63,30 @@ impl ServerCommandHandler for PollMessages {
shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
-
- let stream = shard
- .get_stream(&self.stream_id)
- .with_error_context(|error| {
- format!(
- "{COMPONENT} (error: {error}) - stream not found for stream ID: {}",
- self.stream_id
- )
- })?;
- let topic = stream.get_topic(
- &self.topic_id,
- ).with_error_context(|error| format!(
- "{COMPONENT} (error: {error}) - topic not found for stream ID: {}, topic_id: {}",
- self.stream_id, self.topic_id
- ))?;
-
let PollMessages {
consumer,
partition_id,
strategy,
count,
auto_commit,
- ..
+ stream_id,
+ topic_id,
} = self;
let args = PollingArgs::new(strategy, count, auto_commit);
- shard.permissioner
- .borrow()
- .poll_messages(session.get_user_id(), topic.stream_id, topic.topic_id)
- .with_error_context(|error| format!(
- "{COMPONENT} (error: {error}) - permission denied to poll messages for user {} on stream ID: {}, topic ID: {}",
- session.get_user_id(),
- topic.stream_id,
- topic.topic_id
- ))?;
-
- if !topic.has_partitions() {
- return Err(IggyError::NoPartitions(topic.topic_id, topic.stream_id));
- }
-
- // There might be no partition assigned, if it's the consumer group member without any partitions.
- let Some((consumer, partition_id)) = topic
- .resolve_consumer_with_partition_id(&consumer, session.client_id, partition_id, true)
- .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer with partition id, consumer: {consumer}, client ID: {}, partition ID: {:?}", session.client_id, partition_id))? else {
- todo!("Send early response");
- //return Ok((IggyPollMetadata::new(0, 0), IggyMessagesBatchSet::empty()));
- };
-
- let namespace = IggyNamespace::new(stream.stream_id, topic.topic_id, partition_id);
- let payload = ShardRequestPayload::PollMessages {
- consumer: consumer.clone(),
- args,
- count,
- };
- let request = ShardRequest::new(stream.stream_id, topic.topic_id, partition_id, payload);
- let message = ShardMessage::Request(request);
- let (metadata, batch) = match shard.send_request_to_shard(&namespace, message).await {
- ShardRequestResult::SameShard(message) => match message {
- ShardMessage::Request(request) => match request.payload {
- ShardRequestPayload::PollMessages {
- consumer,
- args,
- count,
- } => {
- topic
- .get_messages(consumer, partition_id, args.strategy, count)
- .await?
- }
- _ => unreachable!(
- "Expected a SendMessages request inside of SendMessages handler, impossible state"
- ),
- },
- _ => unreachable!(
- "Expected a request message inside of an command handler, impossible state"
- ),
- },
- ShardRequestResult::Result(result) => match result? {
- ShardResponse::PollMessages(response) => response,
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a PollMessages response inside of PollMessages handler, impossible state"
- ),
- },
- };
-
+ let user_id = session.get_user_id();
+ let client_id = session.client_id;
+ let (metadata, batch) = shard
+ .poll_messages(
+ client_id,
+ user_id,
+ &stream_id,
+ &topic_id,
+ consumer,
+ partition_id,
+ args,
+ )
+ .await?;
// Collect all chunks first into a Vec to extend their lifetimes.
// This ensures the Bytes (in reality Arc<[u8]>) references from each IggyMessagesBatch stay alive
// throughout the async vectored I/O operation, preventing "borrowed value does not live
diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs
index f4497bf6..47df8270 100644
--- a/core/server/src/binary/handlers/messages/send_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs
@@ -19,22 +19,20 @@
use super::COMPONENT;
use crate::binary::command::{BinaryServerCommand, ServerCommandHandler};
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::shard::namespace::IggyNamespace;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
-use crate::shard::{IggyShard, ShardRequestResult};
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
use crate::streaming::utils::PooledBuffer;
use anyhow::Result;
use bytes::BytesMut;
-use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::Sizeable;
use iggy_common::{INDEX_SIZE, IdKind};
use iggy_common::{IggyError, Partitioning, SendMessages, Validatable};
use std::rc::Rc;
-use std::sync::Arc;
use tracing::instrument;
impl ServerCommandHandler for SendMessages {
@@ -112,80 +110,17 @@ impl ServerCommandHandler for SendMessages {
);
batch.validate()?;
- let stream = shard
- .get_stream(&self.stream_id)
- .with_error_context(|error| {
- format!(
- "Failed to get stream with ID: {} (error: {})",
- self.stream_id, error
- )
- })?;
- let topic = stream
- .get_topic(&self.topic_id)
- .with_error_context(|error| {
- format!(
- "Failed to get topic with ID: {} (error: {})",
- self.topic_id, error
- )
- })?;
- let stream_id = stream.stream_id;
- let topic_id = topic.topic_id;
- let partition_id = topic.calculate_partition_id(&self.partitioning)?;
-
- // Validate permissions for given user on stream and topic.
- shard.permissioner.borrow().append_messages(
- session.get_user_id(),
- stream_id,
- topic_id
- ).with_error_context(|error| format!(
- "{COMPONENT} (error: {error}) - permission denied to append messages for user {} on stream ID: {}, topic ID: {}",
- session.get_user_id(),
- stream_id,
- topic_id
- ))?;
- let messages_count = batch.count();
- shard.metrics.increment_messages(messages_count as u64);
-
- // Encrypt messages if encryptor is enabled in configuration.
- let batch = shard.maybe_encrypt_messages(batch)?;
-
- let namespace = IggyNamespace::new(stream.stream_id, topic.topic_id, partition_id);
- let payload = ShardRequestPayload::SendMessages { batch };
- let request = ShardRequest::new(stream.stream_id, topic.topic_id, partition_id, payload);
- let message = ShardMessage::Request(request);
- // Egh... I don't like those nested match statements,
- // Technically there is only two `request` types that will ever be dispatched
- // to different shards, thus we could get away with generic structs
- // maybe todo ?
- // how to make this code reusable, in a sense that we will have exactly the same code inside of
- // `PollMessages` handler, but with different request....
- match shard.send_request_to_shard(&namespace, message).await {
- ShardRequestResult::SameShard(message) => match message {
- ShardMessage::Request(request) => {
- let partition_id = request.partition_id;
- match request.payload {
- ShardRequestPayload::SendMessages { batch } => {
- topic.append_messages(partition_id, batch).await?
- }
- _ => unreachable!(
- "Expected a SendMessages request inside of SendMessages handler, impossible state"
- ),
- }
- }
- _ => unreachable!(
- "Expected a request message inside of an command handler, impossible state"
- ),
- },
- ShardRequestResult::Result(result) => match result? {
- ShardResponse::SendMessages => (),
- ShardResponse::ErrorResponse(err) => {
- return Err(err);
- }
- _ => unreachable!(
- "Expected a SendMessages response inside of SendMessages handler, impossible state"
- ),
- },
- };
+ let user_id = session.get_user_id();
+ shard
+ .append_messages(
+ user_id,
+ &self.stream_id,
+ &self.topic_id,
+ &self.partitioning,
+ batch,
+ )
+ .await?;
+
sender.send_empty_ok_response().await?;
Ok(())
}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 48bf183a..6c3718f8 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -54,7 +54,7 @@ use crate::{
transmission::{
event::ShardEvent,
frame::{ShardFrame, ShardResponse},
- message::{ShardMessage, ShardRequest, ShardRequestPayload},
+ message::{ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult},
},
},
state::{
@@ -120,11 +120,6 @@ impl ShardInfo {
}
}
-pub enum ShardRequestResult<T, E> {
- SameShard(ShardMessage),
- Result(Result<T, E>),
-}
-
pub struct IggyShard {
pub id: u16,
shards: Vec<Shard>,
@@ -498,13 +493,9 @@ impl IggyShard {
topic.append_messages(partition_id, batch).await?;
Ok(ShardResponse::SendMessages)
}
- ShardRequestPayload::PollMessages {
- args,
- consumer,
- count,
- } => {
+ ShardRequestPayload::PollMessages { args, consumer } => {
let (metadata, batch) = topic
- .get_messages(consumer, partition_id, args.strategy, count)
+ .get_messages(consumer, partition_id, args.strategy, args.count)
.await?;
Ok(ShardResponse::PollMessages((metadata, batch)))
}
@@ -551,14 +542,14 @@ impl IggyShard {
}
}
- pub async fn send_request_to_shard(
+ pub async fn send_request_to_shard_or_recoil(
&self,
namespace: &IggyNamespace,
message: ShardMessage,
- ) -> ShardRequestResult<ShardResponse, IggyError> {
+ ) -> Result<ShardSendRequestResult, IggyError> {
if let Some(shard) = self.find_shard(namespace) {
if shard.id == self.id {
- return ShardRequestResult::SameShard(message);
+ return Ok(ShardSendRequestResult::Recoil(message));
}
let response = match shard.send_request(message).await {
@@ -568,16 +559,16 @@ impl IggyShard {
"{COMPONENT} - failed to send request to shard with ID: {}, error: {err}",
shard.id
);
- return ShardRequestResult::Result(Err(err));
+ return Err(err);
}
};
- ShardRequestResult::Result(Ok(response))
+ Ok(ShardSendRequestResult::Response(response))
} else {
- ShardRequestResult::Result(Err(IggyError::ShardNotFound(
+ Err(IggyError::ShardNotFound(
namespace.stream_id,
namespace.topic_id,
namespace.partition_id,
- )))
+ ))
}
}
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index 36ae234b..b27b034c 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -19,83 +19,192 @@
use super::COMPONENT;
use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
use crate::shard::IggyShard;
+use crate::shard::namespace::IggyNamespace;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
+use crate::streaming::partitions::partition;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet};
use crate::streaming::session::Session;
use crate::streaming::utils::PooledBuffer;
use async_zip::tokio::read::stream;
use error_set::ErrContext;
+use futures::stream_select;
use iggy_common::{
BytesSerializable, Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier, IggyError,
- Partitioning, PollingStrategy,
+ Partitioning, PollingStrategy, UserId,
};
use tracing::{error, trace};
impl IggyShard {
+ pub async fn append_messages(
+ &self,
+ user_id: u32,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partitioning: &Partitioning,
+ batch: IggyMessagesBatchMut,
+ ) -> Result<(), IggyError> {
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "Failed to get stream with ID: {} (error: {})",
+ stream_id, error
+ )
+ })?;
+ let stream_id = stream.stream_id;
+ let numeric_topic_id = stream.get_topic(topic_id).map(|topic| topic.topic_id).with_error_context(|error| {
+ format!(
+ "Failed to get topic with ID: {} (error: {})",
+ topic_id, error
+ )
+ })?;
+ // TODO: We should look into refactoring those permissioners, so they can accept `Identifier` instead of numeric IDs.
+ // Validate permissions for given user on stream and topic.
+ self.permissioner.borrow().append_messages(
+ user_id,
+ stream.stream_id,
+ numeric_topic_id
+ ).with_error_context(|error| format!(
+ "{COMPONENT} (error: {error}) - permission denied to append messages for user {} on stream ID: {}, topic ID: {}",
+ user_id,
+ stream.stream_id,
+ numeric_topic_id
+ ))?;
+ // Encrypt messages if encryptor is enabled in configuration.
+ let batch = self.maybe_encrypt_messages(batch)?;
+ let messages_count = batch.count();
+ stream
+ .append_messages(topic_id, partitioning, async |topic, partition_id| {
+ let namespace = IggyNamespace::new(stream_id, topic.topic_id, partition_id);
+ let payload = ShardRequestPayload::SendMessages { batch };
+ let request = ShardRequest::new(stream_id, numeric_topic_id, partition_id, payload);
+ let message = ShardMessage::Request(request);
+
+ match self
+ .send_request_to_shard_or_recoil(&namespace, message)
+ .await?
+ {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request( ShardRequest { partition_id, payload, .. } ) = message
+ && let ShardRequestPayload::SendMessages { batch } = payload
+ {
+ topic.append_messages(partition_id, batch).await.with_error_context(|error| {
+ format!("{COMPONENT}: Failed to append messages to stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {partition_id}, error: {error})")
+ })
+ } else {
+ unreachable!(
+ "Expected a SendMessages request inside of SendMessages handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => {
+ match response {
+ ShardResponse::SendMessages => { Ok(()) }
+ ShardResponse::ErrorResponse(err) => {
+ Err(err)
+ }
+ _ => unreachable!(
+ "Expected a SendMessages response inside of SendMessages handler, impossible state"
+ ),
+ }
+
+ }
+ }
+ })
+ .await?;
+
+ self.metrics.increment_messages(messages_count as u64);
+ Ok(())
+ }
+
pub async fn poll_messages(
&self,
- session: &Session,
- consumer: &Consumer,
+ client_id: u32,
+ user_id: u32,
stream_id: &Identifier,
topic_id: &Identifier,
- partition_id: Option<u32>,
+ consumer: Consumer,
+ maybe_partition_id: Option<u32>,
args: PollingArgs,
) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> {
- self.ensure_authenticated(session)?;
- if args.count == 0 {
- return Err(IggyError::InvalidMessagesCount);
- }
-
let stream = self.get_stream(stream_id).with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}")
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {}",
+ stream_id
+ )
})?;
let stream_id = stream.stream_id;
- let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
+ let numeric_topic_id = stream.get_topic(topic_id).map(|topic| topic.topic_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - topic not found for stream ID: {}, topic_id: {}",
+ stream_id, topic_id
+ )
+ })?;
+
self.permissioner
.borrow()
- .poll_messages(session.get_user_id(), topic.stream_id, topic.topic_id)
+ .poll_messages(user_id, stream_id, numeric_topic_id)
.with_error_context(|error| format!(
"{COMPONENT} (error: {error}) - permission denied to poll messages for user {} on stream ID: {}, topic ID: {}",
- session.get_user_id(),
- topic.stream_id,
- topic.topic_id
+ user_id,
+ stream_id,
+ numeric_topic_id
))?;
- if !topic.has_partitions() {
- return Err(IggyError::NoPartitions(topic.topic_id, topic.stream_id));
- }
-
// There might be no partition assigned, if it's the consumer group member without any partitions.
- let Some((polling_consumer, partition_id)) = topic
- .resolve_consumer_with_partition_id(consumer, session.client_id, partition_id, true)
- .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer with partition id, consumer: {consumer}, client ID: {}, partition ID: {:?}", session.client_id, partition_id))? else {
- return Ok((IggyPollMetadata::new(0, 0), IggyMessagesBatchSet::empty()));
- };
+ //return Ok((IggyPollMetadata::new(0, 0), IggyMessagesBatchSet::empty()));
- let (metadata, batch_set) = topic
- .get_messages(polling_consumer, partition_id, args.strategy, args.count)
- .await?;
+ let (metadata, batch) = stream.poll_messages(topic_id, client_id, consumer, maybe_partition_id, args.auto_commit, async |topic, consumer, partition_id| {
+ let namespace = IggyNamespace::new(stream.stream_id, topic.topic_id, partition_id);
+ let payload = ShardRequestPayload::PollMessages {
+ consumer,
+ args,
+ };
+ let request = ShardRequest::new(stream.stream_id, topic.topic_id, partition_id, payload);
+ let message = ShardMessage::Request(request);
- if args.auto_commit && !batch_set.is_empty() {
- let offset = batch_set
- .last_offset()
- .expect("Batch set should have at least one batch");
- trace!(
- "Last offset: {} will be automatically stored for {}, stream: {}, topic: {}, partition: {}",
- offset, consumer, stream_id, topic_id, partition_id
- );
- topic
- .store_consumer_offset_internal(polling_consumer, offset, partition_id)
- .await
- .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to store consumer offset internal, polling consumer: {}, offset: {}, partition ID: {}", polling_consumer, offset, partition_id))?;
- }
+ match self
+ .send_request_to_shard_or_recoil(&namespace, message)
+ .await?
+ {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request( ShardRequest { partition_id, payload, .. } ) = message
+ && let ShardRequestPayload::PollMessages { consumer, args } = payload
+ {
+ topic.get_messages(consumer, partition_id, args.strategy, args.count).await.with_error_context(|error| {
+ format!("{COMPONENT}: Failed to get messages for stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {partition_id}, error: {error})")
+ })
+ } else {
+ unreachable!(
+ "Expected a PollMessages request inside of PollMessages handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => {
+ match response {
+ ShardResponse::PollMessages(result) => { Ok(result) }
+ ShardResponse::ErrorResponse(err) => {
+ Err(err)
+ }
+ _ => unreachable!(
+ "Expected a SendMessages response inside of SendMessages handler, impossible state"
+ ),
+ }
- let batch_set = if let Some(encryptor) = &self.encryptor {
- self.decrypt_messages(batch_set, encryptor).await?
+ }
+ }
+ }).await?;
+
+ let batch = if let Some(_encryptor) = &self.encryptor {
+ //TODO: Bring back decryptor
+ todo!();
+ //self.decrypt_messages(batch, encryptor.as_ref()).await?
} else {
- batch_set
+ batch
};
- Ok((metadata, batch_set))
+ Ok((metadata, batch))
}
pub async fn flush_unsaved_buffer(
@@ -126,45 +235,6 @@ impl IggyShard {
Ok(())
}
- pub async fn decrypt_messages(
- &self,
- batches: IggyMessagesBatchSet,
- encryptor: &EncryptorKind,
- ) -> Result<IggyMessagesBatchSet, IggyError> {
- let mut decrypted_batches = Vec::with_capacity(batches.containers_count());
- for batch in batches.iter() {
- let count = batch.count();
-
- let mut indexes = IggyIndexesMut::with_capacity(batch.count() as usize, 0);
- let mut decrypted_messages = PooledBuffer::with_capacity(batch.size() as usize);
- let mut position = 0;
-
- for message in batch.iter() {
- let payload = encryptor.decrypt(message.payload());
- match payload {
- Ok(payload) => {
- message.header().write_to_buffer(&mut decrypted_messages);
- decrypted_messages.extend_from_slice(&payload);
- if let Some(user_headers) = message.user_headers() {
- decrypted_messages.extend_from_slice(user_headers);
- }
- indexes.insert(0, position as u32, 0);
- position += message.size();
- }
- Err(error) => {
- error!("Cannot decrypt the message. Error: {}", error);
- continue;
- }
- }
- }
- let decrypted_batch =
- IggyMessagesBatchMut::from_indexes_and_messages(count, indexes, decrypted_messages);
- decrypted_batches.push(decrypted_batch);
- }
-
- Ok(IggyMessagesBatchSet::from_vec(decrypted_batches))
- }
-
pub fn maybe_encrypt_messages(
&self,
batch: IggyMessagesBatchMut,
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index aa196fde..fe369bad 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -20,10 +20,19 @@ use iggy_common::PollingStrategy;
* under the License.
*/
use crate::{
- shard::{system::messages::PollingArgs, transmission::event::ShardEvent},
+ shard::{
+ system::messages::PollingArgs,
+ transmission::{event::ShardEvent, frame::ShardResponse},
+ },
streaming::{polling_consumer::PollingConsumer, segments::IggyMessagesBatchMut},
};
+pub enum ShardSendRequestResult {
+ // TODO: In the future we can add other variants, for example backpressure from the destination shard,
+ Recoil(ShardMessage),
+ Response(ShardResponse),
+}
+
#[derive(Debug)]
pub enum ShardMessage {
Request(ShardRequest),
@@ -60,9 +69,8 @@ pub enum ShardRequestPayload {
batch: IggyMessagesBatchMut,
},
PollMessages {
- args: PollingArgs,
consumer: PollingConsumer,
- count: u32,
+ args: PollingArgs,
},
}
diff --git a/core/server/src/streaming/streams/messages.rs b/core/server/src/streaming/streams/messages.rs
index 855bda66..bb6ac41e 100644
--- a/core/server/src/streaming/streams/messages.rs
+++ b/core/server/src/streaming/streams/messages.rs
@@ -16,11 +16,99 @@
* under the License.
*/
-use crate::streaming::streams::stream::Stream;
+use error_set::ErrContext;
+use iggy_common::{Consumer, Identifier, IggyError, Partitioning};
+use tracing::trace;
+
+use super::COMPONENT;
+use crate::{
+ binary::handlers::messages::poll_messages_handler::IggyPollMetadata,
+ shard::{
+ namespace::IggyNamespace,
+ transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload},
+ },
+ streaming::{
+ partitions::partition,
+ polling_consumer::PollingConsumer,
+ segments::{IggyMessagesBatchMut, IggyMessagesBatchSet},
+ streams::stream::Stream,
+ topics::topic::Topic,
+ },
+};
use std::sync::atomic::Ordering;
impl Stream {
pub fn get_messages_count(&self) -> u64 {
self.messages_count.load(Ordering::SeqCst)
}
+
+ // TODO: Create a types module, where we will put types of those closures.
+ pub async fn append_messages(
+ &self,
+ topic_id: &Identifier,
+ partitioning: &Partitioning,
+ append_messages: impl AsyncFnOnce(&Topic, u32) -> Result<(), IggyError>,
+ ) -> Result<(), IggyError> {
+ // It's quite empty there, but in the future if we would like to introduce consumer group subscription
+ // based on topic name wildcards, we would have to lift those to stream level,
+ // thus it would be perfect place to perform those calculations here.
+ let topic = self.get_topic(topic_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get topic with ID: {} for stream ID: {}",
+ topic_id, self.stream_id
+ )
+ })?;
+
+ let partition_id = topic.calculate_partition_id(partitioning)?;
+ append_messages(topic, partition_id).await
+ }
+
+ pub async fn poll_messages(
+ &self,
+ topic_id: &Identifier,
+ client_id: u32,
+ consumer: Consumer,
+ maybe_partition_id: Option<u32>,
+ auto_commit: bool,
+ poll_messages: impl AsyncFnOnce(
+ &Topic,
+ PollingConsumer,
+ u32,
+ )
+ -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError>,
+ ) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> {
+ // Same as in `append_messages` method.
+ let topic = self.get_topic(topic_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get topic with ID: {} for stream ID: {}",
+ topic_id, self.stream_id
+ )
+ })?;
+ if !topic.has_partitions() {
+ return Err(IggyError::NoPartitions(topic.topic_id, self.stream_id));
+ }
+
+ let Some((consumer, partition_id)) = topic
+ .resolve_consumer_with_partition_id(&consumer, client_id, maybe_partition_id, true)
+ .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer with partition id, consumer: {consumer}, client ID: {}, partition ID: {:?}", client_id, maybe_partition_id))? else {
+ todo!("Send early response");
+ };
+
+ let (metadata, batch) = poll_messages(topic, consumer, partition_id).await?;
+
+ if auto_commit && !batch.is_empty() {
+ let offset = batch
+ .last_offset()
+ .expect("Batch set should have at least one batch");
+ trace!(
+ "Last offset: {} will be automatically stored for {}, stream: {}, topic: {}, partition: {}",
+ offset, consumer, self.stream_id, topic_id, partition_id
+ );
+ topic
+ .store_consumer_offset_internal(consumer, offset, partition_id)
+ .await
+ .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to store consumer offset internal, polling consumer: {consumer}, offset: {offset}, partition ID: {partition_id}"))?;
+ }
+ Ok((metadata, batch))
+ }
}
(iggy) 04/30: begin shard init
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit a60c99101e6483fba8925c0fee2d9a1774285db6
Author: numinex <g....@gmail.com>
AuthorDate: Sat May 17 14:00:52 2025 +0200
begin shard init
---
core/server/Cargo.toml | 2 +-
core/server/src/bootstrap.rs | 131 +++++++++++++++++++++++++++++++++++--
core/server/src/main.rs | 33 +++++-----
core/server/src/shard/builder.rs | 54 +++++++++++++--
core/server/src/shard/mod.rs | 49 ++++++++------
core/server/src/shard/namespace.rs | 1 +
core/server/src/state/file.rs | 13 +---
7 files changed, 221 insertions(+), 62 deletions(-)
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index d4542eb4..f4295c6c 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -64,7 +64,7 @@ jsonwebtoken = "9.3.1"
lending-iterator = "0.1.7"
mimalloc = { workspace = true, optional = true }
moka = { version = "0.12.10", features = ["future"] }
-monoio = "0.2.4"
+monoio = { version = "0.2.4", features = ["mkdirat", "unlinkat"] }
nix = { version = "0.30", features = ["fs"] }
once_cell = "1.21.3"
opentelemetry = { version = "0.30.0", features = ["trace", "logs"] }
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index fd6a5aa0..d51096a2 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -1,7 +1,24 @@
-use monoio::{Buildable, Driver, Runtime};
+use iggy_common::{
+ defaults::{
+ DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH,
+ MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH,
+ },
+ IggyError,
+};
+use monoio::{
+ fs::create_dir_all,
+ Buildable, Driver, Runtime,
+};
+use tracing::info;
-use crate::shard::{connector::ShardConnector, frame::ShardFrame};
-use std::ops::Range;
+use crate::{
+ configs::{config_provider::ConfigProviderKind, server::ServerConfig, system::SystemConfig},
+ server_error::ServerError,
+ shard::{connector::ShardConnector, frame::ShardFrame},
+ streaming::users::user::User,
+ IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
+};
+use std::{env, fs::remove_dir_all, ops::Range, path::Path};
pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<ShardFrame>> {
let shards_count = shards_set.len();
@@ -13,12 +30,112 @@ pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<
connectors
}
-pub async fn create_directories() {
- todo!();
+pub async fn load_config(
+ config_provider: &ConfigProviderKind,
+) -> Result<ServerConfig, ServerError> {
+ let config = ServerConfig::load(config_provider).await?;
+ Ok(config)
}
-pub async fn create_root_user() {
- todo!();
+pub async fn create_directories(config: &SystemConfig) -> Result<(), IggyError> {
+ let system_path = config.get_system_path();
+ if !Path::new(&system_path).exists() && create_dir_all(&system_path).await.is_err() {
+ return Err(IggyError::CannotCreateBaseDirectory(system_path));
+ }
+
+ let state_path = config.get_state_path();
+ if !Path::new(&state_path).exists() && create_dir_all(&state_path).await.is_err() {
+ return Err(IggyError::CannotCreateStateDirectory(state_path));
+ }
+
+ let streams_path = config.get_streams_path();
+ if !Path::new(&streams_path).exists() && create_dir_all(&streams_path).await.is_err() {
+ return Err(IggyError::CannotCreateStreamsDirectory(streams_path));
+ }
+
+ let runtime_path = config.get_runtime_path();
+ // TODO: Change remove_dir_all to async version, once we implement the dir walk using monoio `remove_dir` method.
+ if Path::new(&runtime_path).exists() && remove_dir_all(&runtime_path).is_err() {
+ return Err(IggyError::CannotRemoveRuntimeDirectory(runtime_path));
+ }
+
+ if create_dir_all(&runtime_path).await.is_err() {
+ return Err(IggyError::CannotCreateRuntimeDirectory(runtime_path));
+ }
+
+ info!(
+ "Initializing system, data will be stored at: {}",
+ config.get_system_path()
+ );
+ Ok(())
+
+ // TODO: Move this to individual shard level
+ /*
+ let state_entries = self.state.init().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to initialize state entries")
+ })?;
+ let system_state = SystemState::init(state_entries)
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to initialize system state")
+ })?;
+ let now = Instant::now();
+ self.load_version().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to load version")
+ })?;
+ self.load_users(system_state.users.into_values().collect())
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to load users")
+ })?;
+ self.load_streams(system_state.streams.into_values().collect())
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to load streams")
+ })?;
+ if let Some(archiver) = self.archiver.as_ref() {
+ archiver
+ .init()
+ .await
+ .expect("Failed to initialize archiver");
+ }
+ info!("Initialized system in {} ms.", now.elapsed().as_millis());
+ Ok(())
+ */
+}
+
+pub fn create_root_user() -> User {
+ info!("Creating root user...");
+ let username = env::var(IGGY_ROOT_USERNAME_ENV);
+ let password = env::var(IGGY_ROOT_PASSWORD_ENV);
+ if (username.is_ok() && password.is_err()) || (username.is_err() && password.is_ok()) {
+ panic!("When providing the custom root user credentials, both username and password must be set.");
+ }
+ if username.is_ok() && password.is_ok() {
+ info!("Using the custom root user credentials.");
+ } else {
+ info!("Using the default root user credentials.");
+ }
+
+ let username = username.unwrap_or(DEFAULT_ROOT_USERNAME.to_string());
+ let password = password.unwrap_or(DEFAULT_ROOT_PASSWORD.to_string());
+ if username.is_empty() || password.is_empty() {
+ panic!("Root user credentials are not set.");
+ }
+ if username.len() < MIN_USERNAME_LENGTH {
+ panic!("Root username is too short.");
+ }
+ if username.len() > MAX_USERNAME_LENGTH {
+ panic!("Root username is too long.");
+ }
+ if password.len() < MIN_PASSWORD_LENGTH {
+ panic!("Root password is too short.");
+ }
+ if password.len() > MAX_PASSWORD_LENGTH {
+ panic!("Root password is too long.");
+ }
+ let user = User::root(&username, &password);
+ user
}
pub fn create_default_executor<D>() -> Runtime<D>
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 3b38700f..bb4ea690 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -21,11 +21,13 @@ use std::thread::available_parallelism;
use anyhow::Result;
use clap::Parser;
use dotenvy::dotenv;
+use error_set::ErrContext;
use figlet_rs::FIGfont;
use monoio::Buildable;
use server::args::Args;
use server::bootstrap::{
create_default_executor, create_directories, create_root_user, create_shard_connections,
+ load_config,
};
use server::channels::commands::archive_state::ArchiveStateExecutor;
use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
@@ -73,17 +75,12 @@ fn main() -> Result<(), ServerError> {
}
let args = Args::parse();
+ // TODO: I think we could get rid of config provider, since we support only TOML
+ // as config provider.
let config_provider = config_provider::resolve(&args.config_provider)?;
let config = std::thread::scope(|scope| {
let config = scope
.spawn(move || {
- async fn load_config(
- config_provider: &ConfigProviderKind,
- ) -> Result<ServerConfig, ServerError> {
- let config = ServerConfig::load(config_provider).await?;
- Ok(config)
- }
-
let mut rt = create_default_executor::<monoio::IoUringDriver>();
rt.block_on(load_config(&config_provider))
})
@@ -93,7 +90,7 @@ fn main() -> Result<(), ServerError> {
})?;
// Create directories and root user.
- // Remove `local_data` directory if run with `--fresh` flag.
+ // Remove `local_data` directory if run with `--fresh` flag.
std::thread::scope(|scope| {
scope
.spawn(|| {
@@ -115,14 +112,15 @@ fn main() -> Result<(), ServerError> {
}
}
- // Create directories and root user
- create_directories().await;
- create_root_user().await;
- });
+ // Create directories.
+ create_directories(&config.system).await?;
+ Ok::<(), ServerError>(())
+ })
})
.join()
- .expect("Failed to create directories and root user");
- });
+ .expect("Failed to create directories and root user")
+ })
+ .with_error_context(|err| format!("Failed to create directories, err: {err}"))?;
// Initialize logging
let mut logging = Logging::new(config.telemetry.clone());
@@ -153,20 +151,21 @@ fn main() -> Result<(), ServerError> {
// let urb = io_uring::IoUring::builder();
// TODO: Shall we make the size of ring be configureable ?
let mut rt = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new()
- //.uring_builder(urb.setup_coop_taskrun())
+ //.uring_builder(urb.setup_coop_taskrun()) // broken shit.
.with_entries(1024) // Default size
.enable_timer()
.build()
.expect(format!("Failed to build monoio runtime for shard-{id}").as_str());
rt.block_on(async move {
let builder = IggyShard::builder();
- let shard = builder
+ let mut shard = builder
.id(id)
.connections(connections)
.server_config(server_config)
- .build_and_init()
+ .build()
.await;
+ shard.init().await;
shard.assert_init();
})
})
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 750e2b71..640ffb57 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -16,7 +16,12 @@
* under the License.
*/
-use crate::{configs::server::ServerConfig, shard::Shard};
+use std::{cell::Cell, rc::Rc, sync::Arc};
+
+use iggy_common::{Aes256GcmEncryptor, EncryptorKind};
+use tracing::info;
+
+use crate::{configs::server::ServerConfig, map_toggle_str, shard::Shard, state::{file::FileState, StateKind}, streaming::{persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind}, storage::SystemStorage}, versioning::SemanticVersion};
use super::{connector::ShardConnector, frame::ShardFrame, IggyShard};
@@ -43,11 +48,12 @@ impl IggyShardBuilder {
self
}
- pub async fn build_and_init(self) -> IggyShard {
+ // TODO: Too much happens in there, some of those bootstrapping logic should be moved outside.
+ pub async fn build(self) -> IggyShard {
let id = self.id.unwrap();
let config = self.config.unwrap();
let connections = self.connections.unwrap();
- let (stop_sender, stop_receiver, receiver) = connections
+ let (stop_sender, stop_receiver, frame_receiver) = connections
.iter()
.filter(|c| c.id == id)
.map(|c| {
@@ -60,7 +66,47 @@ impl IggyShardBuilder {
.next()
.expect("Failed to find connection with the specified ID");
let shards = connections.into_iter().map(Shard::new).collect();
+ let version = SemanticVersion::current().expect("Invalid version");
+
+ info!(
+ "Server-side encryption is {}.",
+ map_toggle_str(config.system.encryption.enabled)
+ );
+ let encryptor: Option<Arc<EncryptorKind>> = match config.system.encryption.enabled {
+ true => Some(Arc::new(EncryptorKind::Aes256Gcm(
+ Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key).unwrap(),
+ ))),
+ false => None,
+ };
+
+ let state_persister = Self::resolve_persister(config.system.state.enforce_fsync);
+ let state = Rc::new(StateKind::File(FileState::new(
+ &config.system.get_state_messages_file_path(),
+ &version,
+ state_persister,
+ encryptor.clone(),
+ )));
+
+ let partition_persister = Self::resolve_persister(config.system.partition.enforce_fsync);
+ let storage = SystemStorage::new(config.system, partition_persister);
+
+ IggyShard {
+ id: id,
+ shards: shards,
+ shards_table: Default::default(),
+ storage: storage,
+ state: state,
+ config: config,
+ stop_receiver: stop_receiver,
+ stop_sender: stop_sender,
+ frame_receiver: Cell::new(Some(frame_receiver)),
+ }
+ }
- IggyShard::new(id, shards, config, stop_receiver, stop_sender, receiver)
+ fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
+ match enforce_fsync {
+ true => Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister)),
+ false => Arc::new(PersisterKind::File(FilePersister)),
+ }
}
}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index dcb089d9..54b88fb1 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -15,6 +15,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
pub mod builder;
pub mod connector;
pub mod frame;
@@ -25,9 +26,9 @@ use builder::IggyShardBuilder;
use connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use frame::ShardFrame;
use namespace::IggyNamespace;
-use std::cell::{Cell, RefCell};
+use std::{cell::{Cell, RefCell}, rc::Rc, sync::Arc};
-use crate::configs::server::ServerConfig;
+use crate::{bootstrap::create_root_user, configs::server::ServerConfig, state::file::FileState, streaming::storage::SystemStorage};
pub(crate) struct Shard {
id: u16,
connection: ShardConnector<ShardFrame>,
@@ -49,21 +50,23 @@ struct ShardInfo {
pub struct IggyShard {
pub id: u16,
shards: Vec<Shard>,
- //shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
+ shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
//pub(crate) permissioner: RefCell<Permissioner>,
//pub(crate) streams: RwLock<HashMap<u32, Stream>>,
//pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
//pub(crate) users: RefCell<HashMap<UserId, User>>,
+ // TODO: Refactor.
+ pub(crate) storage: Arc<SystemStorage>,
// TODO - get rid of this dynamic dispatch.
- //pub(crate) state: Rc<FileState>,
+ pub(crate) state: Rc<FileState>,
//pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
config: ServerConfig,
//pub(crate) client_manager: RefCell<ClientManager>,
//pub(crate) active_sessions: RefCell<Vec<Session>>,
//pub(crate) metrics: Metrics,
- pub message_receiver: Cell<Option<Receiver<ShardFrame>>>,
+ pub frame_receiver: Cell<Option<Receiver<ShardFrame>>>,
stop_receiver: StopReceiver,
stop_sender: StopSender,
}
@@ -73,22 +76,26 @@ impl IggyShard {
Default::default()
}
- pub(crate) fn new(
- id: u16,
- shards: Vec<Shard>,
- config: ServerConfig,
- stop_receiver: StopReceiver,
- stop_sender: StopSender,
- shard_messages_receiver: Receiver<ShardFrame>,
- ) -> Self {
- Self {
- id,
- shards,
- config,
- stop_receiver,
- stop_sender,
- message_receiver: Cell::new(Some(shard_messages_receiver)),
- }
+ pub async fn init(&mut self) {
+ let user = create_root_user();
+ self.load_state().await;
+ self.load_users().await;
+ // Add default root user.
+ todo!();
+ self.load_streams().await;
+
+ }
+
+ async fn load_state(&self) {
+ todo!()
+ }
+
+ async fn load_users(&self) {
+ todo!()
+ }
+
+ async fn load_streams(&self) {
+ todo!()
}
pub fn assert_init(&self) {}
diff --git a/core/server/src/shard/namespace.rs b/core/server/src/shard/namespace.rs
index 4b972125..d9aedfb7 100644
--- a/core/server/src/shard/namespace.rs
+++ b/core/server/src/shard/namespace.rs
@@ -15,6 +15,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct IggyNamespace {
pub(crate) stream_id: u32,
diff --git a/core/server/src/state/file.rs b/core/server/src/state/file.rs
index e739066e..591b8072 100644
--- a/core/server/src/state/file.rs
+++ b/core/server/src/state/file.rs
@@ -84,18 +84,7 @@ impl FileState {
impl State for FileState {
async fn init(&self) -> Result<Vec<StateEntry>, IggyError> {
- if !Path::new(&self.path).exists() {
- info!("State file does not exist, creating a new one");
- self.persister
- .overwrite(&self.path, &[])
- .await
- .with_error_context(|error| {
- format!(
- "{COMPONENT} (error: {error}) - failed to overwrite state file, path: {}",
- self.path
- )
- })?;
- }
+ assert!(Path::new(&self.path).exists());
let entries = self.load_entries().await.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to load entries")
(iggy) 11/30: fixes
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 8566672cc5b26dcd2b0d98781d59dd7acde8dbc0
Author: numinex <g....@gmail.com>
AuthorDate: Wed Jun 25 14:32:26 2025 +0200
fixes
---
core/common/src/locking/mod.rs | 10 +-
core/common/src/locking/tokio_lock.rs | 2 -
core/server/src/shard/system/clients.rs | 2 +-
core/server/src/shard/system/consumer_groups.rs | 3 +-
core/server/src/shard/system/consumer_offsets.rs | 2 -
core/server/src/shard/system/mod.rs | 3 +-
.../src/shard/system/personal_access_tokens.rs | 2 -
core/server/src/shard/system/segments.rs | 3 +-
core/server/src/shard/system/stats.rs | 2 +-
core/server/src/shard/system/system.rs | 291 ---------------------
core/server/src/shard/system/topics.rs | 5 +-
.../src/streaming/partitions/consumer_offsets.rs | 2 +-
core/server/src/streaming/storage.rs | 2 +-
core/server/src/streaming/streams/topics.rs | 3 +-
core/server/src/streaming/topics/consumer_group.rs | 31 ++-
.../server/src/streaming/topics/consumer_groups.rs | 136 ++++++----
.../src/streaming/topics/consumer_offsets.rs | 6 +-
core/server/src/streaming/topics/messages.rs | 5 +-
core/server/src/streaming/topics/partitions.rs | 4 +-
core/server/src/streaming/topics/storage.rs | 9 +-
core/server/src/streaming/topics/topic.rs | 25 +-
21 files changed, 142 insertions(+), 406 deletions(-)
diff --git a/core/common/src/locking/mod.rs b/core/common/src/locking/mod.rs
index 42d72ebb..c2e542c7 100644
--- a/core/common/src/locking/mod.rs
+++ b/core/common/src/locking/mod.rs
@@ -29,19 +29,19 @@ mod fast_async_lock;
#[cfg(feature = "tokio_lock")]
#[cfg(not(any(feature = "fast_async_lock")))]
-pub type IggySharedMut<T> = tokio_lock::IggyTokioRwLock<T>;
+pub type IggyRwLock<T> = tokio_lock::IggyTokioRwLock<T>;
//this can be used in the future to provide different locking mechanisms
#[cfg(feature = "fast_async_lock")]
-pub type IggySharedMut<T> = fast_async_lock::IggyFastAsyncRwLock<T>;
+pub type IggyRwLock<T> = fast_async_lock::IggyFastAsyncRwLock<T>;
#[allow(async_fn_in_trait)]
-pub trait IggySharedMutFn<T>: Send + Sync {
- type ReadGuard<'a>: Deref<Target = T> + Send
+pub trait IggySharedMutFn<T> {
+ type ReadGuard<'a>: Deref<Target = T>
where
T: 'a,
Self: 'a;
- type WriteGuard<'a>: DerefMut<Target = T> + Send
+ type WriteGuard<'a>: DerefMut<Target = T>
where
T: 'a,
Self: 'a;
diff --git a/core/common/src/locking/tokio_lock.rs b/core/common/src/locking/tokio_lock.rs
index 2e26b367..ef5baaad 100644
--- a/core/common/src/locking/tokio_lock.rs
+++ b/core/common/src/locking/tokio_lock.rs
@@ -25,8 +25,6 @@ use tokio::sync::{RwLock as TokioRwLock, RwLockReadGuard, RwLockWriteGuard};
pub struct IggyTokioRwLock<T>(Arc<TokioRwLock<T>>);
impl<T> IggySharedMutFn<T> for IggyTokioRwLock<T>
-where
- T: Send + Sync,
{
type ReadGuard<'a>
= RwLockReadGuard<'a, T>
diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs
index fde182ce..a2debc42 100644
--- a/core/server/src/shard/system/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -22,7 +22,7 @@ use crate::streaming::session::Session;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
-use iggy_common::locking::IggySharedMut;
+use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
use std::net::SocketAddr;
use std::rc::Rc;
diff --git a/core/server/src/shard/system/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs
index 4975a92d..d8d8818e 100644
--- a/core/server/src/shard/system/consumer_groups.rs
+++ b/core/server/src/shard/system/consumer_groups.rs
@@ -16,9 +16,8 @@
* under the License.
*/
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use crate::streaming::topics::consumer_group::ConsumerGroup;
use error_set::ErrContext;
use iggy_common::Identifier;
diff --git a/core/server/src/shard/system/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs
index e63815ff..0c7438a6 100644
--- a/core/server/src/shard/system/consumer_offsets.rs
+++ b/core/server/src/shard/system/consumer_offsets.rs
@@ -17,8 +17,6 @@
*/
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use error_set::ErrContext;
use iggy_common::{Consumer, ConsumerOffsetInfo, Identifier, IggyError};
diff --git a/core/server/src/shard/system/mod.rs b/core/server/src/shard/system/mod.rs
index 1419abb9..fae8ad2b 100644
--- a/core/server/src/shard/system/mod.rs
+++ b/core/server/src/shard/system/mod.rs
@@ -28,8 +28,7 @@ pub mod snapshot;
pub mod stats;
pub mod storage;
pub mod streams;
-pub mod system;
pub mod topics;
pub mod users;
-pub const COMPONENT: &str = "SYSTEM";
+pub const COMPONENT: &str = "SHARD_SYSTEM";
diff --git a/core/server/src/shard/system/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs
index 18125bed..b5452e9a 100644
--- a/core/server/src/shard/system/personal_access_tokens.rs
+++ b/core/server/src/shard/system/personal_access_tokens.rs
@@ -18,8 +18,6 @@
use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use crate::streaming::users::user::User;
use error_set::ErrContext;
use iggy_common::IggyError;
diff --git a/core/server/src/shard/system/segments.rs b/core/server/src/shard/system/segments.rs
index bd4043a2..27abb76b 100644
--- a/core/server/src/shard/system/segments.rs
+++ b/core/server/src/shard/system/segments.rs
@@ -1,3 +1,4 @@
+use crate::shard::IggyShard;
/* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,8 +17,6 @@
* under the License.
*/
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
diff --git a/core/server/src/shard/system/stats.rs b/core/server/src/shard/system/stats.rs
index 2d2d2ea7..8410cac2 100644
--- a/core/server/src/shard/system/stats.rs
+++ b/core/server/src/shard/system/stats.rs
@@ -16,8 +16,8 @@
* under the License.
*/
+use crate::shard::IggyShard;
use crate::VERSION;
-use crate::streaming::systems::system::System;
use crate::versioning::SemanticVersion;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{IggyDuration, IggyError, Stats};
diff --git a/core/server/src/shard/system/system.rs b/core/server/src/shard/system/system.rs
deleted file mode 100644
index 5d830d71..00000000
--- a/core/server/src/shard/system/system.rs
+++ /dev/null
@@ -1,291 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-use crate::archiver::{ArchiverKind, ArchiverKindType};
-use crate::configs::server::{DataMaintenanceConfig, PersonalAccessTokenConfig};
-use crate::configs::system::SystemConfig;
-use crate::map_toggle_str;
-use crate::state::StateKind;
-use crate::state::file::FileState;
-use crate::state::system::SystemState;
-use crate::streaming::clients::client_manager::ClientManager;
-use crate::streaming::diagnostics::metrics::Metrics;
-use crate::streaming::persistence::persister::*;
-use crate::streaming::session::Session;
-use crate::streaming::storage::SystemStorage;
-use crate::streaming::streams::stream::Stream;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::users::permissioner::Permissioner;
-use crate::streaming::users::user::User;
-use crate::versioning::SemanticVersion;
-use ahash::AHashMap;
-use error_set::ErrContext;
-use iggy_common::locking::IggySharedMut;
-use iggy_common::locking::IggySharedMutFn;
-use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError, UserId};
-use std::path::Path;
-use std::sync::Arc;
-use tokio::fs::{create_dir_all, remove_dir_all};
-use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
-use tokio::time::Instant;
-use tracing::{error, info, instrument, trace};
-
-#[derive(Debug)]
-pub struct SharedSystem {
- system: Arc<RwLock<System>>,
-}
-
-impl SharedSystem {
- pub fn new(system: System) -> SharedSystem {
- SharedSystem {
- system: Arc::new(RwLock::new(system)),
- }
- }
-
- pub async fn read(&self) -> RwLockReadGuard<System> {
- self.system.read().await
- }
-
- pub async fn write(&self) -> RwLockWriteGuard<System> {
- self.system.write().await
- }
-}
-
-impl Clone for SharedSystem {
- fn clone(&self) -> Self {
- SharedSystem {
- system: self.system.clone(),
- }
- }
-}
-
-#[derive(Debug)]
-pub struct System {
- pub permissioner: Permissioner,
- pub(crate) storage: Arc<SystemStorage>,
- pub(crate) streams: AHashMap<u32, Stream>,
- pub(crate) streams_ids: AHashMap<String, u32>,
- pub(crate) users: AHashMap<UserId, User>,
- pub(crate) config: Arc<SystemConfig>,
- pub(crate) client_manager: IggySharedMut<ClientManager>,
- pub(crate) encryptor: Option<Arc<EncryptorKind>>,
- pub(crate) metrics: Metrics,
- pub(crate) state: Arc<StateKind>,
- pub(crate) archiver: Option<Arc<ArchiverKind>>,
- pub personal_access_token: PersonalAccessTokenConfig,
-}
-
-impl System {
- pub fn new(
- config: Arc<SystemConfig>,
- data_maintenance_config: DataMaintenanceConfig,
- pat_config: PersonalAccessTokenConfig,
- ) -> System {
- let version = SemanticVersion::current().expect("Invalid version");
- info!(
- "Server-side encryption is {}.",
- map_toggle_str(config.encryption.enabled)
- );
-
- let encryptor: Option<EncryptorKind> = match config.encryption.enabled {
- true => Some(EncryptorKind::Aes256Gcm(
- Aes256GcmEncryptor::from_base64_key(&config.encryption.key).unwrap(),
- )),
- false => None,
- };
-
- let state_persister = Self::resolve_persister(config.state.enforce_fsync);
- let partition_persister = Self::resolve_persister(config.partition.enforce_fsync);
-
- let state = Arc::new(StateKind::File(FileState::new(
- &config.get_state_messages_file_path(),
- &version,
- state_persister,
- encryptor,
- )));
-
- //TODO: Just shut the fuck up rust-analyzer.
- let encryptor: Option<EncryptorKind> = match config.encryption.enabled {
- true => Some(EncryptorKind::Aes256Gcm(
- Aes256GcmEncryptor::from_base64_key(&config.encryption.key).unwrap(),
- )),
- false => None,
- };
- Self::create(
- config.clone(),
- SystemStorage::new(config, partition_persister),
- state,
- encryptor.map(Arc::new),
- data_maintenance_config,
- pat_config,
- )
- }
-
- fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
- match enforce_fsync {
- true => Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister)),
- false => Arc::new(PersisterKind::File(FilePersister)),
- }
- }
-
- pub fn create(
- system_config: Arc<SystemConfig>,
- storage: SystemStorage,
- state: Arc<StateKind>,
- encryptor: Option<Arc<EncryptorKind>>,
- data_maintenance_config: DataMaintenanceConfig,
- pat_config: PersonalAccessTokenConfig,
- ) -> System {
- let archiver_config = data_maintenance_config.archiver;
- let archiver: Option<Arc<ArchiverKind>> = if archiver_config.enabled {
- info!("Archiving is enabled, kind: {}", archiver_config.kind);
- match archiver_config.kind {
- ArchiverKindType::Disk => Some(Arc::new(ArchiverKind::get_disk_archiver(
- archiver_config
- .disk
- .clone()
- .expect("Disk archiver config is missing"),
- ))),
- ArchiverKindType::S3 => Some(Arc::new(
- ArchiverKind::get_s3_archiver(
- archiver_config
- .s3
- .clone()
- .expect("S3 archiver config is missing"),
- )
- .expect("Failed to create S3 archiver"),
- )),
- }
- } else {
- info!("Archiving is disabled.");
- None
- };
-
- System {
- config: system_config,
- streams: AHashMap::new(),
- streams_ids: AHashMap::new(),
- storage: Arc::new(storage),
- encryptor,
- client_manager: IggySharedMut::new(ClientManager::default()),
- permissioner: Permissioner::default(),
- metrics: Metrics::init(),
- users: AHashMap::new(),
- state,
- personal_access_token: pat_config,
- archiver,
- }
- }
-
- #[instrument(skip_all, name = "trace_system_init")]
- pub async fn init(&mut self) -> Result<(), IggyError> {
- let system_path = self.config.get_system_path();
- if !Path::new(&system_path).exists() && create_dir_all(&system_path).await.is_err() {
- return Err(IggyError::CannotCreateBaseDirectory(system_path));
- }
-
- let state_path = self.config.get_state_path();
- if !Path::new(&state_path).exists() && create_dir_all(&state_path).await.is_err() {
- return Err(IggyError::CannotCreateStateDirectory(state_path));
- }
-
- let streams_path = self.config.get_streams_path();
- if !Path::new(&streams_path).exists() && create_dir_all(&streams_path).await.is_err() {
- return Err(IggyError::CannotCreateStreamsDirectory(streams_path));
- }
-
- let runtime_path = self.config.get_runtime_path();
- if Path::new(&runtime_path).exists() && remove_dir_all(&runtime_path).await.is_err() {
- return Err(IggyError::CannotRemoveRuntimeDirectory(runtime_path));
- }
-
- if create_dir_all(&runtime_path).await.is_err() {
- return Err(IggyError::CannotCreateRuntimeDirectory(runtime_path));
- }
-
- info!(
- "Initializing system, data will be stored at: {}",
- self.config.get_system_path()
- );
-
- let state_entries = self.state.init().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to initialize state entries")
- })?;
- let system_state = SystemState::init(state_entries)
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to initialize system state")
- })?;
- let now = Instant::now();
- //DONE
- /*
- self.load_version().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load version")
- })?;
- */
- self.load_users(system_state.users.into_values().collect())
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load users")
- })?;
- self.load_streams(system_state.streams.into_values().collect())
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load streams")
- })?;
- if let Some(archiver) = self.archiver.as_ref() {
- archiver
- .init()
- .await
- .expect("Failed to initialize archiver");
- }
- info!("Initialized system in {} ms.", now.elapsed().as_millis());
- Ok(())
- }
-
- #[instrument(skip_all, name = "trace_shutdown")]
- pub async fn shutdown(&mut self) -> Result<(), IggyError> {
- self.persist_messages().await?;
- Ok(())
- }
-
- #[instrument(skip_all, name = "trace_persist_messages")]
- pub async fn persist_messages(&self) -> Result<usize, IggyError> {
- trace!("Saving buffered messages on disk...");
- let mut saved_messages_number = 0;
- for stream in self.streams.values() {
- saved_messages_number += stream.persist_messages().await?;
- }
-
- Ok(saved_messages_number)
- }
-
- pub fn ensure_authenticated(&self, session: &Session) -> Result<(), IggyError> {
- if !session.is_active() {
- error!("{COMPONENT} - session is inactive, session: {session}");
- return Err(IggyError::StaleClient);
- }
-
- if session.is_authenticated() {
- Ok(())
- } else {
- error!("{COMPONENT} - unauthenticated access attempt, session: {session}");
- Err(IggyError::Unauthenticated)
- }
- }
-}
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index a61f5464..145689af 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -16,15 +16,14 @@
* under the License.
*/
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize};
-impl System {
+impl IggyShard {
pub fn find_topic(
&self,
session: &Session,
diff --git a/core/server/src/streaming/partitions/consumer_offsets.rs b/core/server/src/streaming/partitions/consumer_offsets.rs
index 177793c9..96080c03 100644
--- a/core/server/src/streaming/partitions/consumer_offsets.rs
+++ b/core/server/src/streaming/partitions/consumer_offsets.rs
@@ -26,7 +26,7 @@ use iggy_common::IggyError;
use tracing::trace;
impl Partition {
- pub async fn get_consumer_offset(
+ pub fn get_consumer_offset(
&self,
consumer: PollingConsumer,
) -> Result<Option<u64>, IggyError> {
diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs
index 51e2565e..978b97aa 100644
--- a/core/server/src/streaming/storage.rs
+++ b/core/server/src/streaming/storage.rs
@@ -154,7 +154,7 @@ pub struct SystemStorage {
impl SystemStorage {
pub fn new(config: Rc<SystemConfig>, persister: Arc<PersisterKind>) -> Self {
Self {
- info: Arc::new(SystemInfoStorageKind::File(FileSystemInfoStorage::new(
+ info: Rc::new(SystemInfoStorageKind::File(FileSystemInfoStorage::new(
config.get_state_info_path(),
persister.clone(),
))),
diff --git a/core/server/src/streaming/streams/topics.rs b/core/server/src/streaming/streams/topics.rs
index b975ee83..ed350bee 100644
--- a/core/server/src/streaming/streams/topics.rs
+++ b/core/server/src/streaming/streams/topics.rs
@@ -146,7 +146,8 @@ impl Stream {
topic.name = name.to_owned();
topic.message_expiry = message_expiry;
topic.compression_algorithm = compression_algorithm;
- for partition in topic.partitions.borrow_mut().values_mut() {
+ for partition in topic.partitions.values_mut() {
+ let mut partition = partition.write().await;
partition.message_expiry = message_expiry;
for segment in partition.segments.iter_mut() {
segment.update_message_expiry(message_expiry);
diff --git a/core/server/src/streaming/topics/consumer_group.rs b/core/server/src/streaming/topics/consumer_group.rs
index cc28ef6f..f000ce79 100644
--- a/core/server/src/streaming/topics/consumer_group.rs
+++ b/core/server/src/streaming/topics/consumer_group.rs
@@ -52,12 +52,12 @@ impl ConsumerGroup {
self.members.values().cloned().collect()
}
- pub async fn reassign_partitions(&mut self, partitions_count: u32) {
+ pub fn reassign_partitions(&mut self, partitions_count: u32) {
self.partitions_count = partitions_count;
- self.assign_partitions().await;
+ self.assign_partitions();
}
- pub async fn calculate_partition_id(
+ pub fn calculate_partition_id(
&mut self,
member_id: u32,
) -> Result<Option<u32>, IggyError> {
@@ -72,7 +72,7 @@ impl ConsumerGroup {
))
}
- pub async fn get_current_partition_id(&self, member_id: u32) -> Result<Option<u32>, IggyError> {
+ pub fn get_current_partition_id(&mut self, member_id: u32) -> Result<Option<u32>, IggyError> {
let member = self.members.get(&member_id);
if let Some(member) = member {
return Ok(member.current_partition_id);
@@ -84,7 +84,7 @@ impl ConsumerGroup {
))
}
- pub async fn add_member(&mut self, member_id: u32) {
+ pub fn add_member(&mut self, member_id: u32) {
self.members.insert(
member_id,
ConsumerGroupMember {
@@ -98,20 +98,20 @@ impl ConsumerGroup {
"Added member with ID: {} to consumer group: {} for topic with ID: {}",
member_id, self.group_id, self.topic_id
);
- self.assign_partitions().await;
+ self.assign_partitions();
}
- pub async fn delete_member(&mut self, member_id: u32) {
+ pub fn delete_member(&mut self, member_id: u32) {
if self.members.remove(&member_id).is_some() {
trace!(
"Deleted member with ID: {} in consumer group: {} for topic with ID: {}",
member_id, self.group_id, self.topic_id
);
- self.assign_partitions().await;
+ self.assign_partitions();
}
}
- async fn assign_partitions(&mut self) {
+ fn assign_partitions(&mut self) {
let mut members = self.members.values_mut().collect::<Vec<_>>();
if members.is_empty() {
return;
@@ -189,11 +189,10 @@ mod tests {
members: AHashMap::new(),
};
- consumer_group.add_member(member_id).await;
+ consumer_group.add_member(member_id);
for i in 0..1000 {
let partition_id = consumer_group
.calculate_partition_id(member_id)
- .await
.unwrap()
.expect("Partition ID not found");
assert_eq!(partition_id, (i % consumer_group.partitions_count) + 1);
@@ -211,7 +210,7 @@ mod tests {
members: AHashMap::new(),
};
- consumer_group.add_member(member_id).await;
+ consumer_group.add_member(member_id);
let member = consumer_group.members.get(&member_id).unwrap();
assert_eq!(
member.partitions.len() as u32,
@@ -235,8 +234,8 @@ mod tests {
members: AHashMap::new(),
};
- consumer_group.add_member(member1_id).await;
- consumer_group.add_member(member2_id).await;
+ consumer_group.add_member(member1_id);
+ consumer_group.add_member(member2_id);
let member1 = consumer_group.members.get(&member1_id).unwrap();
let member2 = consumer_group.members.get(&member2_id).unwrap();
assert_eq!(
@@ -270,8 +269,8 @@ mod tests {
members: AHashMap::new(),
};
- consumer_group.add_member(member1_id).await;
- consumer_group.add_member(member2_id).await;
+ consumer_group.add_member(member1_id);
+ consumer_group.add_member(member2_id);
let member1 = consumer_group.members.get(&member1_id).unwrap();
let member2 = consumer_group.members.get(&member2_id).unwrap();
if member1.partitions.len() == 1 {
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index 1f50a1bb..c1371096 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -16,36 +16,39 @@
* under the License.
*/
-use crate::streaming::topics::COMPONENT;
+use crate::binary::handlers::topics::get_topic_handler;
+use crate::streaming::topics::{consumer_group, COMPONENT};
use crate::streaming::topics::consumer_group::ConsumerGroup;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
+use iggy_common::locking::IggySharedMutFn;
use iggy_common::IggyError;
use iggy_common::{IdKind, Identifier};
+use std::cell::{Ref, RefMut};
use std::sync::atomic::Ordering;
use tracing::info;
impl Topic {
pub fn reassign_consumer_groups(&mut self) {
- if self.consumer_groups.is_empty() {
+ if self.consumer_groups.borrow().is_empty() {
return;
}
- let partitions_count = self.partitions.borrow().len() as u32;
+ let partitions_count = self.partitions.len() as u32;
info!(
"Reassigning consumer groups for topic with ID: {} for stream with ID with {}, partitions count: {}",
self.topic_id, self.stream_id, partitions_count
);
- for (_, consumer_group) in self.consumer_groups.iter_mut() {
+ for (_, consumer_group) in self.consumer_groups.borrow_mut().iter_mut() {
consumer_group.reassign_partitions(partitions_count);
}
}
- pub fn get_consumer_groups(&self) -> Vec<&ConsumerGroup> {
- self.consumer_groups.values().collect()
+ pub fn get_consumer_groups(&self) -> Vec<ConsumerGroup> {
+ self.consumer_groups.borrow().values().cloned().collect()
}
- pub fn get_consumer_group(&self, identifier: &Identifier) -> Result<&ConsumerGroup, IggyError> {
+ pub fn get_consumer_group(&self, identifier: &Identifier) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
match identifier.kind {
IdKind::Numeric => self.get_consumer_group_by_id(identifier.get_u32_value().unwrap()),
IdKind::String => self.get_consumer_group_by_name(&identifier.get_cow_str_value()?),
@@ -55,22 +58,42 @@ impl Topic {
pub fn try_get_consumer_group(
&self,
identifier: &Identifier,
- ) -> Result<Option<&ConsumerGroup>, IggyError> {
+ ) -> Result<Option<Ref<'_, ConsumerGroup>>, IggyError> {
match identifier.kind {
- IdKind::Numeric => Ok(self.consumer_groups.get(&identifier.get_u32_value()?)),
+ IdKind::Numeric => Ok(self.try_get_consumer_group_by_id(&identifier.get_u32_value()?)),
IdKind::String => {
Ok(self.try_get_consumer_group_by_name(&identifier.get_cow_str_value()?))
}
}
}
+ fn try_get_consumer_group_by_id(&self, id: &u32) -> Option<Ref<'_, ConsumerGroup>> {
+ let consumer_groups = self.consumer_groups.borrow();
+ let exists = consumer_groups.contains_key(id);
+ if !exists {
+ return None;
+ }
+
+ Some(Ref::map(consumer_groups, |cg| {
+ let consumer_group = cg.get(id);
+ consumer_group.unwrap()
+ }))
+ }
+
+ fn try_get_consumer_group_by_name(&self, name: &str) -> Option<Ref<'_, ConsumerGroup>> {
+ let consumer_groups = self.consumer_groups.borrow();
+ let exists = self.consumer_groups_ids.contains_key(name);
+ let id = self.consumer_groups_ids.get(name).unwrap();
+ if !exists {
+ return None;
+ }
- fn try_get_consumer_group_by_name(&self, name: &str) -> Option<&ConsumerGroup> {
- self.consumer_groups_ids
- .get(name)
- .and_then(|id| self.consumer_groups.get(id))
+ Some(Ref::map(consumer_groups, |cg| {
+ let consumer_group = cg.get(id);
+ consumer_group.unwrap()
+ }))
}
- pub fn get_consumer_group_by_name(&self, name: &str) -> Result<&ConsumerGroup, IggyError> {
+ pub fn get_consumer_group_by_name(&self, name: &str) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
let group_id = self.consumer_groups_ids.get(name);
if group_id.is_none() {
return Err(IggyError::ConsumerGroupNameNotFound(
@@ -82,19 +105,24 @@ impl Topic {
self.get_consumer_group_by_id(*group_id.unwrap())
}
- pub fn get_consumer_group_by_id(&self, id: u32) -> Result<&ConsumerGroup, IggyError> {
- let consumer_group = self.consumer_groups.get(&id);
- if consumer_group.is_none() {
+ pub fn get_consumer_group_by_id(&self, id: u32) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
+ let consumer_groups = self.consumer_groups.borrow();
+ let exists = consumer_groups.contains_key(&id);
+ if !exists {
return Err(IggyError::ConsumerGroupIdNotFound(id, self.topic_id));
}
+ let consumer_group = Ref::map(consumer_groups, |cg| {
+ let consumer_group = cg.get(&id);
+ consumer_group.unwrap()
+ });
- Ok(consumer_group.unwrap())
+ Ok(consumer_group)
}
pub fn get_consumer_group_mut(
- &mut self,
+ &self,
identifier: &Identifier,
- ) -> Result<&mut ConsumerGroup, IggyError> {
+ ) -> Result<RefMut<'_, ConsumerGroup>, IggyError> {
match identifier.kind {
IdKind::Numeric => {
self.get_consumer_group_by_id_mut(identifier.get_u32_value().unwrap())
@@ -104,21 +132,25 @@ impl Topic {
}
pub fn get_consumer_group_by_id_mut(
- &mut self,
+ &self,
id: u32,
- ) -> Result<&mut ConsumerGroup, IggyError> {
- let consumer_group = self.consumer_groups.get_mut(&id);
- if consumer_group.is_none() {
+ ) -> Result<RefMut<'_, ConsumerGroup>, IggyError> {
+ let consumer_groups = self.consumer_groups.borrow_mut();
+ let exists = consumer_groups.contains_key(&id);
+ if !exists {
return Err(IggyError::ConsumerGroupIdNotFound(id, self.topic_id));
}
-
- Ok(consumer_group.unwrap())
+ let consumer_group = RefMut::map(consumer_groups, |cg| {
+ let consumer_group = cg.get_mut(&id);
+ consumer_group.unwrap()
+ });
+ Ok(consumer_group)
}
pub fn get_consumer_group_by_name_mut(
- &mut self,
+ &self,
name: &str,
- ) -> Result<&mut ConsumerGroup, IggyError> {
+ ) -> Result<RefMut<'_, ConsumerGroup>, IggyError> {
let group_id = self.consumer_groups_ids.get(name).copied();
if group_id.is_none() {
return Err(IggyError::ConsumerGroupNameNotFound(
@@ -148,7 +180,7 @@ impl Topic {
.current_consumer_group_id
.fetch_add(1, Ordering::SeqCst);
loop {
- if self.consumer_groups.contains_key(&id) {
+ if self.consumer_groups.borrow().contains_key(&id) {
if id == u32::MAX {
return Err(IggyError::ConsumerGroupIdAlreadyExists(id, self.topic_id));
}
@@ -163,7 +195,7 @@ impl Topic {
id = group_id.unwrap();
}
- if self.consumer_groups.contains_key(&id) {
+ if self.consumer_groups.borrow().contains_key(&id) {
return Err(IggyError::ConsumerGroupIdAlreadyExists(id, self.topic_id));
}
@@ -171,11 +203,11 @@ impl Topic {
self.topic_id,
id,
name,
- self.partitions.borrow().len() as u32,
+ self.partitions.len() as u32,
);
self.consumer_groups_ids.insert(name.to_owned(), id);
let cloned_group = consumer_group.clone();
- self.consumer_groups.insert(id, consumer_group);
+ self.consumer_groups.borrow_mut().insert(id, consumer_group);
info!(
"Created consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
id, self.topic_id, self.stream_id
@@ -195,11 +227,14 @@ impl Topic {
group_id = consumer_group.group_id;
}
- let consumer_group = self.consumer_groups.remove(&group_id);
+ let mut consumer_groups = self.consumer_groups.borrow_mut();
+ let consumer_group = consumer_groups.remove(&group_id);
if consumer_group.is_none() {
return Err(IggyError::ConsumerGroupIdNotFound(group_id, self.topic_id));
}
let consumer_group = consumer_group.unwrap();
+ let consumer_group = consumer_group.clone();
+ drop(consumer_groups);
{
self.consumer_groups_ids.remove(&consumer_group.name);
let current_group_id = self.current_consumer_group_id.load(Ordering::SeqCst);
@@ -208,7 +243,8 @@ impl Topic {
.store(group_id, Ordering::SeqCst);
}
- for (_, partition) in self.partitions.borrow().iter() {
+ for (_, partition) in self.partitions.iter() {
+ let partition = partition.read().await;
if let Some((_, offset)) = partition.consumer_group_offsets.remove(&group_id) {
self.storage
.partition
@@ -231,13 +267,15 @@ impl Topic {
group_id: &Identifier,
member_id: u32,
) -> Result<(), IggyError> {
- let consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
+ let topic_id = self.topic_id;
+ let stream_id = self.stream_id;
+ let mut consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get consumer group with id: {group_id}")
})?;
consumer_group.add_member(member_id);
info!(
"Member with ID: {} has joined consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
- member_id, group_id, self.topic_id, self.stream_id
+ member_id, group_id, topic_id, stream_id
);
Ok(())
}
@@ -247,13 +285,15 @@ impl Topic {
group_id: &Identifier,
member_id: u32,
) -> Result<(), IggyError> {
- let consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
+ let topic_id = self.topic_id;
+ let stream_id = self.stream_id;
+ let mut consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get consumer group with id: {group_id}")
})?;
consumer_group.delete_member(member_id);
info!(
"Member with ID: {} has left consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
- member_id, group_id, self.topic_id, self.stream_id
+ member_id, group_id, topic_id, stream_id
);
Ok(())
}
@@ -286,7 +326,7 @@ mod tests {
assert_eq!(created_consumer_group.topic_id, topic_id);
}
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
let consumer_group = topic
.get_consumer_group(&Identifier::numeric(group_id).unwrap())
.unwrap();
@@ -295,7 +335,7 @@ mod tests {
assert_eq!(consumer_group.topic_id, topic_id);
assert_eq!(
consumer_group.partitions_count,
- topic.partitions.borrow().len() as u32
+ topic.partitions.len() as u32
);
}
@@ -306,12 +346,12 @@ mod tests {
let mut topic = get_topic().await;
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
let result = topic.create_consumer_group(Some(group_id), "test2");
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, IggyError::ConsumerGroupIdAlreadyExists(_, _)));
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
}
#[tokio::test]
@@ -321,7 +361,7 @@ mod tests {
let mut topic = get_topic().await;
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
let group_id = group_id + 1;
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_err());
@@ -330,7 +370,7 @@ mod tests {
err,
IggyError::ConsumerGroupNameAlreadyExists(_, _)
));
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
}
#[tokio::test]
@@ -340,12 +380,12 @@ mod tests {
let mut topic = get_topic().await;
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
let result = topic
.delete_consumer_group(&Identifier::numeric(group_id).unwrap())
.await;
assert!(result.is_ok());
- assert!(topic.consumer_groups.is_empty());
+ assert!(topic.consumer_groups.borrow().is_empty());
}
#[tokio::test]
@@ -355,13 +395,13 @@ mod tests {
let mut topic = get_topic().await;
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
let group_id = group_id + 1;
let result = topic
.delete_consumer_group(&Identifier::numeric(group_id).unwrap())
.await;
assert!(result.is_err());
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
}
#[tokio::test]
diff --git a/core/server/src/streaming/topics/consumer_offsets.rs b/core/server/src/streaming/topics/consumer_offsets.rs
index f59e9276..8748576b 100644
--- a/core/server/src/streaming/topics/consumer_offsets.rs
+++ b/core/server/src/streaming/topics/consumer_offsets.rs
@@ -34,7 +34,6 @@ impl Topic {
) -> Result<(), IggyError> {
let Some((polling_consumer, partition_id)) = self
.resolve_consumer_with_partition_id(&consumer, client_id, partition_id, false)
- .await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer with partition id, consumer ID: {}, client ID: {}, partition ID: {:?}", consumer.id, client_id, partition_id))? else {
return Err(IggyError::ConsumerOffsetNotFound(client_id));
};
@@ -78,8 +77,7 @@ impl Topic {
) -> Result<Option<ConsumerOffsetInfo>, IggyError> {
let Some((polling_consumer, partition_id)) = self
.resolve_consumer_with_partition_id(consumer, client_id, partition_id, false)
- .await
- .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer offset for consumer: {consumer}, client ID: {client_id}, partition ID: {partition_id:#?}"))? else {
+ .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer offset for consumer: {consumer}, client ID: {client_id}, partition ID: {:#?}", partition_id))? else {
return Ok(None);
};
@@ -93,7 +91,6 @@ impl Topic {
let partition = partition.read().await;
let offset = partition
.get_consumer_offset(polling_consumer)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get consumer offset for consumer: {polling_consumer}"
@@ -118,7 +115,6 @@ impl Topic {
) -> Result<(), IggyError> {
let Some((polling_consumer, partition_id)) = self
.resolve_consumer_with_partition_id(&consumer, client_id, partition_id, false)
- .await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer with partition id, consumer ID: {}, client ID: {}, partition ID: {:?}", consumer.id, client_id, partition_id))? else {
return Err(IggyError::ConsumerOffsetNotFound(client_id));
};
diff --git a/core/server/src/streaming/topics/messages.rs b/core/server/src/streaming/topics/messages.rs
index 56d7c1c7..e8c52385 100644
--- a/core/server/src/streaming/topics/messages.rs
+++ b/core/server/src/streaming/topics/messages.rs
@@ -209,6 +209,7 @@ mod tests {
use bytes::Bytes;
use iggy_common::CompressionAlgorithm;
use iggy_common::{IggyMessage, MaxTopicSize};
+ use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::AtomicU32;
use std::sync::atomic::AtomicU64;
@@ -321,11 +322,11 @@ mod tests {
async fn init_topic(partitions_count: u32) -> Topic {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
diff --git a/core/server/src/streaming/topics/partitions.rs b/core/server/src/streaming/topics/partitions.rs
index eb6b4280..dfedf0a9 100644
--- a/core/server/src/streaming/topics/partitions.rs
+++ b/core/server/src/streaming/topics/partitions.rs
@@ -20,7 +20,7 @@ use crate::streaming::partitions::partition::Partition;
use crate::streaming::topics::COMPONENT;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
-use iggy_common::locking::{IggySharedMut, IggySharedMutFn};
+use iggy_common::locking::{IggyRwLock, IggySharedMutFn};
use iggy_common::{IggyError, IggyTimestamp};
const MAX_PARTITIONS_COUNT: u32 = 100_000;
@@ -63,7 +63,7 @@ impl Topic {
)
.await;
self.partitions
- .insert(partition_id, IggySharedMut::new(partition));
+ .insert(partition_id, IggyRwLock::new(partition));
partition_ids.push(partition_id)
}
diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs
index 1cc33310..ae9435e5 100644
--- a/core/server/src/streaming/topics/storage.rs
+++ b/core/server/src/streaming/topics/storage.rs
@@ -27,7 +27,7 @@ use anyhow::Context;
use error_set::ErrContext;
use futures::future::join_all;
use iggy_common::IggyError;
-use iggy_common::locking::IggySharedMut;
+use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
use serde::{Deserialize, Serialize};
use std::path::Path;
@@ -189,7 +189,7 @@ impl TopicStorage for FileTopicStorage {
for mut partition in unloaded_partitions {
let loaded_partitions = loaded_partitions.clone();
let partition_state = state.partitions.remove(&partition.partition_id).unwrap();
- let load_partition = tokio::spawn(async move {
+ let load_partition = monoio::spawn(async move {
match partition.load(partition_state).await {
Ok(_) => {
loaded_partitions.lock().await.push(partition);
@@ -209,7 +209,7 @@ impl TopicStorage for FileTopicStorage {
for partition in loaded_partitions.lock().await.drain(..) {
topic
.partitions
- .insert(partition.partition_id, IggySharedMut::new(partition));
+ .insert(partition.partition_id, IggyRwLock::new(partition));
}
for consumer_group in state.consumer_groups.into_values() {
@@ -224,7 +224,8 @@ impl TopicStorage for FileTopicStorage {
.insert(consumer_group.name.to_owned(), consumer_group.group_id);
topic
.consumer_groups
- .insert(consumer_group.group_id, RwLock::new(consumer_group));
+ .borrow_mut()
+ .insert(consumer_group.group_id, consumer_group);
}
info!("Loaded topic {topic}");
diff --git a/core/server/src/streaming/topics/topic.rs b/core/server/src/streaming/topics/topic.rs
index ad3e25ef..16aa324c 100644
--- a/core/server/src/streaming/topics/topic.rs
+++ b/core/server/src/streaming/topics/topic.rs
@@ -25,7 +25,7 @@ use ahash::AHashMap;
use core::fmt;
use std::cell::RefCell;
use std::rc::Rc;
-use iggy_common::locking::IggySharedMut;
+use iggy_common::locking::IggyRwLock;
use iggy_common::{
CompressionAlgorithm, Consumer, ConsumerKind, IggyByteSize, IggyError, IggyExpiry,
IggyTimestamp, MaxTopicSize, Sizeable,
@@ -33,7 +33,6 @@ use iggy_common::{
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
-use tokio::sync::RwLock;
use tracing::info;
const ALMOST_FULL_THRESHOLD: f64 = 0.9;
@@ -51,9 +50,9 @@ pub struct Topic {
pub(crate) messages_count: Arc<AtomicU64>,
pub(crate) segments_count_of_parent_stream: Arc<AtomicU32>,
pub(crate) config: Rc<SystemConfig>,
- pub(crate) partitions: RefCell<AHashMap<u32, Partition>>,
+ pub(crate) partitions: AHashMap<u32, IggyRwLock<Partition>>,
pub(crate) storage: Rc<SystemStorage>,
- pub(crate) consumer_groups: AHashMap<u32, ConsumerGroup>,
+ pub(crate) consumer_groups: RefCell<AHashMap<u32, ConsumerGroup>>,
pub(crate) consumer_groups_ids: AHashMap<String, u32>,
pub(crate) current_consumer_group_id: AtomicU32,
pub(crate) current_partition_id: AtomicU32,
@@ -126,7 +125,7 @@ impl Topic {
messages_count_of_parent_stream,
messages_count: Arc::new(AtomicU64::new(0)),
segments_count_of_parent_stream,
- consumer_groups: AHashMap::new(),
+ consumer_groups: RefCell::new(AHashMap::new()),
consumer_groups_ids: AHashMap::new(),
current_consumer_group_id: AtomicU32::new(1),
current_partition_id: AtomicU32::new(1),
@@ -172,11 +171,11 @@ impl Topic {
matches!(self.max_topic_size, MaxTopicSize::Unlimited)
}
- pub fn get_partitions(&self) -> Vec<IggySharedMut<Partition>> {
+ pub fn get_partitions(&self) -> Vec<IggyRwLock<Partition>> {
self.partitions.values().cloned().collect()
}
- pub fn get_partition(&self, partition_id: u32) -> Result<IggySharedMut<Partition>, IggyError> {
+ pub fn get_partition(&self, partition_id: u32) -> Result<IggyRwLock<Partition>, IggyError> {
match self.partitions.get(&partition_id) {
Some(partition_arc) => Ok(partition_arc.clone()),
None => Err(IggyError::PartitionNotFound(
@@ -187,7 +186,7 @@ impl Topic {
}
}
- pub async fn resolve_consumer_with_partition_id(
+ pub fn resolve_consumer_with_partition_id(
&self,
consumer: &Consumer,
client_id: u32,
@@ -203,7 +202,7 @@ impl Topic {
)))
}
ConsumerKind::ConsumerGroup => {
- let consumer_group = self.get_consumer_group(&consumer.id)?.read().await;
+ let mut consumer_group = self.get_consumer_group_mut(&consumer.id)?;
if let Some(partition_id) = partition_id {
return Ok(Some((
PollingConsumer::consumer_group(consumer_group.group_id, client_id),
@@ -212,9 +211,9 @@ impl Topic {
}
let partition_id = if calculate_partition_id {
- consumer_group.calculate_partition_id(client_id).await?
+ consumer_group.calculate_partition_id(client_id)?
} else {
- consumer_group.get_current_partition_id(client_id).await?
+ consumer_group.get_current_partition_id(client_id)?
};
let Some(partition_id) = partition_id else {
return Ok(None);
@@ -291,11 +290,11 @@ mod tests {
#[tokio::test]
async fn should_be_created_given_valid_parameters() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
(iggy) 24/30: feat(io_uring): fix metadata crud operations (#1950)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit f44adee78872dca0656c39d3805081db3a40c572
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Tue Jul 1 09:11:12 2025 +0200
feat(io_uring): fix metadata crud operations (#1950)
This PR fixes the metadata crud operations, preventing double borrow_mut
call on collections across awaits
---
core/integration/tests/streaming/stream.rs | 1 -
core/server/src/shard/mod.rs | 1 +
core/server/src/shard/system/messages.rs | 15 +--
core/server/src/shard/system/partitions.rs | 115 ++++++++++++++++------
core/server/src/shard/system/streams.rs | 7 ++
core/server/src/shard/system/topics.rs | 130 ++++++++++++++++---------
core/server/src/streaming/streams/topics.rs | 29 +-----
core/server/src/streaming/topics/partitions.rs | 37 ++-----
8 files changed, 196 insertions(+), 139 deletions(-)
diff --git a/core/integration/tests/streaming/stream.rs b/core/integration/tests/streaming/stream.rs
index 677e2d32..4cfa88ba 100644
--- a/core/integration/tests/streaming/stream.rs
+++ b/core/integration/tests/streaming/stream.rs
@@ -132,7 +132,6 @@ async fn should_purge_existing_stream_on_disk() {
MaxTopicSize::ServerDefault,
1,
)
- .await
.unwrap();
let messages = create_messages();
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 6c3718f8..8ffc6560 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -400,6 +400,7 @@ impl IggyShard {
}
}
+ //TODO: Refactor...
let mut streams_states = streams
.into_iter()
.filter(|s| !missing_ids.contains(&s.id))
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index b27b034c..c5e527d0 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -53,12 +53,15 @@ impl IggyShard {
)
})?;
let stream_id = stream.stream_id;
- let numeric_topic_id = stream.get_topic(topic_id).map(|topic| topic.topic_id).with_error_context(|error| {
- format!(
- "Failed to get topic with ID: {} (error: {})",
- topic_id, error
- )
- })?;
+ let numeric_topic_id = stream
+ .get_topic(topic_id)
+ .map(|topic| topic.topic_id)
+ .with_error_context(|error| {
+ format!(
+ "Failed to get topic with ID: {} (error: {})",
+ topic_id, error
+ )
+ })?;
// TODO: We should look into refactoring those permissioners, so they can accept `Identifier` instead of numeric IDs.
// Validate permissions for given user on stream and topic.
self.permissioner.borrow().append_messages(
diff --git a/core/server/src/shard/system/partitions.rs b/core/server/src/shard/system/partitions.rs
index 78ef2144..a5b59516 100644
--- a/core/server/src/shard/system/partitions.rs
+++ b/core/server/src/shard/system/partitions.rs
@@ -24,7 +24,9 @@ use crate::streaming::session::Session;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
+use iggy_common::locking::IggySharedMutFn;
+// TODO: MAJOR REFACTOR!!!!!!!!!!!!!!!!!
impl IggyShard {
pub async fn create_partitions(
&self,
@@ -33,6 +35,9 @@ impl IggyShard {
topic_id: &Identifier,
partitions_count: u32,
) -> Result<(), IggyError> {
+ // This whole method is yeah....
+ // I don't admit to writing it.
+ // Sorry, not sorry.
self.ensure_authenticated(session)?;
{
let stream = self.get_stream(stream_id).with_error_context(|error| {
@@ -53,35 +58,61 @@ impl IggyShard {
))?;
}
- let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
- })?;
- let stream_id = stream.stream_id;
- let topic = stream
+ let partition_ids = {
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let topic = stream
.get_topic_mut(topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {stream_id}"
)
})?;
- let topic_id = topic.topic_id;
-
- // TODO: Make add persisted partitions to topic sync, and extract the storage persister out of it
- // perform disk i/o outside of the borrow_mut of the stream.
- let partition_ids = topic
+ let partition_ids = topic
.add_persisted_partitions(partitions_count)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to add persisted partitions, topic: {topic}")
})?;
- let records = partition_ids.into_iter().map(|partition_id| {
- let namespace = IggyNamespace::new(stream_id, topic_id, partition_id);
- let hash = namespace.generate_hash();
- let shard_id = hash % self.get_available_shards_count();
- let shard_info = ShardInfo::new(shard_id as u16);
- (namespace, shard_info)
- });
- self.insert_shard_table_records(records);
+ partition_ids
+ };
+
+ {
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let topic = stream.get_topic(topic_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get topic with ID: {topic_id} in stream with ID: {stream_id}")
+ })?;
+ let topic_id = topic.topic_id;
+ for partition_id in &partition_ids {
+ let partition = topic.partitions.get(partition_id).unwrap();
+ let mut partition = partition.write().await;
+ partition.persist().await.with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to persist partition with id: {}",
+ partition.partition_id
+ )
+ })?;
+ }
+ let records = partition_ids.into_iter().map(|partition_id| {
+ let namespace = IggyNamespace::new(stream_id, topic_id, partition_id);
+ let hash = namespace.generate_hash();
+ let shard_id = hash % self.get_available_shards_count();
+ let shard_info = ShardInfo::new(shard_id as u16);
+ (namespace, shard_info)
+ });
+ self.insert_shard_table_records(records);
+ }
+
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let topic = stream.get_topic_mut(topic_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get topic with ID: {topic_id} in stream with ID: {stream_id}")
+ })?;
topic.reassign_consumer_groups();
self.metrics.increment_partitions(partitions_count);
@@ -116,10 +147,11 @@ impl IggyShard {
))?;
}
- let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
- })?;
- let topic = stream
+ let (numeric_topic_id, partitions) = {
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let topic = stream
.get_topic_mut(topic_id)
.with_error_context(|error| {
format!(
@@ -127,19 +159,42 @@ impl IggyShard {
)
})?;
- // TODO: Make delete persisted partitions from topic sync, and extract the storage persister out of it
- // perform disk i/o outside of the borrow_mut of the stream.
- let partitions = topic
+ let partitions = topic
.delete_persisted_partitions(partitions_count)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to delete persisted partitions for topic: {topic}")
})?;
+ (topic.topic_id, partitions)
+ };
+
+ let mut segments_count = 0;
+ let mut messages_count = 0;
+ for partition in &partitions {
+ let mut partition = partition.write().await;
+ let partition_id = partition.partition_id;
+ let partition_messages_count = partition.get_messages_count();
+ segments_count += partition.get_segments_count();
+ messages_count += partition_messages_count;
+ partition.delete().await.with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to delete partition with ID: {} in topic with ID: {}",
+ partition_id,
+ numeric_topic_id
+ )
+ })?;
+ }
+
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let topic = stream.get_topic_mut(topic_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get topic with ID: {topic_id}")
+ })?;
topic.reassign_consumer_groups();
- if let Some(partitions) = partitions {
+ if partitions.len() > 0 {
self.metrics.decrement_partitions(partitions_count);
- self.metrics.decrement_segments(partitions.segments_count);
- self.metrics.decrement_messages(partitions.messages_count);
+ self.metrics.decrement_segments(segments_count);
+ self.metrics.decrement_messages(messages_count);
}
Ok(())
}
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index 47068f11..b4921066 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -298,6 +298,13 @@ impl IggyShard {
})?;
old_name = stream.name.clone();
stream.name = name.to_owned();
+ // Drop the exclusive borrow.
+ drop(stream);
+ // Get it again using inclusive borrow
+ let stream = self.get_stream(id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {id}")
+ })?;
+ // Persist it.
stream.persist().await?;
}
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index c247cf74..0f8155fb 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -26,6 +26,7 @@ use error_set::ErrContext;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize};
use tokio_util::io::StreamReader;
+use tracing::info;
impl IggyShard {
pub fn find_topic<'topic, 'stream>(
@@ -112,28 +113,35 @@ impl IggyShard {
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
) -> Result<(), IggyError> {
- let (stream_id, topic_id, partition_ids) = self
- .create_topic_base(
- stream_id,
- topic_id,
- name,
- partitions_count,
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- )
- .await?;
+ let (topic_id, partition_ids) = self.create_topic_base(
+ stream_id,
+ topic_id,
+ name,
+ partitions_count,
+ message_expiry,
+ compression_algorithm,
+ max_topic_size,
+ replication_factor,
+ )?;
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let topic = stream
+ .get_topic(&Identifier::numeric(topic_id)?)
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get topic with ID: {topic_id} in stream with ID: {stream_id}")
+ })?;
+ topic.persist().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to persist topic: {topic}")
+ })?;
+
// TODO: Figure out a way how to distribute the shards table among different shards,
// without the need to do code from below, everytime we handle a `ShardEvent`.
- // I think we shouldn't be sharing it, maintain a single shard table per shard,
- // but figure out a way how to distribute it smarter (maybe move the broadcast inside of the `insert_shard_table_records`) method.
+ // I think we shouldn't be sharing it tho and still maintain a single shard table per shard,
+ // but find a way how to distribute it smarter (maybe move the broadcast inside of the `insert_shard_table_records`) method.
let records = partition_ids.into_iter().map(|partition_id| {
let namespace = IggyNamespace::new(stream_id, topic_id, partition_id);
- // TODO: This setup isn't deterministic.
- // Imagine a scenario where client creates partition using `String` identifiers,
- // but then for poll_messages requests uses numeric ones.
- // the namespace wouldn't match, therefore we would get miss in the shard table.
let hash = namespace.generate_hash();
let shard_id = hash % self.get_available_shards_count();
let shard_info = ShardInfo::new(shard_id as u16);
@@ -176,24 +184,32 @@ impl IggyShard {
})?;
}
- let (stream_id, topic_id, partition_ids) = self
- .create_topic_base(
- stream_id,
- topic_id,
- name,
- partitions_count,
- message_expiry,
- compression_algorithm,
- max_topic_size,
- replication_factor,
- )
- .await?;
+ let (topic_id, partition_ids) = self.create_topic_base(
+ stream_id,
+ topic_id,
+ name,
+ partitions_count,
+ message_expiry,
+ compression_algorithm,
+ max_topic_size,
+ replication_factor,
+ )?;
+
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let topic = stream
+ .get_topic(&Identifier::numeric(topic_id)?)
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get topic with ID: {topic_id} in stream with ID: {stream_id}")
+ })?;
+ topic.persist().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to persist topic: {topic}")
+ })?;
+
let records = partition_ids.into_iter().map(|partition_id| {
let namespace = IggyNamespace::new(stream_id, topic_id, partition_id);
- // TODO: This setup isn't deterministic.
- // Imagine a scenario where client creates partition using `String` identifiers,
- // but then for poll_messages requests uses numeric ones.
- // the namespace wouldn't match, therefore we would get miss in the shard table.
let hash = namespace.generate_hash();
let shard_id = hash % self.get_available_shards_count();
let shard_info = ShardInfo::new(shard_id as u16);
@@ -208,7 +224,7 @@ impl IggyShard {
Ok(Identifier::numeric(topic_id)?)
}
- async fn create_topic_base(
+ fn create_topic_base(
&self,
stream_id: &Identifier,
topic_id: Option<u32>,
@@ -218,9 +234,7 @@ impl IggyShard {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
- ) -> Result<(u32, u32, Vec<u32>), IggyError> {
- // TODO: Make create topic sync, and extract the storage persister out of it
- // perform disk i/o outside of the borrow_mut of the stream.
+ ) -> Result<(u32, Vec<u32>), IggyError> {
let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}")
})?;
@@ -235,11 +249,10 @@ impl IggyShard {
max_topic_size,
replication_factor.unwrap_or(1),
)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to create topic with name: {name} in stream ID: {stream_id}")
})?;
- Ok((stream_id, topic_id, partition_ids))
+ Ok((topic_id, partition_ids))
}
#[allow(clippy::too_many_arguments)]
@@ -280,8 +293,6 @@ impl IggyShard {
})?;
}
- // TODO: Make update topic sync, and extract the storage persister out of it
- // perform disk i/o outside of the borrow_mut of the stream.
self.get_stream_mut(stream_id)?
.update_topic(
topic_id,
@@ -291,12 +302,32 @@ impl IggyShard {
max_topic_size,
replication_factor.unwrap_or(1),
)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to update topic with ID: {topic_id} in stream with ID: {stream_id}",
)
})?;
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let topic = stream
+ .get_topic(topic_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get topic with ID: {topic_id} in stream with ID: {stream_id}",
+ )
+ })?;
+ for partition in topic.partitions.values() {
+ let mut partition = partition.write().await;
+ partition.message_expiry = message_expiry;
+ for segment in partition.segments.iter_mut() {
+ segment.update_message_expiry(message_expiry);
+ }
+ }
+ topic.persist().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to persist topic: {topic}")
+ })?;
+ info!("Updated topic: {topic}");
// TODO: if message_expiry is changed, we need to check if we need to purge messages based on the new expiry
// TODO: if max_size_bytes is changed, we need to check if we need to purge messages based on the new size
@@ -334,13 +365,20 @@ impl IggyShard {
stream_id_value = topic.stream_id;
}
- // TODO: Make delete topic sync, and extract the storage persister out of it
- // perform disk i/o outside of the borrow_mut of the stream.
let topic = self
.get_stream_mut(stream_id)?
.delete_topic(topic_id)
- .await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to delete topic with ID: {topic_id} in stream with ID: {stream_id}"))?;
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}")
+ })?;
+ topic
+ .delete()
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to delete topic: {topic}")
+ })
+ .map_err(|_| IggyError::CannotDeleteTopic(topic.topic_id, stream.stream_id))?;
self.metrics.decrement_topics(1);
self.metrics
diff --git a/core/server/src/streaming/streams/topics.rs b/core/server/src/streaming/streams/topics.rs
index 6a201c7a..7833efa2 100644
--- a/core/server/src/streaming/streams/topics.rs
+++ b/core/server/src/streaming/streams/topics.rs
@@ -36,7 +36,7 @@ impl Stream {
}
#[allow(clippy::too_many_arguments)]
- pub async fn create_topic(
+ pub fn create_topic(
&mut self,
topic_id: Option<u32>,
name: &str,
@@ -94,16 +94,13 @@ impl Stream {
replication_factor,
)?;
- topic.persist().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to persist topic: {topic}")
- })?;
info!("Created topic {}", topic);
self.topics_ids.insert(name.to_owned(), id);
self.topics.insert(id, topic);
Ok((id, partition_ids))
}
- pub async fn update_topic(
+ pub fn update_topic(
&mut self,
id: &Identifier,
name: &str,
@@ -150,21 +147,9 @@ impl Stream {
topic.name = name.to_owned();
topic.message_expiry = message_expiry;
topic.compression_algorithm = compression_algorithm;
- for partition in topic.partitions.values_mut() {
- let mut partition = partition.write().await;
- partition.message_expiry = message_expiry;
- for segment in partition.segments.iter_mut() {
- segment.update_message_expiry(message_expiry);
- }
- }
topic.max_topic_size = max_topic_size;
topic.replication_factor = replication_factor;
- topic.persist().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to persist topic: {topic}")
- })?;
- info!("Updated topic: {topic}");
}
-
Ok(())
}
@@ -253,7 +238,7 @@ impl Stream {
.ok_or(IggyError::TopicIdNotFound(topic_id, self.stream_id))
}
- pub async fn delete_topic(&mut self, id: &Identifier) -> Result<Topic, IggyError> {
+ pub fn delete_topic(&mut self, id: &Identifier) -> Result<Topic, IggyError> {
let topic = self.remove_topic(id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to remove topic with id: {id}")
})?;
@@ -263,13 +248,6 @@ impl Stream {
self.current_topic_id.store(topic_id, Ordering::SeqCst);
}
- topic
- .delete()
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to delete topic: {topic}")
- })
- .map_err(|_| IggyError::CannotDeleteTopic(topic.topic_id, self.stream_id))?;
Ok(topic)
}
}
@@ -319,7 +297,6 @@ mod tests {
max_topic_size,
1,
)
- .await
.unwrap();
let topic = stream.get_topic(&Identifier::numeric(topic_id).unwrap());
diff --git a/core/server/src/streaming/topics/partitions.rs b/core/server/src/streaming/topics/partitions.rs
index b9e56331..cbbe15e5 100644
--- a/core/server/src/streaming/topics/partitions.rs
+++ b/core/server/src/streaming/topics/partitions.rs
@@ -69,29 +69,19 @@ impl Topic {
Ok(partition_ids)
}
- pub async fn add_persisted_partitions(&mut self, count: u32) -> Result<Vec<u32>, IggyError> {
+ pub fn add_persisted_partitions(&mut self, count: u32) -> Result<Vec<u32>, IggyError> {
let partition_ids = self.add_partitions(count).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to add partitions, count: {count}")
})?;
- for partition_id in &partition_ids {
- let partition = self.partitions.get(partition_id).unwrap();
- let mut partition = partition.write().await;
- partition.persist().await.with_error_context(|error| {
- format!(
- "{COMPONENT} (error: {error}) - failed to persist partition with id: {}",
- partition.partition_id
- )
- })?;
- }
Ok(partition_ids)
}
- pub async fn delete_persisted_partitions(
+ pub fn delete_persisted_partitions(
&mut self,
mut count: u32,
- ) -> Result<Option<DeletedPartitions>, IggyError> {
+ ) -> Result<Vec<IggyRwLock<Partition>>, IggyError> {
if count == 0 {
- return Ok(None);
+ return Ok(vec![]);
}
let current_partitions_count = self.partitions.len() as u32;
@@ -99,25 +89,12 @@ impl Topic {
count = current_partitions_count;
}
- let mut segments_count = 0;
- let mut messages_count = 0;
+ let mut partitions = Vec::with_capacity(count as usize);
for partition_id in current_partitions_count - count + 1..=current_partitions_count {
let partition = self.partitions.remove(&partition_id).unwrap();
- let mut partition = partition.write().await;
- let partition_messages_count = partition.get_messages_count();
- segments_count += partition.get_segments_count();
- messages_count += partition_messages_count;
- partition.delete().await.with_error_context(|error| {
- format!(
- "{COMPONENT} (error: {error}) - failed to delete partition with ID: {partition_id} in topic with ID: {}",
- self.topic_id
- )
- })?;
+ partitions.push(partition);
}
- Ok(Some(DeletedPartitions {
- segments_count,
- messages_count,
- }))
+ Ok(partitions)
}
}
(iggy) 06/30: refactors
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 044f59fbcca5ac5df8ce3a85718f373ccbf5c475
Author: numinex <g....@gmail.com>
AuthorDate: Fri Jun 20 10:52:31 2025 +0200
refactors
---
Cargo.lock | 13 +++++
Cargo.toml | 1 +
core/server/Cargo.toml | 1 +
core/server/src/bootstrap.rs | 7 +--
core/server/src/main.rs | 11 ++--
core/server/src/shard/builder.rs | 8 ++-
core/server/src/shard/frame.rs | 67 ----------------------
core/server/src/shard/mod.rs | 17 +++---
.../src/shard/{ => transmission}/connector.rs | 49 ++++++++++++----
core/server/src/shard/transmission/frame.rs | 42 ++++++++++++++
core/server/src/shard/transmission/message.rs | 22 +++++++
core/server/src/shard/transmission/mod.rs | 3 +
12 files changed, 142 insertions(+), 99 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 40849f7d..267da703 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -467,6 +467,18 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "async-channel"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a"
+dependencies = [
+ "concurrent-queue",
+ "event-listener-strategy",
+ "futures-core",
+ "pin-project-lite",
+]
+
[[package]]
name = "async-compression"
version = "0.4.25"
@@ -6940,6 +6952,7 @@ version = "0.5.0"
dependencies = [
"ahash 0.8.12",
"anyhow",
+ "async-channel",
"async_zip",
"axum 0.8.4",
"axum-server",
diff --git a/Cargo.toml b/Cargo.toml
index f910068b..6a2c27f4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -88,6 +88,7 @@ dotenvy = "0.15.7"
enum_dispatch = "0.3.13"
figlet-rs = "0.1.5"
flume = "0.11.1"
+async-channel = "2.3.1"
futures = "0.3.31"
futures-util = "0.3.31"
human-repr = "1.1.0"
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index f4295c6c..138bdc81 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -56,6 +56,7 @@ error_set = { version = "0.8.5", features = ["tracing"] }
figlet-rs = { workspace = true }
figment = { version = "0.10.19", features = ["toml", "env"] }
flume = { workspace = true }
+async-channel = { workspace = true }
futures = { workspace = true }
human-repr = { workspace = true }
iggy_common = { workspace = true }
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index d70d0317..c6108568 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -5,14 +5,14 @@ use iggy_common::{
MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH,
},
};
-use monoio::{fs::create_dir_all, time::TimeDriver, Buildable, Driver, Runtime};
+use monoio::{Buildable, Driver, Runtime, fs::create_dir_all, time::TimeDriver};
use tracing::info;
use crate::{
IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
configs::{config_provider::ConfigProviderKind, server::ServerConfig, system::SystemConfig},
server_error::ServerError,
- shard::{connector::ShardConnector, frame::ShardFrame},
+ shard::{transmission::connector::ShardConnector, transmission::frame::ShardFrame},
streaming::users::user::User,
};
use std::{env, fs::remove_dir_all, ops::Range, path::Path};
@@ -146,8 +146,7 @@ where
rt
}
-pub fn create_shard_executor() -> Runtime<TimeDriver<monoio::IoUringDriver>>
-{
+pub fn create_shard_executor() -> Runtime<TimeDriver<monoio::IoUringDriver>> {
// TODO: Figure out what else we could tweak there
// We for sure want to disable the userspace interrupts on new cq entry (set_coop_taskrun)
// let urb = io_uring::IoUring::builder();
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 06ffec35..871ff2f1 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -25,7 +25,8 @@ use error_set::ErrContext;
use figlet_rs::FIGfont;
use server::args::Args;
use server::bootstrap::{
- create_default_executor, create_directories, create_root_user, create_shard_connections, create_shard_executor, load_config
+ create_default_executor, create_directories, create_root_user, create_shard_connections,
+ create_shard_executor, load_config,
};
use server::channels::commands::archive_state::ArchiveStateExecutor;
use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
@@ -126,14 +127,10 @@ fn main() -> Result<(), ServerError> {
// TODO: Make this configurable from config as a range
// for example this instance of Iggy will use cores from 0..4
- let available_cpus = available_parallelism()
- .expect("Failed to get num of cores")
- .into();
- let shards_count = available_cpus;
+ let available_cpus = available_parallelism().expect("Failed to get num of cores");
+ let shards_count = available_cpus.into();
let shards_set = 0..shards_count;
-
let connections = create_shard_connections(shards_set.clone());
-
for shard_id in shards_set {
let id = shard_id as u16;
let connections = connections.clone();
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 06366ebf..36fe0e54 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -33,7 +33,7 @@ use crate::{
versioning::SemanticVersion,
};
-use super::{IggyShard, connector::ShardConnector, frame::ShardFrame};
+use super::{IggyShard, transmission::connector::ShardConnector, transmission::frame::ShardFrame};
#[derive(Default)]
pub struct IggyShardBuilder {
@@ -76,6 +76,7 @@ impl IggyShardBuilder {
.next()
.expect("Failed to find connection with the specified ID");
let shards = connections.into_iter().map(Shard::new).collect();
+
//TODO: This can be discrete step in the builder bootstrapped from main function.
let version = SemanticVersion::current().expect("Invalid version");
@@ -102,7 +103,10 @@ impl IggyShardBuilder {
//TODO: This can be discrete step in the builder bootstrapped from main function.
let partition_persister = Self::resolve_persister(config.system.partition.enforce_fsync);
- let storage = Rc::new(SystemStorage::new(config.system.clone(), partition_persister));
+ let storage = Rc::new(SystemStorage::new(
+ config.system.clone(),
+ partition_persister,
+ ));
IggyShard {
id: id,
diff --git a/core/server/src/shard/frame.rs b/core/server/src/shard/frame.rs
deleted file mode 100644
index a2c1220f..00000000
--- a/core/server/src/shard/frame.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-use bytes::Bytes;
-use flume::Sender;
-use iggy_common::IggyError;
-
-#[derive(Debug, Clone)]
-pub enum ShardMessage {
- //TODO: Fixme
- //Command(ServerCommand),
- Event(ShardEvent),
-}
-
-#[derive(Debug, Clone)]
-pub enum ShardEvent {}
-
-#[derive(Debug)]
-pub enum ShardResponse {
- BinaryResponse(Bytes),
- ErrorResponse(IggyError),
-}
-
-#[derive(Debug, Clone)]
-pub struct ShardFrame {
- pub client_id: u32,
- pub message: ShardMessage,
- pub response_sender: Option<Sender<ShardResponse>>,
-}
-
-impl ShardFrame {
- pub fn new(
- client_id: u32,
- message: ShardMessage,
- response_sender: Option<Sender<ShardResponse>>,
- ) -> Self {
- Self {
- client_id,
- message,
- response_sender,
- }
- }
-}
-
-#[macro_export]
-macro_rules! handle_response {
- ($sender:expr, $response:expr) => {
- match $response {
- ShardResponse::BinaryResponse(payload) => $sender.send_ok_response(&payload).await?,
- ShardResponse::ErrorResponse(err) => $sender.send_error_response(err).await?,
- }
- };
-}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index b98f6381..6a6d35b8 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -17,25 +17,27 @@
*/
pub mod builder;
-pub mod connector;
-pub mod frame;
pub mod namespace;
+pub mod transmission;
use ahash::HashMap;
use builder::IggyShardBuilder;
-use connector::{Receiver, ShardConnector, StopReceiver, StopSender};
-use frame::ShardFrame;
use iggy_common::IggyError;
use namespace::IggyNamespace;
-use tracing::info;
use std::{
cell::{Cell, RefCell},
rc::Rc,
- sync::Arc, time::Instant,
+ sync::Arc,
+ time::Instant,
};
+use tracing::info;
+use transmission::connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use crate::{
- bootstrap::create_root_user, configs::server::ServerConfig, state::{file::FileState, StateKind},
+ bootstrap::create_root_user,
+ configs::server::ServerConfig,
+ shard::transmission::frame::ShardFrame,
+ state::{StateKind, file::FileState},
streaming::storage::SystemStorage,
};
pub(crate) struct Shard {
@@ -68,7 +70,6 @@ pub struct IggyShard {
// TODO: Refactor.
pub(crate) storage: Rc<SystemStorage>,
- // TODO - get rid of this dynamic dispatch.
pub(crate) state: Rc<StateKind>,
//pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
config: ServerConfig,
diff --git a/core/server/src/shard/connector.rs b/core/server/src/shard/transmission/connector.rs
similarity index 74%
rename from core/server/src/shard/connector.rs
rename to core/server/src/shard/transmission/connector.rs
index 95aedc73..01ca4147 100644
--- a/core/server/src/shard/connector.rs
+++ b/core/server/src/shard/transmission/connector.rs
@@ -22,10 +22,9 @@ use std::{
task::Poll,
};
-pub type StopSender = flume::Sender<()>;
-pub type StopReceiver = flume::Receiver<()>;
+pub type StopSender = async_channel::Sender<()>;
+pub type StopReceiver = async_channel::Receiver<()>;
-#[derive(Clone)]
pub struct ShardConnector<T> {
pub id: u16,
pub sender: Sender<T>,
@@ -34,12 +33,25 @@ pub struct ShardConnector<T> {
pub stop_sender: StopSender,
}
-// TODO(numinex) - replace flume with async_channel
-impl<T: Clone> ShardConnector<T> {
+impl<T> Clone for ShardConnector<T> {
+ fn clone(&self) -> Self {
+ Self {
+ id: self.id,
+ sender: self.sender.clone(),
+ receiver: self.receiver.clone(),
+ stop_receiver: self.stop_receiver.clone(),
+ stop_sender: self.stop_sender.clone(),
+ }
+ }
+}
+
+// TODO(numinex) - replace async_channel with some other form of one shot async channel.
+// !!!!!IMPORTANT!!!! the one shot channel Sender/Receiver has to be Cloneable!!!!!
+impl<T> ShardConnector<T> {
pub fn new(id: u16, max_concurrent_thread_count: usize) -> Self {
let channel = Arc::new(ShardedChannel::new(max_concurrent_thread_count));
let (sender, receiver) = channel.unbounded();
- let (stop_sender, stop_receiver) = flume::bounded(1);
+ let (stop_sender, stop_receiver) = async_channel::bounded(1);
Self {
id,
receiver,
@@ -55,17 +67,32 @@ impl<T: Clone> ShardConnector<T> {
}
// TODO: I think those Arcs can be replaced with 'static lifetimes...
-// Those shards will live for the entire duration of the application.
-#[derive(Clone)]
+// Those connectors will live for the duration of the shard itself
+// and the shard lives for the duration of the entire application.
pub struct Receiver<T> {
channel: Arc<ShardedChannel<T>>,
}
-#[derive(Clone)]
+impl<T> Clone for Receiver<T> {
+ fn clone(&self) -> Self {
+ Self {
+ channel: self.channel.clone(),
+ }
+ }
+}
+
pub struct Sender<T> {
channel: Arc<ShardedChannel<T>>,
}
+impl<T> Clone for Sender<T> {
+ fn clone(&self) -> Self {
+ Self {
+ channel: self.channel.clone(),
+ }
+ }
+}
+
impl<T> Sender<T> {
pub fn send(&self, data: T) {
self.channel
@@ -94,13 +121,13 @@ impl<T> ShardedChannel<T> {
}
}
-pub trait ShardedChannelsSplit<T: Clone> {
+pub trait ShardedChannelsSplit<T> {
fn unbounded(&self) -> (Sender<T>, Receiver<T>);
fn sender(&self) -> Sender<T>;
}
-impl<T: Clone> ShardedChannelsSplit<T> for Arc<ShardedChannel<T>> {
+impl<T> ShardedChannelsSplit<T> for Arc<ShardedChannel<T>> {
fn unbounded(&self) -> (Sender<T>, Receiver<T>) {
let tx = self.sender();
let rx = Receiver {
diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs
new file mode 100644
index 00000000..ac60863d
--- /dev/null
+++ b/core/server/src/shard/transmission/frame.rs
@@ -0,0 +1,42 @@
+use async_channel::Sender;
+use bytes::Bytes;
+use iggy_common::IggyError;
+
+use crate::shard::transmission::message::ShardMessage;
+
+#[derive(Debug)]
+pub struct ShardFrame {
+ pub client_id: u32,
+ pub message: ShardMessage,
+ pub response_sender: Option<Sender<ShardResponse>>,
+}
+
+impl ShardFrame {
+ pub fn new(
+ client_id: u32,
+ message: ShardMessage,
+ response_sender: Option<Sender<ShardResponse>>,
+ ) -> Self {
+ Self {
+ client_id,
+ message,
+ response_sender,
+ }
+ }
+}
+
+#[derive(Debug)]
+pub enum ShardResponse {
+ BinaryResponse(Bytes),
+ ErrorResponse(IggyError),
+}
+
+#[macro_export]
+macro_rules! handle_response {
+ ($sender:expr, $response:expr) => {
+ match $response {
+ ShardResponse::BinaryResponse(payload) => $sender.send_ok_response(&payload).await?,
+ ShardResponse::ErrorResponse(err) => $sender.send_error_response(err).await?,
+ }
+ };
+}
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
new file mode 100644
index 00000000..e9f1dd4b
--- /dev/null
+++ b/core/server/src/shard/transmission/message.rs
@@ -0,0 +1,22 @@
+use crate::binary::command::ServerCommand;
+
+#[derive(Debug)]
+pub enum ShardMessage {
+ Command(ServerCommand),
+ Event(ShardEvent),
+}
+
+#[derive(Debug)]
+pub enum ShardEvent {}
+
+impl From<ServerCommand> for ShardMessage {
+ fn from(command: ServerCommand) -> Self {
+ ShardMessage::Command(command)
+ }
+}
+
+impl From<ShardEvent> for ShardMessage {
+ fn from(event: ShardEvent) -> Self {
+ ShardMessage::Event(event)
+ }
+}
diff --git a/core/server/src/shard/transmission/mod.rs b/core/server/src/shard/transmission/mod.rs
new file mode 100644
index 00000000..b871152a
--- /dev/null
+++ b/core/server/src/shard/transmission/mod.rs
@@ -0,0 +1,3 @@
+pub mod connector;
+pub mod frame;
+pub mod message;
(iggy) 21/30: feat(tpc): implement cooperative shutdown (#1941)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 2bcf4de7809cbeaefe3072a78a3bb2417461d6a3
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Mon Jun 30 17:04:45 2025 +0200
feat(tpc): implement cooperative shutdown (#1941)
This closes #1932.
---
Cargo.lock | 11 +++
core/server/Cargo.toml | 1 +
core/server/src/bootstrap.rs | 18 +++--
core/server/src/main.rs | 37 +++++++---
core/server/src/shard/builder.rs | 10 ++-
core/server/src/shard/mod.rs | 60 +++++++++++------
core/server/src/shard/task_registry.rs | 108 ++++++++++++++++++++++++++++++
core/server/src/shard/tasks/messages.rs | 66 +++++++++++-------
core/server/src/tcp/connection_handler.rs | 30 ++++++---
core/server/src/tcp/tcp_listener.rs | 97 +++++++++++++++++----------
core/server/src/tcp/tcp_server.rs | 8 +--
11 files changed, 340 insertions(+), 106 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 8a3b9e16..6898bb77 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1856,6 +1856,16 @@ dependencies = [
"cipher",
]
+[[package]]
+name = "ctrlc"
+version = "3.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73"
+dependencies = [
+ "nix 0.30.1",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "cucumber"
version = "0.21.1"
@@ -7025,6 +7035,7 @@ dependencies = [
"clap",
"console-subscriber",
"crossbeam",
+ "ctrlc",
"dashmap",
"derive_more 2.0.1",
"dotenvy",
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 8da2f495..3e6c801e 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -48,6 +48,7 @@ chrono = { workspace = true }
clap = { workspace = true }
console-subscriber = { workspace = true, optional = true }
crossbeam = { workspace = true }
+ctrlc = { version = "3.4", features = ["termination"] }
dashmap = { workspace = true }
derive_more = { workspace = true }
dotenvy = { workspace = true }
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index 2d9f7eaa..6f046fa8 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -12,7 +12,10 @@ use crate::{
IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
configs::{config_provider::ConfigProviderKind, server::ServerConfig, system::SystemConfig},
server_error::ServerError,
- shard::transmission::{connector::ShardConnector, frame::ShardFrame},
+ shard::transmission::{
+ connector::{ShardConnector, StopSender},
+ frame::ShardFrame,
+ },
streaming::{
persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind},
users::user::User,
@@ -21,14 +24,21 @@ use crate::{
};
use std::{env, fs::remove_dir_all, ops::Range, path::Path, sync::Arc};
-pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<ShardFrame>> {
+pub fn create_shard_connections(
+ shards_set: Range<usize>,
+) -> (Vec<ShardConnector<ShardFrame>>, Vec<(u16, StopSender)>) {
let shards_count = shards_set.len();
- let connectors = shards_set
+ let connectors: Vec<ShardConnector<ShardFrame>> = shards_set
.into_iter()
.map(|id| ShardConnector::new(id as u16, shards_count))
.collect();
- connectors
+ let shutdown_handles = connectors
+ .iter()
+ .map(|conn| (conn.id, conn.stop_sender.clone()))
+ .collect();
+
+ (connectors, shutdown_handles)
}
pub async fn load_config(
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 1b9331a1..2cd70c4d 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -127,21 +127,23 @@ fn main() -> Result<(), ServerError> {
let available_cpus = available_parallelism().expect("Failed to get num of cores");
let shards_count = available_cpus.into();
let shards_set = 0..shards_count;
- let connections = create_shard_connections(shards_set.clone());
+ let (connections, shutdown_handles) = create_shard_connections(shards_set.clone());
let gate = Arc::new(Gate::new());
let mut handles = Vec::with_capacity(shards_set.len());
+
for shard_id in shards_set {
let id = shard_id as u16;
let gate = gate.clone();
let connections = connections.clone();
let config = config.clone();
let state_persister = resolve_persister(config.system.state.enforce_fsync);
+
let handle = std::thread::Builder::new()
.name(format!("shard-{id}"))
.spawn(move || {
MemoryPool::init_pool(config.system.clone());
monoio::utils::bind_to_cpu_set(Some(shard_id))
- .expect(format!("Failed to set CPU affinity for shard-{id}").as_str());
+ .unwrap_or_else(|e| panic!("Failed to set CPU affinity for shard-{id}: {e}"));
let mut rt = create_shard_executor();
rt.block_on(async move {
@@ -242,21 +244,40 @@ fn main() -> Result<(), ServerError> {
.build()
.into();
- //TODO: If one of the shards fails to initialize, we should crash the whole program;
if let Err(e) = shard.run().await {
error!("Failed to run shard-{id}: {e}");
}
- //TODO: If one of the shards fails to initialize, we should crash the whole program;
- //shard.assert_init();
+ info!("Shard {} run completed", id);
})
})
- .expect(format!("Failed to spawn thread for shard-{id}").as_str());
+ .unwrap_or_else(|e| panic!("Failed to spawn thread for shard-{id}: {e}"));
handles.push(handle);
}
- handles.into_iter().for_each(|handle| {
+ let shutdown_handles_for_signal = shutdown_handles.clone();
+ ctrlc::set_handler(move || {
+ info!("Received shutdown signal (SIGTERM/SIGINT), initiating graceful shutdown...");
+
+ for (shard_id, stop_sender) in &shutdown_handles_for_signal {
+ info!("Sending shutdown signal to shard {}", shard_id);
+ if let Err(e) = stop_sender.send_blocking(()) {
+ error!(
+ "Failed to send shutdown signal to shard {}: {}",
+ shard_id, e
+ );
+ }
+ }
+ })
+ .expect("Error setting Ctrl-C handler");
+
+ info!("Iggy server is running. Press Ctrl+C or send SIGTERM to shutdown.");
+ for (idx, handle) in handles.into_iter().enumerate() {
+ info!("Waiting for shard thread {} to complete...", idx);
handle.join().expect("Failed to join shard thread");
- });
+ info!("Shard thread {} completed", idx);
+ }
+
+ info!("All shards have shut down. Iggy server is exiting.");
/*
#[cfg(feature = "disable-mimalloc")]
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index beda9ae2..eb851271 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -16,7 +16,11 @@
* under the License.
*/
-use std::{cell::Cell, rc::Rc, sync::Arc};
+use std::{
+ cell::Cell,
+ rc::Rc,
+ sync::{Arc, atomic::AtomicBool},
+};
use iggy_common::{Aes256GcmEncryptor, EncryptorKind};
use tracing::info;
@@ -25,7 +29,7 @@ use crate::{
bootstrap::resolve_persister,
configs::server::ServerConfig,
map_toggle_str,
- shard::Shard,
+ shard::{Shard, task_registry::TaskRegistry},
state::{StateKind, file::FileState},
streaming::{diagnostics::metrics::Metrics, storage::SystemStorage},
versioning::SemanticVersion,
@@ -115,6 +119,8 @@ impl IggyShardBuilder {
stop_sender: stop_sender,
messages_receiver: Cell::new(Some(frame_receiver)),
metrics: Metrics::init(),
+ task_registry: TaskRegistry::new(),
+ is_shutting_down: AtomicBool::new(false),
users: Default::default(),
permissioner: Default::default(),
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index cfbd6eac..cc5023e2 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -20,6 +20,7 @@ pub mod builder;
pub mod gate;
pub mod namespace;
pub mod system;
+pub mod task_registry;
pub mod tasks;
pub mod transmission;
@@ -36,9 +37,9 @@ use std::{
str::FromStr,
sync::{
Arc, RwLock,
- atomic::{AtomicU32, Ordering},
+ atomic::{AtomicBool, AtomicU32, Ordering},
},
- time::Instant,
+ time::{Duration, Instant},
};
use tracing::{error, info, instrument, trace, warn};
use transmission::connector::{Receiver, ShardConnector, StopReceiver, StopSender};
@@ -47,6 +48,7 @@ use crate::{
configs::server::ServerConfig,
shard::{
system::info::SystemInfo,
+ task_registry::TaskRegistry,
tasks::messages::spawn_shard_message_task,
transmission::{
event::ShardEvent,
@@ -74,6 +76,8 @@ use crate::{
};
pub const COMPONENT: &str = "SHARD";
+pub const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10);
+
static USER_ID: AtomicU32 = AtomicU32::new(1);
type Task = Pin<Box<dyn Future<Output = Result<(), IggyError>>>>;
@@ -142,8 +146,10 @@ pub struct IggyShard {
pub(crate) metrics: Metrics,
pub messages_receiver: Cell<Option<Receiver<ShardFrame>>>,
- stop_receiver: StopReceiver,
- stop_sender: StopSender,
+ pub(crate) stop_receiver: StopReceiver,
+ pub(crate) stop_sender: StopSender,
+ pub(crate) task_registry: TaskRegistry,
+ pub(crate) is_shutting_down: AtomicBool,
}
impl IggyShard {
@@ -181,11 +187,31 @@ impl IggyShard {
// TODO: Fixme
//self.assert_init();
info!("Initiated shard with ID: {}", self.id);
+
// Create all tasks (tcp listener, http listener, command processor, in the future also the background jobs).
let mut tasks: Vec<Task> = vec![Box::pin(spawn_shard_message_task(self.clone()))];
if self.config.tcp.enabled {
tasks.push(Box::pin(spawn_tcp_server(self.clone())));
}
+
+ let stop_receiver = self.get_stop_receiver();
+ let shard_for_shutdown = self.clone();
+
+ monoio::spawn(async move {
+ let _ = stop_receiver.recv().await;
+ info!("Shard {} received shutdown signal", shard_for_shutdown.id);
+
+ let shutdown_success = shard_for_shutdown.trigger_shutdown().await;
+ if !shutdown_success {
+ error!("Shard {} shutdown timed out", shard_for_shutdown.id);
+ } else {
+ info!(
+ "Shard {} shutdown completed successfully",
+ shard_for_shutdown.id
+ );
+ }
+ });
+
let result = try_join_all(tasks).await;
result?;
@@ -430,25 +456,19 @@ impl IggyShard {
Ok(())
}
- #[instrument(skip_all, name = "trace_shutdown")]
- pub async fn shutdown(&mut self) -> Result<(), IggyError> {
- //TODO: Fixme, impl cooperative shutdown.
- self.persist_messages().await?;
- Ok(())
+ pub fn is_shutting_down(&self) -> bool {
+ self.is_shutting_down.load(Ordering::Relaxed)
}
- #[instrument(skip_all, name = "trace_persist_messages")]
- pub async fn persist_messages(&self) -> Result<usize, IggyError> {
- trace!("Saving buffered messages on disk...");
- let mut saved_messages_number = 0;
- //TODO: Fixme
- /*
- for stream in self.streams.values() {
- saved_messages_number += stream.persist_messages().await?;
- }
- */
+ pub fn get_stop_receiver(&self) -> StopReceiver {
+ self.stop_receiver.clone()
+ }
- Ok(saved_messages_number)
+ #[instrument(skip_all, name = "trace_shutdown")]
+ pub async fn trigger_shutdown(&self) -> bool {
+ self.is_shutting_down.store(true, Ordering::SeqCst);
+ info!("Shard {} shutdown state set", self.id);
+ self.task_registry.shutdown_all(SHUTDOWN_TIMEOUT).await
}
pub fn get_available_shards_count(&self) -> u32 {
diff --git a/core/server/src/shard/task_registry.rs b/core/server/src/shard/task_registry.rs
new file mode 100644
index 00000000..1a201d3b
--- /dev/null
+++ b/core/server/src/shard/task_registry.rs
@@ -0,0 +1,108 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use async_channel::{Receiver, Sender, bounded};
+use futures::future::join_all;
+use monoio::task::JoinHandle;
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::future::Future;
+use std::time::Duration;
+use tracing::{error, info, warn};
+
+pub struct TaskRegistry {
+ tasks: RefCell<Vec<JoinHandle<()>>>,
+ active_connections: RefCell<HashMap<u32, Sender<()>>>,
+}
+
+impl TaskRegistry {
+ pub fn new() -> Self {
+ Self {
+ tasks: RefCell::new(Vec::new()),
+ active_connections: RefCell::new(HashMap::new()),
+ }
+ }
+
+ pub fn spawn_tracked<F>(&self, future: F)
+ where
+ F: Future<Output = ()> + 'static,
+ {
+ let handle = monoio::spawn(future);
+ self.tasks.borrow_mut().push(handle);
+ }
+
+ pub fn add_connection(&self, client_id: u32) -> Receiver<()> {
+ let (stop_sender, stop_receiver) = bounded(1);
+ self.active_connections
+ .borrow_mut()
+ .insert(client_id, stop_sender);
+ stop_receiver
+ }
+
+ pub fn remove_connection(&self, client_id: &u32) {
+ self.active_connections.borrow_mut().remove(client_id);
+ }
+
+ pub async fn shutdown_all(&self, timeout: Duration) -> bool {
+ info!("Initiating task registry shutdown");
+
+ let connections = self.active_connections.borrow();
+ for (client_id, stop_sender) in connections.iter() {
+ info!("Sending shutdown signal to client {}", client_id);
+ if let Err(e) = stop_sender.send(()).await {
+ warn!(
+ "Failed to send shutdown signal to client {}: {}",
+ client_id, e
+ );
+ }
+ }
+ drop(connections);
+
+ let tasks = self.tasks.take();
+ let total = tasks.len();
+
+ if total == 0 {
+ info!("No tasks to shut down");
+ return true;
+ }
+
+ let timeout_futures: Vec<_> = tasks
+ .into_iter()
+ .enumerate()
+ .map(|(idx, handle)| async move {
+ match monoio::time::timeout(timeout, handle).await {
+ Ok(()) => (idx, true),
+ Err(_) => {
+ warn!("Task {} did not complete within timeout", idx);
+ (idx, false)
+ }
+ }
+ })
+ .collect();
+
+ let results = join_all(timeout_futures).await;
+ let completed = results.iter().filter(|(_, success)| *success).count();
+
+ info!(
+ "Task registry shutdown complete. {} of {} tasks completed",
+ completed, total
+ );
+
+ completed == total
+ }
+}
diff --git a/core/server/src/shard/tasks/messages.rs b/core/server/src/shard/tasks/messages.rs
index 87660290..ec4018c0 100644
--- a/core/server/src/shard/tasks/messages.rs
+++ b/core/server/src/shard/tasks/messages.rs
@@ -1,38 +1,56 @@
-use futures::StreamExt;
+use futures::{FutureExt, StreamExt};
use iggy_common::IggyError;
-use std::rc::Rc;
-use tracing::error;
+use std::{rc::Rc, time::Duration};
+use tracing::{error, info};
use crate::shard::{IggyShard, transmission::frame::ShardFrame};
async fn run_shard_messages_receiver(shard: Rc<IggyShard>) -> Result<(), IggyError> {
let mut messages_receiver = shard.messages_receiver.take().unwrap();
+
loop {
- if let Some(frame) = messages_receiver.next().await {
- let ShardFrame {
- message,
- response_sender,
- } = frame;
- match (shard.handle_shard_message(message).await, response_sender) {
- (Some(response), Some(response_sender)) => {
- response_sender
- .send(response)
- .await
- .expect("Failed to send response back to origin shard.");
+ let shutdown_check = async {
+ loop {
+ if shard.is_shutting_down() {
+ return;
+ }
+ monoio::time::sleep(Duration::from_millis(100)).await;
+ }
+ };
+
+ futures::select! {
+ _ = shutdown_check.fuse() => {
+ info!("Shard {} message receiver shutting down", shard.id);
+ break;
+ }
+ frame = messages_receiver.next().fuse() => {
+ if let Some(frame) = frame {
+ let ShardFrame {
+ message,
+ response_sender,
+ } = frame;
+ match (shard.handle_shard_message(message).await, response_sender) {
+ (Some(response), Some(response_sender)) => {
+ response_sender
+ .send(response)
+ .await
+ .expect("Failed to send response back to origin shard.");
+ }
+ _ => {}
+ };
}
- _ => {}
- };
+ }
}
}
+
+ Ok(())
}
pub async fn spawn_shard_message_task(shard: Rc<IggyShard>) -> Result<(), IggyError> {
- monoio::spawn(async move {
- let result = run_shard_messages_receiver(shard).await;
- if let Err(err) = &result {
- error!("Error running shard: {err}");
- }
- result
- })
- .await
+ let result = run_shard_messages_receiver(shard).await;
+ if let Err(err) = result {
+ error!("Error running shard message receiver: {err}");
+ return Err(err);
+ }
+ Ok(())
}
diff --git a/core/server/src/tcp/connection_handler.rs b/core/server/src/tcp/connection_handler.rs
index 5622d2a8..418af387 100644
--- a/core/server/src/tcp/connection_handler.rs
+++ b/core/server/src/tcp/connection_handler.rs
@@ -22,7 +22,9 @@ use crate::server_error::ConnectionError;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
use crate::tcp::connection_handler::command::ServerCommand;
+use async_channel::Receiver;
use bytes::BytesMut;
+use futures::FutureExt;
use iggy_common::IggyError;
use std::io::ErrorKind;
use std::rc::Rc;
@@ -34,6 +36,7 @@ pub(crate) async fn handle_connection(
session: &Rc<Session>,
sender: &mut SenderKind,
shard: &Rc<IggyShard>,
+ stop_receiver: Receiver<()>,
) -> Result<(), ConnectionError> {
let mut length_buffer = BytesMut::with_capacity(INITIAL_BYTES_LENGTH);
unsafe {
@@ -44,14 +47,25 @@ pub(crate) async fn handle_connection(
code_buffer.set_len(INITIAL_BYTES_LENGTH);
}
loop {
- let (read_length, initial_buffer) = match sender.read(length_buffer.clone()).await {
- (Ok(read_length), initial_buffer) => (read_length, initial_buffer),
- (Err(error), _) => {
- if error.as_code() == IggyError::ConnectionClosed.as_code() {
- return Err(ConnectionError::from(error));
- } else {
- sender.send_error_response(error).await?;
- continue;
+ let read_future = sender.read(length_buffer.clone());
+
+ let (read_length, initial_buffer) = futures::select! {
+ _ = stop_receiver.recv().fuse() => {
+ info!("Connection stop signal received for session: {}", session);
+ let _ = sender.send_error_response(IggyError::Disconnected).await;
+ return Ok(());
+ }
+ result = read_future.fuse() => {
+ match result {
+ (Ok(read_length), initial_buffer) => (read_length, initial_buffer),
+ (Err(error), _) => {
+ if error.as_code() == IggyError::ConnectionClosed.as_code() {
+ return Err(ConnectionError::from(error));
+ } else {
+ sender.send_error_response(error).await?;
+ continue;
+ }
+ }
}
}
};
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index c0080edf..90b1eabb 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -22,9 +22,11 @@ use crate::shard::transmission::event::ShardEvent;
use crate::streaming::clients::client_manager::Transport;
use crate::tcp::connection_handler::{handle_connection, handle_error};
use crate::tcp::tcp_socket;
+use futures::FutureExt;
use iggy_common::IggyError;
use std::net::SocketAddr;
use std::rc::Rc;
+use std::time::Duration;
use tracing::{error, info};
pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) -> Result<(), IggyError> {
@@ -38,37 +40,61 @@ pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) -> Result<()
.expect("Failed to parse TCP address");
let socket = tcp_socket::build(ip_v6, socket_config);
- monoio::spawn(async move {
- socket
- .bind(&addr.into())
- .expect("Failed to bind TCP listener");
- socket.listen(1024).unwrap();
- let listener: std::net::TcpListener = socket.into();
- let listener = monoio::net::TcpListener::from_std(listener).unwrap();
- info!("{server_name} server has started on: {:?}", addr);
- loop {
- match listener.accept().await {
- Ok((stream, address)) => {
- let shard = shard.clone();
- info!("Accepted new TCP connection: {address}");
- let transport = Transport::Tcp;
- let session = shard.add_client(&address, transport);
- //TODO: Those can be shared with other shards.
- shard.add_active_session(session.clone());
- // Broadcast session to all shards.
- let event = ShardEvent::NewSession { address, transport };
- // TODO: Fixme look inside of broadcast_event_to_all_shards method.
- let _responses = shard.broadcast_event_to_all_shards(event.into());
+ socket
+ .bind(&addr.into())
+ .expect("Failed to bind TCP listener");
+ socket.listen(1024).unwrap();
+ let listener: std::net::TcpListener = socket.into();
+ let listener = monoio::net::TcpListener::from_std(listener).unwrap();
+ info!("{server_name} server has started on: {:?}", addr);
+
+ loop {
+ let shutdown_check = async {
+ loop {
+ if shard.is_shutting_down() {
+ return;
+ }
+ monoio::time::sleep(Duration::from_millis(100)).await;
+ }
+ };
+
+ let accept_future = listener.accept();
+ futures::select! {
+ _ = shutdown_check.fuse() => {
+ info!("{server_name} detected shutdown flag, no longer accepting connections");
+ break;
+ }
+ result = accept_future.fuse() => {
+ match result {
+ Ok((stream, address)) => {
+ if shard.is_shutting_down() {
+ info!("Rejecting new connection from {} during shutdown", address);
+ continue;
+ }
+ let shard_clone = shard.clone();
+ info!("Accepted new TCP connection: {address}");
+ let transport = Transport::Tcp;
+ let session = shard_clone.add_client(&address, transport);
+ //TODO: Those can be shared with other shards.
+ shard_clone.add_active_session(session.clone());
+ // Broadcast session to all shards.
+ let event = ShardEvent::NewSession { address, transport };
+ // TODO: Fixme look inside of broadcast_event_to_all_shards method.
+ let _responses = shard_clone.broadcast_event_to_all_shards(event.into());
+
+ let client_id = session.client_id;
+ info!("Created new session: {session}");
+ let mut sender = SenderKind::get_tcp_sender(stream);
+
+ let conn_stop_receiver = shard_clone.task_registry.add_connection(client_id);
+
+ let shard_for_conn = shard_clone.clone();
+ shard_clone.task_registry.spawn_tracked(async move {
+ if let Err(error) = handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver).await {
+ handle_error(error);
+ }
+ shard_for_conn.task_registry.remove_connection(&client_id);
- let _client_id = session.client_id;
- info!("Created new session: {session}");
- let mut sender = SenderKind::get_tcp_sender(stream);
- monoio::spawn(async move {
- if let Err(error) = handle_connection(&session, &mut sender, &shard).await {
- handle_error(error);
- //TODO: Fixme
- /*
- //system.read().await.delete_client(client_id).await;
if let Err(error) = sender.shutdown().await {
error!(
"Failed to shutdown TCP stream for client: {client_id}, address: {address}. {error}"
@@ -78,13 +104,12 @@ pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) -> Result<()
"Successfully closed TCP stream for client: {client_id}, address: {address}."
);
}
- */
- }
- });
+ });
+ }
+ Err(error) => error!("Unable to accept TCP socket. {error}"),
}
- Err(error) => error!("Unable to accept TCP socket. {error}"),
}
}
- })
- .await
+ }
+ Ok(())
}
diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs
index 0430f773..e558d08f 100644
--- a/core/server/src/tcp/tcp_server.rs
+++ b/core/server/src/tcp/tcp_server.rs
@@ -32,10 +32,10 @@ pub async fn spawn_tcp_server(shard: Rc<IggyShard>) -> Result<(), IggyError> {
};
info!("Initializing {server_name} server...");
// TODO: Fixme -- storing addr of the server inside of the config for integration tests...
- let result = match shard.config.tcp.tls.enabled {
+ match shard.config.tcp.tls.enabled {
true => unimplemented!("TLS support is not implemented yet"),
- false => tcp_listener::start(server_name, shard).await,
+ false => tcp_listener::start(server_name, shard.clone()).await?,
};
- //info!("{server_name} server has started on: {:?}", addr);
- result
+
+ Ok(())
}
(iggy) 16/30: fix sdk
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit e98ed53194ac0ef43f12baed180bc1bc9b9de449
Author: numinex <g....@gmail.com>
AuthorDate: Fri Jun 27 16:50:29 2025 +0200
fix sdk
---
core/sdk/src/clients/client.rs | 10 +++++-----
core/sdk/src/clients/consumer.rs | 10 +++++-----
core/sdk/src/clients/consumer_builder.rs | 7 ++++---
core/sdk/src/clients/producer.rs | 6 +++---
core/sdk/src/clients/producer_builder.rs | 7 ++++---
core/sdk/src/http/http_client.rs | 6 +++---
6 files changed, 24 insertions(+), 22 deletions(-)
diff --git a/core/sdk/src/clients/client.rs b/core/sdk/src/clients/client.rs
index 659cd3ba..7fd798fa 100644
--- a/core/sdk/src/clients/client.rs
+++ b/core/sdk/src/clients/client.rs
@@ -17,7 +17,7 @@
*/
use crate::clients::client_builder::IggyClientBuilder;
-use iggy_common::locking::{IggySharedMut, IggySharedMutFn};
+use iggy_common::locking::{IggyRwLock, IggySharedMutFn};
use crate::client_wrappers::client_wrapper::ClientWrapper;
use crate::http::http_client::HttpClient;
@@ -46,7 +46,7 @@ use tracing::{debug, error, info};
#[derive(Debug)]
#[allow(dead_code)]
pub struct IggyClient {
- pub(crate) client: IggySharedMut<ClientWrapper>,
+ pub(crate) client: IggyRwLock<ClientWrapper>,
partitioner: Option<Arc<dyn Partitioner>>,
pub(crate) encryptor: Option<Arc<EncryptorKind>>,
}
@@ -72,7 +72,7 @@ impl IggyClient {
/// Creates a new `IggyClient` with the provided client implementation for the specific transport.
pub fn new(client: ClientWrapper) -> Self {
- let client = IggySharedMut::new(client);
+ let client = IggyRwLock::new(client);
IggyClient {
client,
partitioner: None,
@@ -107,7 +107,7 @@ impl IggyClient {
info!("Client-side encryption is enabled.");
}
- let client = IggySharedMut::new(client);
+ let client = IggyRwLock::new(client);
IggyClient {
client,
partitioner,
@@ -116,7 +116,7 @@ impl IggyClient {
}
/// Returns the underlying client implementation for the specific transport.
- pub fn client(&self) -> IggySharedMut<ClientWrapper> {
+ pub fn client(&self) -> IggyRwLock<ClientWrapper> {
self.client.clone()
}
diff --git a/core/sdk/src/clients/consumer.rs b/core/sdk/src/clients/consumer.rs
index 1c5fb728..4124ccc6 100644
--- a/core/sdk/src/clients/consumer.rs
+++ b/core/sdk/src/clients/consumer.rs
@@ -24,7 +24,7 @@ use futures_util::{FutureExt, StreamExt};
use iggy_binary_protocol::{
Client, ConsumerGroupClient, ConsumerOffsetClient, MessageClient, StreamClient, TopicClient,
};
-use iggy_common::locking::{IggySharedMut, IggySharedMutFn};
+use iggy_common::locking::{IggyRwLock, IggySharedMutFn};
use iggy_common::{
Consumer, ConsumerKind, DiagnosticEvent, EncryptorKind, IdKind, Identifier, IggyDuration,
IggyError, IggyMessage, IggyTimestamp, PolledMessages, PollingKind, PollingStrategy,
@@ -96,7 +96,7 @@ unsafe impl Sync for IggyConsumer {}
pub struct IggyConsumer {
initialized: bool,
can_poll: Arc<AtomicBool>,
- client: IggySharedMut<ClientWrapper>,
+ client: IggyRwLock<ClientWrapper>,
consumer_name: String,
consumer: Arc<Consumer>,
is_consumer_group: bool,
@@ -132,7 +132,7 @@ pub struct IggyConsumer {
impl IggyConsumer {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
- client: IggySharedMut<ClientWrapper>,
+ client: IggyRwLock<ClientWrapper>,
consumer_name: String,
consumer: Consumer,
stream_id: Identifier,
@@ -408,7 +408,7 @@ impl IggyConsumer {
#[allow(clippy::too_many_arguments)]
async fn store_consumer_offset(
- client: &IggySharedMut<ClientWrapper>,
+ client: &IggyRwLock<ClientWrapper>,
consumer: &Consumer,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -784,7 +784,7 @@ impl IggyConsumer {
}
async fn initialize_consumer_group(
- client: IggySharedMut<ClientWrapper>,
+ client: IggyRwLock<ClientWrapper>,
create_consumer_group_if_not_exists: bool,
stream_id: Arc<Identifier>,
topic_id: Arc<Identifier>,
diff --git a/core/sdk/src/clients/consumer_builder.rs b/core/sdk/src/clients/consumer_builder.rs
index 576d841e..3246b06e 100644
--- a/core/sdk/src/clients/consumer_builder.rs
+++ b/core/sdk/src/clients/consumer_builder.rs
@@ -18,13 +18,14 @@
use crate::client_wrappers::client_wrapper::ClientWrapper;
use crate::prelude::{AutoCommit, AutoCommitWhen, IggyConsumer};
-use iggy_common::locking::IggySharedMut;
+use iggy_binary_protocol::Client;
+use iggy_common::locking::IggyRwLock;
use iggy_common::{Consumer, EncryptorKind, Identifier, IggyDuration, PollingStrategy};
use std::sync::Arc;
#[derive(Debug)]
pub struct IggyConsumerBuilder {
- client: IggySharedMut<ClientWrapper>,
+ client: IggyRwLock<ClientWrapper>,
consumer_name: String,
consumer: Consumer,
stream: Identifier,
@@ -46,7 +47,7 @@ pub struct IggyConsumerBuilder {
impl IggyConsumerBuilder {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
- client: IggySharedMut<ClientWrapper>,
+ client: IggyRwLock<ClientWrapper>,
consumer_name: String,
consumer: Consumer,
stream_id: Identifier,
diff --git a/core/sdk/src/clients/producer.rs b/core/sdk/src/clients/producer.rs
index 5eee0e25..c4db03b0 100644
--- a/core/sdk/src/clients/producer.rs
+++ b/core/sdk/src/clients/producer.rs
@@ -24,7 +24,7 @@ use crate::clients::producer_dispatcher::ProducerDispatcher;
use bytes::Bytes;
use futures_util::StreamExt;
use iggy_binary_protocol::{Client, MessageClient, StreamClient, TopicClient};
-use iggy_common::locking::{IggySharedMut, IggySharedMutFn};
+use iggy_common::locking::{IggyRwLock, IggySharedMutFn};
use iggy_common::{
CompressionAlgorithm, DiagnosticEvent, EncryptorKind, IdKind, Identifier, IggyDuration,
IggyError, IggyExpiry, IggyMessage, IggyTimestamp, MaxTopicSize, Partitioner, Partitioning,
@@ -53,7 +53,7 @@ pub trait ProducerCoreBackend: Send + Sync + 'static {
pub struct ProducerCore {
initialized: AtomicBool,
can_send: Arc<AtomicBool>,
- client: Arc<IggySharedMut<ClientWrapper>>,
+ client: Arc<IggyRwLock<ClientWrapper>>,
stream_id: Arc<Identifier>,
stream_name: String,
topic_id: Arc<Identifier>,
@@ -432,7 +432,7 @@ pub struct IggyProducer {
impl IggyProducer {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
- client: IggySharedMut<ClientWrapper>,
+ client: IggyRwLock<ClientWrapper>,
stream: Identifier,
stream_name: String,
topic: Identifier,
diff --git a/core/sdk/src/clients/producer_builder.rs b/core/sdk/src/clients/producer_builder.rs
index ae8bb368..9a0a20cf 100644
--- a/core/sdk/src/clients/producer_builder.rs
+++ b/core/sdk/src/clients/producer_builder.rs
@@ -18,7 +18,8 @@
use crate::client_wrappers::client_wrapper::ClientWrapper;
use crate::clients::producer_config::{BackgroundConfig, DirectConfig};
use crate::prelude::IggyProducer;
-use iggy_common::locking::IggySharedMut;
+use iggy_binary_protocol::Client;
+use iggy_common::locking::IggyRwLock;
use iggy_common::{
EncryptorKind, Identifier, IggyDuration, IggyExpiry, MaxTopicSize, Partitioner, Partitioning,
};
@@ -36,7 +37,7 @@ impl Default for SendMode {
}
pub struct IggyProducerBuilder {
- client: IggySharedMut<ClientWrapper>,
+ client: IggyRwLock<ClientWrapper>,
stream: Identifier,
stream_name: String,
topic: Identifier,
@@ -58,7 +59,7 @@ pub struct IggyProducerBuilder {
impl IggyProducerBuilder {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
- client: IggySharedMut<ClientWrapper>,
+ client: IggyRwLock<ClientWrapper>,
stream: Identifier,
stream_name: String,
topic: Identifier,
diff --git a/core/sdk/src/http/http_client.rs b/core/sdk/src/http/http_client.rs
index be619dad..6c6eb066 100644
--- a/core/sdk/src/http/http_client.rs
+++ b/core/sdk/src/http/http_client.rs
@@ -20,7 +20,7 @@ use crate::http::http_transport::HttpTransport;
use crate::prelude::{Client, HttpClientConfig, IggyDuration, IggyError};
use async_broadcast::{Receiver, Sender, broadcast};
use async_trait::async_trait;
-use iggy_common::locking::{IggySharedMut, IggySharedMutFn};
+use iggy_common::locking::{IggyRwLock, IggySharedMutFn};
use iggy_common::{
ConnectionString, ConnectionStringUtils, DiagnosticEvent, HttpConnectionStringOptions,
IdentityInfo, TransportProtocol,
@@ -51,7 +51,7 @@ pub struct HttpClient {
pub api_url: Url,
pub(crate) heartbeat_interval: IggyDuration,
client: ClientWithMiddleware,
- access_token: IggySharedMut<String>,
+ access_token: IggyRwLock<String>,
events: (Sender<DiagnosticEvent>, Receiver<DiagnosticEvent>),
}
@@ -280,7 +280,7 @@ impl HttpClient {
api_url,
client,
heartbeat_interval: IggyDuration::from_str("5s").unwrap(),
- access_token: IggySharedMut::new("".to_string()),
+ access_token: IggyRwLock::new("".to_string()),
events: broadcast(1000),
})
}
(iggy) 28/30: fix(io_uring): skip thread_pool_limit(0) on macOS (#1963)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 384b969b15feef26b9b86e1981ea40498bcde4e6
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Sat Jul 5 13:18:37 2025 +0200
fix(io_uring): skip thread_pool_limit(0) on macOS (#1963)
---
core/server/src/bootstrap.rs | 25 ++++++++++++++++---------
1 file changed, 16 insertions(+), 9 deletions(-)
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index 6a2eb8af..ead6242a 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -122,21 +122,28 @@ pub fn create_root_user() -> User {
}
pub fn create_shard_executor() -> Runtime {
- //TODO: The event intererval tick, could be configureed based on the fact
+ // TODO: The event intererval tick, could be configured based on the fact
// How many clients we expect to have connected.
// This roughly estimates the number of tasks we will create.
- let proactor = compio::driver::ProactorBuilder::new()
+
+ let mut proactor = compio::driver::ProactorBuilder::new();
+
+ proactor
.capacity(4096)
.coop_taskrun(true)
- .taskrun_flag(false) //TODO: Try enabling this.
- .thread_pool_limit(0)
- .to_owned();
- let rt = compio::runtime::RuntimeBuilder::new()
- .with_proactor(proactor)
+ .taskrun_flag(false); // TODO: Try enabling this.
+
+ // FIXME(hubcio): Only set thread_pool_limit(0) on non-macOS platforms
+ // This causes a freeze on macOS with compio fs operations
+ // see https://github.com/compio-rs/compio/issues/446
+ #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
+ proactor.thread_pool_limit(0);
+
+ compio::runtime::RuntimeBuilder::new()
+ .with_proactor(proactor.to_owned())
.event_interval(69)
.build()
- .unwrap();
- rt
+ .unwrap()
}
pub fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
(iggy) 22/30: feat(tpc): implement async walk_dir and remove_dir_all (#1942)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit c7b623508a08000c38ca943a079cb06589a4924f
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Mon Jun 30 19:22:40 2025 +0200
feat(tpc): implement async walk_dir and remove_dir_all (#1942)
This closes #1938.
---
core/server/src/bootstrap.rs | 12 ++--
core/server/src/io/fs_utils.rs | 93 +++++++++++++++++++++++++
core/server/src/io/mod.rs | 1 +
core/server/src/main.rs | 6 +-
core/server/src/shard/mod.rs | 3 +-
core/server/src/streaming/partitions/storage.rs | 9 ++-
core/server/src/streaming/streams/storage.rs | 6 +-
core/server/src/streaming/topics/storage.rs | 6 +-
8 files changed, 115 insertions(+), 21 deletions(-)
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index 6f046fa8..ddd0599c 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -11,6 +11,7 @@ use tracing::info;
use crate::{
IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
configs::{config_provider::ConfigProviderKind, server::ServerConfig, system::SystemConfig},
+ io::fs_utils,
server_error::ServerError,
shard::transmission::{
connector::{ShardConnector, StopSender},
@@ -22,7 +23,7 @@ use crate::{
utils::file::overwrite,
},
};
-use std::{env, fs::remove_dir_all, ops::Range, path::Path, sync::Arc};
+use std::{env, ops::Range, path::Path, sync::Arc};
pub fn create_shard_connections(
shards_set: Range<usize>,
@@ -59,10 +60,8 @@ pub async fn create_directories(config: &SystemConfig) -> Result<(), IggyError>
return Err(IggyError::CannotCreateStateDirectory(state_path));
}
let state_log = config.get_state_messages_file_path();
- if !Path::new(&state_log).exists() {
- if let Err(_) = overwrite(&state_log).await {
- return Err(IggyError::CannotCreateStateDirectory(state_log));
- }
+ if !Path::new(&state_log).exists() && (overwrite(&state_log).await).is_err() {
+ return Err(IggyError::CannotCreateStateDirectory(state_log));
}
let streams_path = config.get_streams_path();
@@ -71,8 +70,7 @@ pub async fn create_directories(config: &SystemConfig) -> Result<(), IggyError>
}
let runtime_path = config.get_runtime_path();
- // TODO: Change remove_dir_all to async version, once we implement the dir walk using monoio `remove_dir` method.
- if Path::new(&runtime_path).exists() && remove_dir_all(&runtime_path).is_err() {
+ if Path::new(&runtime_path).exists() && fs_utils::remove_dir_all(&runtime_path).await.is_err() {
return Err(IggyError::CannotRemoveRuntimeDirectory(runtime_path));
}
diff --git a/core/server/src/io/fs_utils.rs b/core/server/src/io/fs_utils.rs
new file mode 100644
index 00000000..d9be21e7
--- /dev/null
+++ b/core/server/src/io/fs_utils.rs
@@ -0,0 +1,93 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use monoio::fs;
+use std::io;
+use std::path::{Path, PathBuf};
+
+#[derive(Debug, Clone)]
+pub struct DirEntry {
+ pub path: PathBuf,
+ pub is_dir: bool,
+}
+
+impl DirEntry {
+ fn file(path: PathBuf) -> Self {
+ Self {
+ path,
+ is_dir: false,
+ }
+ }
+
+ fn dir(path: PathBuf) -> Self {
+ Self { path, is_dir: true }
+ }
+}
+
+/// Asynchronously walks a directory tree iteratively (without recursion).
+/// Returns all entries with directories listed after their contents to enable
+/// safe deletion (contents before containers).
+/// Symlinks are treated as files and not followed.
+pub async fn walk_dir(root: impl AsRef<Path>) -> io::Result<Vec<DirEntry>> {
+ let root = root.as_ref();
+
+ let metadata = fs::metadata(root).await?;
+ if !metadata.is_dir() {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidInput,
+ "path is not a directory",
+ ));
+ }
+
+ let mut files = Vec::new();
+ let mut directories = Vec::new();
+ let mut stack = vec![root.to_path_buf()];
+
+ while let Some(current_dir) = stack.pop() {
+ directories.push(DirEntry::dir(current_dir.clone()));
+
+ for entry in std::fs::read_dir(¤t_dir)? {
+ let entry = entry?;
+ let entry_path = entry.path();
+ let metadata = fs::symlink_metadata(&entry_path).await?;
+
+ if metadata.is_dir() {
+ stack.push(entry_path);
+ } else {
+ files.push(DirEntry::file(entry_path));
+ }
+ }
+ }
+
+ directories.reverse();
+ files.extend(directories);
+ Ok(files)
+}
+
+/// Removes a directory and all its contents.
+/// This is the equivalent of `tokio::fs::remove_dir_all` for monoio.
+/// Uses walk_dir to traverse the directory tree without recursion.
+pub async fn remove_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
+ for entry in walk_dir(path).await? {
+ match entry.is_dir {
+ true => fs::remove_dir(&entry.path).await?,
+ false => fs::remove_file(&entry.path).await?,
+ }
+ }
+ Ok(())
+}
diff --git a/core/server/src/io/mod.rs b/core/server/src/io/mod.rs
index ea2bfab5..686717cd 100644
--- a/core/server/src/io/mod.rs
+++ b/core/server/src/io/mod.rs
@@ -1,3 +1,4 @@
pub mod file;
+pub mod fs_utils;
pub mod reader;
pub mod writer;
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 2cd70c4d..53fa2d7b 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -35,6 +35,7 @@ use server::bootstrap::{
};
use server::configs::config_provider::{self};
use server::configs::server::ServerConfig;
+use server::io::fs_utils;
#[cfg(not(feature = "tokio-console"))]
use server::log::logger::Logging;
#[cfg(feature = "tokio-console")]
@@ -98,12 +99,9 @@ fn main() -> Result<(), ServerError> {
"Removing system path at: {} because `--fresh` flag was set",
system_path
);
- //TODO: Impl dir walk and remove the files
- /*
- if let Err(e) = tokio::fs::remove_dir_all(&system_path).await {
+ if let Err(e) = fs_utils::remove_dir_all(&system_path).await {
eprintln!("Failed to remove system path at {}: {}", system_path, e);
}
- */
}
}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index cc5023e2..48bf183a 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -46,6 +46,7 @@ use transmission::connector::{Receiver, ShardConnector, StopReceiver, StopSender
use crate::{
configs::server::ServerConfig,
+ io::fs_utils,
shard::{
system::info::SystemInfo,
task_registry::TaskRegistry,
@@ -341,7 +342,7 @@ impl IggyShard {
error!(
"Stream with ID: '{stream_id}' was not found in state, but exists on disk and will be removed."
);
- if let Err(error) = std::fs::remove_dir_all(&dir_entry.path()) {
+ if let Err(error) = fs_utils::remove_dir_all(&dir_entry.path()).await {
error!("Cannot remove stream directory: {error}");
} else {
warn!("Stream with ID: '{stream_id}' was removed.");
diff --git a/core/server/src/streaming/partitions/storage.rs b/core/server/src/streaming/partitions/storage.rs
index 954025d1..6621ea19 100644
--- a/core/server/src/streaming/partitions/storage.rs
+++ b/core/server/src/streaming/partitions/storage.rs
@@ -19,6 +19,7 @@
use crate::compat::index_rebuilding::index_rebuilder::IndexRebuilder;
use crate::configs::cache_indexes::CacheIndexesConfig;
use crate::io::file::IggyFile;
+use crate::io::fs_utils;
use crate::state::system::PartitionState;
use crate::streaming::partitions::COMPONENT;
use crate::streaming::partitions::partition::{ConsumerOffset, Partition};
@@ -366,7 +367,10 @@ impl PartitionStorage for FilePartitionStorage {
));
}
- if std::fs::remove_dir_all(&partition.partition_path).is_err() {
+ if fs_utils::remove_dir_all(&partition.partition_path)
+ .await
+ .is_err()
+ {
error!(
"Cannot delete partition directory: {} for partition with ID: {} for topic with ID: {} for stream with ID: {}.",
partition.partition_path,
@@ -474,8 +478,7 @@ impl PartitionStorage for FilePartitionStorage {
return Ok(());
}
- //TODO: replace with async once its there
- if std::fs::remove_dir_all(path).is_err() {
+ if fs_utils::remove_dir_all(path).await.is_err() {
error!("Cannot delete consumer offsets directory: {}.", path);
return Err(IggyError::CannotDeleteConsumerOffsetsDirectory(
path.to_owned(),
diff --git a/core/server/src/streaming/streams/storage.rs b/core/server/src/streaming/streams/storage.rs
index 50212014..d9848f03 100644
--- a/core/server/src/streaming/streams/storage.rs
+++ b/core/server/src/streaming/streams/storage.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use crate::io::fs_utils;
use crate::state::system::StreamState;
use crate::streaming::storage::StreamStorage;
use crate::streaming::streams::COMPONENT;
@@ -72,8 +73,7 @@ impl StreamStorage for FileStreamStorage {
error!(
"Topic with ID: '{topic_id}' for stream with ID: '{stream_id}' was not found in state, but exists on disk and will be removed."
);
- // TODO: Replace this with the dir walk impl that is mentioed in main function.
- if let Err(error) = std::fs::remove_dir_all(&dir_entry.path()) {
+ if let Err(error) = fs_utils::remove_dir_all(&dir_entry.path()).await {
error!("Cannot remove topic directory: {error}");
} else {
warn!(
@@ -223,7 +223,7 @@ impl StreamStorage for FileStreamStorage {
async fn delete(&self, stream: &Stream) -> Result<(), IggyError> {
info!("Deleting stream with ID: {}...", stream.stream_id);
- if std::fs::remove_dir_all(&stream.path).is_err() {
+ if fs_utils::remove_dir_all(&stream.path).await.is_err() {
return Err(IggyError::CannotDeleteStreamDirectory(stream.stream_id));
}
info!("Deleted stream with ID: {}.", stream.stream_id);
diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs
index 46894d20..8d92b9d1 100644
--- a/core/server/src/streaming/topics/storage.rs
+++ b/core/server/src/streaming/topics/storage.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use crate::io::fs_utils;
use crate::state::system::TopicState;
use crate::streaming::partitions::partition::Partition;
use crate::streaming::storage::TopicStorage;
@@ -88,8 +89,7 @@ impl TopicStorage for FileTopicStorage {
error!(
"Partition with ID: '{partition_id}' for stream with ID: '{stream_id}' and topic with ID: '{topic_id}' was not found in state, but exists on disk and will be removed."
);
- // TODO: Replace this with the dir walk impl that is mentioed in main function.
- if let Err(error) = std::fs::remove_dir_all(&dir_entry.path()) {
+ if let Err(error) = fs_utils::remove_dir_all(&dir_entry.path()).await {
error!("Cannot remove partition directory: {error}");
} else {
warn!(
@@ -270,7 +270,7 @@ impl TopicStorage for FileTopicStorage {
async fn delete(&self, topic: &Topic) -> Result<(), IggyError> {
info!("Deleting topic {topic}...");
- if std::fs::remove_dir_all(&topic.path).is_err() {
+ if fs_utils::remove_dir_all(&topic.path).await.is_err() {
return Err(IggyError::CannotDeleteTopicDirectory(
topic.topic_id,
topic.stream_id,
(iggy) 30/30: add cargo lock
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit af1b1ae721a2ee1b4ad54610df77bb2c3e71cb0a
Author: numinex <g....@gmail.com>
AuthorDate: Sat Jul 5 20:25:53 2025 +0200
add cargo lock
---
Cargo.lock | 100 +++++++++++++++++++++++++------------------------------------
1 file changed, 41 insertions(+), 59 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index cc838a3a..a8e41f6b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -477,9 +477,9 @@ dependencies = [
[[package]]
name = "async-channel"
-version = "2.3.1"
+version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a"
+checksum = "16c74e56284d2188cabb6ad99603d1ace887a5d7e7b695d01b728155ed9ed427"
dependencies = [
"concurrent-queue",
"event-listener-strategy",
@@ -1322,9 +1322,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.27"
+version = "1.2.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc"
+checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362"
dependencies = [
"jobserver",
"libc",
@@ -2681,25 +2681,13 @@ dependencies = [
[[package]]
name = "file-operation"
-version = "0.5.12"
+version = "0.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1b81e9a35a431f57f3304aec4ed6a59c7f1fb82af62b02e18afe197e8e3769d"
+checksum = "5ed4567f6a714792534e876e3830b00402ffc59b74d60e6ade3df2c6c442b377"
dependencies = [
"tokio",
]
-[[package]]
-name = "filetime"
-version = "0.2.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586"
-dependencies = [
- "cfg-if",
- "libc",
- "libredox",
- "windows-sys 0.59.0",
-]
-
[[package]]
name = "flatbuffers"
version = "24.12.23"
@@ -3503,9 +3491,9 @@ dependencies = [
[[package]]
name = "h2"
-version = "0.4.10"
+version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5"
+checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785"
dependencies = [
"atomic-waker",
"bytes",
@@ -3760,7 +3748,7 @@ dependencies = [
"bytes",
"futures-channel",
"futures-util",
- "h2 0.4.10",
+ "h2 0.4.11",
"http 1.3.1",
"http-body",
"httparse",
@@ -4739,7 +4727,6 @@ checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638"
dependencies = [
"bitflags 2.9.1",
"libc",
- "redox_syscall 0.5.13",
]
[[package]]
@@ -4944,8 +4931,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
[[package]]
-<<<<<<< HEAD
-<<<<<<< HEAD
name = "miette"
version = "7.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4965,19 +4950,9 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
-=======
-name = "memoffset"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
-dependencies = [
- "autocfg",
->>>>>>> 4d172830 (feat(io_uring): load server_config (#1798))
]
[[package]]
-=======
->>>>>>> 4f07d68c (feat(io_uring): move to compio runtime (#1960))
name = "mimalloc"
version = "0.1.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5154,12 +5129,11 @@ checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
[[package]]
name = "notify"
-version = "8.0.0"
+version = "8.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943"
+checksum = "3163f59cd3fa0e9ef8c32f242966a7b9994fd7378366099593e0e73077cd8c97"
dependencies = [
"bitflags 2.9.1",
- "filetime",
"fsevent-sys",
"inotify",
"kqueue",
@@ -5168,7 +5142,7 @@ dependencies = [
"mio",
"notify-types",
"walkdir",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
]
[[package]]
@@ -5450,9 +5424,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
[[package]]
name = "openssl-src"
-version = "300.5.0+3.5.0"
+version = "300.5.1+3.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8ce546f549326b0e6052b649198487d91320875da901e7bd11a06d1ee3f9c2f"
+checksum = "735230c832b28c000e3bc117119e6466a663ec73506bc0a9907ea4187508e42a"
dependencies = [
"cc",
]
@@ -6493,9 +6467,9 @@ dependencies = [
[[package]]
name = "reqwest"
-version = "0.12.20"
+version = "0.12.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813"
+checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -6642,13 +6616,12 @@ checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746"
[[package]]
name = "rust-ini"
-version = "0.21.1"
+version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f"
+checksum = "e7295b7ce3bf4806b419dc3420745998b447178b7005e2011947b38fc5aa6791"
dependencies = [
"cfg-if",
"ordered-multimap",
- "trim-in-place",
]
[[package]]
@@ -6919,6 +6892,18 @@ dependencies = [
"serde_json",
]
+[[package]]
+name = "schemars"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1375ba8ef45a6f15d83fa8748f1079428295d403d6ea991d09ab100155fbc06d"
+dependencies = [
+ "dyn-clone",
+ "ref-cast",
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "scoped-tls"
version = "1.0.1"
@@ -7083,16 +7068,17 @@ dependencies = [
[[package]]
name = "serde_with"
-version = "3.13.0"
+version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf65a400f8f66fb7b0552869ad70157166676db75ed8181f8104ea91cf9d0b42"
+checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5"
dependencies = [
"base64 0.22.1",
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.10.0",
- "schemars",
+ "schemars 0.9.0",
+ "schemars 1.0.3",
"serde",
"serde_derive",
"serde_json",
@@ -7102,9 +7088,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
-version = "3.13.0"
+version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81679d9ed988d5e9a5e6531dc3f2c28efbd639cbd1dfb628df08edea6004da77"
+checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f"
dependencies = [
"darling",
"proc-macro2",
@@ -7772,17 +7758,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.45.1"
+version = "1.46.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779"
+checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17"
dependencies = [
"backtrace",
"bytes",
+ "io-uring",
"libc",
"mio",
"parking_lot 0.12.4",
"pin-project-lite",
"signal-hook-registry",
+ "slab",
"socket2",
"tokio-macros",
"tracing",
@@ -7898,7 +7886,7 @@ dependencies = [
"axum 0.7.9",
"base64 0.22.1",
"bytes",
- "h2 0.4.10",
+ "h2 0.4.11",
"http 1.3.1",
"http-body",
"http-body-util",
@@ -8129,12 +8117,6 @@ dependencies = [
"syn 2.0.104",
]
-[[package]]
-name = "trim-in-place"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc"
-
[[package]]
name = "try-lock"
version = "0.2.5"
(iggy) 20/30: chore: reformat code on tpc branch (#1939)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 170b02a1d32d12a8992cd7152f6239989b4065e5
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Mon Jun 30 13:26:27 2025 +0200
chore: reformat code on tpc branch (#1939)
---
core/server/src/main.rs | 2 +-
.../src/streaming/segments/indexes/index_reader.rs | 28 +++++++++++-----------
.../src/streaming/segments/indexes/index_writer.rs | 4 ++--
core/server/src/streaming/segments/messages/mod.rs | 20 +++++++++-------
.../streaming/segments/types/messages_batch_mut.rs | 2 +-
.../src/streaming/segments/writing_messages.rs | 5 +---
core/server/src/streaming/streams/storage.rs | 6 ++---
core/server/src/streaming/topics/storage.rs | 2 +-
8 files changed, 34 insertions(+), 35 deletions(-)
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 5875e6bd..1b9331a1 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -245,7 +245,7 @@ fn main() -> Result<(), ServerError> {
//TODO: If one of the shards fails to initialize, we should crash the whole program;
if let Err(e) = shard.run().await {
error!("Failed to run shard-{id}: {e}");
- }
+ }
//TODO: If one of the shards fails to initialize, we should crash the whole program;
//shard.assert_init();
})
diff --git a/core/server/src/streaming/segments/indexes/index_reader.rs b/core/server/src/streaming/segments/indexes/index_reader.rs
index 240e62a1..8b09666a 100644
--- a/core/server/src/streaming/segments/indexes/index_reader.rs
+++ b/core/server/src/streaming/segments/indexes/index_reader.rs
@@ -21,6 +21,7 @@ use crate::{io::file::IggyFile, streaming::utils::PooledBuffer};
use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::{INDEX_SIZE, IggyError, IggyIndex, IggyIndexView};
+use monoio::fs::OpenOptions;
use std::{
fs::File as StdFile,
io::ErrorKind,
@@ -30,7 +31,6 @@ use std::{
atomic::{AtomicU64, Ordering},
},
};
-use monoio::fs::OpenOptions;
use tracing::{error, trace};
/// A dedicated struct for reading from the index file.
@@ -334,19 +334,19 @@ impl IndexReader {
len: u32,
use_pool: bool,
) -> Result<PooledBuffer, std::io::Error> {
- if use_pool {
- let mut buf = PooledBuffer::with_capacity(len as usize);
- unsafe { buf.set_len(len as usize) };
- let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
- result?;
- Ok(buf)
- } else {
- let mut buf = BytesMut::with_capacity(len as usize);
- unsafe { buf.set_len(len as usize) };
- let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
- result?;
- Ok(PooledBuffer::from_existing(buf))
- }
+ if use_pool {
+ let mut buf = PooledBuffer::with_capacity(len as usize);
+ unsafe { buf.set_len(len as usize) };
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ result?;
+ Ok(buf)
+ } else {
+ let mut buf = BytesMut::with_capacity(len as usize);
+ unsafe { buf.set_len(len as usize) };
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ result?;
+ Ok(PooledBuffer::from_existing(buf))
+ }
}
/// Gets the nth index from the index file.
diff --git a/core/server/src/streaming/segments/indexes/index_writer.rs b/core/server/src/streaming/segments/indexes/index_writer.rs
index 120bfe4e..4b79f9d2 100644
--- a/core/server/src/streaming/segments/indexes/index_writer.rs
+++ b/core/server/src/streaming/segments/indexes/index_writer.rs
@@ -94,7 +94,7 @@ impl IndexWriter {
}
let count = indexes.len() / INDEX_SIZE;
- let len = indexes.len();
+ let len = indexes.len();
self.file
.write_all(indexes)
@@ -109,7 +109,7 @@ impl IndexWriter {
.map_err(|_| IggyError::CannotSaveIndexToSegment)?;
self.index_size_bytes
- .fetch_add(len as u64, Ordering::Release);
+ .fetch_add(len as u64, Ordering::Release);
if self.fsync {
let _ = self.fsync().await;
diff --git a/core/server/src/streaming/segments/messages/mod.rs b/core/server/src/streaming/segments/messages/mod.rs
index b79b1af0..9799608f 100644
--- a/core/server/src/streaming/segments/messages/mod.rs
+++ b/core/server/src/streaming/segments/messages/mod.rs
@@ -19,7 +19,7 @@
mod messages_reader;
mod messages_writer;
-use crate::{io::file::IggyFile};
+use crate::io::file::IggyFile;
use super::IggyMessagesBatchSet;
use error_set::ErrContext;
@@ -29,7 +29,6 @@ use monoio::io::AsyncWriteRentExt;
pub use messages_reader::MessagesReader;
pub use messages_writer::MessagesWriter;
-
/// Vectored write a batches of messages to file
async fn write_batch(
file: &mut IggyFile,
@@ -39,14 +38,17 @@ async fn write_batch(
//let mut slices = batches.iter().map(|b| to_iovec(&b)).collect::<Vec<iovec>>();
let mut total_written = 0;
// TODO: Fork monoio, piece of shit runtime.
- for batch in batches.iter_mut() {
+ for batch in batches.iter_mut() {
let messages = batch.take_messages();
- let writen = file.write_all(messages).await.0.with_error_context(|error| {
- format!(
- "Failed to write messages to file: {file_path}, error: {error}",
- )
- // TODO: Better error variant.
- }).map_err(|_| IggyError::CannotAppendMessage)?;
+ let writen = file
+ .write_all(messages)
+ .await
+ .0
+ .with_error_context(|error| {
+ format!("Failed to write messages to file: {file_path}, error: {error}",)
+ // TODO: Better error variant.
+ })
+ .map_err(|_| IggyError::CannotAppendMessage)?;
total_written += writen;
}
Ok(total_written)
diff --git a/core/server/src/streaming/segments/types/messages_batch_mut.rs b/core/server/src/streaming/segments/types/messages_batch_mut.rs
index 6a76b582..764faa11 100644
--- a/core/server/src/streaming/segments/types/messages_batch_mut.rs
+++ b/core/server/src/streaming/segments/types/messages_batch_mut.rs
@@ -264,7 +264,7 @@ impl IggyMessagesBatchMut {
(indexes, messages)
}
- pub fn take_messages(&mut self) -> PooledBuffer {
+ pub fn take_messages(&mut self) -> PooledBuffer {
std::mem::take(&mut self.messages)
}
diff --git a/core/server/src/streaming/segments/writing_messages.rs b/core/server/src/streaming/segments/writing_messages.rs
index 8313f589..e95d1e2c 100644
--- a/core/server/src/streaming/segments/writing_messages.rs
+++ b/core/server/src/streaming/segments/writing_messages.rs
@@ -106,10 +106,7 @@ impl Segment {
.save_indexes(unsaved_indexes_slice)
.await
.with_error_context(|error| {
- format!(
- "Failed to save index of {} indexes to {self}. {error}",
- len
- )
+ format!("Failed to save index of {} indexes to {self}. {error}", len)
})?;
self.indexes.mark_saved();
diff --git a/core/server/src/streaming/streams/storage.rs b/core/server/src/streaming/streams/storage.rs
index 5bf4c554..50212014 100644
--- a/core/server/src/streaming/streams/storage.rs
+++ b/core/server/src/streaming/streams/storage.rs
@@ -26,12 +26,12 @@ use error_set::ErrContext;
use futures::future::join_all;
use iggy_common::IggyError;
use iggy_common::IggyTimestamp;
+use monoio::fs;
+use monoio::fs::create_dir_all;
use serde::{Deserialize, Serialize};
-use tokio::sync::Mutex;
use std::path::Path;
use std::sync::Arc;
-use monoio::fs;
-use monoio::fs::create_dir_all;
+use tokio::sync::Mutex;
use tracing::{error, info, warn};
#[derive(Debug)]
diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs
index 1025dbd3..46894d20 100644
--- a/core/server/src/streaming/topics/storage.rs
+++ b/core/server/src/streaming/topics/storage.rs
@@ -29,11 +29,11 @@ use futures::future::join_all;
use iggy_common::IggyError;
use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
+use monoio::fs;
use monoio::fs::create_dir_all;
use serde::{Deserialize, Serialize};
use std::path::Path;
use std::sync::Arc;
-use monoio::fs;
use tokio::sync::Mutex;
use tracing::{error, info, warn};
(iggy) 29/30: fix(io_uring): fix buffer reading with memory pool enabled (#1964)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 9a634fd46639cd63a3fa860def6df7c5a49b2643
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Sat Jul 5 19:25:59 2025 +0200
fix(io_uring): fix buffer reading with memory pool enabled (#1964)
Replace unsafe set_len() with slice() for reading exact buffer sizes.
This fixes hangs when memory pool returns larger buffers than requested.
Following compio's recommended pattern: buffer.slice(0..exact_size).
---
.../handlers/messages/send_messages_handler.rs | 29 +++++++++++-----------
core/server/src/streaming/utils/memory_pool.rs | 15 +++--------
core/server/src/streaming/utils/pooled_buffer.rs | 2 +-
3 files changed, 19 insertions(+), 27 deletions(-)
diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs
index 47df8270..bb536c61 100644
--- a/core/server/src/binary/handlers/messages/send_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs
@@ -27,7 +27,7 @@ use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
use crate::streaming::utils::PooledBuffer;
use anyhow::Result;
-use bytes::BytesMut;
+use compio::buf::{IntoInner as _, IoBuf};
use iggy_common::Identifier;
use iggy_common::Sizeable;
use iggy_common::{INDEX_SIZE, IdKind};
@@ -57,17 +57,18 @@ impl ServerCommandHandler for SendMessages {
let total_payload_size = length as usize - std::mem::size_of::<u32>();
let metadata_len_field_size = std::mem::size_of::<u32>();
- let mut metadata_length_buffer = BytesMut::with_capacity(4);
- unsafe { metadata_length_buffer.set_len(4) };
- let (result, metadata_len_buf) = sender.read(metadata_length_buffer).await;
+ let metadata_length_buffer = PooledBuffer::with_capacity(4);
+ let (result, metadata_len_buf) = sender.read(metadata_length_buffer.slice(0..4)).await;
+ let metadata_len_buf = metadata_len_buf.into_inner();
result?;
- let metadata_len_buf = metadata_len_buf.freeze();
let metadata_size = u32::from_le_bytes(metadata_len_buf[..].try_into().unwrap());
- let mut metadata_buffer = PooledBuffer::with_capacity(metadata_size as usize);
- unsafe { metadata_buffer.set_len(metadata_size as usize) };
- let (result, metadata_buf) = sender.read(metadata_buffer).await;
+ let metadata_buffer = PooledBuffer::with_capacity(metadata_size as usize);
+ let (result, metadata_buf) = sender
+ .read(metadata_buffer.slice(0..metadata_size as usize))
+ .await;
result?;
+ let metadata_buf = metadata_buf.into_inner();
let mut element_size = 0;
@@ -90,17 +91,17 @@ impl ServerCommandHandler for SendMessages {
);
let indexes_size = messages_count as usize * INDEX_SIZE;
- let mut indexes_buffer = PooledBuffer::with_capacity(indexes_size);
- unsafe { indexes_buffer.set_len(indexes_size) };
- let (result, indexes_buffer) = sender.read(indexes_buffer).await;
+ let indexes_buffer = PooledBuffer::with_capacity(indexes_size);
+ let (result, indexes_buffer) = sender.read(indexes_buffer.slice(0..indexes_size)).await;
result?;
+ let indexes_buffer = indexes_buffer.into_inner();
let messages_size =
total_payload_size - metadata_size as usize - indexes_size - metadata_len_field_size;
- let mut messages_buffer = PooledBuffer::with_capacity(messages_size);
- unsafe { messages_buffer.set_len(messages_size) };
- let (result, messages_buffer) = sender.read(messages_buffer).await;
+ let messages_buffer = PooledBuffer::with_capacity(messages_size);
+ let (result, messages_buffer) = sender.read(messages_buffer.slice(0..messages_size)).await;
result?;
+ let messages_buffer = messages_buffer.into_inner();
let indexes = IggyIndexesMut::from_bytes(indexes_buffer, 0);
let batch = IggyMessagesBatchMut::from_indexes_and_messages(
diff --git a/core/server/src/streaming/utils/memory_pool.rs b/core/server/src/streaming/utils/memory_pool.rs
index 16ad2087..675c0076 100644
--- a/core/server/src/streaming/utils/memory_pool.rs
+++ b/core/server/src/streaming/utils/memory_pool.rs
@@ -155,21 +155,12 @@ impl MemoryPool {
/// Initialize the global pool from the given config.
pub fn init_pool(config: Arc<SystemConfig>) {
- // TODO: Fixme (stary napraw).
- let is_enabled = false;
+ let is_enabled = config.memory_pool.enabled;
let memory_limit = config.memory_pool.size.as_bytes_usize();
let bucket_capacity = config.memory_pool.bucket_capacity as usize;
- let pool = MemoryPool::new(is_enabled, memory_limit, bucket_capacity);
- if MEMORY_POOL.set(pool).is_err() {
- warn!("Memory pool already initialized.");
- // This shouldn't ever happen in production code, only in tests
- // if someone forgets to add #[serial] tag to tests that have different
- // memory pool limits (different instances are created within same executable).
- if memory_pool().memory_limit != memory_limit {
- panic!("Previously initialized memory pool has a different limit.");
- }
- }
+ let _ =
+ MEMORY_POOL.get_or_init(|| MemoryPool::new(is_enabled, memory_limit, bucket_capacity));
}
/// Acquire a `BytesMut` buffer with at least `capacity` bytes.
diff --git a/core/server/src/streaming/utils/pooled_buffer.rs b/core/server/src/streaming/utils/pooled_buffer.rs
index 39ef25be..6bdaa357 100644
--- a/core/server/src/streaming/utils/pooled_buffer.rs
+++ b/core/server/src/streaming/utils/pooled_buffer.rs
@@ -229,7 +229,7 @@ impl SetBufInit for PooledBuffer {
unsafe impl IoBufMut for PooledBuffer {
fn as_buf_mut_ptr(&mut self) -> *mut u8 {
- self.inner.as_buf_mut_ptr()
+ self.inner.as_mut_ptr()
}
}
(iggy) 18/30: finish send/poll messages
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch rebase_master
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 05ab5a7705ce86a3f08aa95759617c754cc3aec9
Author: numinex <g....@gmail.com>
AuthorDate: Sun Jun 29 10:01:36 2025 +0200
finish send/poll messages
---
core/common/src/error/iggy_error.rs | 5 +
core/common/src/lib.rs | 1 -
core/common/src/types/confirmation/mod.rs | 53 ---------
core/common/src/types/mod.rs | 1 -
.../tests/streaming/get_by_timestamp.rs | 5 +-
core/integration/tests/streaming/topic_messages.rs | 5 +-
core/sdk/src/prelude.rs | 2 +-
.../handlers/messages/poll_messages_handler.rs | 128 +++++++++++++++++----
.../handlers/messages/send_messages_handler.rs | 111 +++++++++++++++---
core/server/src/configs/defaults.rs | 6 -
core/server/src/configs/displays.rs | 3 +-
core/server/src/configs/system.rs | 3 -
core/server/src/shard/mod.rs | 80 ++++++++++++-
core/server/src/shard/namespace.rs | 11 +-
core/server/src/shard/system/messages.rs | 52 ++-------
core/server/src/shard/system/partitions.rs | 15 ++-
core/server/src/shard/system/streams.rs | 9 ++
core/server/src/shard/system/topics.rs | 24 +++-
core/server/src/shard/transmission/event.rs | 60 ++++++++++
core/server/src/shard/transmission/frame.rs | 25 ++--
core/server/src/shard/transmission/message.rs | 28 ++++-
core/server/src/shard/transmission/mod.rs | 1 +
core/server/src/streaming/partitions/messages.rs | 20 +---
core/server/src/streaming/partitions/partition.rs | 8 +-
.../streaming/segments/messages/messages_reader.rs | 31 +++--
.../streaming/segments/messages/messages_writer.rs | 2 +-
core/server/src/streaming/streams/storage.rs | 14 +--
core/server/src/streaming/streams/topics.rs | 14 ++-
.../server/src/streaming/topics/consumer_groups.rs | 6 +-
core/server/src/streaming/topics/messages.rs | 55 +++++----
core/server/src/streaming/topics/partitions.rs | 14 +--
core/server/src/streaming/topics/storage.rs | 6 +-
core/server/src/streaming/topics/topic.rs | 26 +++--
33 files changed, 530 insertions(+), 294 deletions(-)
diff --git a/core/common/src/error/iggy_error.rs b/core/common/src/error/iggy_error.rs
index cc8a284b..334e14c7 100644
--- a/core/common/src/error/iggy_error.rs
+++ b/core/common/src/error/iggy_error.rs
@@ -459,6 +459,11 @@ pub enum IggyError {
CannotReadIndexPosition = 10011,
#[error("Cannot read index timestamp")]
CannotReadIndexTimestamp = 10012,
+
+ #[error("Shard not found for stream ID: {0}, topic ID: {1}, partition ID: {2}")]
+ ShardNotFound(u32, u32, u32) = 11000,
+ #[error("Shard communication error, shard ID: {0}")]
+ ShardCommunicationError(u16) = 11001,
}
impl IggyError {
diff --git a/core/common/src/lib.rs b/core/common/src/lib.rs
index 6603a7d5..f055476e 100644
--- a/core/common/src/lib.rs
+++ b/core/common/src/lib.rs
@@ -64,7 +64,6 @@ pub use types::configuration::tcp_config::tcp_client_config::*;
pub use types::configuration::tcp_config::tcp_client_config_builder::*;
pub use types::configuration::tcp_config::tcp_client_reconnection_config::*;
pub use types::configuration::tcp_config::tcp_connection_string_options::*;
-pub use types::confirmation::*;
pub use types::consumer::consumer_group::*;
pub use types::consumer::consumer_kind::*;
pub use types::consumer::consumer_offset_info::*;
diff --git a/core/common/src/types/confirmation/mod.rs b/core/common/src/types/confirmation/mod.rs
deleted file mode 100644
index bc26cffa..00000000
--- a/core/common/src/types/confirmation/mod.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-use serde::{Deserialize, Serialize};
-use strum::{Display, EnumString};
-
-#[derive(Clone, Copy, Debug, Default, Display, Serialize, Deserialize, EnumString, PartialEq)]
-#[strum(serialize_all = "snake_case")]
-pub enum Confirmation {
- #[default]
- Wait,
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::str::FromStr;
-
- #[test]
- fn test_to_string() {
- assert_eq!(Confirmation::Wait.to_string(), "wait");
- assert_eq!(Confirmation::NoWait.to_string(), "no_wait");
- }
-
- #[test]
- fn test_from_str() {
- assert_eq!(Confirmation::from_str("wait").unwrap(), Confirmation::Wait);
- assert_eq!(
- Confirmation::from_str("no_wait").unwrap(),
- Confirmation::NoWait
- );
- }
-
- #[test]
- fn test_default() {
- assert_eq!(Confirmation::default(), Confirmation::Wait);
- }
-}
diff --git a/core/common/src/types/mod.rs b/core/common/src/types/mod.rs
index 2376abd6..6940f065 100644
--- a/core/common/src/types/mod.rs
+++ b/core/common/src/types/mod.rs
@@ -21,7 +21,6 @@ pub(crate) mod client_state;
pub(crate) mod command;
pub(crate) mod compression;
pub(crate) mod configuration;
-pub(crate) mod confirmation;
pub(crate) mod consumer;
pub(crate) mod diagnostic;
pub(crate) mod identifier;
diff --git a/core/integration/tests/streaming/get_by_timestamp.rs b/core/integration/tests/streaming/get_by_timestamp.rs
index d0cc2e5b..93a178aa 100644
--- a/core/integration/tests/streaming/get_by_timestamp.rs
+++ b/core/integration/tests/streaming/get_by_timestamp.rs
@@ -202,10 +202,7 @@ async fn test_get_messages_by_timestamp(
let batch = IggyMessagesBatchMut::from_messages(messages_slice_to_append, messages_size);
assert_eq!(batch.count(), batch_len);
- partition
- .append_messages(batch)
- .await
- .unwrap();
+ partition.append_messages(batch).await.unwrap();
// Capture the timestamp of this batch
batch_timestamps.push(IggyTimestamp::now());
diff --git a/core/integration/tests/streaming/topic_messages.rs b/core/integration/tests/streaming/topic_messages.rs
index f21263c4..19de255f 100644
--- a/core/integration/tests/streaming/topic_messages.rs
+++ b/core/integration/tests/streaming/topic_messages.rs
@@ -62,10 +62,7 @@ async fn assert_polling_messages() {
.map(|m| m.get_size_bytes())
.sum::<IggyByteSize>();
let batch = IggyMessagesBatchMut::from_messages(&messages, batch_size.as_bytes_u32());
- topic
- .append_messages(&partitioning, batch)
- .await
- .unwrap();
+ topic.append_messages(&partitioning, batch).await.unwrap();
let consumer = PollingConsumer::Consumer(1, partition_id);
let (_, polled_messages) = topic
diff --git a/core/sdk/src/prelude.rs b/core/sdk/src/prelude.rs
index 1e1ee111..e04164fe 100644
--- a/core/sdk/src/prelude.rs
+++ b/core/sdk/src/prelude.rs
@@ -51,7 +51,7 @@ pub use iggy_binary_protocol::{
};
pub use iggy_common::{
Aes256GcmEncryptor, Args, ArgsOptional, AutoLogin, BytesSerializable, CacheMetrics,
- CacheMetricsKey, ClientError, ClientInfoDetails, CompressionAlgorithm, Confirmation, Consumer,
+ CacheMetricsKey, ClientError, ClientInfoDetails, CompressionAlgorithm, Consumer,
ConsumerGroupDetails, ConsumerKind, EncryptorKind, FlushUnsavedBuffer, GlobalPermissions,
HeaderKey, HeaderValue, HttpClientConfig, HttpClientConfigBuilder, IdKind, Identifier,
IdentityInfo, IggyByteSize, IggyDuration, IggyError, IggyExpiry, IggyIndexView, IggyMessage,
diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
index 9d6d526b..c590dcf2 100644
--- a/core/server/src/binary/handlers/messages/poll_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
@@ -20,8 +20,12 @@ use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHa
use crate::binary::handlers::messages::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::sender::SenderKind;
-use crate::shard::IggyShard;
+use crate::shard::namespace::IggyNamespace;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{ShardMessage, ShardRequest};
+use crate::shard::{IggyShard, ShardRequestResult};
use crate::shard::system::messages::PollingArgs;
+use crate::streaming::segments::IggyMessagesBatchSet;
use crate::streaming::session::Session;
use crate::to_iovec;
use anyhow::Result;
@@ -60,48 +64,128 @@ impl ServerCommandHandler for PollMessages {
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let (metadata, messages) = shard
- .poll_messages(
- session,
- &self.consumer,
- &self.stream_id,
- &self.topic_id,
- self.partition_id,
- PollingArgs::new(self.strategy, self.count, self.auto_commit),
- )
- .await
+ let stream = shard
+ .get_stream(&self.stream_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {}",
+ self.stream_id
+ )
+ })?;
+ let topic = stream.get_topic(
+ &self.topic_id,
+ ).with_error_context(|error| format!(
+ "{COMPONENT} (error: {error}) - topic not found for stream ID: {}, topic_id: {}",
+ self.stream_id, self.topic_id
+ ))?;
+
+ let PollMessages {
+ consumer,
+ partition_id,
+ strategy,
+ count,
+ auto_commit,
+ ..
+ } = self;
+ let args = PollingArgs::new(strategy, count, auto_commit);
+
+ shard.permissioner
+ .borrow()
+ .poll_messages(session.get_user_id(), topic.stream_id, topic.topic_id)
.with_error_context(|error| format!(
- "{COMPONENT} (error: {error}) - failed to poll messages for consumer: {}, stream_id: {}, topic_id: {}, partition_id: {:?}, session: {session}.",
- self.consumer, self.stream_id, self.topic_id, self.partition_id
+ "{COMPONENT} (error: {error}) - permission denied to poll messages for user {} on stream ID: {}, topic ID: {}",
+ session.get_user_id(),
+ topic.stream_id,
+ topic.topic_id
))?;
+ if !topic.has_partitions() {
+ return Err(IggyError::NoPartitions(topic.topic_id, topic.stream_id));
+ }
+
+ // There might be no partition assigned, if it's the consumer group member without any partitions.
+ let Some((consumer, partition_id)) = topic
+ .resolve_consumer_with_partition_id(&consumer, session.client_id, partition_id, true)
+ .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer with partition id, consumer: {consumer}, client ID: {}, partition ID: {:?}", session.client_id, partition_id))? else {
+ todo!("Send early response");
+ //return Ok((IggyPollMetadata::new(0, 0), IggyMessagesBatchSet::empty()));
+ };
+
+ let namespace = IggyNamespace::new(stream.stream_id, topic.topic_id, partition_id);
+ let request = ShardRequest::PollMessages {
+ consumer,
+ partition_id,
+ args,
+ count,
+ };
+ let message = ShardMessage::Request(request);
+ let (metadata, batch) = match shard.send_request_to_shard(&namespace, message).await {
+ ShardRequestResult::SameShard(message) => {
+ match message {
+ ShardMessage::Request(request) => {
+ match request {
+ ShardRequest::PollMessages {
+ consumer,
+ partition_id,
+ args,
+ count,
+ } => {
+ topic.get_messages(consumer, partition_id, args.strategy, count).await?
+
+ }
+ _ => unreachable!(
+ "Expected a SendMessages request inside of SendMessages handler, impossible state"
+ ),
+ }
+ }
+ _ => unreachable!(
+ "Expected a request message inside of an command handler, impossible state"
+ ),
+ }
+ }
+ ShardRequestResult::Result(result) => {
+ match result? {
+ ShardResponse::PollMessages(response) => {
+ response
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!(
+ "Expected a PollMessages response inside of PollMessages handler, impossible state"
+ ),
+ }
+ }
+ };
+
+
// Collect all chunks first into a Vec to extend their lifetimes.
// This ensures the Bytes (in reality Arc<[u8]>) references from each IggyMessagesBatch stay alive
// throughout the async vectored I/O operation, preventing "borrowed value does not live
// long enough" errors while optimizing transmission by using larger chunks.
// 4 bytes for partition_id + 8 bytes for current_offset + 4 bytes for messages_count + size of all batches.
- let response_length = 4 + 8 + 4 + messages.size();
+ let response_length = 4 + 8 + 4 + batch.size();
let response_length_bytes = response_length.to_le_bytes();
let partition_id = metadata.partition_id.to_le_bytes();
let current_offset = metadata.current_offset.to_le_bytes();
- let count = messages.count().to_le_bytes();
+ let count = batch.count().to_le_bytes();
- let mut io_slices = Vec::with_capacity(messages.containers_count() + 3);
- io_slices.push(to_iovec(&partition_id));
- io_slices.push(to_iovec(¤t_offset));
- io_slices.push(to_iovec(&count));
+ let mut iovecs = Vec::with_capacity(batch.containers_count() + 3);
+ iovecs.push(to_iovec(&partition_id));
+ iovecs.push(to_iovec(¤t_offset));
+ iovecs.push(to_iovec(&count));
- io_slices.extend(messages.iter().map(|m| to_iovec(&m)));
+ iovecs.extend(batch.iter().map(|m| to_iovec(&m)));
trace!(
"Sending {} messages to client ({} bytes) to client",
- messages.count(),
+ batch.count(),
response_length
);
sender
- .send_ok_response_vectored(&response_length_bytes, io_slices)
+ .send_ok_response_vectored(&response_length_bytes, iovecs)
.await?;
Ok(())
}
diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs
index d9a4770f..61e2c39c 100644
--- a/core/server/src/binary/handlers/messages/send_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs
@@ -16,19 +16,25 @@
* under the License.
*/
+use super::COMPONENT;
use crate::binary::command::{BinaryServerCommand, ServerCommandHandler};
use crate::binary::sender::SenderKind;
-use crate::shard::IggyShard;
+use crate::shard::namespace::IggyNamespace;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{ShardMessage, ShardRequest};
+use crate::shard::{IggyShard, ShardRequestResult};
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
use crate::streaming::utils::PooledBuffer;
use anyhow::Result;
use bytes::BytesMut;
-use iggy_common::INDEX_SIZE;
+use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::Sizeable;
+use iggy_common::{INDEX_SIZE, IdKind};
use iggy_common::{IggyError, Partitioning, SendMessages, Validatable};
use std::rc::Rc;
+use std::sync::Arc;
use tracing::instrument;
impl ServerCommandHandler for SendMessages {
@@ -104,19 +110,98 @@ impl ServerCommandHandler for SendMessages {
indexes,
messages_buffer,
);
-
batch.validate()?;
- shard
- .append_messages(
- session,
- &self.stream_id,
- &self.topic_id,
- &self.partitioning,
- batch,
- )
- .await?;
-
+ let stream = shard
+ .get_stream(&self.stream_id)
+ .with_error_context(|error| {
+ format!(
+ "Failed to get stream with ID: {} (error: {})",
+ self.stream_id, error
+ )
+ })?;
+ let topic = stream
+ .get_topic(&self.topic_id)
+ .with_error_context(|error| {
+ format!(
+ "Failed to get topic with ID: {} (error: {})",
+ self.topic_id, error
+ )
+ })?;
+ let stream_id = stream.stream_id;
+ let topic_id = topic.topic_id;
+ let partition_id = topic.calculate_partition_id(&self.partitioning)?;
+
+ // Validate permissions for given user on stream and topic.
+ shard.permissioner.borrow().append_messages(
+ session.get_user_id(),
+ stream_id,
+ topic_id
+ ).with_error_context(|error| format!(
+ "{COMPONENT} (error: {error}) - permission denied to append messages for user {} on stream ID: {}, topic ID: {}",
+ session.get_user_id(),
+ stream_id,
+ topic_id
+ ))?;
+ let messages_count = batch.count();
+ shard.metrics.increment_messages(messages_count as u64);
+
+ // Encrypt messages if encryptor is enabled in configuration.
+ let batch = shard.maybe_encrypt_messages(batch)?;
+
+ let namespace = IggyNamespace::new(stream.stream_id, topic.topic_id, partition_id);
+ let request = ShardRequest::SendMessages {
+ stream_id,
+ topic_id,
+ partition_id,
+ batch,
+ };
+ let message = ShardMessage::Request(request);
+ // Egh... I don't like those nested match statements,
+ // Technically there is only two `request` types that will ever be dispatched
+ // to different shards, thus we could get away with generic structs
+ // maybe todo ?
+ // how to make this code reusable, in a sense that we will have exactly the same code inside of
+ // `PollMessages` handler, but with different request....
+ match shard.send_request_to_shard(&namespace, message).await {
+ ShardRequestResult::SameShard(message) => {
+ match message {
+ ShardMessage::Request(request) => {
+ match request {
+ ShardRequest::SendMessages {
+ stream_id,
+ topic_id,
+ partition_id,
+ batch,
+ } => {
+ // Just shut up rust analyzer.
+ let _stream_id = stream_id;
+ let _topic_id = topic_id;
+ topic.append_messages(partition_id, batch).await?
+ }
+ _ => unreachable!(
+ "Expected a SendMessages request inside of SendMessages handler, impossible state"
+ ),
+ }
+ }
+ _ => unreachable!(
+ "Expected a request message inside of an command handler, impossible state"
+ ),
+ }
+ }
+ ShardRequestResult::Result(result) => {
+ match result? {
+ ShardResponse::SendMessages => {
+ ()
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!("Expected a SendMessages response inside of SendMessages handler, impossible state"),
+ }
+
+ }
+ };
sender.send_empty_ok_response().await?;
Ok(())
}
diff --git a/core/server/src/configs/defaults.rs b/core/server/src/configs/defaults.rs
index 82b560cf..dd74ade7 100644
--- a/core/server/src/configs/defaults.rs
+++ b/core/server/src/configs/defaults.rs
@@ -451,12 +451,6 @@ impl Default for SegmentConfig {
cache_indexes: SERVER_CONFIG.system.segment.cache_indexes.parse().unwrap(),
message_expiry: SERVER_CONFIG.system.segment.message_expiry.parse().unwrap(),
archive_expired: SERVER_CONFIG.system.segment.archive_expired,
- server_confirmation: SERVER_CONFIG
- .system
- .segment
- .server_confirmation
- .parse()
- .unwrap(),
}
}
}
diff --git a/core/server/src/configs/displays.rs b/core/server/src/configs/displays.rs
index de4805a8..51873421 100644
--- a/core/server/src/configs/displays.rs
+++ b/core/server/src/configs/displays.rs
@@ -289,12 +289,11 @@ impl Display for SegmentConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
- "{{ size_bytes: {}, cache_indexes: {}, message_expiry: {}, archive_expired: {}, server_confirmation: {} }}",
+ "{{ size_bytes: {}, cache_indexes: {}, message_expiry: {}, archive_expired: {} }}",
self.size,
self.cache_indexes,
self.message_expiry,
self.archive_expired,
- self.server_confirmation,
)
}
}
diff --git a/core/server/src/configs/system.rs b/core/server/src/configs/system.rs
index fc263a26..a5193294 100644
--- a/core/server/src/configs/system.rs
+++ b/core/server/src/configs/system.rs
@@ -17,7 +17,6 @@
*/
use super::cache_indexes::CacheIndexesConfig;
-use iggy_common::Confirmation;
use iggy_common::IggyByteSize;
use iggy_common::IggyExpiry;
use iggy_common::MaxTopicSize;
@@ -141,8 +140,6 @@ pub struct SegmentConfig {
#[serde_as(as = "DisplayFromStr")]
pub message_expiry: IggyExpiry,
pub archive_expired: bool,
- #[serde_as(as = "DisplayFromStr")]
- pub server_confirmation: Confirmation,
}
#[serde_as]
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index dbd94b68..7976cca3 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -47,14 +47,12 @@ use crate::{
shard::{
system::info::SystemInfo,
transmission::{
- frame::ShardFrame,
- message::{ShardEvent, ShardMessage},
+ frame::{ShardFrame, ShardResponse},
+ message::{ShardEvent, ShardMessage, ShardRequest},
},
},
state::{
- StateKind,
- file::FileState,
- system::{StreamState, SystemState, UserState},
+ file::FileState, system::{StreamState, SystemState, UserState}, StateKind
},
streaming::{
clients::client_manager::ClientManager,
@@ -85,12 +83,37 @@ impl Shard {
connection,
}
}
+
+ pub async fn send_request(&self, message: ShardMessage) -> Result<ShardResponse, IggyError> {
+ let (sender, receiver) = async_channel::bounded(1);
+ self.connection
+ .sender
+ .send(ShardFrame::new(message, Some(sender.clone()))); // Apparently sender needs to be cloned, otherwise channel will close...
+ //TODO: Fixme
+ let response = receiver.recv().await
+ .map_err(|err| {
+ error!("Failed to receive response from shard: {err}");
+ IggyError::ShardCommunicationError(self.id)
+ })?;
+ Ok(response)
+ }
}
struct ShardInfo {
id: u16,
}
+impl ShardInfo {
+ pub fn new(id: u16) -> Self {
+ Self { id }
+ }
+}
+
+pub enum ShardRequestResult<T, E> {
+ SameShard(ShardMessage),
+ Result(Result<T, E>),
+}
+
pub struct IggyShard {
pub id: u16,
shards: Vec<Shard>,
@@ -426,6 +449,53 @@ impl IggyShard {
self.shards.len() as u32
}
+ pub async fn send_request_to_shard(
+ &self,
+ namespace: &IggyNamespace,
+ message: ShardMessage,
+ ) -> ShardRequestResult<ShardResponse, IggyError> {
+ if let Some(shard) = self.find_shard(namespace) {
+ if shard.id == self.id {
+ return ShardRequestResult::SameShard(message);
+ }
+
+ let response = match shard.send_request(message).await {
+ Ok(response) => response,
+ Err(err) => {
+ error!(
+ "{COMPONENT} - failed to send request to shard with ID: {}, error: {err}",
+ shard.id
+ );
+ return ShardRequestResult::Result(Err(err));
+ }
+ };
+ ShardRequestResult::Result(Ok(response))
+ } else {
+ ShardRequestResult::Result(Err(IggyError::ShardNotFound(
+ namespace.stream_id,
+ namespace.topic_id,
+ namespace.partition_id,
+ )))
+ }
+ }
+
+ fn find_shard(&self, namespace: &IggyNamespace) -> Option<&Shard> {
+ let shards_table = self.shards_table.borrow();
+ shards_table.get(namespace).map(|shard_info| {
+ self.shards
+ .iter()
+ .find(|shard| shard.id == shard_info.id)
+ .expect("Shard not found in the shards table.")
+ })
+ }
+
+ pub fn insert_shard_table_records(
+ &self,
+ records: impl Iterator<Item = (IggyNamespace, ShardInfo)>,
+ ) {
+ self.shards_table.borrow_mut().extend(records);
+ }
+
pub fn broadcast_event_to_all_shards(&self, client_id: u32, event: ShardEvent) {
self.shards
.iter()
diff --git a/core/server/src/shard/namespace.rs b/core/server/src/shard/namespace.rs
index a7c899aa..6b3b5b7d 100644
--- a/core/server/src/shard/namespace.rs
+++ b/core/server/src/shard/namespace.rs
@@ -17,19 +17,18 @@
*/
use hash32::{Hasher, Murmur3Hasher};
-use iggy_common::Identifier;
use std::hash::Hasher as _;
//TODO: Will probably want to move it to separate crate so we can share it with sdk.
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct IggyNamespace {
- pub(crate) stream_id: Identifier,
- pub(crate) topic_id: Identifier,
+ pub(crate) stream_id: u32,
+ pub(crate) topic_id: u32,
pub(crate) partition_id: u32,
}
impl IggyNamespace {
- pub fn new(stream_id: Identifier, topic_id: Identifier, partition_id: u32) -> Self {
+ pub fn new(stream_id: u32, topic_id: u32, partition_id: u32) -> Self {
Self {
stream_id,
topic_id,
@@ -39,8 +38,8 @@ impl IggyNamespace {
pub fn generate_hash(&self) -> u32 {
let mut hasher = Murmur3Hasher::default();
- hasher.write(&self.stream_id.value);
- hasher.write(&self.topic_id.value);
+ hasher.write_u32(self.stream_id);
+ hasher.write_u32(self.topic_id);
hasher.write_u32(self.partition_id);
hasher.finish32()
}
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index 9a23a9bb..76337a1a 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -25,7 +25,7 @@ use crate::streaming::utils::PooledBuffer;
use async_zip::tokio::read::stream;
use error_set::ErrContext;
use iggy_common::{
- BytesSerializable, Confirmation, Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier,
+ BytesSerializable, Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier,
IggyError, Partitioning, PollingStrategy,
};
use tracing::{error, trace};
@@ -98,47 +98,6 @@ impl IggyShard {
Ok((metadata, batch_set))
}
- pub async fn append_messages(
- &self,
- session: &Session,
- stream_id: &Identifier,
- topic_id: &Identifier,
- partitioning: &Partitioning,
- messages: IggyMessagesBatchMut,
- ) -> Result<(), IggyError> {
- self.ensure_authenticated(session)?;
- let stream = self.get_stream(stream_id).with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - stream not found for stream_id: {stream_id}")
- })?;
- let stream_id = stream.stream_id;
- let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream_id: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.borrow().append_messages(
- session.get_user_id(),
- topic.stream_id,
- topic.topic_id
- ).with_error_context(|error| format!(
- "{COMPONENT} (error: {error}) - permission denied to append messages for user {} on stream ID: {}, topic ID: {}",
- session.get_user_id(),
- topic.stream_id,
- topic.topic_id
- ))?;
- let messages_count = messages.count();
-
- // Encrypt messages if encryptor is configured
- let messages = if let Some(encryptor) = &self.encryptor {
- self.encrypt_messages(messages, encryptor)?
- } else {
- messages
- };
-
- topic
- .append_messages(partitioning, messages)
- .await?;
-
- self.metrics.increment_messages(messages_count as u64);
- Ok(())
- }
-
pub async fn flush_unsaved_buffer(
&self,
session: &Session,
@@ -167,7 +126,7 @@ impl IggyShard {
Ok(())
}
- async fn decrypt_messages(
+ pub async fn decrypt_messages(
&self,
batches: IggyMessagesBatchSet,
encryptor: &EncryptorKind,
@@ -206,11 +165,14 @@ impl IggyShard {
Ok(IggyMessagesBatchSet::from_vec(decrypted_batches))
}
- fn encrypt_messages(
+ pub fn maybe_encrypt_messages(
&self,
batch: IggyMessagesBatchMut,
- encryptor: &EncryptorKind,
) -> Result<IggyMessagesBatchMut, IggyError> {
+ let encryptor = match self.encryptor.as_ref() {
+ Some(encryptor) => encryptor,
+ None => return Ok(batch),
+ };
let mut encrypted_messages = PooledBuffer::with_capacity(batch.size() as usize * 2);
let count = batch.count();
let mut indexes = IggyIndexesMut::with_capacity(batch.count() as usize, 0);
diff --git a/core/server/src/shard/system/partitions.rs b/core/server/src/shard/system/partitions.rs
index 01e02968..78ef2144 100644
--- a/core/server/src/shard/system/partitions.rs
+++ b/core/server/src/shard/system/partitions.rs
@@ -18,6 +18,8 @@
use super::COMPONENT;
use crate::shard::IggyShard;
+use crate::shard::ShardInfo;
+use crate::shard::namespace::IggyNamespace;
use crate::streaming::session::Session;
use error_set::ErrContext;
use iggy_common::Identifier;
@@ -54,6 +56,7 @@ impl IggyShard {
let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
})?;
+ let stream_id = stream.stream_id;
let topic = stream
.get_topic_mut(topic_id)
.with_error_context(|error| {
@@ -61,15 +64,25 @@ impl IggyShard {
"{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {stream_id}"
)
})?;
+ let topic_id = topic.topic_id;
// TODO: Make add persisted partitions to topic sync, and extract the storage persister out of it
// perform disk i/o outside of the borrow_mut of the stream.
- topic
+ let partition_ids = topic
.add_persisted_partitions(partitions_count)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to add persisted partitions, topic: {topic}")
})?;
+ let records = partition_ids.into_iter().map(|partition_id| {
+ let namespace = IggyNamespace::new(stream_id, topic_id, partition_id);
+ let hash = namespace.generate_hash();
+ let shard_id = hash % self.get_available_shards_count();
+ let shard_info = ShardInfo::new(shard_id as u16);
+ (namespace, shard_info)
+ });
+ self.insert_shard_table_records(records);
+
topic.reassign_consumer_groups();
self.metrics.increment_partitions(partitions_count);
self.metrics.increment_segments(partitions_count);
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index 3cb0da5b..fecec536 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -112,6 +112,15 @@ impl IggyShard {
}
}
+ pub fn try_get_topic_id(&self, stream_id: &Identifier, name: &str) -> Option<u32> {
+ let stream = self.get_stream(stream_id).ok()?;
+ stream.topics_ids.get(name).and_then(|id| Some(*id))
+ }
+
+ pub fn try_get_stream_id(&self, name: &str) -> Option<u32> {
+ self.streams_ids.borrow().get(name).and_then(|id| Some(*id))
+ }
+
fn try_get_stream_by_name(&self, name: &str) -> Option<Ref<'_, Stream>> {
self.streams_ids
.borrow()
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index 477308bc..252dae30 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -17,7 +17,8 @@
*/
use super::COMPONENT;
-use crate::shard::IggyShard;
+use crate::shard::namespace::IggyNamespace;
+use crate::shard::{IggyShard, ShardInfo};
use crate::streaming::session::Session;
use crate::streaming::streams::stream::Stream;
use crate::streaming::topics::topic::Topic;
@@ -131,8 +132,11 @@ impl IggyShard {
// TODO: Make create topic sync, and extract the storage persister out of it
// perform disk i/o outside of the borrow_mut of the stream.
- let created_topic_id = self
- .get_stream_mut(stream_id)?
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let (topic_id, partition_ids) = stream
.create_topic(
topic_id,
name,
@@ -146,12 +150,24 @@ impl IggyShard {
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to create topic with name: {name} in stream ID: {stream_id}")
})?;
+ let records = partition_ids.into_iter().map(|partition_id| {
+ let namespace = IggyNamespace::new(stream_id, topic_id, partition_id);
+ // TODO: This setup isn't deterministic.
+ // Imagine a scenario where client creates partition using `String` identifiers,
+ // but then for poll_messages requests uses numeric ones.
+ // the namespace wouldn't match, therefore we would get miss in the shard table.
+ let hash = namespace.generate_hash();
+ let shard_id = hash % self.get_available_shards_count();
+ let shard_info = ShardInfo::new(shard_id as u16);
+ (namespace, shard_info)
+ });
+ self.insert_shard_table_records(records);
self.metrics.increment_topics(1);
self.metrics.increment_partitions(partitions_count);
self.metrics.increment_segments(partitions_count);
- Ok(Identifier::numeric(created_topic_id)?)
+ Ok(Identifier::numeric(topic_id)?)
}
#[allow(clippy::too_many_arguments)]
diff --git a/core/server/src/shard/transmission/event.rs b/core/server/src/shard/transmission/event.rs
new file mode 100644
index 00000000..45815a50
--- /dev/null
+++ b/core/server/src/shard/transmission/event.rs
@@ -0,0 +1,60 @@
+use std::net::SocketAddr;
+
+use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize};
+
+use crate::streaming::clients::client_manager::Transport;
+
+pub enum ShardEvent {
+ CreatedStream {
+ stream_id: Option<u32>,
+ topic_id: Identifier
+ },
+ //DeletedStream(Identifier),
+ //UpdatedStream(Identifier, String),
+ //PurgedStream(Identifier),
+ //CreatedPartitions(Identifier, Identifier, u32),
+ //DeletedPartitions(Identifier, Identifier, u32),
+ CreatedTopic(
+ Identifier,
+ Option<u32>,
+ String,
+ u32,
+ IggyExpiry,
+ CompressionAlgorithm,
+ MaxTopicSize,
+ Option<u8>,
+ ),
+ //CreatedConsumerGroup(Identifier, Identifier, Option<u32>, String),
+ //DeletedConsumerGroup(Identifier, Identifier, Identifier),
+ /*
+ UpdatedTopic(
+ Identifier,
+ Identifier,
+ String,
+ IggyExpiry,
+ CompressionAlgorithm,
+ MaxTopicSize,
+ Option<u8>,
+ ),
+ */
+ //PurgedTopic(Identifier, Identifier),
+ //DeletedTopic(Identifier, Identifier),
+ //CreatedUser(String, String, UserStatus, Option<Permissions>),
+ //DeletedUser(Identifier),
+ LoginUser {
+ username: String,
+ password: String,
+ },
+ //LogoutUser,
+ //UpdatedUser(Identifier, Option<String>, Option<UserStatus>),
+ //ChangedPassword(Identifier, String, String),
+ //CreatedPersonalAccessToken(String, IggyExpiry),
+ //DeletedPersonalAccessToken(String),
+ //LoginWithPersonalAccessToken(String),
+ //StoredConsumerOffset(Identifier, Identifier, PollingConsumer, u64),
+ NewSession {
+ user_id: u32,
+ socket_addr: SocketAddr,
+ transport: Transport,
+ },
+}
\ No newline at end of file
diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs
index ec63daf0..ff519ab4 100644
--- a/core/server/src/shard/transmission/frame.rs
+++ b/core/server/src/shard/transmission/frame.rs
@@ -16,38 +16,33 @@
* under the License.
*/
use async_channel::Sender;
-use bytes::Bytes;
use iggy_common::IggyError;
-use crate::shard::transmission::message::ShardMessage;
+use crate::{binary::handlers::messages::poll_messages_handler::IggyPollMetadata, shard::transmission::message::ShardMessage, streaming::segments::IggyMessagesBatchSet};
+
+#[derive(Debug)]
+pub enum ShardResponse {
+ PollMessages((IggyPollMetadata, IggyMessagesBatchSet)),
+ SendMessages,
+ ShardEvent,
+ ErrorResponse(IggyError),
+}
#[derive(Debug)]
pub struct ShardFrame {
- pub client_id: u32,
pub message: ShardMessage,
pub response_sender: Option<Sender<ShardResponse>>,
}
impl ShardFrame {
- pub fn new(
- client_id: u32,
- message: ShardMessage,
- response_sender: Option<Sender<ShardResponse>>,
- ) -> Self {
+ pub fn new(message: ShardMessage, response_sender: Option<Sender<ShardResponse>>) -> Self {
Self {
- client_id,
message,
response_sender,
}
}
}
-#[derive(Debug)]
-pub enum ShardResponse {
- BinaryResponse(Bytes),
- ErrorResponse(IggyError),
-}
-
#[macro_export]
macro_rules! handle_response {
($sender:expr, $response:expr) => {
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index cbf4fdd4..360e7a3c 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -1,5 +1,7 @@
use std::rc::Rc;
+use iggy_common::PollingStrategy;
+
/* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,11 +19,11 @@ use std::rc::Rc;
* specific language governing permissions and limitations
* under the License.
*/
-use crate::{binary::command::ServerCommand, streaming::session::Session};
+use crate::{shard::system::messages::PollingArgs, streaming::{polling_consumer::PollingConsumer, segments::IggyMessagesBatchMut, session::Session}};
#[derive(Debug)]
pub enum ShardMessage {
- Command(ServerCommand),
+ Request(ShardRequest),
Event(ShardEvent),
}
@@ -30,9 +32,25 @@ pub enum ShardEvent {
NewSession(),
}
-impl From<ServerCommand> for ShardMessage {
- fn from(command: ServerCommand) -> Self {
- ShardMessage::Command(command)
+#[derive(Debug)]
+pub enum ShardRequest {
+ SendMessages {
+ stream_id: u32,
+ topic_id: u32,
+ partition_id: u32,
+ batch: IggyMessagesBatchMut,
+ },
+ PollMessages {
+ partition_id: u32,
+ args: PollingArgs,
+ consumer: PollingConsumer,
+ count: u32,
+ },
+}
+
+impl From<ShardRequest> for ShardMessage {
+ fn from(request: ShardRequest) -> Self {
+ ShardMessage::Request(request)
}
}
diff --git a/core/server/src/shard/transmission/mod.rs b/core/server/src/shard/transmission/mod.rs
index 1f9a79ed..4ed9bd1e 100644
--- a/core/server/src/shard/transmission/mod.rs
+++ b/core/server/src/shard/transmission/mod.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+pub mod event;
pub mod connector;
pub mod frame;
pub mod message;
diff --git a/core/server/src/streaming/partitions/messages.rs b/core/server/src/streaming/partitions/messages.rs
index cc961370..3b2d942b 100644
--- a/core/server/src/streaming/partitions/messages.rs
+++ b/core/server/src/streaming/partitions/messages.rs
@@ -21,7 +21,7 @@ use crate::streaming::partitions::partition::Partition;
use crate::streaming::polling_consumer::PollingConsumer;
use crate::streaming::segments::*;
use error_set::ErrContext;
-use iggy_common::{Confirmation, IggyError, IggyTimestamp, Sizeable};
+use iggy_common::{IggyError, IggyTimestamp, Sizeable};
use std::sync::atomic::Ordering;
use tracing::trace;
@@ -218,10 +218,7 @@ impl Partition {
Ok(batches)
}
- pub async fn append_messages(
- &mut self,
- batch: IggyMessagesBatchMut,
- ) -> Result<(), IggyError> {
+ pub async fn append_messages(&mut self, batch: IggyMessagesBatchMut) -> Result<(), IggyError> {
if batch.count() == 0 {
return Ok(());
}
@@ -564,10 +561,7 @@ mod tests {
.map(|m| m.get_size_bytes().as_bytes_u32())
.sum();
let initial_batch = IggyMessagesBatchMut::from_messages(&initial_messages, initial_size);
- partition
- .append_messages(initial_batch)
- .await
- .unwrap();
+ partition.append_messages(initial_batch).await.unwrap();
// Now try to add only duplicates
let duplicate_messages = vec![
@@ -582,10 +576,7 @@ mod tests {
.sum();
let duplicate_batch =
IggyMessagesBatchMut::from_messages(&duplicate_messages, duplicate_size);
- partition
- .append_messages(duplicate_batch)
- .await
- .unwrap();
+ partition.append_messages(duplicate_batch).await.unwrap();
let loaded_messages = partition.get_messages_by_offset(0, 10).await.unwrap();
@@ -739,8 +730,7 @@ mod tests {
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
IggyTimestamp::now(),
- )
- .await,
+ ),
temp_dir,
)
}
diff --git a/core/server/src/streaming/partitions/partition.rs b/core/server/src/streaming/partitions/partition.rs
index 1b8bacf8..4a573d14 100644
--- a/core/server/src/streaming/partitions/partition.rs
+++ b/core/server/src/streaming/partitions/partition.rs
@@ -84,7 +84,7 @@ impl ConsumerOffset {
impl Partition {
#[allow(clippy::too_many_arguments)]
- pub async fn create(
+ pub fn create(
stream_id: u32,
topic_id: u32,
partition_id: u32,
@@ -247,8 +247,7 @@ mod tests {
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
IggyTimestamp::now(),
- )
- .await;
+ );
assert_eq!(partition.stream_id, stream_id);
assert_eq!(partition.topic_id, topic_id);
@@ -290,8 +289,7 @@ mod tests {
Arc::new(AtomicU64::new(0)),
Arc::new(AtomicU32::new(0)),
IggyTimestamp::now(),
- )
- .await;
+ );
assert!(partition.segments.is_empty());
}
}
diff --git a/core/server/src/streaming/segments/messages/messages_reader.rs b/core/server/src/streaming/segments/messages/messages_reader.rs
index 0f413650..4091e8e3 100644
--- a/core/server/src/streaming/segments/messages/messages_reader.rs
+++ b/core/server/src/streaming/segments/messages/messages_reader.rs
@@ -16,9 +16,9 @@
* under the License.
*/
-use crate::io::file::{IggyFile};
+use crate::io::file::IggyFile;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
-use crate::streaming::utils::{file, PooledBuffer};
+use crate::streaming::utils::{PooledBuffer, file};
use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::IggyError;
@@ -174,19 +174,18 @@ impl MessagesReader {
len: u32,
use_pool: bool,
) -> Result<PooledBuffer, std::io::Error> {
- if use_pool {
- let mut buf = PooledBuffer::with_capacity(len as usize);
- unsafe { buf.set_len(len as usize) };
- let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
- result?;
- Ok(buf)
-
- } else {
- let mut buf = BytesMut::with_capacity(len as usize);
- unsafe { buf.set_len(len as usize) };
- let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
- result?;
- Ok(PooledBuffer::from_existing(buf))
- }
+ if use_pool {
+ let mut buf = PooledBuffer::with_capacity(len as usize);
+ unsafe { buf.set_len(len as usize) };
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ result?;
+ Ok(buf)
+ } else {
+ let mut buf = BytesMut::with_capacity(len as usize);
+ unsafe { buf.set_len(len as usize) };
+ let (result, buf) = self.file.read_exact_at(buf, offset as u64).await;
+ result?;
+ Ok(PooledBuffer::from_existing(buf))
+ }
}
}
diff --git a/core/server/src/streaming/segments/messages/messages_writer.rs b/core/server/src/streaming/segments/messages/messages_writer.rs
index 0429817f..63e424a9 100644
--- a/core/server/src/streaming/segments/messages/messages_writer.rs
+++ b/core/server/src/streaming/segments/messages/messages_writer.rs
@@ -21,7 +21,7 @@ use crate::{
streaming::segments::{IggyMessagesBatchSet, messages::write_batch},
};
use error_set::ErrContext;
-use iggy_common::{Confirmation, IggyByteSize, IggyError};
+use iggy_common::{IggyByteSize, IggyError};
use monoio::fs::{File, OpenOptions};
use std::sync::{
Arc,
diff --git a/core/server/src/streaming/streams/storage.rs b/core/server/src/streaming/streams/storage.rs
index 53639de9..ebd4946a 100644
--- a/core/server/src/streaming/streams/storage.rs
+++ b/core/server/src/streaming/streams/storage.rs
@@ -20,6 +20,7 @@ use crate::state::system::StreamState;
use crate::streaming::storage::StreamStorage;
use crate::streaming::streams::COMPONENT;
use crate::streaming::streams::stream::Stream;
+use crate::streaming::topics::topic::CreatedTopicInfo;
use crate::streaming::topics::topic::Topic;
use ahash::AHashSet;
use error_set::ErrContext;
@@ -83,7 +84,7 @@ impl StreamStorage for FileStreamStorage {
}
let topic_state = topic_state.unwrap();
- let topic = Topic::empty(
+ let topic_info = Topic::empty(
stream.stream_id,
topic_id,
&topic_state.name,
@@ -92,9 +93,8 @@ impl StreamStorage for FileStreamStorage {
stream.segments_count.clone(),
stream.config.clone(),
stream.storage.clone(),
- )
- .await;
- unloaded_topics.push(topic);
+ );
+ unloaded_topics.push(topic_info.topic);
}
let state_topic_ids = state.topics.keys().copied().collect::<AHashSet<u32>>();
@@ -123,7 +123,7 @@ impl StreamStorage for FileStreamStorage {
);
for topic_id in missing_ids {
let topic_state = state.topics.get(&topic_id).unwrap();
- let topic = Topic::empty(
+ let topic_info = Topic::empty(
stream.stream_id,
topic_id,
&topic_state.name,
@@ -132,8 +132,8 @@ impl StreamStorage for FileStreamStorage {
stream.segments_count.clone(),
stream.config.clone(),
stream.storage.clone(),
- )
- .await;
+ );
+ let topic = topic_info.topic;
topic.persist().await.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to persist topic: {topic}")
})?;
diff --git a/core/server/src/streaming/streams/topics.rs b/core/server/src/streaming/streams/topics.rs
index 1d47cc36..6a201c7a 100644
--- a/core/server/src/streaming/streams/topics.rs
+++ b/core/server/src/streaming/streams/topics.rs
@@ -18,6 +18,7 @@
use crate::streaming::streams::COMPONENT;
use crate::streaming::streams::stream::Stream;
+use crate::streaming::topics::topic::CreatedTopicInfo;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
use iggy_common::CompressionAlgorithm;
@@ -44,7 +45,7 @@ impl Stream {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: u8,
- ) -> Result<u32, IggyError> {
+ ) -> Result<(u32, Vec<u32>), IggyError> {
let max_topic_size = Topic::get_max_topic_size(max_topic_size, &self.config)?;
if self.topics_ids.contains_key(name) {
return Err(IggyError::TopicNameAlreadyExists(
@@ -74,7 +75,10 @@ impl Stream {
return Err(IggyError::TopicIdAlreadyExists(id, self.stream_id));
}
- let topic = Topic::create(
+ let CreatedTopicInfo {
+ topic,
+ partition_ids,
+ } = Topic::create(
self.stream_id,
id,
name,
@@ -88,15 +92,15 @@ impl Stream {
compression_algorithm,
max_topic_size,
replication_factor,
- )
- .await?;
+ )?;
+
topic.persist().await.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to persist topic: {topic}")
})?;
info!("Created topic {}", topic);
self.topics_ids.insert(name.to_owned(), id);
self.topics.insert(id, topic);
- Ok(id)
+ Ok((id, partition_ids))
}
pub async fn update_topic(
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index 28a8705c..79c7f9f3 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -458,7 +458,7 @@ mod tests {
let messages_count_of_parent_stream = Arc::new(AtomicU64::new(0));
let segments_count_of_parent_stream = Arc::new(AtomicU32::new(0));
- Topic::create(
+ let topic_info = Topic::create(
stream_id,
id,
name,
@@ -473,7 +473,7 @@ mod tests {
MaxTopicSize::ServerDefault,
1,
)
- .await
- .unwrap()
+ .unwrap();
+ topic_info.topic
}
}
diff --git a/core/server/src/streaming/topics/messages.rs b/core/server/src/streaming/topics/messages.rs
index acbe8c28..2f8f0557 100644
--- a/core/server/src/streaming/topics/messages.rs
+++ b/core/server/src/streaming/topics/messages.rs
@@ -25,7 +25,7 @@ use crate::streaming::utils::hash;
use ahash::AHashMap;
use error_set::ErrContext;
use iggy_common::locking::IggySharedMutFn;
-use iggy_common::{Confirmation, IggyTimestamp, PollingStrategy};
+use iggy_common::{IggyTimestamp, PollingStrategy};
use iggy_common::{IggyError, IggyExpiry, Partitioning, PartitioningKind, PollingKind};
use std::sync::atomic::Ordering;
use tracing::trace;
@@ -78,8 +78,8 @@ impl Topic {
pub async fn append_messages(
&self,
- partitioning: &Partitioning,
- messages: IggyMessagesBatchMut,
+ partition_id: u32,
+ batch: IggyMessagesBatchMut,
) -> Result<(), IggyError> {
if !self.has_partitions() {
return Err(IggyError::NoPartitions(self.topic_id, self.stream_id));
@@ -91,24 +91,11 @@ impl Topic {
return Err(IggyError::TopicFull(self.topic_id, self.stream_id));
}
- if messages.is_empty() {
+ if batch.is_empty() {
return Ok(());
}
- let partition_id = match partitioning.kind {
- PartitioningKind::Balanced => self.get_next_partition_id(),
- PartitioningKind::PartitionId => u32::from_le_bytes(
- partitioning.value[..partitioning.length as usize]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- ),
- PartitioningKind::MessagesKey => {
- self.calculate_partition_id_by_messages_key_hash(&partitioning.value)
- }
- };
-
- self.append_messages_to_partition(messages, partition_id)
- .await
+ self.append_messages_to_partition(batch, partition_id).await
}
pub async fn flush_unsaved_buffer(
@@ -152,6 +139,20 @@ impl Topic {
Ok(())
}
+ pub fn calculate_partition_id(&self, partitioning: &Partitioning) -> Result<u32, IggyError> {
+ match partitioning.kind {
+ PartitioningKind::Balanced => Ok(self.get_next_partition_id()),
+ PartitioningKind::PartitionId => Ok(u32::from_le_bytes(
+ partitioning.value[..partitioning.length as usize]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ )),
+ PartitioningKind::MessagesKey => {
+ Ok(self.calculate_partition_id_by_messages_key_hash(&partitioning.value))
+ }
+ }
+ }
+
fn get_next_partition_id(&self) -> u32 {
let mut partition_id = self.current_partition_id.fetch_add(1, Ordering::SeqCst);
let partitions_count = self.partitions.len() as u32;
@@ -227,10 +228,8 @@ mod tests {
.build()
.expect("Failed to create message with valid payload and headers");
let messages = IggyMessagesBatchMut::from_messages(&[message], 1);
- topic
- .append_messages(&partitioning, messages)
- .await
- .unwrap();
+ let partition = topic.calculate_partition_id(&partitioning).unwrap();
+ topic.append_messages(partition, messages).await.unwrap();
}
let partitions = topic.get_partitions();
@@ -260,10 +259,10 @@ mod tests {
.build()
.expect("Failed to create message with valid payload and headers");
let messages = IggyMessagesBatchMut::from_messages(&[message], 1);
- topic
- .append_messages(&partitioning, messages)
- .await
- .unwrap();
+ let partition_id = topic
+ .calculate_partition_id(&partitioning)
+ .expect("Failed to calculate partition ID");
+ topic.append_messages(partition_id, messages).await.unwrap();
}
let mut read_messages_count = 0;
@@ -337,7 +336,7 @@ mod tests {
let messages_count_of_parent_stream = Arc::new(AtomicU64::new(0));
let segments_count_of_parent_stream = Arc::new(AtomicU32::new(0));
- let topic = Topic::create(
+ let created_info = Topic::create(
stream_id,
id,
name,
@@ -352,8 +351,8 @@ mod tests {
MaxTopicSize::ServerDefault,
1,
)
- .await
.unwrap();
+ let topic = created_info.topic;
topic.persist().await.unwrap();
topic
}
diff --git a/core/server/src/streaming/topics/partitions.rs b/core/server/src/streaming/topics/partitions.rs
index dfedf0a9..b9e56331 100644
--- a/core/server/src/streaming/topics/partitions.rs
+++ b/core/server/src/streaming/topics/partitions.rs
@@ -34,7 +34,7 @@ impl Topic {
self.partitions.len() as u32
}
- pub async fn add_partitions(&mut self, count: u32) -> Result<Vec<u32>, IggyError> {
+ pub fn add_partitions(&mut self, count: u32) -> Result<Vec<u32>, IggyError> {
if count == 0 {
return Ok(vec![]);
}
@@ -60,8 +60,7 @@ impl Topic {
self.size_bytes.clone(),
self.segments_count_of_parent_stream.clone(),
IggyTimestamp::now(),
- )
- .await;
+ );
self.partitions
.insert(partition_id, IggyRwLock::new(partition));
partition_ids.push(partition_id)
@@ -71,12 +70,9 @@ impl Topic {
}
pub async fn add_persisted_partitions(&mut self, count: u32) -> Result<Vec<u32>, IggyError> {
- let partition_ids = self
- .add_partitions(count)
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to add partitions, count: {count}")
- })?;
+ let partition_ids = self.add_partitions(count).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to add partitions, count: {count}")
+ })?;
for partition_id in &partition_ids {
let partition = self.partitions.get(partition_id).unwrap();
let mut partition = partition.write().await;
diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs
index db74fc98..827ba6c9 100644
--- a/core/server/src/streaming/topics/storage.rs
+++ b/core/server/src/streaming/topics/storage.rs
@@ -113,8 +113,7 @@ impl TopicStorage for FileTopicStorage {
topic.size_bytes.clone(),
topic.segments_count_of_parent_stream.clone(),
partition_state.created_at,
- )
- .await;
+ );
unloaded_partitions.push(partition);
}
@@ -160,8 +159,7 @@ impl TopicStorage for FileTopicStorage {
topic.size_bytes.clone(),
topic.segments_count_of_parent_stream.clone(),
partition_state.created_at,
- )
- .await;
+ );
partition.persist().await.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to persist partition: {partition}"
diff --git a/core/server/src/streaming/topics/topic.rs b/core/server/src/streaming/topics/topic.rs
index 2e9e38c9..43b28c49 100644
--- a/core/server/src/streaming/topics/topic.rs
+++ b/core/server/src/streaming/topics/topic.rs
@@ -63,9 +63,14 @@ pub struct Topic {
pub created_at: IggyTimestamp,
}
+pub struct CreatedTopicInfo {
+ pub topic: Topic,
+ pub partition_ids: Vec<u32>,
+}
+
impl Topic {
#[allow(clippy::too_many_arguments)]
- pub async fn empty(
+ pub fn empty(
stream_id: u32,
topic_id: u32,
name: &str,
@@ -74,7 +79,7 @@ impl Topic {
segments_count_of_parent_stream: Arc<AtomicU32>,
config: Arc<SystemConfig>,
storage: Rc<SystemStorage>,
- ) -> Topic {
+ ) -> CreatedTopicInfo {
Topic::create(
stream_id,
topic_id,
@@ -90,12 +95,11 @@ impl Topic {
MaxTopicSize::ServerDefault,
1,
)
- .await
.unwrap()
}
#[allow(clippy::too_many_arguments)]
- pub async fn create(
+ pub fn create(
stream_id: u32,
topic_id: u32,
name: &str,
@@ -109,7 +113,7 @@ impl Topic {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: u8,
- ) -> Result<Topic, IggyError> {
+ ) -> Result<CreatedTopicInfo, IggyError> {
let path = config.get_topic_path(stream_id, topic_id);
let partitions_path = config.get_partitions_path(stream_id, topic_id);
let mut topic = Topic {
@@ -142,8 +146,11 @@ impl Topic {
message_expiry, topic.message_expiry
);
- topic.add_partitions(partitions_count).await?;
- Ok(topic)
+ let partition_ids = topic.add_partitions(partitions_count)?;
+ Ok(CreatedTopicInfo {
+ topic,
+ partition_ids,
+ })
}
pub fn is_full(&self) -> bool {
@@ -313,7 +320,7 @@ mod tests {
let messages_count_of_parent_stream = Arc::new(AtomicU64::new(0));
let segments_count_of_parent_stream = Arc::new(AtomicU32::new(0));
- let topic = Topic::create(
+ let topic_info = Topic::create(
stream_id,
topic_id,
name,
@@ -328,9 +335,8 @@ mod tests {
max_topic_size,
replication_factor,
)
- .await
.unwrap();
-
+ let topic = topic_info.topic;
assert_eq!(topic.stream_id, stream_id);
assert_eq!(topic.topic_id, topic_id);
assert_eq!(topic.path, path);