You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2025/09/30 16:35:24 UTC
(iggy) branch io_uring_tpc updated: feat(io_uring): implement robust task supervision system, fix shutdown (#2219)
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/io_uring_tpc by this push:
new d2c88e4c feat(io_uring): implement robust task supervision system, fix shutdown (#2219)
d2c88e4c is described below
commit d2c88e4c8869728e486a0fa1fe388a5c27b92ad1
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Tue Sep 30 18:35:19 2025 +0200
feat(io_uring): implement robust task supervision system, fix shutdown (#2219)
Replace ad-hoc channel commands with structured task registry that
provides lifecycle management, graceful shutdown, and proper resource
cleanup for all async tasks.
---
Cargo.lock | 4 -
core/bench/src/actors/consumer/client/low_level.rs | 2 +-
core/bench/src/benchmarks/benchmark.rs | 7 +-
core/bench/src/benchmarks/common.rs | 10 +-
core/common/src/error/iggy_error.rs | 2 +
core/integration/tests/streaming/get_by_offset.rs | 3 +-
.../tests/streaming/get_by_timestamp.rs | 3 +-
core/integration/tests/streaming/mod.rs | 7 +
core/server/Cargo.toml | 1 -
.../commands/clean_personal_access_tokens.rs | 75 ---
core/server/src/channels/commands/print_sysinfo.rs | 70 ---
core/server/src/channels/commands/save_messages.rs | 86 ---
.../src/channels/commands/verify_heartbeats.rs | 80 ---
core/server/src/http/http_server.rs | 83 ++-
core/server/src/http/jwt/mod.rs | 1 -
core/server/src/http/mod.rs | 2 +-
core/server/src/http/partitions.rs | 1 -
core/server/src/lib.rs | 1 -
core/server/src/log/logger.rs | 4 +-
core/server/src/log/runtime.rs | 22 +-
core/server/src/main.rs | 89 ++--
core/server/src/quic/listener.rs | 122 +++--
core/server/src/quic/quic_server.rs | 8 +-
core/server/src/shard/builder.rs | 41 +-
core/server/src/shard/mod.rs | 183 ++++---
core/server/src/shard/stats.rs | 1 -
core/server/src/shard/system/messages.rs | 16 +-
core/server/src/shard/task_registry.rs | 108 ----
core/server/src/shard/task_registry/builders.rs | 25 +
.../src/shard/task_registry/builders/continuous.rs | 92 ++++
.../src/shard/task_registry/builders/oneshot.rs | 103 ++++
.../src/shard/task_registry/builders/periodic.rs | 118 +++++
core/server/src/shard/task_registry/mod.rs | 6 +
core/server/src/shard/task_registry/registry.rs | 581 +++++++++++++++++++++
core/server/src/shard/task_registry/shutdown.rs | 233 +++++++++
.../tasks/continuous/http_server.rs} | 41 +-
.../src/shard/tasks/continuous/message_pump.rs | 72 +++
.../commands => shard/tasks/continuous}/mod.rs | 13 +-
.../tasks/continuous/quic_server.rs} | 35 +-
.../tasks/continuous/tcp_server.rs} | 35 +-
core/server/src/shard/tasks/messages.rs | 59 ---
core/server/src/shard/tasks/mod.rs | 21 +-
.../src/shard/tasks/periodic/heartbeat_verifier.rs | 90 ++++
.../src/shard/tasks/periodic/jwt_token_cleaner.rs | 61 +++
.../src/shard/tasks/periodic/message_saver.rs | 130 +++++
.../src/{channels => shard/tasks/periodic}/mod.rs | 12 +-
.../periodic/personal_access_token_cleaner.rs | 76 +++
.../src/shard/tasks/periodic/sysinfo_printer.rs | 75 +++
core/server/src/shard/transmission/message.rs | 5 +-
core/server/src/slab/mod.rs | 23 +-
core/server/src/slab/streams.rs | 128 ++++-
core/server/src/streaming/storage.rs | 2 +-
core/server/src/streaming/traits.rs | 4 +
core/server/src/streaming/utils/head_tail_buf.rs | 55 --
core/server/src/streaming/utils/mod.rs | 1 -
core/server/src/tcp/connection_handler.rs | 3 +
core/server/src/tcp/tcp_listener.rs | 29 +-
core/server/src/tcp/tcp_server.rs | 15 +-
core/server/src/tcp/tcp_tls_listener.rs | 26 +-
59 files changed, 2282 insertions(+), 919 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 876935f9..0b2edd0b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -614,8 +614,6 @@ dependencies = [
"futures-lite",
"pin-project",
"thiserror 2.0.16",
- "tokio",
- "tokio-util",
]
[[package]]
@@ -5716,8 +5714,6 @@ dependencies = [
"rand 0.9.2",
"serde_json",
"thiserror 2.0.16",
- "tokio",
- "tokio-stream",
]
[[package]]
diff --git a/core/bench/src/actors/consumer/client/low_level.rs b/core/bench/src/actors/consumer/client/low_level.rs
index 088ad42f..4c80c4ce 100644
--- a/core/bench/src/actors/consumer/client/low_level.rs
+++ b/core/bench/src/actors/consumer/client/low_level.rs
@@ -109,7 +109,7 @@ impl ConsumerClient for LowLevelConsumerClient {
}
Ok(Some(BatchMetrics {
- messages: messages_count as u32,
+ messages: messages_count.try_into().unwrap(),
user_data_bytes: user_bytes,
total_bytes,
latency,
diff --git a/core/bench/src/benchmarks/benchmark.rs b/core/bench/src/benchmarks/benchmark.rs
index 37e65b78..31f1828c 100644
--- a/core/bench/src/benchmarks/benchmark.rs
+++ b/core/bench/src/benchmarks/benchmark.rs
@@ -102,7 +102,7 @@ pub trait Benchmarkable: Send {
login_root(&client).await;
let streams = client.get_streams().await?;
for i in 1..=number_of_streams {
- let stream_name = format!("bench-stream-{}", i);
+ let stream_name = format!("bench-stream-{i}");
let stream_id: Identifier = stream_name.as_str().try_into()?;
if streams.iter().all(|s| s.name != stream_name) {
info!("Creating the test stream '{}'", stream_name);
@@ -141,11 +141,10 @@ pub trait Benchmarkable: Send {
login_root(&client).await;
let streams = client.get_streams().await?;
for i in 1..=number_of_streams {
- let stream_name = format!("bench-stream-{}", i);
+ let stream_name = format!("bench-stream-{i}");
if streams.iter().all(|s| s.name != stream_name) {
return Err(IggyError::ResourceNotFound(format!(
- "Streams for testing are not properly initialized. Stream '{}' is missing.",
- stream_name
+ "Streams for testing are not properly initialized. Stream '{stream_name}' is missing."
)));
}
}
diff --git a/core/bench/src/benchmarks/common.rs b/core/bench/src/benchmarks/common.rs
index 3ee4f686..5aa2c458 100644
--- a/core/bench/src/benchmarks/common.rs
+++ b/core/bench/src/benchmarks/common.rs
@@ -79,7 +79,7 @@ pub async fn init_consumer_groups(
login_root(&client).await;
for i in 1..=cg_count {
let consumer_group_id = CONSUMER_GROUP_BASE_ID + i;
- let stream_name = format!("bench-stream-{}", i);
+ let stream_name = format!("bench-stream-{i}");
let stream_id: Identifier = stream_name.as_str().try_into()?;
let topic_id: Identifier = "topic-1".try_into()?;
let consumer_group_name = format!("{CONSUMER_GROUP_NAME_PREFIX}-{consumer_group_id}");
@@ -136,7 +136,7 @@ pub fn build_producer_futures(
};
let stream_idx = 1 + ((producer_id - 1) % streams);
- let stream_id = format!("bench-stream-{}", stream_idx);
+ let stream_id = format!("bench-stream-{stream_idx}");
async move {
let producer = TypedBenchmarkProducer::new(
@@ -203,7 +203,7 @@ pub fn build_consumer_futures(
} else {
consumer_id
};
- let stream_id = format!("bench-stream-{}", stream_idx);
+ let stream_id = format!("bench-stream-{stream_idx}");
let consumer_group_id = if cg_count > 0 {
Some(CONSUMER_GROUP_BASE_ID + 1 + (consumer_id % cg_count))
} else {
@@ -251,7 +251,7 @@ pub fn build_producing_consumers_futures(
let client_factory_clone = client_factory.clone();
let args_clone = args.clone();
let stream_idx = 1 + ((actor_id - 1) % streams);
- let stream_id = format!("bench-stream-{}", stream_idx);
+ let stream_id = format!("bench-stream-{stream_idx}");
let send_finish_condition = BenchmarkFinishCondition::new(
&args,
@@ -329,7 +329,7 @@ pub fn build_producing_consumer_groups_futures(
let client_factory_clone = client_factory.clone();
let args_clone = args.clone();
let stream_idx = 1 + ((actor_id - 1) % cg_count);
- let stream_id = format!("bench-stream-{}", stream_idx);
+ let stream_id = format!("bench-stream-{stream_idx}");
let should_produce = actor_id <= producers;
let should_consume = actor_id <= consumers;
diff --git a/core/common/src/error/iggy_error.rs b/core/common/src/error/iggy_error.rs
index 614757f1..19dd41a9 100644
--- a/core/common/src/error/iggy_error.rs
+++ b/core/common/src/error/iggy_error.rs
@@ -476,6 +476,8 @@ pub enum IggyError {
#[error("Cannot bind to socket with addr: {0}")]
CannotBindToSocket(String) = 12000,
+ #[error("Task execution timeout")]
+ TaskTimeout = 12001,
}
impl IggyError {
diff --git a/core/integration/tests/streaming/get_by_offset.rs b/core/integration/tests/streaming/get_by_offset.rs
index 8044284a..7fd903bb 100644
--- a/core/integration/tests/streaming/get_by_offset.rs
+++ b/core/integration/tests/streaming/get_by_offset.rs
@@ -124,6 +124,7 @@ async fn test_get_messages_by_offset(
let stream_identifier = bootstrap_result.stream_id;
let topic_identifier = bootstrap_result.topic_id;
let partition_id = bootstrap_result.partition_id;
+ let task_registry = bootstrap_result.task_registry;
// Create namespace for MainOps calls
let namespace = IggyFullNamespace::new(
@@ -195,7 +196,7 @@ async fn test_get_messages_by_offset(
let batch = IggyMessagesBatchMut::from_messages(messages_slice_to_append, messages_size);
assert_eq!(batch.count(), batch_len);
streams
- .append_messages(shard_id as u16, &config, &namespace, batch)
+ .append_messages(shard_id as u16, &config, &task_registry, &namespace, batch)
.await
.unwrap();
diff --git a/core/integration/tests/streaming/get_by_timestamp.rs b/core/integration/tests/streaming/get_by_timestamp.rs
index de30345a..bf7428f9 100644
--- a/core/integration/tests/streaming/get_by_timestamp.rs
+++ b/core/integration/tests/streaming/get_by_timestamp.rs
@@ -124,6 +124,7 @@ async fn test_get_messages_by_timestamp(
let stream_identifier = bootstrap_result.stream_id;
let topic_identifier = bootstrap_result.topic_id;
let partition_id = bootstrap_result.partition_id;
+ let task_registry = bootstrap_result.task_registry;
// Create namespace for MainOps calls
let namespace = IggyFullNamespace::new(
@@ -200,7 +201,7 @@ async fn test_get_messages_by_timestamp(
let batch = IggyMessagesBatchMut::from_messages(messages_slice_to_append, messages_size);
assert_eq!(batch.count(), batch_len);
streams
- .append_messages(shard_id as u16, &config, &namespace, batch)
+ .append_messages(shard_id as u16, &config, &task_registry, &namespace, batch)
.await
.unwrap();
diff --git a/core/integration/tests/streaming/mod.rs b/core/integration/tests/streaming/mod.rs
index 4d547be7..c759c165 100644
--- a/core/integration/tests/streaming/mod.rs
+++ b/core/integration/tests/streaming/mod.rs
@@ -21,6 +21,7 @@ use iggy::prelude::IggyMessage;
use iggy_common::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize};
use server::{
configs::system::SystemConfig,
+ shard::task_registry::TaskRegistry,
slab::{streams::Streams, traits_ext::EntityMarker},
streaming::{
self,
@@ -30,6 +31,7 @@ use server::{
topics::{storage2::create_topic_file_hierarchy, topic2},
},
};
+use std::rc::Rc;
mod common;
mod get_by_offset;
@@ -41,6 +43,7 @@ struct BootstrapResult {
stream_id: Identifier,
topic_id: Identifier,
partition_id: usize,
+ task_registry: Rc<TaskRegistry>,
}
async fn bootstrap_test_environment(
@@ -118,11 +121,15 @@ async fn bootstrap_test_environment(
});
}
+ // Create a test task registry
+ let task_registry = Rc::new(TaskRegistry::new(shard_id));
+
Ok(BootstrapResult {
streams,
stream_id,
topic_id,
partition_id: 0,
+ task_registry,
})
}
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 64f71a31..49b6b66c 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -34,7 +34,6 @@ path = "src/main.rs"
[features]
default = ["mimalloc"]
-tokio-console = ["dep:console-subscriber", "tokio/tracing"]
disable-mimalloc = []
mimalloc = ["dep:mimalloc"]
diff --git a/core/server/src/channels/commands/clean_personal_access_tokens.rs b/core/server/src/channels/commands/clean_personal_access_tokens.rs
deleted file mode 100644
index d846aa27..00000000
--- a/core/server/src/channels/commands/clean_personal_access_tokens.rs
+++ /dev/null
@@ -1,75 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-use std::rc::Rc;
-
-use crate::shard::IggyShard;
-use iggy_common::{IggyError, IggyTimestamp};
-use tracing::{debug, info, trace};
-
-pub async fn clear_personal_access_tokens(shard: Rc<IggyShard>) -> Result<(), IggyError> {
- let config = &shard.config.personal_access_token.cleaner;
- if !config.enabled {
- info!("Personal access token cleaner is disabled.");
- return Ok(());
- }
-
- info!(
- "Personal access token cleaner is enabled, expired tokens will be deleted every: {}.",
- config.interval
- );
-
- let interval = config.interval.get_duration();
- let mut interval_timer = compio::time::interval(interval);
-
- loop {
- interval_timer.tick().await;
- trace!("Cleaning expired personal access tokens...");
-
- let users = shard.users.borrow();
- let now = IggyTimestamp::now();
- let mut deleted_tokens_count = 0;
-
- for (_, user) in users.iter() {
- let expired_tokens = user
- .personal_access_tokens
- .iter()
- .filter(|token| token.is_expired(now))
- .map(|token| token.token.clone())
- .collect::<Vec<_>>();
-
- for token in expired_tokens {
- debug!(
- "Personal access token: {} for user with ID: {} is expired.",
- token, user.id
- );
- deleted_tokens_count += 1;
- user.personal_access_tokens.remove(&token);
- debug!(
- "Deleted personal access token: {} for user with ID: {}.",
- token, user.id
- );
- }
- }
-
- info!(
- "Deleted {} expired personal access tokens.",
- deleted_tokens_count
- );
- }
-}
diff --git a/core/server/src/channels/commands/print_sysinfo.rs b/core/server/src/channels/commands/print_sysinfo.rs
deleted file mode 100644
index 0c3992d2..00000000
--- a/core/server/src/channels/commands/print_sysinfo.rs
+++ /dev/null
@@ -1,70 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-use std::rc::Rc;
-
-use crate::shard::IggyShard;
-use crate::streaming::utils::memory_pool;
-use human_repr::HumanCount;
-use iggy_common::IggyError;
-use tracing::{error, info, trace};
-
-pub async fn print_sys_info(shard: Rc<IggyShard>) -> Result<(), IggyError> {
- let config = &shard.config.system.logging;
- let interval = config.sysinfo_print_interval;
-
- if interval.is_zero() {
- info!("SysInfoPrinter is disabled.");
- return Ok(());
- }
- info!("SysInfoPrinter is enabled, system information will be printed every {interval}.");
- let mut interval_timer = compio::time::interval(interval.get_duration());
- loop {
- interval_timer.tick().await;
- trace!("Printing system information...");
-
- let stats = match shard.get_stats().await {
- Ok(stats) => stats,
- Err(e) => {
- error!("Failed to get system information. Error: {e}");
- continue;
- }
- };
-
- let free_memory_percent = (stats.available_memory.as_bytes_u64() as f64
- / stats.total_memory.as_bytes_u64() as f64)
- * 100f64;
-
- info!(
- "CPU: {:.2}%/{:.2}% (IggyUsage/Total), Mem: {:.2}%/{}/{}/{} (Free/IggyUsage/TotalUsed/Total), Clients: {}, Messages processed: {}, Read: {}, Written: {}, Uptime: {}",
- stats.cpu_usage,
- stats.total_cpu_usage,
- free_memory_percent,
- stats.memory_usage,
- stats.total_memory - stats.available_memory,
- stats.total_memory,
- stats.clients_count,
- stats.messages_count.human_count_bare(),
- stats.read_bytes,
- stats.written_bytes,
- stats.run_time
- );
-
- memory_pool().log_stats();
- }
-}
diff --git a/core/server/src/channels/commands/save_messages.rs b/core/server/src/channels/commands/save_messages.rs
deleted file mode 100644
index 84330563..00000000
--- a/core/server/src/channels/commands/save_messages.rs
+++ /dev/null
@@ -1,86 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-use crate::{shard::IggyShard, shard_info};
-use iggy_common::{Identifier, IggyError};
-use std::rc::Rc;
-use tracing::{error, info, trace};
-
-pub async fn save_messages(shard: Rc<IggyShard>) -> Result<(), IggyError> {
- let config = &shard.config.message_saver;
- if !config.enabled {
- info!("Message saver is disabled.");
- return Ok(());
- }
-
- // TODO: Maybe we should get rid of it in order to not complicate, and use the fsync settings per partition from config.
- let enforce_fsync = config.enforce_fsync;
- let interval = config.interval;
- info!(
- "Message saver is enabled, buffered messages will be automatically saved every: {interval}, enforce fsync: {enforce_fsync}."
- );
-
- let mut interval_timer = compio::time::interval(interval.get_duration());
- loop {
- interval_timer.tick().await;
- trace!("Saving buffered messages...");
-
- let namespaces = shard.get_current_shard_namespaces();
- let mut total_saved_messages = 0u32;
- let reason = "background saver triggered".to_string();
-
- for ns in namespaces {
- let stream_id = Identifier::numeric(ns.stream_id() as u32).unwrap();
- let topic_id = Identifier::numeric(ns.topic_id() as u32).unwrap();
- let partition_id = ns.partition_id();
-
- match shard
- .streams2
- .persist_messages(
- shard.id,
- &stream_id,
- &topic_id,
- partition_id,
- reason.clone(),
- &shard.config.system,
- )
- .await
- {
- Ok(batch_count) => {
- total_saved_messages += batch_count;
- }
- Err(err) => {
- error!(
- "Failed to save messages for partition {}: {}",
- partition_id, err
- );
- }
- }
- }
-
- if total_saved_messages > 0 {
- shard_info!(
- shard.id,
- "Saved {} buffered messages on disk.",
- total_saved_messages
- );
- }
-
- trace!("Finished saving buffered messages.");
- }
-}
diff --git a/core/server/src/channels/commands/verify_heartbeats.rs b/core/server/src/channels/commands/verify_heartbeats.rs
deleted file mode 100644
index 123c174f..00000000
--- a/core/server/src/channels/commands/verify_heartbeats.rs
+++ /dev/null
@@ -1,80 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-use crate::shard::IggyShard;
-use iggy_common::{IggyDuration, IggyError, IggyTimestamp};
-use std::rc::Rc;
-use tracing::{debug, info, trace, warn};
-
-const MAX_THRESHOLD: f64 = 1.2;
-
-pub async fn verify_heartbeats(shard: Rc<IggyShard>) -> Result<(), IggyError> {
- let config = &shard.config.heartbeat;
- if !config.enabled {
- info!("Heartbeats verification is disabled.");
- return Ok(());
- }
-
- let interval = config.interval;
- let max_interval = IggyDuration::from((MAX_THRESHOLD * interval.as_micros() as f64) as u64);
- info!("Heartbeats will be verified every: {interval}. Max allowed interval: {max_interval}.");
-
- let mut interval_timer = compio::time::interval(interval.get_duration());
-
- loop {
- interval_timer.tick().await;
- trace!("Verifying heartbeats...");
-
- let clients = {
- let client_manager = shard.client_manager.borrow();
- client_manager.get_clients()
- };
-
- let now = IggyTimestamp::now();
- let heartbeat_to = IggyTimestamp::from(now.as_micros() - max_interval.as_micros());
- debug!("Verifying heartbeats at: {now}, max allowed timestamp: {heartbeat_to}");
-
- let mut stale_clients = Vec::new();
- for client in clients {
- if client.last_heartbeat.as_micros() < heartbeat_to.as_micros() {
- warn!(
- "Stale client session: {}, last heartbeat at: {}, max allowed timestamp: {heartbeat_to}",
- client.session, client.last_heartbeat,
- );
- client.session.set_stale();
- stale_clients.push(client.session.client_id);
- } else {
- debug!(
- "Valid heartbeat at: {} for client session: {}, max allowed timestamp: {heartbeat_to}",
- client.last_heartbeat, client.session,
- );
- }
- }
-
- if stale_clients.is_empty() {
- continue;
- }
-
- let count = stale_clients.len();
- info!("Removing {count} stale clients...");
- for client_id in stale_clients {
- shard.delete_client(client_id);
- }
- info!("Removed {count} stale clients.");
- }
-}
diff --git a/core/server/src/http/http_server.rs b/core/server/src/http/http_server.rs
index 2ae46271..61ebe091 100644
--- a/core/server/src/http/http_server.rs
+++ b/core/server/src/http/http_server.rs
@@ -19,21 +19,22 @@
use crate::configs::http::{HttpConfig, HttpCorsConfig};
use crate::http::diagnostics::request_diagnostics;
use crate::http::http_shard_wrapper::HttpSafeShard;
-use crate::http::jwt::cleaner::start_expired_tokens_cleaner;
use crate::http::jwt::jwt_manager::JwtManager;
use crate::http::jwt::middleware::jwt_auth;
use crate::http::metrics::metrics;
use crate::http::shared::AppState;
use crate::http::*;
use crate::shard::IggyShard;
+use crate::shard::task_registry::ShutdownToken;
+use crate::shard::tasks::periodic::spawn_jwt_token_cleaner;
use crate::streaming::persistence::persister::PersisterKind;
-// use crate::streaming::systems::system::SharedSystem;
use axum::extract::DefaultBodyLimit;
use axum::extract::connect_info::Connected;
use axum::http::Method;
use axum::{Router, middleware};
use axum_server::tls_rustls::RustlsConfig;
use compio_net::TcpListener;
+use futures::FutureExt;
use iggy_common::IggyError;
use std::net::SocketAddr;
use std::path::PathBuf;
@@ -66,7 +67,12 @@ impl<'a> Connected<cyper_axum::IncomingStream<'a, TcpListener>> for CompioSocket
/// Starts the HTTP API server.
/// Returns the address the server is listening on.
-pub async fn start(config: HttpConfig, persister: Arc<PersisterKind>, shard: Rc<IggyShard>) -> Result<(), IggyError> {
+pub async fn start_http_server(
+ config: HttpConfig,
+ persister: Arc<PersisterKind>,
+ shard: Rc<IggyShard>,
+ shutdown: ShutdownToken,
+) -> Result<(), IggyError> {
if shard.id != 0 {
info!(
"HTTP server disabled for shard {} (only runs on shard 0)",
@@ -81,7 +87,7 @@ pub async fn start(config: HttpConfig, persister: Arc<PersisterKind>, shard: Rc<
"HTTP API"
};
- let app_state = build_app_state(&config, persister, shard).await;
+ let app_state = build_app_state(&config, persister, shard.clone()).await;
let mut app = Router::new()
.merge(system::router(app_state.clone(), &config.metrics))
.merge(personal_access_tokens::router(app_state.clone()))
@@ -105,7 +111,8 @@ pub async fn start(config: HttpConfig, persister: Arc<PersisterKind>, shard: Rc<
app = app.layer(middleware::from_fn_with_state(app_state.clone(), metrics));
}
- start_expired_tokens_cleaner(app_state.clone());
+ spawn_jwt_token_cleaner(shard.clone(), app_state.clone());
+
app = app.layer(middleware::from_fn(request_diagnostics));
if !config.tls.enabled {
@@ -116,18 +123,19 @@ pub async fn start(config: HttpConfig, persister: Arc<PersisterKind>, shard: Rc<
.local_addr()
.expect("Failed to get local address for HTTP server");
info!("Started {api_name} on: {address}");
- compio::runtime::spawn(async move {
- if let Err(error) = cyper_axum::serve(
- listener,
- app.into_make_service_with_connect_info::<CompioSocketAddr>(),
- )
- .await
- {
- error!("Failed to start {api_name} server, error {}", error);
- }
- })
- .detach();
- // address
+
+ let service = app.into_make_service_with_connect_info::<CompioSocketAddr>();
+
+ // Spawn the server in a task so we can handle shutdown
+ let server_task =
+ compio::runtime::spawn(async move { cyper_axum::serve(listener, service).await });
+
+ // Wait for shutdown signal
+ shutdown.wait().await;
+ info!("{api_name} received shutdown signal, stopping server");
+
+ // The server task will complete when all connections are closed
+ // For now, we just return as cyper_axum doesn't have a graceful shutdown API
Ok(())
} else {
let tls_config = RustlsConfig::from_pem_file(
@@ -138,27 +146,48 @@ pub async fn start(config: HttpConfig, persister: Arc<PersisterKind>, shard: Rc<
.unwrap();
let listener = std::net::TcpListener::bind(config.address).unwrap();
+ listener
+ .set_nonblocking(true)
+ .expect("Failed to set TLS listener to non-blocking");
let address = listener
.local_addr()
.expect("Failed to get local address for HTTPS / TLS server");
info!("Started {api_name} on: {address}");
- tokio::task::spawn(async move {
- if let Err(error) = axum_server::from_tcp_rustls(listener, tls_config)
- .serve(app.into_make_service_with_connect_info::<SocketAddr>())
- .await
- {
- error!("Failed to start {api_name} server, error: {}", error);
- }
- });
+ let service = app.into_make_service_with_connect_info::<SocketAddr>();
+
+ // Create a handle for graceful shutdown
+ let handle = axum_server::Handle::new();
+ let shutdown_handle = handle.clone();
+
+ // Spawn a task to handle shutdown
+ let shutdown_clone = shutdown.clone();
+ compio::runtime::spawn(async move {
+ shutdown_clone.wait().await;
+ info!("Initiating graceful shutdown for {api_name}");
+ shutdown_handle.graceful_shutdown(None);
+ })
+ .detach();
+
+ // Run the server with the handle
+ let server = axum_server::from_tcp_rustls(listener, tls_config).handle(handle);
+
+ if let Err(error) = server.serve(service).await {
+ error!("Failed to start {api_name} server, error: {}", error);
+ return Err(IggyError::CannotBindToSocket(format!("HTTPS: {}", error)));
+ }
- // address
+ info!("{api_name} server stopped gracefully");
Ok(())
}
}
-async fn build_app_state(config: &HttpConfig, persister: Arc<PersisterKind>, shard: Rc<IggyShard>) -> Arc<AppState> {
+async fn build_app_state(
+ config: &HttpConfig,
+ persister: Arc<PersisterKind>,
+ shard: Rc<IggyShard>,
+) -> Arc<AppState> {
let tokens_path;
{
tokens_path = shard.config.system.get_state_tokens_path();
diff --git a/core/server/src/http/jwt/mod.rs b/core/server/src/http/jwt/mod.rs
index a60b270b..5481f110 100644
--- a/core/server/src/http/jwt/mod.rs
+++ b/core/server/src/http/jwt/mod.rs
@@ -16,7 +16,6 @@
* under the License.
*/
-pub mod cleaner;
pub mod json_web_token;
pub mod jwt_manager;
pub mod middleware;
diff --git a/core/server/src/http/mod.rs b/core/server/src/http/mod.rs
index ba369815..4cf663d3 100644
--- a/core/server/src/http/mod.rs
+++ b/core/server/src/http/mod.rs
@@ -23,7 +23,7 @@ mod http_shard_wrapper;
pub mod jwt;
mod mapper;
pub mod metrics;
-mod shared;
+pub mod shared;
pub mod consumer_groups;
pub mod consumer_offsets;
diff --git a/core/server/src/http/partitions.rs b/core/server/src/http/partitions.rs
index 11a1c4a1..b54539ea 100644
--- a/core/server/src/http/partitions.rs
+++ b/core/server/src/http/partitions.rs
@@ -33,7 +33,6 @@ use iggy_common::Identifier;
use iggy_common::Validatable;
use iggy_common::create_partitions::CreatePartitions;
use iggy_common::delete_partitions::DeletePartitions;
-use iggy_common::locking::IggyRwLockFn;
use send_wrapper::SendWrapper;
use std::sync::Arc;
use tracing::instrument;
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index 38263692..735a2e51 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -31,7 +31,6 @@ compile_error!("iggy-server doesn't support windows.");
pub mod args;
pub mod binary;
pub mod bootstrap;
-pub mod channels;
pub(crate) mod compat;
pub mod configs;
pub mod http;
diff --git a/core/server/src/log/logger.rs b/core/server/src/log/logger.rs
index 59725685..b400233a 100644
--- a/core/server/src/log/logger.rs
+++ b/core/server/src/log/logger.rs
@@ -16,10 +16,10 @@
* under the License.
*/
-use crate::log::runtime::CompioRuntime;
use crate::VERSION;
use crate::configs::server::{TelemetryConfig, TelemetryTransport};
use crate::configs::system::LoggingConfig;
+use crate::log::runtime::CompioRuntime;
use crate::server_error::LogError;
use opentelemetry::KeyValue;
use opentelemetry::global;
@@ -250,7 +250,7 @@ impl Logging {
.with_span_processor(
span_processor_with_async_runtime::BatchSpanProcessor::builder(
trace_exporter,
- CompioRuntime
+ CompioRuntime,
)
.build(),
)
diff --git a/core/server/src/log/runtime.rs b/core/server/src/log/runtime.rs
index 1683b34a..1cbec8ce 100644
--- a/core/server/src/log/runtime.rs
+++ b/core/server/src/log/runtime.rs
@@ -1,7 +1,6 @@
-use std::{pin::Pin, task::Poll, time::Duration};
-
-use futures::{channel::mpsc, future::poll_fn, FutureExt, SinkExt, Stream, StreamExt};
+use futures::{FutureExt, SinkExt, Stream, StreamExt, channel::mpsc, future::poll_fn};
use opentelemetry_sdk::runtime::{Runtime, RuntimeChannel, TrySend};
+use std::{pin::Pin, task::Poll, time::Duration};
#[derive(Clone)]
pub struct CompioRuntime;
@@ -17,6 +16,7 @@ impl Runtime for CompioRuntime {
// "This is mainly used to run batch span processing in the background. Note, that the function
// does not return a handle. OpenTelemetry will use a different way to wait for the future to
// finish when the caller shuts down.""
+ // TODO(hubcio): investigate if we can use TaskRegistry API for task spawn
compio::runtime::spawn(future).detach();
}
@@ -34,17 +34,20 @@ impl<T> CompioSender<T> {
pub fn new(sender: mpsc::UnboundedSender<T>) -> Self {
Self { sender }
}
-}
+}
-// Safety: Since we use compio runtime which is single-threaded, or rather the Future: !Send + !Sync,
+// Safety: Since we use compio runtime which is single-threaded, or rather the Future: !Send + !Sync,
// we can implement those traits, to satisfy the trait bounds from `Runtime` and `RuntimeChannel` traits.
unsafe impl<T> Send for CompioSender<T> {}
unsafe impl<T> Sync for CompioSender<T> {}
-impl<T: std::fmt::Debug + Send> TrySend for CompioSender<T> {
+impl<T: std::fmt::Debug + Send> TrySend for CompioSender<T> {
type Message = T;
- fn try_send(&self, item: Self::Message) -> Result<(), opentelemetry_sdk::runtime::TrySendError> {
+ fn try_send(
+ &self,
+ item: Self::Message,
+ ) -> Result<(), opentelemetry_sdk::runtime::TrySendError> {
self.sender.unbounded_send(item).map_err(|_err| {
// Unbounded channels can only fail if disconnected, never full
opentelemetry_sdk::runtime::TrySendError::ChannelClosed
@@ -65,7 +68,10 @@ impl<T> CompioReceiver<T> {
impl<T: std::fmt::Debug + Send> Stream for CompioReceiver<T> {
type Item = T;
- fn poll_next(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<Option<Self::Item>> {
+ fn poll_next(
+ mut self: Pin<&mut Self>,
+ cx: &mut std::task::Context<'_>,
+ ) -> std::task::Poll<Option<Self::Item>> {
self.receiver.poll_next_unpin(cx)
}
}
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 095ba4c2..35a37fee 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -16,37 +16,23 @@
* under the License.
*/
-use std::collections::HashSet;
-use std::rc::Rc;
-use std::str::FromStr;
-use std::sync::Arc;
-use std::sync::atomic::AtomicBool;
-
use anyhow::Result;
use clap::Parser;
use dashmap::DashMap;
use dotenvy::dotenv;
use error_set::ErrContext;
use figlet_rs::FIGfont;
-use iggy_common::create_user::CreateUser;
-use iggy_common::defaults::DEFAULT_ROOT_USER_ID;
-use iggy_common::{Aes256GcmEncryptor, EncryptorKind, Identifier, IggyError};
-use lending_iterator::lending_iterator::constructors::into_lending_iter;
+use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError};
use server::args::Args;
-use server::binary::handlers::streams;
use server::bootstrap::{
- create_directories, create_root_user, create_shard_connections, create_shard_executor,
- load_config, load_streams, load_users, resolve_persister, update_system_info,
+ create_directories, create_shard_connections, create_shard_executor, load_config, load_streams,
+ load_users, resolve_persister, update_system_info,
};
use server::configs::config_provider::{self};
-use server::configs::server::ServerConfig;
use server::configs::sharding::CpuAllocation;
use server::io::fs_utils;
-#[cfg(not(feature = "tokio-console"))]
use server::log::logger::Logging;
-#[cfg(feature = "tokio-console")]
-use server::log::tokio_console::Logging;
-use server::server_error::{ConfigError, ServerError};
+use server::server_error::ServerError;
use server::shard::namespace::IggyNamespace;
use server::shard::system::info::SystemInfo;
use server::shard::{IggyShard, ShardInfo, calculate_shard_assignment};
@@ -54,22 +40,26 @@ use server::slab::traits_ext::{
EntityComponentSystem, EntityComponentSystemMutCell, IntoComponents,
};
use server::state::StateKind;
-use server::state::command::EntryCommand;
use server::state::file::FileState;
-use server::state::models::CreateUserWithId;
use server::state::system::SystemState;
use server::streaming::diagnostics::metrics::Metrics;
use server::streaming::storage::SystemStorage;
use server::streaming::utils::MemoryPool;
use server::streaming::utils::ptr::EternalPtr;
use server::versioning::SemanticVersion;
-use server::{IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV, map_toggle_str, shard_info};
+use server::{map_toggle_str, shard_info};
+use std::collections::HashSet;
+use std::rc::Rc;
+use std::str::FromStr;
+use std::sync::atomic::{AtomicU64, Ordering};
use tokio::time::Instant;
use tracing::{error, info, instrument, warn};
const COMPONENT: &str = "MAIN";
const SHARDS_TABLE_CAPACITY: usize = 16384;
+static SHUTDOWN_START_TIME: AtomicU64 = AtomicU64::new(0);
+
#[instrument(skip_all, name = "trace_start_server")]
#[compio::main]
async fn main() -> Result<(), ServerError> {
@@ -286,7 +276,6 @@ async fn main() -> Result<(), ServerError> {
let streams = streams.clone();
let shards_table = shards_table.clone();
let users = users.clone();
- let persister = storage.persister.clone();
let connections = connections.clone();
let config = config.clone();
let encryptor = encryptor.clone();
@@ -343,7 +332,7 @@ async fn main() -> Result<(), ServerError> {
.build();
let shard = Rc::new(shard);
- if let Err(e) = shard.run(persister).await {
+ if let Err(e) = shard.run().await {
error!("Failed to run shard-{id}: {e}");
}
shard_info!(shard.id, "Run completed");
@@ -354,29 +343,51 @@ async fn main() -> Result<(), ServerError> {
}
let shutdown_handles_for_signal = shutdown_handles.clone();
- /*
- ::set_handler(move || {
- info!("Received shutdown signal (SIGTERM/SIGINT), initiating graceful shutdown...");
-
- for (shard_id, stop_sender) in &shutdown_handles_for_signal {
- info!("Sending shutdown signal to shard {}", shard_id);
- if let Err(e) = stop_sender.send_blocking(()) {
- error!(
- "Failed to send shutdown signal to shard {}: {}",
- shard_id, e
- );
- }
+
+ ctrlc::set_handler(move || {
+ let now = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_millis() as u64;
+ SHUTDOWN_START_TIME.store(now, Ordering::SeqCst);
+
+ info!("Received shutdown signal (SIGTERM/SIGINT), initiating graceful shutdown...");
+
+ for (shard_id, stop_sender) in &shutdown_handles_for_signal {
+ if let Err(e) = stop_sender.try_send(()) {
+ error!(
+ "Failed to send shutdown signal to shard {}: {}",
+ shard_id, e
+ );
}
- })
- .expect("Error setting Ctrl-C handler");
- */
+ }
+ })
+ .expect("Error setting Ctrl-C handler");
info!("Iggy server is running. Press Ctrl+C or send SIGTERM to shutdown.");
+
for (idx, handle) in handles.into_iter().enumerate() {
handle.join().expect("Failed to join shard thread");
}
- info!("All shards have shut down. Iggy server is exiting.");
+ let shutdown_duration_msg = {
+ let start_time = SHUTDOWN_START_TIME.load(Ordering::SeqCst);
+ if start_time > 0 {
+ let now = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_millis() as u64;
+ let elapsed = now - start_time;
+ format!(" (shutdown took {} ms)", elapsed)
+ } else {
+ String::new()
+ }
+ };
+
+ info!(
+ "All shards have shut down. Iggy server is exiting.{}",
+ shutdown_duration_msg
+ );
/*
#[cfg(feature = "disable-mimalloc")]
diff --git a/core/server/src/quic/listener.rs b/core/server/src/quic/listener.rs
index 5fce5c84..19e9e9f5 100644
--- a/core/server/src/quic/listener.rs
+++ b/core/server/src/quic/listener.rs
@@ -16,56 +16,76 @@
* under the License.
*/
-use std::rc::Rc;
-
use crate::binary::command::{ServerCommand, ServerCommandHandler};
use crate::binary::sender::SenderKind;
use crate::server_error::ConnectionError;
use crate::shard::IggyShard;
+use crate::shard::task_registry::ShutdownToken;
use crate::shard::transmission::event::ShardEvent;
use crate::streaming::session::Session;
use crate::{shard_debug, shard_info};
use anyhow::anyhow;
use compio_quic::{Connection, Endpoint, RecvStream, SendStream};
+use futures::FutureExt;
use iggy_common::IggyError;
use iggy_common::TransportProtocol;
+use std::rc::Rc;
use tracing::{error, info, trace};
const INITIAL_BYTES_LENGTH: usize = 4;
-pub async fn start(endpoint: Endpoint, shard: Rc<IggyShard>) -> Result<(), IggyError> {
- info!("Starting QUIC listener for shard {}", shard.id);
-
- // Since the QUIC Endpoint is internally Arc-wrapped and can be shared,
- // we only need one worker per shard rather than multiple workers per endpoint.
- // This avoids the N×workers multiplication when multiple shards are used.
- while let Some(incoming_conn) = endpoint.wait_incoming().await {
- let remote_addr = incoming_conn.remote_address();
- trace!("Incoming connection from client: {}", remote_addr);
- let shard = shard.clone();
-
- // Spawn each connection handler independently to maintain concurrency
- compio::runtime::spawn(async move {
- trace!("Accepting connection from {}", remote_addr);
- match incoming_conn.await {
- Ok(connection) => {
- trace!("Connection established from {}", remote_addr);
- if let Err(error) = handle_connection(connection, shard).await {
- error!("QUIC connection from {} has failed: {error}", remote_addr);
+pub async fn start(
+ endpoint: Endpoint,
+ shard: Rc<IggyShard>,
+ shutdown: ShutdownToken,
+) -> Result<(), IggyError> {
+ loop {
+ let accept_future = endpoint.wait_incoming();
+
+ futures::select! {
+ _ = shutdown.wait().fuse() => {
+ shard_debug!(shard.id, "QUIC listener received shutdown signal, no longer accepting connections");
+ break;
+ }
+ incoming_conn = accept_future.fuse() => {
+ match incoming_conn {
+ Some(incoming_conn) => {
+ let remote_addr = incoming_conn.remote_address();
+
+ if shard.is_shutting_down() {
+ shard_info!(shard.id, "Rejecting new QUIC connection from {} during shutdown", remote_addr);
+ continue;
+ }
+
+ trace!("Incoming connection from client: {}", remote_addr);
+ let shard_for_conn = shard.clone();
+
+ shard.task_registry.spawn_connection(async move {
+ trace!("Accepting connection from {}", remote_addr);
+ match incoming_conn.await {
+ Ok(connection) => {
+ trace!("Connection established from {}", remote_addr);
+ if let Err(error) = handle_connection(connection, shard_for_conn).await {
+ error!("QUIC connection from {} has failed: {error}", remote_addr);
+ }
+ }
+ Err(error) => {
+ error!(
+ "Error when accepting incoming connection from {}: {:?}",
+ remote_addr, error
+ );
+ }
+ }
+ });
+ }
+ None => {
+ info!("QUIC endpoint closed for shard {}", shard.id);
+ break;
}
- }
- Err(error) => {
- error!(
- "Error when accepting incoming connection from {}: {:?}",
- remote_addr, error
- );
}
}
- })
- .detach();
+ }
}
-
- info!("QUIC listener for shard {} stopped", shard.id);
Ok(())
}
@@ -77,7 +97,6 @@ async fn handle_connection(
info!("Client has connected: {address}");
let session = shard.add_client(&address, TransportProtocol::Quic);
- let session = shard.add_client(&address, TransportProtocol::Quic);
let client_id = session.client_id;
shard_debug!(
shard.id,
@@ -93,19 +112,40 @@ async fn handle_connection(
address,
transport: TransportProtocol::Quic,
};
- let _responses = shard.broadcast_event_to_all_shards(event.into()).await;
- while let Some(stream) = accept_stream(&connection, &shard, client_id).await? {
- let shard = shard.clone();
- let session = session.clone();
+ // TODO(hubcio): unused?
+ let _responses = shard.broadcast_event_to_all_shards(event).await;
- let handle_stream_task = async move {
- if let Err(err) = handle_stream(stream, shard, session).await {
- error!("Error when handling QUIC stream: {:?}", err)
+ let conn_stop_receiver = shard.task_registry.add_connection(client_id);
+
+ loop {
+ futures::select! {
+ // Check for shutdown signal
+ _ = conn_stop_receiver.recv().fuse() => {
+ info!("QUIC connection {} shutting down gracefully", client_id);
+ break;
+ }
+ // Accept new connection
+ stream_result = accept_stream(&connection, &shard, client_id).fuse() => {
+ match stream_result? {
+ Some(stream) => {
+ let shard_clone = shard.clone();
+ let session_clone = session.clone();
+
+ shard.task_registry.spawn_connection(async move {
+ if let Err(err) = handle_stream(stream, shard_clone, session_clone).await {
+ error!("Error when handling QUIC stream: {:?}", err)
+ }
+ });
+ }
+ None => break, // Connection closed
+ }
}
- };
- let _handle = compio::runtime::spawn(handle_stream_task).detach();
+ }
}
+
+ shard.task_registry.remove_connection(&client_id);
+ info!("QUIC connection {} closed", client_id);
Ok(())
}
diff --git a/core/server/src/quic/quic_server.rs b/core/server/src/quic/quic_server.rs
index 6a0255ad..6f29965c 100644
--- a/core/server/src/quic/quic_server.rs
+++ b/core/server/src/quic/quic_server.rs
@@ -35,10 +35,14 @@ use crate::configs::quic::QuicConfig;
use crate::quic::{COMPONENT, listener, quic_socket};
use crate::server_error::QuicError;
use crate::shard::IggyShard;
+use crate::shard::task_registry::ShutdownToken;
/// Starts the QUIC server.
/// Returns the address the server is listening on.
-pub async fn span_quic_server(shard: Rc<IggyShard>) -> Result<(), iggy_common::IggyError> {
+pub async fn spawn_quic_server(
+ shard: Rc<IggyShard>,
+ shutdown: ShutdownToken,
+) -> Result<(), iggy_common::IggyError> {
// Ensure rustls crypto provider is installed (thread-safe, idempotent)
if rustls::crypto::CryptoProvider::get_default().is_none() {
if let Err(e) = default_provider().install_default() {
@@ -102,7 +106,7 @@ pub async fn span_quic_server(shard: Rc<IggyShard>) -> Result<(), iggy_common::I
shard.id, actual_addr
);
shard.quic_bound_address.set(Some(actual_addr));
- listener::start(endpoint, shard).await
+ listener::start(endpoint, shard, shutdown).await
}
fn configure_quic(config: &QuicConfig) -> Result<ServerConfig, QuicError> {
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 287f31bb..ce9cf7de 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -16,31 +16,27 @@
* under the License.
*/
-use std::{
- cell::{Cell, RefCell},
- rc::Rc,
- sync::{Arc, atomic::AtomicBool},
-};
-
-use ahash::HashMap;
-use dashmap::DashMap;
-use iggy_common::{Aes256GcmEncryptor, EncryptorKind, UserId};
-use tracing::info;
-
use crate::{
configs::server::ServerConfig,
- io::storage::Storage,
- shard::{Shard, ShardInfo, namespace::IggyNamespace, task_registry::TaskRegistry},
+ shard::{Shard, ShardInfo, namespace::IggyNamespace},
slab::streams::Streams,
- state::{StateKind, system::SystemState},
- streaming::{
- diagnostics::metrics::Metrics, storage::SystemStorage, users::user::User,
- utils::ptr::EternalPtr,
- },
+ state::StateKind,
+ streaming::{diagnostics::metrics::Metrics, users::user::User, utils::ptr::EternalPtr},
versioning::SemanticVersion,
};
+use ahash::HashMap;
+use dashmap::DashMap;
+use iggy_common::{EncryptorKind, UserId};
+use std::{
+ cell::{Cell, RefCell},
+ rc::Rc,
+ sync::atomic::AtomicBool,
+};
-use super::{IggyShard, transmission::connector::ShardConnector, transmission::frame::ShardFrame};
+use super::{
+ IggyShard, TaskRegistry, transmission::connector::ShardConnector,
+ transmission::frame::ShardFrame,
+};
#[derive(Default)]
pub struct IggyShardBuilder {
@@ -137,6 +133,10 @@ impl IggyShardBuilder {
// Initialize metrics
let metrics = self.metrics.unwrap_or_else(|| Metrics::init());
+
+ // Create TaskRegistry for this shard
+ let task_registry = Rc::new(TaskRegistry::new(id));
+
IggyShard {
id: id,
shards: shards,
@@ -151,11 +151,10 @@ impl IggyShardBuilder {
stop_sender: stop_sender,
messages_receiver: Cell::new(Some(frame_receiver)),
metrics: metrics,
- task_registry: TaskRegistry::new(),
is_shutting_down: AtomicBool::new(false),
tcp_bound_address: Cell::new(None),
quic_bound_address: Cell::new(None),
-
+ task_registry,
permissioner: Default::default(),
client_manager: Default::default(),
active_sessions: Default::default(),
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 534ddfe3..035cdaac 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -19,12 +19,48 @@ inner() * or more contributor license agreements. See the NOTICE file
pub mod builder;
pub mod logging;
pub mod namespace;
-pub mod stats;
pub mod system;
pub mod task_registry;
pub mod tasks;
pub mod transmission;
+use self::tasks::{continuous, periodic};
+use crate::{
+ binary::handlers::messages::poll_messages_handler::IggyPollMetadata,
+ configs::server::ServerConfig,
+ io::fs_utils,
+ shard::{
+ namespace::{IggyFullNamespace, IggyNamespace},
+ task_registry::TaskRegistry,
+ transmission::{
+ event::ShardEvent,
+ frame::{ShardFrame, ShardResponse},
+ message::{ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult},
+ },
+ },
+ shard_error, shard_info, shard_warn,
+ slab::{
+ streams::Streams,
+ traits_ext::{EntityComponentSystem, EntityMarker, Insert},
+ },
+ state::{
+ StateKind,
+ file::FileState,
+ system::{StreamState, SystemState, UserState},
+ },
+ streaming::{
+ clients::client_manager::ClientManager,
+ diagnostics::metrics::Metrics,
+ partitions,
+ persistence::persister::PersisterKind,
+ polling_consumer::PollingConsumer,
+ session::Session,
+ traits::MainOps,
+ users::{permissioner::Permissioner, user::User},
+ utils::ptr::EternalPtr,
+ },
+ versioning::SemanticVersion,
+};
use ahash::{AHashMap, AHashSet, HashMap};
use builder::IggyShardBuilder;
use dashmap::DashMap;
@@ -51,46 +87,14 @@ use std::{
},
time::{Duration, Instant},
};
-use tracing::{error, info, instrument, trace, warn};
+use tracing::{debug, error, info, instrument, trace, warn};
use transmission::connector::{Receiver, ShardConnector, StopReceiver, StopSender};
-use crate::{
- binary::handlers::messages::poll_messages_handler::IggyPollMetadata,
- configs::server::ServerConfig,
- http::http_server,
- io::fs_utils,
- shard::{
- namespace::{IggyFullNamespace, IggyNamespace},
- task_registry::TaskRegistry,
- tasks::messages::spawn_shard_message_task,
- transmission::{
- event::ShardEvent,
- frame::{ShardFrame, ShardResponse},
- message::{ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult},
- },
- },
- shard_error, shard_info, shard_warn,
- slab::{
- streams::Streams,
- traits_ext::{EntityComponentSystem, EntityMarker, Insert},
- },
- state::{
- file::FileState, system::{StreamState, SystemState, UserState}, StateKind
- },
- streaming::{
- clients::client_manager::ClientManager, diagnostics::metrics::Metrics, partitions, persistence::persister::PersisterKind, polling_consumer::PollingConsumer, session::Session, traits::MainOps, users::{permissioner::Permissioner, user::User}, utils::ptr::EternalPtr
- },
- tcp::tcp_server::spawn_tcp_server,
- versioning::SemanticVersion,
-};
-
pub const COMPONENT: &str = "SHARD";
pub const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10);
static USER_ID: AtomicU32 = AtomicU32::new(1);
-type Task = Pin<Box<dyn Future<Output = Result<(), IggyError>>>>;
-
pub(crate) struct Shard {
id: u16,
connection: ShardConnector<ShardFrame>,
@@ -157,10 +161,10 @@ pub struct IggyShard {
pub messages_receiver: Cell<Option<Receiver<ShardFrame>>>,
pub(crate) stop_receiver: StopReceiver,
pub(crate) stop_sender: StopSender,
- pub(crate) task_registry: TaskRegistry,
pub(crate) is_shutting_down: AtomicBool,
pub(crate) tcp_bound_address: Cell<Option<SocketAddr>>,
pub(crate) quic_bound_address: Cell<Option<SocketAddr>>,
+ pub(crate) task_registry: Rc<TaskRegistry>,
}
impl IggyShard {
@@ -174,78 +178,81 @@ impl IggyShard {
Ok(())
}
- pub async fn run(self: &Rc<Self>, persister: Arc<PersisterKind>) -> Result<(), IggyError> {
- // Workaround to ensure that the statistics are initialized before the server
- // loads streams and starts accepting connections. This is necessary to
- // have the correct statistics when the server starts.
- let now = Instant::now();
- self.get_stats().await?;
- shard_info!(self.id, "Starting...");
- self.init().await?;
- // TODO: Fixme
- //self.assert_init();
+ fn init_tasks(self: &Rc<Self>) {
+ continuous::spawn_message_pump(self.clone());
- // Create all tasks (tcp listener, http listener, command processor, in the future also the background jobs).
- let mut tasks: Vec<Task> = vec![Box::pin(spawn_shard_message_task(self.clone()))];
if self.config.tcp.enabled {
- tasks.push(Box::pin(spawn_tcp_server(self.clone())));
+ continuous::spawn_tcp_server(self.clone());
}
if self.config.http.enabled && self.id == 0 {
- println!("Starting HTTP server on shard: {}", self.id);
- tasks.push(Box::pin(http_server::start(
- self.config.http.clone(),
- persister,
- self.clone(),
- )));
+ continuous::spawn_http_server(self.clone());
}
+ // JWT token cleaner task is spawned inside HTTP server because it needs `AppState`.
+
if self.config.quic.enabled {
- tasks.push(Box::pin(crate::quic::quic_server::span_quic_server(
- self.clone(),
- )));
+ continuous::spawn_quic_server(self.clone());
}
- tasks.push(Box::pin(
- crate::channels::commands::clean_personal_access_tokens::clear_personal_access_tokens(
- self.clone(),
- ),
- ));
- // TOOD: Fixme, not always id 0 is the first shard.
- if self.id == 0 {
- tasks.push(Box::pin(
- crate::channels::commands::print_sysinfo::print_sys_info(self.clone()),
- ));
+ if self.config.message_saver.enabled {
+ periodic::spawn_message_saver(self.clone());
}
- tasks.push(Box::pin(
- crate::channels::commands::verify_heartbeats::verify_heartbeats(self.clone()),
- ));
- tasks.push(Box::pin(
- crate::channels::commands::save_messages::save_messages(self.clone()),
- ));
+ if self.config.heartbeat.enabled {
+ periodic::spawn_heartbeat_verifier(self.clone());
+ }
+ if self.config.personal_access_token.cleaner.enabled {
+ periodic::spawn_personal_access_token_cleaner(self.clone());
+ }
+
+ if self
+ .config
+ .system
+ .logging
+ .sysinfo_print_interval
+ .as_micros()
+ > 0
+ && self.id == 0
+ {
+ periodic::spawn_sysinfo_printer(self.clone());
+ }
+ }
+
+ pub async fn run(self: &Rc<Self>) -> Result<(), IggyError> {
+ let now = Instant::now();
+
+ // Workaround to ensure that the statistics are initialized before the server
+ // loads streams and starts accepting connections. This is necessary to
+ // have the correct statistics when the server starts.
+ self.get_stats().await?;
+ shard_info!(self.id, "Starting...");
+ self.init().await?;
+
+ // TODO: Fixme
+ //self.assert_init();
+
+ self.init_tasks();
+ let (shutdown_complete_tx, shutdown_complete_rx) = async_channel::bounded(1);
let stop_receiver = self.get_stop_receiver();
let shard_for_shutdown = self.clone();
- /*
+ // Spawn shutdown handler - only this task consumes the stop signal
compio::runtime::spawn(async move {
let _ = stop_receiver.recv().await;
- info!("Shard {} received shutdown signal", shard_for_shutdown.id);
-
let shutdown_success = shard_for_shutdown.trigger_shutdown().await;
if !shutdown_success {
shard_error!(shard_for_shutdown.id, "shutdown timed out");
- } else {
- shard_info!(shard_for_shutdown.id, "shutdown completed successfully");
}
- });
- */
+ let _ = shutdown_complete_tx.send(()).await;
+ })
+ .detach();
let elapsed = now.elapsed();
shard_info!(self.id, "Initialized in {} ms.", elapsed.as_millis());
- let result = try_join_all(tasks).await;
- result?;
+
+ shutdown_complete_rx.recv().await.ok();
Ok(())
}
@@ -352,8 +359,8 @@ impl IggyShard {
#[instrument(skip_all, name = "trace_shutdown")]
pub async fn trigger_shutdown(&self) -> bool {
self.is_shutting_down.store(true, Ordering::SeqCst);
- info!("Shard {} shutdown state set", self.id);
- self.task_registry.shutdown_all(SHUTDOWN_TIMEOUT).await
+ debug!("Shard {} shutdown state set", self.id);
+ self.task_registry.graceful_shutdown(SHUTDOWN_TIMEOUT).await
}
pub fn get_available_shards_count(&self) -> u32 {
@@ -383,7 +390,13 @@ impl IggyShard {
let batch = self.maybe_encrypt_messages(batch)?;
let messages_count = batch.count();
self.streams2
- .append_messages(self.id, &self.config.system, &ns, batch)
+ .append_messages(
+ self.id,
+ &self.config.system,
+ &self.task_registry,
+ &ns,
+ batch,
+ )
.await?;
self.metrics.increment_messages(messages_count as u64);
Ok(ShardResponse::SendMessages)
diff --git a/core/server/src/shard/stats.rs b/core/server/src/shard/stats.rs
deleted file mode 100644
index 8b137891..00000000
--- a/core/server/src/shard/stats.rs
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index a64602d0..320ce902 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -16,8 +16,6 @@
* under the License.
*/
-use std::sync::atomic::Ordering;
-
use super::COMPONENT;
use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
use crate::shard::IggyShard;
@@ -34,10 +32,11 @@ use crate::streaming::traits::MainOps;
use crate::streaming::utils::{PooledBuffer, hash};
use crate::streaming::{partitions, streams, topics};
use error_set::ErrContext;
-
use iggy_common::{
- BytesSerializable, Consumer, EncryptorKind, Identifier, IggyError, IggyTimestamp, Partitioning, PartitioningKind, PollingKind, PollingStrategy, IGGY_MESSAGE_HEADER_SIZE
+ BytesSerializable, Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier, IggyError,
+ IggyTimestamp, Partitioning, PartitioningKind, PollingKind, PollingStrategy,
};
+use std::sync::atomic::Ordering;
use tracing::{error, trace};
impl IggyShard {
@@ -142,7 +141,13 @@ impl IggyShard {
let batch = self.maybe_encrypt_messages(batch)?;
let messages_count = batch.count();
self.streams2
- .append_messages(self.id, &self.config.system, &ns, batch)
+ .append_messages(
+ self.id,
+ &self.config.system,
+ &self.task_registry,
+ &ns,
+ batch,
+ )
.await?;
self.metrics.increment_messages(messages_count as u64);
Ok(())
@@ -352,7 +357,6 @@ impl IggyShard {
todo!();
}
-
async fn decrypt_messages(
&self,
batches: IggyMessagesBatchSet,
diff --git a/core/server/src/shard/task_registry.rs b/core/server/src/shard/task_registry.rs
deleted file mode 100644
index ff674a8e..00000000
--- a/core/server/src/shard/task_registry.rs
+++ /dev/null
@@ -1,108 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-use async_channel::{Receiver, Sender, bounded};
-use compio::runtime::JoinHandle;
-use futures::future::join_all;
-use std::cell::RefCell;
-use std::collections::HashMap;
-use std::future::Future;
-use std::time::Duration;
-use tracing::{error, info, warn};
-
-pub struct TaskRegistry {
- tasks: RefCell<Vec<JoinHandle<()>>>,
- active_connections: RefCell<HashMap<u32, Sender<()>>>,
-}
-
-impl TaskRegistry {
- pub fn new() -> Self {
- Self {
- tasks: RefCell::new(Vec::new()),
- active_connections: RefCell::new(HashMap::new()),
- }
- }
-
- pub fn spawn_tracked<F>(&self, future: F)
- where
- F: Future<Output = ()> + 'static,
- {
- let handle = compio::runtime::spawn(future);
- self.tasks.borrow_mut().push(handle);
- }
-
- pub fn add_connection(&self, client_id: u32) -> Receiver<()> {
- let (stop_sender, stop_receiver) = bounded(1);
- self.active_connections
- .borrow_mut()
- .insert(client_id, stop_sender);
- stop_receiver
- }
-
- pub fn remove_connection(&self, client_id: &u32) {
- self.active_connections.borrow_mut().remove(client_id);
- }
-
- pub async fn shutdown_all(&self, timeout: Duration) -> bool {
- info!("Initiating task registry shutdown");
-
- let connections = self.active_connections.borrow();
- for (client_id, stop_sender) in connections.iter() {
- info!("Sending shutdown signal to client {}", client_id);
- if let Err(e) = stop_sender.send(()).await {
- warn!(
- "Failed to send shutdown signal to client {}: {}",
- client_id, e
- );
- }
- }
- drop(connections);
-
- let tasks = self.tasks.take();
- let total = tasks.len();
-
- if total == 0 {
- info!("No tasks to shut down");
- return true;
- }
-
- let timeout_futures: Vec<_> = tasks
- .into_iter()
- .enumerate()
- .map(|(idx, handle)| async move {
- match compio::time::timeout(timeout, handle).await {
- Ok(_) => (idx, true),
- Err(_) => {
- warn!("Task {} did not complete within timeout", idx);
- (idx, false)
- }
- }
- })
- .collect();
-
- let results = join_all(timeout_futures).await;
- let completed = results.iter().filter(|(_, success)| *success).count();
-
- info!(
- "Task registry shutdown complete. {} of {} tasks completed",
- completed, total
- );
-
- completed == total
- }
-}
diff --git a/core/server/src/shard/task_registry/builders.rs b/core/server/src/shard/task_registry/builders.rs
new file mode 100644
index 00000000..a59e7813
--- /dev/null
+++ b/core/server/src/shard/task_registry/builders.rs
@@ -0,0 +1,25 @@
+pub mod continuous;
+pub mod oneshot;
+pub mod periodic;
+
+use super::registry::TaskRegistry;
+
+// Marker type for when no shutdown callback is provided
+pub struct NoShutdown;
+
+impl TaskRegistry {
+ pub fn periodic(&self, name: &'static str) -> periodic::PeriodicBuilder<'_, (), NoShutdown> {
+ periodic::PeriodicBuilder::new(self, name)
+ }
+
+ pub fn continuous(
+ &self,
+ name: &'static str,
+ ) -> continuous::ContinuousBuilder<'_, (), NoShutdown> {
+ continuous::ContinuousBuilder::new(self, name)
+ }
+
+ pub fn oneshot(&self, name: &'static str) -> oneshot::OneShotBuilder<'_, (), NoShutdown> {
+ oneshot::OneShotBuilder::new(self, name)
+ }
+}
diff --git a/core/server/src/shard/task_registry/builders/continuous.rs b/core/server/src/shard/task_registry/builders/continuous.rs
new file mode 100644
index 00000000..63c59fb6
--- /dev/null
+++ b/core/server/src/shard/task_registry/builders/continuous.rs
@@ -0,0 +1,92 @@
+use super::NoShutdown;
+use crate::shard::task_registry::ShutdownToken;
+use crate::shard::task_registry::registry::TaskRegistry;
+use iggy_common::IggyError;
+use std::ops::AsyncFnOnce;
+
+pub struct ContinuousBuilder<'a, Task, OnShutdown = NoShutdown> {
+ reg: &'a TaskRegistry,
+ name: &'static str,
+ critical: bool,
+ run_fn: Option<Task>,
+ on_shutdown: Option<OnShutdown>,
+}
+
+impl<'a> ContinuousBuilder<'a, (), NoShutdown> {
+ pub fn new(reg: &'a TaskRegistry, name: &'static str) -> Self {
+ Self {
+ reg,
+ name,
+ critical: false,
+ run_fn: None,
+ on_shutdown: None,
+ }
+ }
+}
+
+impl<'a, Task, OnShutdown> ContinuousBuilder<'a, Task, OnShutdown> {
+ pub fn critical(mut self, c: bool) -> Self {
+ self.critical = c;
+ self
+ }
+
+ pub fn on_shutdown<NewShutdown>(
+ self,
+ f: NewShutdown,
+ ) -> ContinuousBuilder<'a, Task, NewShutdown>
+ where
+ NewShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static,
+ {
+ ContinuousBuilder {
+ reg: self.reg,
+ name: self.name,
+ critical: self.critical,
+ run_fn: self.run_fn,
+ on_shutdown: Some(f),
+ }
+ }
+}
+
+impl<'a, OnShutdown> ContinuousBuilder<'a, (), OnShutdown> {
+ pub fn run<NewTask>(self, f: NewTask) -> ContinuousBuilder<'a, NewTask, OnShutdown>
+ where
+ NewTask: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static,
+ {
+ ContinuousBuilder {
+ reg: self.reg,
+ name: self.name,
+ critical: self.critical,
+ run_fn: Some(f),
+ on_shutdown: self.on_shutdown,
+ }
+ }
+}
+
+impl<'a, Task> ContinuousBuilder<'a, Task, NoShutdown>
+where
+ Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static,
+{
+ pub fn spawn(self) {
+ if let Some(f) = self.run_fn {
+ self.reg
+ .spawn_continuous_closure(self.name, self.critical, f, Some(|_| async {}));
+ } else {
+ panic!("run() must be called before spawn()");
+ }
+ }
+}
+
+impl<'a, Task, OnShutdown> ContinuousBuilder<'a, Task, OnShutdown>
+where
+ Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static,
+ OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static,
+{
+ pub fn spawn(self) {
+ if let Some(f) = self.run_fn {
+ self.reg
+ .spawn_continuous_closure(self.name, self.critical, f, self.on_shutdown);
+ } else {
+ panic!("run() must be called before spawn()");
+ }
+ }
+}
diff --git a/core/server/src/shard/task_registry/builders/oneshot.rs b/core/server/src/shard/task_registry/builders/oneshot.rs
new file mode 100644
index 00000000..fd8580e5
--- /dev/null
+++ b/core/server/src/shard/task_registry/builders/oneshot.rs
@@ -0,0 +1,103 @@
+use super::NoShutdown;
+use crate::shard::task_registry::ShutdownToken;
+use crate::shard::task_registry::registry::TaskRegistry;
+use iggy_common::IggyError;
+use std::ops::AsyncFnOnce;
+use std::time::Duration;
+
+pub struct OneShotBuilder<'a, Task, OnShutdown = NoShutdown> {
+ reg: &'a TaskRegistry,
+ name: &'static str,
+ critical: bool,
+ timeout: Option<Duration>,
+ run_fn: Option<Task>,
+ on_shutdown: Option<OnShutdown>,
+}
+
+impl<'a> OneShotBuilder<'a, (), NoShutdown> {
+ pub fn new(reg: &'a TaskRegistry, name: &'static str) -> Self {
+ Self {
+ reg,
+ name,
+ critical: false,
+ timeout: None,
+ run_fn: None,
+ on_shutdown: None,
+ }
+ }
+}
+
+impl<'a, Task, OnShutdown> OneShotBuilder<'a, Task, OnShutdown> {
+ pub fn critical(mut self, c: bool) -> Self {
+ self.critical = c;
+ self
+ }
+
+ pub fn timeout(mut self, d: Duration) -> Self {
+ self.timeout = Some(d);
+ self
+ }
+
+ pub fn on_shutdown<NewShutdown>(self, f: NewShutdown) -> OneShotBuilder<'a, Task, NewShutdown>
+ where
+ NewShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static,
+ {
+ OneShotBuilder {
+ reg: self.reg,
+ name: self.name,
+ critical: self.critical,
+ timeout: self.timeout,
+ run_fn: self.run_fn,
+ on_shutdown: Some(f),
+ }
+ }
+}
+
+impl<'a, OnShutdown> OneShotBuilder<'a, (), OnShutdown> {
+ pub fn run<NewTask>(self, f: NewTask) -> OneShotBuilder<'a, NewTask, OnShutdown>
+ where
+ NewTask: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static,
+ {
+ OneShotBuilder {
+ reg: self.reg,
+ name: self.name,
+ critical: self.critical,
+ timeout: self.timeout,
+ run_fn: Some(f),
+ on_shutdown: self.on_shutdown,
+ }
+ }
+}
+
+impl<'a, Task> OneShotBuilder<'a, Task, NoShutdown>
+where
+ Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static,
+{
+ pub fn spawn(self) {
+ let run_fn = self.run_fn.expect("run() must be called before spawn()");
+ self.reg.spawn_oneshot_closure(
+ self.name,
+ self.critical,
+ self.timeout,
+ run_fn,
+ Some(|_| async {}),
+ );
+ }
+}
+
+impl<'a, Task, OnShutdown> OneShotBuilder<'a, Task, OnShutdown>
+where
+ Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static,
+ OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static,
+{
+ pub fn spawn(self) {
+ let run_fn = self.run_fn.expect("run() must be called before spawn()");
+ self.reg.spawn_oneshot_closure(
+ self.name,
+ self.critical,
+ self.timeout,
+ run_fn,
+ self.on_shutdown,
+ );
+ }
+}
diff --git a/core/server/src/shard/task_registry/builders/periodic.rs b/core/server/src/shard/task_registry/builders/periodic.rs
new file mode 100644
index 00000000..c21c5025
--- /dev/null
+++ b/core/server/src/shard/task_registry/builders/periodic.rs
@@ -0,0 +1,118 @@
+use super::NoShutdown;
+use crate::shard::task_registry::ShutdownToken;
+use crate::shard::task_registry::registry::TaskRegistry;
+use iggy_common::IggyError;
+use std::ops::{AsyncFn, AsyncFnOnce};
+use std::time::Duration;
+
+pub struct PeriodicBuilder<'a, Tick, OnShutdown = NoShutdown> {
+ reg: &'a TaskRegistry,
+ name: &'static str,
+ critical: bool,
+ period: Option<Duration>,
+ last_on_shutdown: bool,
+ tick_fn: Option<Tick>,
+ on_shutdown: Option<OnShutdown>,
+}
+
+impl<'a> PeriodicBuilder<'a, (), NoShutdown> {
+ pub fn new(reg: &'a TaskRegistry, name: &'static str) -> Self {
+ Self {
+ reg,
+ name,
+ critical: false,
+ period: None,
+ last_on_shutdown: false,
+ tick_fn: None,
+ on_shutdown: None,
+ }
+ }
+}
+
+impl<'a, Tick, OnShutdown> PeriodicBuilder<'a, Tick, OnShutdown> {
+ pub fn every(mut self, d: Duration) -> Self {
+ self.period = Some(d);
+ self
+ }
+
+ pub fn critical(mut self, c: bool) -> Self {
+ self.critical = c;
+ self
+ }
+
+ pub fn last_tick_on_shutdown(mut self, v: bool) -> Self {
+ self.last_on_shutdown = v;
+ self
+ }
+
+ pub fn on_shutdown<NewShutdown>(self, f: NewShutdown) -> PeriodicBuilder<'a, Tick, NewShutdown>
+ where
+ NewShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static,
+ {
+ PeriodicBuilder {
+ reg: self.reg,
+ name: self.name,
+ critical: self.critical,
+ period: self.period,
+ last_on_shutdown: self.last_on_shutdown,
+ tick_fn: self.tick_fn,
+ on_shutdown: Some(f),
+ }
+ }
+}
+
+impl<'a> PeriodicBuilder<'a, ()> {
+ pub fn tick<NewTick>(self, f: NewTick) -> PeriodicBuilder<'a, NewTick>
+ where
+ NewTick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static,
+ {
+ PeriodicBuilder {
+ reg: self.reg,
+ name: self.name,
+ critical: self.critical,
+ period: self.period,
+ last_on_shutdown: self.last_on_shutdown,
+ tick_fn: Some(f),
+ on_shutdown: self.on_shutdown,
+ }
+ }
+}
+
+impl<'a, Tick> PeriodicBuilder<'a, Tick, NoShutdown>
+where
+ Tick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static,
+{
+ pub fn spawn(self) {
+ let period = self.period.expect("period required - use .every()");
+ let tick_fn = self.tick_fn.expect("tick function required - use .tick()");
+
+ self.reg.spawn_periodic_closure(
+ self.name,
+ period,
+ self.critical,
+ self.last_on_shutdown,
+ tick_fn,
+ Some(|_| async {}),
+ );
+ }
+}
+
+impl<'a, Tick, OnShutdown> PeriodicBuilder<'a, Tick, OnShutdown>
+where
+ Tick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static,
+ OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static,
+{
+ pub fn spawn(self) {
+ let period = self.period.expect("period required - use .every()");
+ let tick_fn = self.tick_fn.expect("tick function required - use .tick()");
+
+ self.reg.spawn_periodic_closure(
+ self.name,
+ period,
+ self.critical,
+ self.last_on_shutdown,
+ tick_fn,
+ self.on_shutdown,
+ );
+ }
+}
diff --git a/core/server/src/shard/task_registry/mod.rs b/core/server/src/shard/task_registry/mod.rs
new file mode 100644
index 00000000..c6abd875
--- /dev/null
+++ b/core/server/src/shard/task_registry/mod.rs
@@ -0,0 +1,6 @@
+pub mod builders;
+pub mod registry;
+pub mod shutdown;
+
+pub use registry::TaskRegistry;
+pub use shutdown::{Shutdown, ShutdownToken};
diff --git a/core/server/src/shard/task_registry/registry.rs b/core/server/src/shard/task_registry/registry.rs
new file mode 100644
index 00000000..b43b718c
--- /dev/null
+++ b/core/server/src/shard/task_registry/registry.rs
@@ -0,0 +1,581 @@
+use super::shutdown::{Shutdown, ShutdownToken};
+use compio::runtime::JoinHandle;
+use futures::future::join_all;
+use iggy_common::IggyError;
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::ops::{AsyncFn, AsyncFnOnce};
+use std::time::{Duration, Instant};
+use tracing::{debug, error, trace, warn};
+
+#[derive(Debug)]
+enum Kind {
+ Continuous,
+ Periodic,
+ OneShot,
+}
+
+#[derive(Debug)]
+struct TaskHandle {
+ name: String,
+ kind: Kind,
+ handle: JoinHandle<Result<(), IggyError>>,
+ critical: bool,
+}
+
+pub struct TaskRegistry {
+ shard_id: u16,
+ shutdown: Shutdown,
+ shutdown_token: ShutdownToken,
+ long_running: RefCell<Vec<TaskHandle>>,
+ oneshots: RefCell<Vec<TaskHandle>>,
+ connections: RefCell<HashMap<u32, async_channel::Sender<()>>>,
+ shutting_down: RefCell<bool>,
+}
+
+impl TaskRegistry {
+ pub fn new(shard_id: u16) -> Self {
+ let (s, t) = Shutdown::new();
+ Self {
+ shard_id,
+ shutdown: s,
+ shutdown_token: t,
+ long_running: RefCell::new(vec![]),
+ oneshots: RefCell::new(vec![]),
+ connections: RefCell::new(HashMap::new()),
+ shutting_down: RefCell::new(false),
+ }
+ }
+
+ pub fn shutdown_token(&self) -> ShutdownToken {
+ self.shutdown_token.clone()
+ }
+
+ pub(crate) fn spawn_continuous_closure<Task, OnShutdown>(
+ &self,
+ name: &'static str,
+ critical: bool,
+ f: Task,
+ on_shutdown: Option<OnShutdown>,
+ ) where
+ Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static,
+ OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static,
+ {
+ if *self.shutting_down.borrow() {
+ warn!(
+ "Attempted to spawn continuous task '{}' during shutdown",
+ name
+ );
+ return;
+ }
+
+ let shutdown = self.shutdown_token.clone();
+ let shard_id = self.shard_id;
+
+ let handle = compio::runtime::spawn(async move {
+ trace!("continuous '{}' starting on shard {}", name, shard_id);
+ let fut = f(shutdown);
+ let r = fut.await;
+ match &r {
+ Ok(()) => debug!("continuous '{}' completed on shard {}", name, shard_id),
+ Err(e) => error!("continuous '{}' failed on shard {}: {}", name, shard_id, e),
+ }
+
+ // Execute on_shutdown callback if provided
+ if let Some(shutdown_fn) = on_shutdown {
+ trace!("continuous '{}' executing on_shutdown callback", name);
+ shutdown_fn(r.clone()).await;
+ }
+
+ r
+ });
+
+ self.long_running.borrow_mut().push(TaskHandle {
+ name: name.into(),
+ kind: Kind::Continuous,
+ handle,
+ critical,
+ });
+ }
+
+ pub(crate) fn spawn_periodic_closure<Tick, OnShutdown>(
+ &self,
+ name: &'static str,
+ period: Duration,
+ critical: bool,
+ last_on_shutdown: bool,
+ tick_fn: Tick,
+ on_shutdown: Option<OnShutdown>,
+ ) where
+ Tick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static,
+ OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static,
+ {
+ if *self.shutting_down.borrow() {
+ warn!(
+ "Attempted to spawn periodic task '{}' during shutdown",
+ name
+ );
+ return;
+ }
+
+ let shutdown = self.shutdown_token.clone();
+ let shutdown_for_task = self.shutdown_token.clone();
+ let shard_id = self.shard_id;
+
+ let handle = compio::runtime::spawn(async move {
+ trace!(
+ "periodic '{}' every {:?} on shard {}",
+ name, period, shard_id
+ );
+
+ loop {
+ if !shutdown.sleep_or_shutdown(period).await {
+ break;
+ }
+
+ let fut = tick_fn(shutdown_for_task.clone());
+ if let Err(e) = fut.await {
+ error!(
+ "periodic '{}' tick failed on shard {}: {}",
+ name, shard_id, e
+ );
+ }
+ }
+
+ if last_on_shutdown {
+ const FINAL_TICK_TIMEOUT: Duration = Duration::from_secs(5);
+ trace!(
+ "periodic '{}' executing final tick on shutdown (timeout: {:?})",
+ name, FINAL_TICK_TIMEOUT
+ );
+
+ let fut = tick_fn(shutdown_for_task);
+ match compio::time::timeout(FINAL_TICK_TIMEOUT, fut).await {
+ Ok(Ok(())) => trace!("periodic '{}' final tick completed", name),
+ Ok(Err(e)) => error!("periodic '{}' final tick failed: {}", name, e),
+ Err(_) => error!(
+ "periodic '{}' final tick timed out after {:?}",
+ name, FINAL_TICK_TIMEOUT
+ ),
+ }
+ }
+
+ let result = Ok(());
+
+ if let Some(on_shutdown) = on_shutdown {
+ on_shutdown(result.clone()).await;
+ }
+
+ result
+ });
+
+ self.long_running.borrow_mut().push(TaskHandle {
+ name: name.into(),
+ kind: Kind::Periodic,
+ handle,
+ critical,
+ });
+ }
+
+ pub(crate) fn spawn_oneshot_closure<Task, OnShutdown>(
+ &self,
+ name: &'static str,
+ critical: bool,
+ timeout: Option<Duration>,
+ f: Task,
+ on_shutdown: Option<OnShutdown>,
+ ) where
+ Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static,
+ OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static,
+ {
+ if *self.shutting_down.borrow() {
+ warn!("Attempted to spawn oneshot task '{}' during shutdown", name);
+ return;
+ }
+
+ let shutdown = self.shutdown_token.clone();
+ let shard_id = self.shard_id;
+
+ let handle = compio::runtime::spawn(async move {
+ trace!("oneshot '{}' starting on shard {}", name, shard_id);
+ let fut = f(shutdown);
+
+ let r = if let Some(d) = timeout {
+ match compio::time::timeout(d, fut).await {
+ Ok(r) => r,
+ Err(_) => Err(IggyError::TaskTimeout),
+ }
+ } else {
+ fut.await
+ };
+
+ match &r {
+ Ok(()) => trace!("oneshot '{}' completed on shard {}", name, shard_id),
+ Err(e) => error!("oneshot '{}' failed on shard {}: {}", name, shard_id, e),
+ }
+
+ if let Some(on_shutdown) = on_shutdown {
+ on_shutdown(r.clone()).await;
+ }
+
+ r
+ });
+
+ self.oneshots.borrow_mut().push(TaskHandle {
+ name: name.into(),
+ kind: Kind::OneShot,
+ handle,
+ critical,
+ });
+ }
+
+ pub async fn graceful_shutdown(&self, timeout: Duration) -> bool {
+ let start = Instant::now();
+ *self.shutting_down.borrow_mut() = true;
+ self.shutdown_connections();
+ self.shutdown.trigger();
+
+ // First shutdown long-running tasks (continuous and periodic)
+ let long = self.long_running.take();
+ let long_ok = if !long.is_empty() {
+ debug!(
+ "Shutting down {} long-running task(s) on shard {}",
+ long.len(),
+ self.shard_id
+ );
+ self.await_with_timeout(long, timeout).await
+ } else {
+ true
+ };
+
+ // Calculate remaining time for oneshots
+ let elapsed = start.elapsed();
+ let remaining = timeout.saturating_sub(elapsed);
+
+ // Then shutdown oneshot tasks with remaining time
+ let ones = self.oneshots.take();
+ let ones_ok = if !ones.is_empty() {
+ if remaining.is_zero() {
+ warn!(
+ "No time remaining for {} oneshot task(s) on shard {}, they will be cancelled",
+ ones.len(),
+ self.shard_id
+ );
+ false
+ } else {
+ debug!(
+ "Shutting down {} oneshot task(s) on shard {} with {:?} remaining",
+ ones.len(),
+ self.shard_id,
+ remaining
+ );
+ self.await_with_timeout(ones, remaining).await
+ }
+ } else {
+ true
+ };
+
+ let total_elapsed = start.elapsed();
+ if long_ok && ones_ok {
+ debug!(
+ "Graceful shutdown completed successfully on shard {} in {:?}",
+ self.shard_id, total_elapsed
+ );
+ } else {
+ warn!(
+ "Graceful shutdown completed with failures on shard {} in {:?}",
+ self.shard_id, total_elapsed
+ );
+ }
+
+ long_ok && ones_ok
+ }
+
+ async fn await_with_timeout(&self, tasks: Vec<TaskHandle>, timeout: Duration) -> bool {
+ if tasks.is_empty() {
+ return true;
+ }
+ let results = join_all(tasks.into_iter().map(|t| async move {
+ match compio::time::timeout(timeout, t.handle).await {
+ Ok(Ok(Ok(()))) => true,
+ Ok(Ok(Err(e))) => {
+ error!("task '{}' of kind {:?} failed: {}", t.name, t.kind, e);
+ !t.critical
+ }
+ Ok(Err(_)) => {
+ error!("task '{}' of kind {:?} panicked", t.name, t.kind);
+ !t.critical
+ }
+ Err(_) => {
+ error!(
+ "task '{}' of kind {:?} timed out after {:?}",
+ t.name, t.kind, timeout
+ );
+ !t.critical
+ }
+ }
+ }))
+ .await;
+
+ results.into_iter().all(|x| x)
+ }
+
+ #[cfg(test)]
+ async fn await_all(&self, tasks: Vec<TaskHandle>) -> bool {
+ if tasks.is_empty() {
+ return true;
+ }
+ let results = join_all(tasks.into_iter().map(|t| async move {
+ match t.handle.await {
+ Ok(Ok(())) => true,
+ Ok(Err(e)) => {
+ error!("task '{}' failed: {}", t.name, e);
+ !t.critical
+ }
+ Err(_) => {
+ error!("task '{}' panicked", t.name);
+ !t.critical
+ }
+ }
+ }))
+ .await;
+ results.into_iter().all(|x| x)
+ }
+
+ pub fn add_connection(&self, client_id: u32) -> async_channel::Receiver<()> {
+ let (tx, rx) = async_channel::bounded(1);
+ self.connections.borrow_mut().insert(client_id, tx);
+ rx
+ }
+
+ pub fn remove_connection(&self, client_id: &u32) {
+ self.connections.borrow_mut().remove(client_id);
+ }
+
+ fn shutdown_connections(&self) {
+ // Close all connection channels to signal shutdown
+ // We use close() instead of send_blocking() to avoid potential blocking
+ for tx in self.connections.borrow().values() {
+ tx.close();
+ }
+ }
+
+ /// Spawn a connection handler that doesn't need to be tracked for shutdown.
+ /// These handlers have their own shutdown mechanism via connection channels.
+ pub fn spawn_connection<F>(&self, future: F)
+ where
+ F: Future<Output = ()> + 'static,
+ {
+ compio::runtime::spawn(future).detach();
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[compio::test]
+ async fn test_oneshot_completion_detection() {
+ let registry = TaskRegistry::new(1);
+
+ // Spawn a failing non-critical task
+ registry
+ .oneshot("failing_non_critical")
+ .run(|_shutdown| async { Err(IggyError::Error) })
+ .spawn();
+
+ // Spawn a successful task
+ registry
+ .oneshot("successful")
+ .run(|_shutdown| async { Ok(()) })
+ .spawn();
+
+ // Wait for all tasks
+ let all_ok = registry.await_all(registry.oneshots.take()).await;
+
+ // Should return true because the failing task is not critical
+ assert!(all_ok);
+ }
+
+ #[compio::test]
+ async fn test_oneshot_critical_failure() {
+ let registry = TaskRegistry::new(1);
+
+ // Spawn a failing critical task
+ registry
+ .oneshot("failing_critical")
+ .critical(true)
+ .run(|_shutdown| async { Err(IggyError::Error) })
+ .spawn();
+
+ // Wait for all tasks
+ let all_ok = registry.await_all(registry.oneshots.take()).await;
+
+ // Should return false because the failing task is critical
+ assert!(!all_ok);
+ }
+
+ #[compio::test]
+ async fn test_shutdown_prevents_spawning() {
+ let registry = TaskRegistry::new(1);
+
+ // Trigger shutdown
+ *registry.shutting_down.borrow_mut() = true;
+
+ let initial_count = registry.oneshots.borrow().len();
+
+ // Try to spawn after shutdown
+ registry
+ .oneshot("should_not_spawn")
+ .run(|_shutdown| async { Ok(()) })
+ .spawn();
+
+ // Task should not be added
+ assert_eq!(registry.oneshots.borrow().len(), initial_count);
+ }
+
+ #[compio::test]
+ async fn test_timeout_error() {
+ let registry = TaskRegistry::new(1);
+
+ // Create a task that will timeout
+ let handle = compio::runtime::spawn(async move {
+ compio::time::sleep(Duration::from_secs(10)).await;
+ Ok(())
+ });
+
+ let task_handle = TaskHandle {
+ name: "timeout_test".to_string(),
+ kind: Kind::OneShot,
+ handle,
+ critical: false,
+ };
+
+ let tasks = vec![task_handle];
+ let all_ok = registry
+ .await_with_timeout(tasks, Duration::from_millis(50))
+ .await;
+
+ // Should return true because the task is not critical
+ assert!(all_ok);
+ }
+
+ #[compio::test]
+ async fn test_composite_timeout() {
+ let registry = TaskRegistry::new(1);
+
+ // Create a long-running task that takes 100ms
+ let long_handle = compio::runtime::spawn(async move {
+ compio::time::sleep(Duration::from_millis(100)).await;
+ Ok(())
+ });
+
+ registry.long_running.borrow_mut().push(TaskHandle {
+ name: "long_task".to_string(),
+ kind: Kind::Continuous,
+ handle: long_handle,
+ critical: false,
+ });
+
+ // Create a oneshot that would succeed quickly
+ let oneshot_handle = compio::runtime::spawn(async move {
+ compio::time::sleep(Duration::from_millis(10)).await;
+ Ok(())
+ });
+
+ registry.oneshots.borrow_mut().push(TaskHandle {
+ name: "quick_oneshot".to_string(),
+ kind: Kind::OneShot,
+ handle: oneshot_handle,
+ critical: false,
+ });
+
+ // Give total timeout of 150ms
+ // Long-running should complete in ~100ms
+ // Oneshot should have ~50ms remaining, which is enough
+ let all_ok = registry.graceful_shutdown(Duration::from_millis(150)).await;
+ assert!(all_ok);
+ }
+
+ #[compio::test]
+ async fn test_composite_timeout_insufficient() {
+ let registry = TaskRegistry::new(1);
+
+ // Create a long-running task that takes 50ms
+ let long_handle = compio::runtime::spawn(async move {
+ compio::time::sleep(Duration::from_millis(50)).await;
+ Ok(())
+ });
+
+ registry.long_running.borrow_mut().push(TaskHandle {
+ name: "long_task".to_string(),
+ kind: Kind::Continuous,
+ handle: long_handle,
+ critical: false,
+ });
+
+ // Create a oneshot that would take 100ms (much longer)
+ let oneshot_handle = compio::runtime::spawn(async move {
+ compio::time::sleep(Duration::from_millis(100)).await;
+ Ok(())
+ });
+
+ registry.oneshots.borrow_mut().push(TaskHandle {
+ name: "slow_oneshot".to_string(),
+ kind: Kind::OneShot,
+ handle: oneshot_handle,
+ critical: true, // Make it critical so failure is detected
+ });
+
+ // Give total timeout of 60ms
+ // Long-running should complete in ~50ms
+ // Oneshot would need 100ms but only has ~10ms, so it should definitely fail
+ let all_ok = registry.graceful_shutdown(Duration::from_millis(60)).await;
+ assert!(!all_ok); // Should fail because critical oneshot times out
+ }
+
+ #[compio::test]
+ async fn test_periodic_last_tick_timeout() {
+ // This test verifies that periodic tasks with last_tick_on_shutdown
+ // don't hang shutdown if the final tick takes too long
+ let registry = TaskRegistry::new(1);
+
+ // Create a handle that simulates a periodic task whose final tick will hang
+ let handle = compio::runtime::spawn(async move {
+ // Simulate the periodic task loop that already exited
+ // Now simulate the last_tick_on_shutdown logic with a hanging tick
+ const FINAL_TICK_TIMEOUT: Duration = Duration::from_millis(100);
+ let fut = async {
+ // This would hang for 500ms without timeout
+ compio::time::sleep(Duration::from_millis(500)).await;
+ Ok::<(), IggyError>(())
+ };
+
+ match compio::time::timeout(FINAL_TICK_TIMEOUT, fut).await {
+ Ok(Ok(())) => {}
+ Ok(Err(_)) => {}
+ Err(_) => {
+ // Timeout occurred as expected
+ }
+ }
+ Ok(())
+ });
+
+ registry.long_running.borrow_mut().push(TaskHandle {
+ name: "periodic_with_slow_final".to_string(),
+ kind: Kind::Periodic,
+ handle,
+ critical: false,
+ });
+
+ // Shutdown should complete in ~100ms (the FINAL_TICK_TIMEOUT), not 500ms
+ let start = std::time::Instant::now();
+ let all_ok = registry.graceful_shutdown(Duration::from_secs(1)).await;
+ let elapsed = start.elapsed();
+
+ // Should complete in about 100ms due to the timeout, not hang for 500ms
+ assert!(elapsed >= Duration::from_millis(80)); // At least 80ms
+ assert!(elapsed < Duration::from_millis(200)); // But less than 200ms (not the full 500ms)
+ assert!(all_ok);
+ }
+}
diff --git a/core/server/src/shard/task_registry/shutdown.rs b/core/server/src/shard/task_registry/shutdown.rs
new file mode 100644
index 00000000..fb57b5a9
--- /dev/null
+++ b/core/server/src/shard/task_registry/shutdown.rs
@@ -0,0 +1,233 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use async_channel::{Receiver, Sender, bounded};
+use futures::FutureExt;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+use tracing::trace;
+
+/// Coordinates graceful shutdown across multiple tasks
+#[derive(Clone)]
+pub struct Shutdown {
+ sender: Sender<()>,
+ is_triggered: Arc<AtomicBool>,
+}
+
+impl Shutdown {
+ pub fn new() -> (Self, ShutdownToken) {
+ let (sender, receiver) = bounded(1);
+ let is_triggered = Arc::new(AtomicBool::new(false));
+
+ let shutdown = Self {
+ sender,
+ is_triggered: is_triggered.clone(),
+ };
+
+ let token = ShutdownToken {
+ receiver,
+ is_triggered,
+ };
+
+ (shutdown, token)
+ }
+
+ pub fn trigger(&self) {
+ if self.is_triggered.swap(true, Ordering::SeqCst) {
+ return;
+ }
+
+ trace!("Triggering shutdown signal");
+ let _ = self.sender.close();
+ }
+
+ pub fn is_triggered(&self) -> bool {
+ self.is_triggered.load(Ordering::Relaxed)
+ }
+}
+
+/// Token held by tasks to receive shutdown signals
+#[derive(Clone)]
+pub struct ShutdownToken {
+ receiver: Receiver<()>,
+ is_triggered: Arc<AtomicBool>,
+}
+
+impl ShutdownToken {
+ /// Wait for shutdown signal
+ pub async fn wait(&self) {
+ let _ = self.receiver.recv().await;
+ }
+
+ /// Check if shutdown has been triggered (non-blocking)
+ pub fn is_triggered(&self) -> bool {
+ self.is_triggered.load(Ordering::Relaxed)
+ }
+
+ /// Sleep for the specified duration or until shutdown is triggered
+ /// Returns true if the full duration elapsed, false if shutdown was triggered
+ pub async fn sleep_or_shutdown(&self, duration: Duration) -> bool {
+ futures::select! {
+ _ = self.wait().fuse() => false,
+ _ = compio::time::sleep(duration).fuse() => !self.is_triggered(),
+ }
+ }
+
+ /// Creates a scoped shutdown pair (child `Shutdown`, combined `ShutdownToken`).
+ ///
+ /// This is a bit complicated, but it needs to be this way to avoid deadlocks.
+ ///
+ /// The returned token fires when EITHER the parent or the child is triggered,
+ /// while a child trigger does NOT propagate back to the parent.
+ /// Internally spawns a tiny forwarder to merge both signals into one channel,
+ /// so callers can await a single `wait()` and use fast `is_triggered()` checks
+ /// without writing `select!` at every call site.
+ /// Use when a subtree needs cancelation that respects parent cancelation,
+ /// but can also be canceled locally.
+ pub fn child(&self) -> (Shutdown, ShutdownToken) {
+ let (child_shutdown, child_token) = Shutdown::new();
+ let parent_receiver = self.receiver.clone();
+ let child_receiver = child_token.receiver.clone();
+
+ let (combined_sender, combined_receiver) = bounded(1);
+ let combined_is_triggered = Arc::new(AtomicBool::new(false));
+
+ let parent_triggered = self.is_triggered.clone();
+ let child_triggered = child_token.is_triggered.clone();
+ let combined_flag_for_task = combined_is_triggered.clone();
+
+ compio::runtime::spawn(async move {
+ futures::select! {
+ _ = parent_receiver.recv().fuse() => {
+ trace!("Child token triggered by parent shutdown");
+ },
+ _ = child_receiver.recv().fuse() => {
+ trace!("Child token triggered by child shutdown");
+ },
+ }
+
+ if parent_triggered.load(Ordering::Relaxed) || child_triggered.load(Ordering::Relaxed) {
+ combined_flag_for_task.store(true, Ordering::SeqCst);
+ }
+
+ let _ = combined_sender.close();
+ })
+ .detach();
+
+ let combined_token = ShutdownToken {
+ receiver: combined_receiver,
+ is_triggered: combined_is_triggered,
+ };
+
+ (child_shutdown, combined_token)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[compio::test]
+ async fn test_shutdown_trigger() {
+ let (shutdown, token) = Shutdown::new();
+
+ assert!(!token.is_triggered());
+
+ shutdown.trigger();
+
+ assert!(token.is_triggered());
+
+ token.wait().await;
+ }
+
+ #[compio::test]
+ async fn test_sleep_or_shutdown_completes() {
+ let (_shutdown, token) = Shutdown::new();
+
+ let completed = token.sleep_or_shutdown(Duration::from_millis(10)).await;
+ assert!(completed);
+ }
+
+ #[compio::test]
+ async fn test_sleep_or_shutdown_interrupted() {
+ let (shutdown, token) = Shutdown::new();
+
+ // Trigger shutdown after a short delay
+ let shutdown_clone = shutdown.clone();
+ compio::runtime::spawn(async move {
+ compio::time::sleep(Duration::from_millis(10)).await;
+ shutdown_clone.trigger();
+ })
+ .detach();
+
+ // Should be interrupted
+ let completed = token.sleep_or_shutdown(Duration::from_secs(10)).await;
+ assert!(!completed);
+ }
+
+ #[compio::test]
+ async fn test_child_token_parent_trigger() {
+ let (parent_shutdown, parent_token) = Shutdown::new();
+ let (_child_shutdown, combined_token) = parent_token.child();
+
+ assert!(!combined_token.is_triggered());
+
+ // Trigger parent shutdown
+ parent_shutdown.trigger();
+
+ // Combined token should be triggered
+ combined_token.wait().await;
+ assert!(combined_token.is_triggered());
+ }
+
+ #[compio::test]
+ async fn test_child_token_child_trigger() {
+ let (_parent_shutdown, parent_token) = Shutdown::new();
+ let (child_shutdown, combined_token) = parent_token.child();
+
+ assert!(!combined_token.is_triggered());
+
+ // Trigger child shutdown
+ child_shutdown.trigger();
+
+ // Combined token should be triggered
+ combined_token.wait().await;
+ assert!(combined_token.is_triggered());
+ }
+
+ #[compio::test]
+ async fn test_child_token_no_polling_overhead() {
+ let (_parent_shutdown, parent_token) = Shutdown::new();
+ let (_child_shutdown, combined_token) = parent_token.child();
+
+ // Test that we can create many child tokens without performance issues
+ let start = std::time::Instant::now();
+ for _ in 0..100 {
+ let _ = combined_token.child();
+ }
+ let elapsed = start.elapsed();
+
+ // Should complete very quickly since there's no polling
+ assert!(
+ elapsed.as_millis() < 100,
+ "Creating child tokens took too long: {:?}",
+ elapsed
+ );
+ }
+}
diff --git a/core/server/src/http/jwt/cleaner.rs b/core/server/src/shard/tasks/continuous/http_server.rs
similarity index 50%
rename from core/server/src/http/jwt/cleaner.rs
rename to core/server/src/shard/tasks/continuous/http_server.rs
index 6705d955..24518426 100644
--- a/core/server/src/http/jwt/cleaner.rs
+++ b/core/server/src/shard/tasks/continuous/http_server.rs
@@ -16,26 +16,25 @@
* under the License.
*/
-use crate::http::shared::AppState;
-use iggy_common::IggyTimestamp;
-use std::{sync::Arc, time::Duration};
-use tracing::{error, trace};
+use crate::bootstrap::resolve_persister;
+use crate::http::http_server::start_http_server;
+use crate::shard::IggyShard;
+use crate::shard::task_registry::ShutdownToken;
+use iggy_common::IggyError;
+use std::rc::Rc;
+use tracing::info;
-pub fn start_expired_tokens_cleaner(app_state: Arc<AppState>) {
- compio::runtime::spawn(async move {
- let mut interval_timer = compio::time::interval(Duration::from_secs(300));
- loop {
- interval_timer.tick().await;
- trace!("Deleting expired tokens...");
- let now = IggyTimestamp::now().to_secs();
- app_state
- .jwt_manager
- .delete_expired_revoked_tokens(now)
- .await
- .unwrap_or_else(|err| {
- error!("Failed to delete expired revoked access tokens. Error: {err}");
- });
- }
- })
- .detach();
+pub fn spawn_http_server(shard: Rc<IggyShard>) {
+ let shard_clone = shard.clone();
+ shard
+ .task_registry
+ .continuous("http_server")
+ .run(move |shutdown| http_server(shard_clone, shutdown))
+ .spawn();
+}
+
+async fn http_server(shard: Rc<IggyShard>, shutdown: ShutdownToken) -> Result<(), IggyError> {
+ info!("Starting HTTP server on shard: {}", shard.id);
+ let persister = resolve_persister(shard.config.system.partition.enforce_fsync);
+ start_http_server(shard.config.http.clone(), persister, shard, shutdown).await
}
diff --git a/core/server/src/shard/tasks/continuous/message_pump.rs b/core/server/src/shard/tasks/continuous/message_pump.rs
new file mode 100644
index 00000000..c6ae7375
--- /dev/null
+++ b/core/server/src/shard/tasks/continuous/message_pump.rs
@@ -0,0 +1,72 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::shard::IggyShard;
+use crate::shard::task_registry::ShutdownToken;
+use crate::shard::transmission::frame::ShardFrame;
+use crate::{shard_debug, shard_info};
+use futures::{FutureExt, StreamExt};
+use std::rc::Rc;
+
+pub fn spawn_message_pump(shard: Rc<IggyShard>) {
+ let shard_clone = shard.clone();
+ shard
+ .task_registry
+ .continuous("message_pump")
+ .critical(true)
+ .run(move |shutdown| message_pump(shard_clone, shutdown))
+ .spawn();
+}
+
+async fn message_pump(
+ shard: Rc<IggyShard>,
+ shutdown: ShutdownToken,
+) -> Result<(), iggy_common::IggyError> {
+ let Some(mut messages_receiver) = shard.messages_receiver.take() else {
+ shard_info!(shard.id, "Message receiver already taken; pump not started");
+ return Ok(());
+ };
+
+ shard_info!(shard.id, "Starting message passing task");
+
+ loop {
+ futures::select! {
+ _ = shutdown.wait().fuse() => {
+ shard_debug!(shard.id, "Message receiver shutting down");
+ break;
+ }
+ frame = messages_receiver.next().fuse() => {
+ match frame {
+ Some(ShardFrame { message, response_sender }) => {
+ if let (Some(response), Some(tx)) =
+ (shard.handle_shard_message(message).await, response_sender)
+ {
+ let _ = tx.send(response).await;
+ }
+ }
+ None => {
+ shard_debug!(shard.id, "Message receiver closed; exiting pump");
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ Ok(())
+}
diff --git a/core/server/src/channels/commands/mod.rs b/core/server/src/shard/tasks/continuous/mod.rs
similarity index 77%
rename from core/server/src/channels/commands/mod.rs
rename to core/server/src/shard/tasks/continuous/mod.rs
index 0130170b..f8ae5fbc 100644
--- a/core/server/src/channels/commands/mod.rs
+++ b/core/server/src/shard/tasks/continuous/mod.rs
@@ -16,7 +16,12 @@
* under the License.
*/
-pub mod clean_personal_access_tokens;
-pub mod print_sysinfo;
-pub mod save_messages;
-pub mod verify_heartbeats;
+mod http_server;
+mod message_pump;
+mod quic_server;
+mod tcp_server;
+
+pub use http_server::spawn_http_server;
+pub use message_pump::spawn_message_pump;
+pub use quic_server::spawn_quic_server;
+pub use tcp_server::spawn_tcp_server;
diff --git a/core/server/src/http/mod.rs b/core/server/src/shard/tasks/continuous/quic_server.rs
similarity index 58%
copy from core/server/src/http/mod.rs
copy to core/server/src/shard/tasks/continuous/quic_server.rs
index ba369815..713cbc9d 100644
--- a/core/server/src/http/mod.rs
+++ b/core/server/src/shard/tasks/continuous/quic_server.rs
@@ -16,23 +16,22 @@
* under the License.
*/
-pub mod diagnostics;
-pub mod error;
-pub mod http_server;
-mod http_shard_wrapper;
-pub mod jwt;
-mod mapper;
-pub mod metrics;
-mod shared;
+use crate::quic::quic_server;
+use crate::shard::IggyShard;
+use crate::shard::task_registry::ShutdownToken;
+use iggy_common::IggyError;
+use std::rc::Rc;
-pub mod consumer_groups;
-pub mod consumer_offsets;
-pub mod messages;
-pub mod partitions;
-pub mod personal_access_tokens;
-pub mod streams;
-pub mod system;
-pub mod topics;
-pub mod users;
+pub fn spawn_quic_server(shard: Rc<IggyShard>) {
+ let shard_clone = shard.clone();
+ shard
+ .task_registry
+ .continuous("quic_server")
+ .critical(true)
+ .run(move |shutdown| quic_server_task(shard_clone, shutdown))
+ .spawn();
+}
-pub const COMPONENT: &str = "HTTP";
+async fn quic_server_task(shard: Rc<IggyShard>, shutdown: ShutdownToken) -> Result<(), IggyError> {
+ quic_server::spawn_quic_server(shard, shutdown).await
+}
diff --git a/core/server/src/http/mod.rs b/core/server/src/shard/tasks/continuous/tcp_server.rs
similarity index 58%
copy from core/server/src/http/mod.rs
copy to core/server/src/shard/tasks/continuous/tcp_server.rs
index ba369815..70ff7137 100644
--- a/core/server/src/http/mod.rs
+++ b/core/server/src/shard/tasks/continuous/tcp_server.rs
@@ -16,23 +16,22 @@
* under the License.
*/
-pub mod diagnostics;
-pub mod error;
-pub mod http_server;
-mod http_shard_wrapper;
-pub mod jwt;
-mod mapper;
-pub mod metrics;
-mod shared;
+use crate::shard::IggyShard;
+use crate::shard::task_registry::ShutdownToken;
+use crate::tcp::tcp_server;
+use iggy_common::IggyError;
+use std::rc::Rc;
-pub mod consumer_groups;
-pub mod consumer_offsets;
-pub mod messages;
-pub mod partitions;
-pub mod personal_access_tokens;
-pub mod streams;
-pub mod system;
-pub mod topics;
-pub mod users;
+pub fn spawn_tcp_server(shard: Rc<IggyShard>) {
+ let shard_clone = shard.clone();
+ shard
+ .task_registry
+ .continuous("tcp_server")
+ .critical(true)
+ .run(move |shutdown| tcp_server_task(shard_clone, shutdown))
+ .spawn();
+}
-pub const COMPONENT: &str = "HTTP";
+async fn tcp_server_task(shard: Rc<IggyShard>, shutdown: ShutdownToken) -> Result<(), IggyError> {
+ tcp_server::spawn_tcp_server(shard, shutdown).await
+}
diff --git a/core/server/src/shard/tasks/messages.rs b/core/server/src/shard/tasks/messages.rs
deleted file mode 100644
index 2feadd1c..00000000
--- a/core/server/src/shard/tasks/messages.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-use futures::{FutureExt, StreamExt};
-use iggy_common::IggyError;
-use std::{rc::Rc, time::Duration};
-
-use crate::{
- shard::{IggyShard, transmission::frame::ShardFrame},
- shard_error, shard_info,
-};
-
-async fn run_shard_messages_receiver(shard: Rc<IggyShard>) -> Result<(), IggyError> {
- let mut messages_receiver = shard.messages_receiver.take().unwrap();
-
- shard_info!(shard.id, "Starting message passing task");
- loop {
- let shutdown_check = async {
- loop {
- if shard.is_shutting_down() {
- return;
- }
- compio::time::sleep(Duration::from_millis(100)).await;
- }
- };
-
- futures::select! {
- _ = shutdown_check.fuse() => {
- shard_info!(shard.id, "Message receiver shutting down");
- break;
- }
- frame = messages_receiver.next().fuse() => {
- if let Some(frame) = frame {
- let ShardFrame {
- message,
- response_sender,
- } = frame;
- match (shard.handle_shard_message(message).await, response_sender) {
- (Some(response), Some(response_sender)) => {
- response_sender
- .send(response)
- .await
- .expect("Failed to send response back to origin shard.");
- }
- _ => {}
- };
- }
- }
- }
- }
-
- Ok(())
-}
-
-pub async fn spawn_shard_message_task(shard: Rc<IggyShard>) -> Result<(), IggyError> {
- let result = run_shard_messages_receiver(shard.clone()).await;
- if let Err(err) = result {
- shard_error!(shard.id, "Error running shard message receiver: {err}");
- return Err(err);
- }
- Ok(())
-}
diff --git a/core/server/src/shard/tasks/mod.rs b/core/server/src/shard/tasks/mod.rs
index ba63992f..6fe9cf1f 100644
--- a/core/server/src/shard/tasks/mod.rs
+++ b/core/server/src/shard/tasks/mod.rs
@@ -1 +1,20 @@
-pub mod messages;
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+pub mod continuous;
+pub mod periodic;
diff --git a/core/server/src/shard/tasks/periodic/heartbeat_verifier.rs b/core/server/src/shard/tasks/periodic/heartbeat_verifier.rs
new file mode 100644
index 00000000..9b2e06bb
--- /dev/null
+++ b/core/server/src/shard/tasks/periodic/heartbeat_verifier.rs
@@ -0,0 +1,90 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::shard::IggyShard;
+use iggy_common::{IggyDuration, IggyError, IggyTimestamp};
+use std::rc::Rc;
+use tracing::{debug, info, trace, warn};
+
+const MAX_THRESHOLD: f64 = 1.2;
+
+pub fn spawn_heartbeat_verifier(shard: Rc<IggyShard>) {
+ let period = shard.config.heartbeat.interval.get_duration();
+ let interval = iggy_common::IggyDuration::from(period);
+ let max_interval =
+ iggy_common::IggyDuration::from((MAX_THRESHOLD * interval.as_micros() as f64) as u64);
+ info!(
+ "Heartbeats will be verified every: {}. Max allowed interval: {}.",
+ interval, max_interval
+ );
+ let shard_clone = shard.clone();
+ shard
+ .task_registry
+ .periodic("verify_heartbeats")
+ .every(period)
+ .tick(move |_shutdown| verify_heartbeats(shard_clone.clone()))
+ .spawn();
+}
+
+async fn verify_heartbeats(shard: Rc<IggyShard>) -> Result<(), IggyError> {
+ trace!("Verifying heartbeats...");
+
+ // Get the period from config to compute max_interval
+ let period = shard.config.heartbeat.interval.get_duration();
+ let interval = IggyDuration::from(period);
+ let max_interval = IggyDuration::from((MAX_THRESHOLD * interval.as_micros() as f64) as u64);
+
+ let clients = {
+ let client_manager = shard.client_manager.borrow();
+ client_manager.get_clients()
+ };
+
+ let now = IggyTimestamp::now();
+ let heartbeat_to = IggyTimestamp::from(now.as_micros() - max_interval.as_micros());
+ debug!("Verifying heartbeats at: {now}, max allowed timestamp: {heartbeat_to}");
+
+ let mut stale_clients = Vec::new();
+ for client in clients {
+ if client.last_heartbeat.as_micros() < heartbeat_to.as_micros() {
+ warn!(
+ "Stale client session: {}, last heartbeat at: {}, max allowed timestamp: {heartbeat_to}",
+ client.session, client.last_heartbeat,
+ );
+ client.session.set_stale();
+ stale_clients.push(client.session.client_id);
+ } else {
+ debug!(
+ "Valid heartbeat at: {} for client session: {}, max allowed timestamp: {heartbeat_to}",
+ client.last_heartbeat, client.session,
+ );
+ }
+ }
+
+ if stale_clients.is_empty() {
+ return Ok(());
+ }
+
+ let count = stale_clients.len();
+
+ for client_id in stale_clients {
+ shard.delete_client(client_id);
+ }
+ info!("Removed {count} stale clients.");
+
+ Ok(())
+}
diff --git a/core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs b/core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs
new file mode 100644
index 00000000..59015c73
--- /dev/null
+++ b/core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs
@@ -0,0 +1,61 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::http::shared::AppState;
+use crate::shard::IggyShard;
+use iggy_common::{IggyError, IggyTimestamp};
+use std::rc::Rc;
+use std::sync::Arc;
+use std::time::Duration;
+use tracing::{error, info, trace};
+
+const JWT_TOKENS_CLEANER_PERIOD: Duration = Duration::from_secs(300);
+
+pub fn spawn_jwt_token_cleaner(shard: Rc<IggyShard>, app_state: Arc<AppState>) {
+ info!(
+ "JWT token cleaner is enabled, expired revoked tokens will be deleted every: {} seconds.",
+ JWT_TOKENS_CLEANER_PERIOD.as_secs()
+ );
+ shard
+ .task_registry
+ .periodic("clear_jwt_tokens")
+ .every(JWT_TOKENS_CLEANER_PERIOD)
+ .tick(move |_shutdown| clear_jwt_tokens(app_state.clone()))
+ .spawn();
+}
+
+async fn clear_jwt_tokens(app_state: Arc<AppState>) -> Result<(), IggyError> {
+ trace!("Checking for expired revoked JWT tokens...");
+
+ let now = IggyTimestamp::now().to_secs();
+
+ match app_state
+ .jwt_manager
+ .delete_expired_revoked_tokens(now)
+ .await
+ {
+ Ok(()) => {
+ trace!("Successfully cleaned up expired revoked JWT tokens");
+ }
+ Err(err) => {
+ error!("Failed to delete expired revoked JWT tokens: {}", err);
+ }
+ }
+
+ Ok(())
+}
diff --git a/core/server/src/shard/tasks/periodic/message_saver.rs b/core/server/src/shard/tasks/periodic/message_saver.rs
new file mode 100644
index 00000000..9bd8ccc6
--- /dev/null
+++ b/core/server/src/shard/tasks/periodic/message_saver.rs
@@ -0,0 +1,130 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::shard::IggyShard;
+use crate::shard_info;
+use iggy_common::{Identifier, IggyError};
+use std::rc::Rc;
+use tracing::{error, info, trace};
+
+pub fn spawn_message_saver(shard: Rc<IggyShard>) {
+ let period = shard.config.message_saver.interval.get_duration();
+ let enforce_fsync = shard.config.message_saver.enforce_fsync;
+ info!(
+ "Message saver is enabled, buffered messages will be automatically saved every: {:?}, enforce fsync: {enforce_fsync}.",
+ period
+ );
+ let shard_clone = shard.clone();
+ let shard_for_shutdown = shard.clone();
+ shard
+ .task_registry
+ .periodic("save_messages")
+ .every(period)
+ .last_tick_on_shutdown(true)
+ .tick(move |_shutdown| save_messages(shard_clone.clone()))
+ .on_shutdown(move |result| {
+ fsync_all_segments_on_shutdown(shard_for_shutdown.clone(), result)
+ })
+ .spawn();
+}
+
+async fn save_messages(shard: Rc<IggyShard>) -> Result<(), IggyError> {
+ trace!("Saving buffered messages...");
+
+ let namespaces = shard.get_current_shard_namespaces();
+ let mut total_saved_messages = 0u32;
+ const REASON: &str = "background saver triggered";
+
+ for ns in namespaces {
+ let stream_id = Identifier::numeric(ns.stream_id() as u32).unwrap();
+ let topic_id = Identifier::numeric(ns.topic_id() as u32).unwrap();
+ let partition_id = ns.partition_id();
+
+ match shard
+ .streams2
+ .persist_messages(
+ shard.id,
+ &stream_id,
+ &topic_id,
+ partition_id,
+ REASON,
+ &shard.config.system,
+ )
+ .await
+ {
+ Ok(batch_count) => {
+ total_saved_messages += batch_count;
+ }
+ Err(err) => {
+ error!(
+ "Failed to save messages for partition {}: {}",
+ partition_id, err
+ );
+ }
+ }
+ }
+
+ if total_saved_messages > 0 {
+ shard_info!(
+ shard.id,
+ "Saved {} buffered messages on disk.",
+ total_saved_messages
+ );
+ }
+ Ok(())
+}
+
+async fn fsync_all_segments_on_shutdown(shard: Rc<IggyShard>, result: Result<(), IggyError>) {
+ // Only fsync if the last save_messages tick succeeded
+ if result.is_err() {
+ error!(
+ "Last save_messages tick failed, skipping fsync: {:?}",
+ result
+ );
+ return;
+ }
+
+ trace!("Performing fsync on all segments during shutdown...");
+
+ let namespaces = shard.get_current_shard_namespaces();
+
+ for ns in namespaces {
+ let stream_id = Identifier::numeric(ns.stream_id() as u32).unwrap();
+ let topic_id = Identifier::numeric(ns.topic_id() as u32).unwrap();
+ let partition_id = ns.partition_id();
+
+ match shard
+ .streams2
+ .fsync_all_messages(&stream_id, &topic_id, partition_id)
+ .await
+ {
+ Ok(()) => {
+ trace!(
+ "Successfully fsynced segment for stream: {}, topic: {}, partition: {} during shutdown",
+ stream_id, topic_id, partition_id
+ );
+ }
+ Err(err) => {
+ error!(
+ "Failed to fsync segment for stream: {}, topic: {}, partition: {} during shutdown: {}",
+ stream_id, topic_id, partition_id, err
+ );
+ }
+ }
+ }
+}
diff --git a/core/server/src/channels/mod.rs b/core/server/src/shard/tasks/periodic/mod.rs
similarity index 66%
rename from core/server/src/channels/mod.rs
rename to core/server/src/shard/tasks/periodic/mod.rs
index d147f7e0..9689060a 100644
--- a/core/server/src/channels/mod.rs
+++ b/core/server/src/shard/tasks/periodic/mod.rs
@@ -16,4 +16,14 @@
* under the License.
*/
-pub mod commands;
+mod heartbeat_verifier;
+mod jwt_token_cleaner;
+mod message_saver;
+mod personal_access_token_cleaner;
+mod sysinfo_printer;
+
+pub use heartbeat_verifier::spawn_heartbeat_verifier;
+pub use jwt_token_cleaner::spawn_jwt_token_cleaner;
+pub use message_saver::spawn_message_saver;
+pub use personal_access_token_cleaner::spawn_personal_access_token_cleaner;
+pub use sysinfo_printer::spawn_sysinfo_printer;
diff --git a/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs b/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs
new file mode 100644
index 00000000..670b04ef
--- /dev/null
+++ b/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs
@@ -0,0 +1,76 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::shard::IggyShard;
+use iggy_common::{IggyError, IggyTimestamp};
+use std::rc::Rc;
+use std::sync::Arc;
+use tracing::{info, trace};
+
+pub fn spawn_personal_access_token_cleaner(shard: Rc<IggyShard>) {
+ let period = shard
+ .config
+ .personal_access_token
+ .cleaner
+ .interval
+ .get_duration();
+ info!(
+ "Personal access token cleaner is enabled, expired tokens will be deleted every: {:?}.",
+ period
+ );
+ let shard_clone = shard.clone();
+ shard
+ .task_registry
+ .periodic("clear_personal_access_tokens")
+ .every(period)
+ .tick(move |_shutdown| clear_personal_access_tokens(shard_clone.clone()))
+ .spawn();
+}
+
+async fn clear_personal_access_tokens(shard: Rc<IggyShard>) -> Result<(), IggyError> {
+ trace!("Checking for expired personal access tokens...");
+
+ let now = IggyTimestamp::now();
+ let mut total_removed = 0;
+
+ let users = shard.users.borrow();
+ for user in users.values() {
+ let expired_tokens: Vec<Arc<String>> = user
+ .personal_access_tokens
+ .iter()
+ .filter(|entry| entry.value().is_expired(now))
+ .map(|entry| entry.key().clone())
+ .collect();
+
+ for token_hash in expired_tokens {
+ if let Some((_, pat)) = user.personal_access_tokens.remove(&token_hash) {
+ info!(
+ "Removed expired personal access token '{}' for user ID {}",
+ pat.name, user.id
+ );
+ total_removed += 1;
+ }
+ }
+ }
+
+ if total_removed > 0 {
+ info!("Removed {total_removed} expired personal access tokens");
+ }
+
+ Ok(())
+}
diff --git a/core/server/src/shard/tasks/periodic/sysinfo_printer.rs b/core/server/src/shard/tasks/periodic/sysinfo_printer.rs
new file mode 100644
index 00000000..d79a8d99
--- /dev/null
+++ b/core/server/src/shard/tasks/periodic/sysinfo_printer.rs
@@ -0,0 +1,75 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::shard::IggyShard;
+use human_repr::HumanCount;
+use iggy_common::IggyError;
+use std::rc::Rc;
+use tracing::{error, info, trace};
+
+pub fn spawn_sysinfo_printer(shard: Rc<IggyShard>) {
+ let period = shard
+ .config
+ .system
+ .logging
+ .sysinfo_print_interval
+ .get_duration();
+ info!(
+ "System info logger is enabled, OS info will be printed every: {:?}",
+ period
+ );
+ let shard_clone = shard.clone();
+ shard
+ .task_registry
+ .periodic("print_sysinfo")
+ .every(period)
+ .tick(move |_shutdown| print_sysinfo(shard_clone.clone()))
+ .spawn();
+}
+
+async fn print_sysinfo(shard: Rc<IggyShard>) -> Result<(), IggyError> {
+ trace!("Printing system information...");
+
+ let stats = match shard.get_stats().await {
+ Ok(stats) => stats,
+ Err(e) => {
+ error!("Failed to get system information. Error: {e}");
+ return Ok(());
+ }
+ };
+
+ let free_memory_percent = (stats.available_memory.as_bytes_u64() as f64
+ / stats.total_memory.as_bytes_u64() as f64)
+ * 100f64;
+
+ info!(
+ "CPU: {:.2}%/{:.2}% (IggyUsage/Total), Mem: {:.2}%/{}/{}/{} (Free/IggyUsage/TotalUsed/Total), Clients: {}, Messages: {}, Read: {}, Written: {}",
+ stats.cpu_usage,
+ stats.total_cpu_usage,
+ free_memory_percent,
+ stats.memory_usage,
+ stats.total_memory - stats.available_memory,
+ stats.total_memory,
+ stats.clients_count.human_count_bare().to_string(),
+ stats.messages_count.human_count_bare().to_string(),
+ stats.read_bytes,
+ stats.written_bytes,
+ );
+
+ Ok(())
+}
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index a4e4bb3a..14bf075b 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -1,7 +1,3 @@
-use std::{rc::Rc, sync::Arc};
-
-use iggy_common::{Identifier, PollingStrategy};
-
/* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -27,6 +23,7 @@ use crate::{
slab::partitions,
streaming::{polling_consumer::PollingConsumer, segments::IggyMessagesBatchMut},
};
+use iggy_common::Identifier;
pub enum ShardSendRequestResult {
// TODO: In the future we can add other variants, for example backpressure from the destination shard,
diff --git a/core/server/src/slab/mod.rs b/core/server/src/slab/mod.rs
index c4b7b4da..13c8f472 100644
--- a/core/server/src/slab/mod.rs
+++ b/core/server/src/slab/mod.rs
@@ -1,15 +1,4 @@
pub mod consumer_groups;
-/// General rules how to implement `with_*` methods on any slab"
-/// 1. When implementing method that accepts closure f, make sure that the caller can supply closure only with 1 depth of callbacks.
-/// for example, observe following code snippet:
-/// ```rust
-/// let topic_id = self.streams2.with_topic_by_id(stream_id, topic_id, get_topic_id());
-/// ```
-/// if we would not provide a `with_topic_by_id` method and purely relied only ony `with_topics`, we would have to write:
-/// ```rust
-/// let topic_id = self.streams2.with_topics(stream_id, get_topic_by_id(topic_id, get_topic_id())); // `get_topic_id` is a closure that retrieves the topic id.
-/// ```
-/// we need to supply a nested closure to `get_topic_by_id`.
pub mod helpers;
pub mod partitions;
pub mod streams;
@@ -18,6 +7,18 @@ pub mod traits_ext;
use std::fmt::{Debug, Display};
+// General rules how to implement `with_*` methods on any slab"
+// 1. When implementing method that accepts closure f, make sure that the caller can supply closure only with 1 depth of callbacks.
+// for example, observe following code snippet:
+// ```rust
+// let topic_id = self.streams2.with_topic_by_id(stream_id, topic_id, get_topic_id());
+// ```
+// if we would not provide a `with_topic_by_id` method and purely relied only ony `with_topics`, we would have to write:
+// ```rust
+// let topic_id = self.streams2.with_topics(stream_id, get_topic_by_id(topic_id, get_topic_id())); // `get_topic_id` is a closure that retrieves the topic id.
+// ```
+// we need to supply a nested closure to `get_topic_by_id`.
+
pub trait Keyed {
type Key: Eq + std::hash::Hash + Clone + Debug + Display;
fn key(&self) -> &Self::Key;
diff --git a/core/server/src/slab/streams.rs b/core/server/src/slab/streams.rs
index d733e04e..7fadce50 100644
--- a/core/server/src/slab/streams.rs
+++ b/core/server/src/slab/streams.rs
@@ -1,3 +1,5 @@
+use crate::shard::task_registry::TaskRegistry;
+use crate::streaming::partitions as streaming_partitions;
use crate::{
binary::handlers::messages::poll_messages_handler::IggyPollMetadata,
configs::{cache_indexes::CacheIndexesConfig, system::SystemConfig},
@@ -41,12 +43,10 @@ use iggy_common::{Identifier, IggyError, IggyTimestamp, PollingKind};
use slab::Slab;
use std::{
cell::RefCell,
+ rc::Rc,
sync::{Arc, atomic::Ordering},
};
-use tracing::trace;
-
-// Import streaming partitions helpers for the persist_messages method
-use crate::streaming::partitions as streaming_partitions;
+use tracing::{error, trace};
const CAPACITY: usize = 1024;
pub type ContainerId = usize;
@@ -173,6 +173,7 @@ impl MainOps for Streams {
&self,
shard_id: u16,
config: &SystemConfig,
+ registry: &Rc<TaskRegistry>,
ns: &Self::Namespace,
mut input: Self::In,
) -> Result<(), Self::Error> {
@@ -241,12 +242,19 @@ impl MainOps for Streams {
);
let _batch_count = self
- .persist_messages(shard_id, stream_id, topic_id, partition_id, reason, config)
+ .persist_messages(shard_id, stream_id, topic_id, partition_id, &reason, config)
.await?;
if is_full {
- self.handle_full_segment(shard_id, stream_id, topic_id, partition_id, config)
- .await?;
+ self.handle_full_segment(
+ shard_id,
+ registry,
+ stream_id,
+ topic_id,
+ partition_id,
+ config,
+ )
+ .await?;
}
}
Ok(())
@@ -708,6 +716,7 @@ impl Streams {
pub async fn handle_full_segment(
&self,
shard_id: u16,
+ registry: &Rc<TaskRegistry>,
stream_id: &Identifier,
topic_id: &Identifier,
partition_id: partitions::ContainerId,
@@ -735,15 +744,37 @@ impl Streams {
(msg.unwrap(), index.unwrap())
});
- compio::runtime::spawn(async move {
- let _ = log_writer.fsync().await;
- })
- .detach();
- compio::runtime::spawn(async move {
- let _ = index_writer.fsync().await;
- drop(index_writer)
- })
- .detach();
+ registry
+ .oneshot("fsync:segment-close-log")
+ .critical(true)
+ .run(move |_shutdown| async move {
+ match log_writer.fsync().await {
+ Ok(_) => Ok(()),
+ Err(e) => {
+ error!("Failed to fsync log writer on segment close: {}", e);
+ Err(e)
+ }
+ }
+ })
+ .spawn();
+
+ registry
+ .oneshot("fsync:segment-close-index")
+ .critical(true)
+ .run(move |_shutdown| async move {
+ match index_writer.fsync().await {
+ Ok(_) => {
+ drop(index_writer);
+ Ok(())
+ }
+ Err(e) => {
+ error!("Failed to fsync index writer on segment close: {}", e);
+ drop(index_writer);
+ Err(e)
+ }
+ }
+ })
+ .spawn();
let (start_offset, size, end_offset) =
self.with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| {
@@ -796,7 +827,7 @@ impl Streams {
stream_id: &Identifier,
topic_id: &Identifier,
partition_id: usize,
- reason: String,
+ reason: &str,
config: &SystemConfig,
) -> Result<u32, IggyError> {
let is_empty = self.with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| {
@@ -824,7 +855,7 @@ impl Streams {
topic_id,
partition_id,
batches,
- reason,
+ reason.to_string(),
),
)
.await?;
@@ -842,4 +873,65 @@ impl Streams {
Ok(batch_count)
}
+
+ pub async fn fsync_all_messages(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: usize,
+ ) -> Result<(), IggyError> {
+ let has_storage =
+ self.with_partition_by_id(stream_id, topic_id, partition_id, |(.., log)| {
+ let storage = log.active_storage();
+ storage.messages_writer.is_some() && storage.index_writer.is_some()
+ });
+
+ if !has_storage {
+ return Ok(());
+ }
+
+ self.with_partition_by_id_async(
+ stream_id,
+ topic_id,
+ partition_id,
+ async move |(.., log)| {
+ let storage = log.active_storage();
+
+ if let Some(ref messages_writer) = storage.messages_writer {
+ if let Err(e) = messages_writer.fsync().await {
+ tracing::error!(
+ "Failed to fsync messages writer for partition {}: {}",
+ partition_id,
+ e
+ );
+ return Err(e);
+ }
+ }
+ Ok(())
+ },
+ )
+ .await?;
+
+ self.with_partition_by_id_async(
+ stream_id,
+ topic_id,
+ partition_id,
+ async move |(.., log)| {
+ if let Some(ref index_writer) = log.active_storage().index_writer {
+ if let Err(e) = index_writer.fsync().await {
+ tracing::error!(
+ "Failed to fsync index writer for partition {}: {}",
+ partition_id,
+ e
+ );
+ return Err(e);
+ }
+ }
+ Ok(())
+ },
+ )
+ .await?;
+
+ Ok(())
+ }
}
diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs
index 03199d70..d64845a2 100644
--- a/core/server/src/streaming/storage.rs
+++ b/core/server/src/streaming/storage.rs
@@ -47,7 +47,7 @@ macro_rules! forward_async_methods {
}
}
-// TODO: Tech debt, how to get rid of this ?
+// TODO: Tech debt, how to get rid of this ?
#[derive(Debug)]
pub enum SystemInfoStorageKind {
File(FileSystemInfoStorage),
diff --git a/core/server/src/streaming/traits.rs b/core/server/src/streaming/traits.rs
index 478c7f50..7502f15e 100644
--- a/core/server/src/streaming/traits.rs
+++ b/core/server/src/streaming/traits.rs
@@ -1,4 +1,7 @@
use crate::configs::system::SystemConfig;
+use crate::shard::task_registry::TaskRegistry;
+use std::future::Future;
+use std::rc::Rc;
// TODO: Major revision of this trait.
pub trait MainOps {
@@ -13,6 +16,7 @@ pub trait MainOps {
&self,
shard_id: u16,
config: &SystemConfig,
+ registry: &Rc<TaskRegistry>,
ns: &Self::Namespace,
input: Self::In,
) -> impl Future<Output = Result<(), Self::Error>>;
diff --git a/core/server/src/streaming/utils/head_tail_buf.rs b/core/server/src/streaming/utils/head_tail_buf.rs
deleted file mode 100644
index 02b11ec5..00000000
--- a/core/server/src/streaming/utils/head_tail_buf.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#[derive(Default)]
-pub struct HeadTailBuffer<T>
-where
- T: Default + Eq + Copy,
-{
- data: [T; 2],
- head_index: usize,
-}
-
-impl<T> HeadTailBuffer<T>
-where
- T: Default + Eq + Copy,
-{
- pub fn new() -> Self {
- HeadTailBuffer {
- data: [Default::default(); 2],
- head_index: 0,
- }
- }
-
- pub fn push(&mut self, element: T) {
- self.data[(self.head_index + 1) % 2] = element;
- self.head_index = (self.head_index + 1) % 2;
- }
-
- pub fn tail(&self) -> Option<T> {
- if self.head_index == 0 && self.data[1] == Default::default() {
- None
- } else {
- // If the tail is empty return None
- if self.data[self.head_index] == Default::default() {
- return None;
- }
- Some(self.data[(self.head_index + 1) % 2])
- }
- }
-}
diff --git a/core/server/src/streaming/utils/mod.rs b/core/server/src/streaming/utils/mod.rs
index 834119bb..74f5b7cf 100644
--- a/core/server/src/streaming/utils/mod.rs
+++ b/core/server/src/streaming/utils/mod.rs
@@ -19,7 +19,6 @@
pub mod crypto;
pub mod file;
pub mod hash;
-pub mod head_tail_buf;
pub mod ptr;
pub mod random_id;
diff --git a/core/server/src/tcp/connection_handler.rs b/core/server/src/tcp/connection_handler.rs
index f331a6b7..597e6162 100644
--- a/core/server/src/tcp/connection_handler.rs
+++ b/core/server/src/tcp/connection_handler.rs
@@ -42,6 +42,9 @@ pub(crate) async fn handle_connection(
let mut code_buffer = BytesMut::with_capacity(INITIAL_BYTES_LENGTH);
loop {
let read_future = sender.read(length_buffer);
+ // TODO(hubcio): this futures::select! call is translated to epoll_wait syscall for every
+ // message, which adds around 100 us median latency. We could instead just call sender.shutdown()
+ // if some atomic bool is set, since this is all happenng within single thread.
let (_, mut initial_buffer) = futures::select! {
_ = stop_receiver.recv().fuse() => {
info!("Connection stop signal received for session: {}", session);
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index 18565ae6..d3aa84a6 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -19,9 +19,10 @@
use crate::binary::sender::SenderKind;
use crate::configs::tcp::TcpSocketConfig;
use crate::shard::IggyShard;
+use crate::shard::task_registry::ShutdownToken;
use crate::shard::transmission::event::ShardEvent;
use crate::tcp::connection_handler::{handle_connection, handle_error};
-use crate::{shard_error, shard_info};
+use crate::{shard_debug, shard_error, shard_info};
use compio::net::{TcpListener, TcpOpts};
use error_set::ErrContext;
use futures::FutureExt;
@@ -68,6 +69,7 @@ pub async fn start(
mut addr: SocketAddr,
config: &TcpSocketConfig,
shard: Rc<IggyShard>,
+ shutdown: ShutdownToken,
) -> Result<(), IggyError> {
//TODO: Fix me, this needs to take into account that first shard id potentially can be greater than 0.
if shard.id != 0 && addr.port() == 0 {
@@ -99,7 +101,6 @@ pub async fn start(
actual_addr
);
- //TODO: Fix me, this needs to take into account that first shard id potentially can be greater than 0.
if shard.id == 0 {
if addr.port() == 0 {
let event = ShardEvent::TcpBound {
@@ -132,29 +133,21 @@ pub async fn start(
}
}
- accept_loop(server_name, listener, shard).await
+ accept_loop(server_name, listener, shard, shutdown).await
}
async fn accept_loop(
server_name: &'static str,
listener: TcpListener,
shard: Rc<IggyShard>,
+ shutdown: ShutdownToken,
) -> Result<(), IggyError> {
loop {
let shard = shard.clone();
- let shutdown_check = async {
- loop {
- if shard.is_shutting_down() {
- return;
- }
- compio::time::sleep(Duration::from_millis(100)).await;
- }
- };
-
let accept_future = listener.accept();
futures::select! {
- _ = shutdown_check.fuse() => {
- shard_info!(shard.id, "{} detected shutdown flag, no longer accepting connections", server_name);
+ _ = shutdown.wait().fuse() => {
+ shard_debug!(shard.id, "{} received shutdown signal, no longer accepting connections", server_name);
break;
}
result = accept_future.fuse() => {
@@ -180,14 +173,16 @@ async fn accept_loop(
shard_info!(shard.id, "Created new session: {}", session);
let mut sender = SenderKind::get_tcp_sender(stream);
- let conn_stop_receiver = shard_clone.task_registry.add_connection(client_id);
+ let conn_stop_receiver = shard.task_registry.add_connection(client_id);
let shard_for_conn = shard_clone.clone();
- shard_clone.task_registry.spawn_tracked(async move {
+ let registry = shard.task_registry.clone();
+ let registry_clone = registry.clone();
+ registry.spawn_connection(async move {
if let Err(error) = handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver).await {
handle_error(error);
}
- shard_for_conn.task_registry.remove_connection(&client_id);
+ registry_clone.remove_connection(&client_id);
if let Err(error) = sender.shutdown().await {
shard_error!(shard.id, "Failed to shutdown TCP stream for client: {}, address: {}. {}", client_id, address, error);
diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs
index cfec3d6f..180388f7 100644
--- a/core/server/src/tcp/tcp_server.rs
+++ b/core/server/src/tcp/tcp_server.rs
@@ -17,6 +17,7 @@
*/
use crate::shard::IggyShard;
+use crate::shard::task_registry::ShutdownToken;
use crate::shard_info;
use crate::tcp::{tcp_listener, tcp_tls_listener};
use iggy_common::IggyError;
@@ -24,7 +25,10 @@ use std::net::SocketAddr;
use std::rc::Rc;
/// Starts the TCP server.
-pub async fn spawn_tcp_server(shard: Rc<IggyShard>) -> Result<(), IggyError> {
+pub async fn spawn_tcp_server(
+ shard: Rc<IggyShard>,
+ shutdown: ShutdownToken,
+) -> Result<(), IggyError> {
let server_name = if shard.config.tcp.tls.enabled {
"Iggy TCP TLS"
} else {
@@ -41,8 +45,13 @@ pub async fn spawn_tcp_server(shard: Rc<IggyShard>) -> Result<(), IggyError> {
shard_info!(shard.id, "Initializing {} server...", server_name);
match shard.config.tcp.tls.enabled {
- true => tcp_tls_listener::start(server_name, addr, socket_config, shard.clone()).await?,
- false => tcp_listener::start(server_name, addr, socket_config, shard.clone()).await?,
+ true => {
+ tcp_tls_listener::start(server_name, addr, socket_config, shard.clone(), shutdown)
+ .await?
+ }
+ false => {
+ tcp_listener::start(server_name, addr, socket_config, shard.clone(), shutdown).await?
+ }
};
Ok(())
diff --git a/core/server/src/tcp/tcp_tls_listener.rs b/core/server/src/tcp/tcp_tls_listener.rs
index ec69dbcd..dd028fb9 100644
--- a/core/server/src/tcp/tcp_tls_listener.rs
+++ b/core/server/src/tcp/tcp_tls_listener.rs
@@ -19,6 +19,7 @@
use crate::binary::sender::SenderKind;
use crate::configs::tcp::TcpSocketConfig;
use crate::shard::IggyShard;
+use crate::shard::task_registry::ShutdownToken;
use crate::shard::transmission::event::ShardEvent;
use crate::tcp::connection_handler::{handle_connection, handle_error};
use crate::{shard_error, shard_info, shard_warn};
@@ -42,6 +43,7 @@ pub(crate) async fn start(
mut addr: SocketAddr,
config: &TcpSocketConfig,
shard: Rc<IggyShard>,
+ shutdown: ShutdownToken,
) -> Result<(), IggyError> {
//TODO: Fix me, this needs to take into account that first shard id potentially can be greater than 0.
if shard.id != 0 && addr.port() == 0 {
@@ -149,7 +151,7 @@ pub(crate) async fn start(
actual_addr
);
- accept_loop(server_name, listener, acceptor, shard).await
+ accept_loop(server_name, listener, acceptor, shard, shutdown).await
}
async fn create_listener(
@@ -188,22 +190,14 @@ async fn accept_loop(
listener: TcpListener,
acceptor: TlsAcceptor,
shard: Rc<IggyShard>,
+ shutdown: ShutdownToken,
) -> Result<(), IggyError> {
loop {
let shard = shard.clone();
- let shutdown_check = async {
- loop {
- if shard.is_shutting_down() {
- return;
- }
- compio::time::sleep(Duration::from_millis(100)).await;
- }
- };
-
let accept_future = listener.accept();
futures::select! {
- _ = shutdown_check.fuse() => {
- shard_info!(shard.id, "{} detected shutdown flag, no longer accepting connections", server_name);
+ _ = shutdown.wait().fuse() => {
+ shard_info!(shard.id, "{} received shutdown signal, no longer accepting connections", server_name);
break;
}
result = accept_future.fuse() => {
@@ -219,7 +213,9 @@ async fn accept_loop(
// Perform TLS handshake in a separate task to avoid blocking the accept loop
let task_shard = shard_clone.clone();
- task_shard.task_registry.spawn_tracked(async move {
+ let registry = shard.task_registry.clone();
+ let registry_clone = registry.clone();
+ registry.spawn_connection(async move {
match acceptor.accept(stream).await {
Ok(tls_stream) => {
// TLS handshake successful, now create session
@@ -237,13 +233,13 @@ async fn accept_loop(
let client_id = session.client_id;
shard_info!(shard_clone.id, "Created new session: {}", session);
- let conn_stop_receiver = shard_clone.task_registry.add_connection(client_id);
+ let conn_stop_receiver = registry_clone.add_connection(client_id);
let shard_for_conn = shard_clone.clone();
let mut sender = SenderKind::get_tcp_tls_sender(tls_stream);
if let Err(error) = handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver).await {
handle_error(error);
}
- shard_for_conn.task_registry.remove_connection(&client_id);
+ registry_clone.remove_connection(&client_id);
if let Err(error) = sender.shutdown().await {
shard_error!(shard.id, "Failed to shutdown TCP TLS stream for client: {}, address: {}. {}", client_id, address, error);