You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2025/09/30 11:01:42 UTC
(iggy) 01/01: szpont fix
This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a commit to branch io_uring_tpc_szpont_fix
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 5852d05ffbd4c9d5ca3000a74bd002753d5ac591
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Tue Sep 30 13:00:57 2025 +0200
szpont fix
---
core/configs/server.toml | 2 +-
core/server/src/shard/mod.rs | 59 +++++---
core/server/src/shard/system/messages.rs | 151 ++-------------------
.../src/shard/tasks/periodic/message_saver.rs | 66 ++++++---
core/server/src/shard/transmission/frame.rs | 2 +
core/server/src/shard/transmission/message.rs | 10 +-
core/server/src/streaming/topics/topic2.rs | 4 +-
7 files changed, 101 insertions(+), 193 deletions(-)
diff --git a/core/configs/server.toml b/core/configs/server.toml
index 3d25b0ce..b0312c13 100644
--- a/core/configs/server.toml
+++ b/core/configs/server.toml
@@ -599,4 +599,4 @@ address = "127.0.0.1:8091"
# - "all": Use all available CPU cores (default)
# - numeric value (e.g. 4): Use 4 shards (4 threads pinned to cores 0, 1, 2, 3)
# - range (e.g. "5..8"): Use 3 shards with affinity to cores 5, 6, 7
-cpu_allocation = "all"
+cpu_allocation = "1"
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 035cdaac..af6d88fa 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -35,7 +35,7 @@ use crate::{
transmission::{
event::ShardEvent,
frame::{ShardFrame, ShardResponse},
- message::{ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult},
+ message::{ShardMessage, ShardRequest, ShardRequestPayload},
},
},
shard_error, shard_info, shard_warn,
@@ -626,6 +626,26 @@ impl IggyShard {
}
Ok(ShardResponse::PollMessages((metadata, batches)))
}
+ ShardRequestPayload::PersistMessages { reason } => {
+ let batch_count = self
+ .streams2
+ .persist_messages(
+ self.id,
+ &stream_id,
+ &topic_id,
+ partition_id,
+ &reason,
+ &self.config.system,
+ )
+ .await?;
+ Ok(ShardResponse::PersistMessages(batch_count))
+ }
+ ShardRequestPayload::FsyncMessages => {
+ self.streams2
+ .fsync_all_messages(&stream_id, &topic_id, partition_id)
+ .await?;
+ Ok(ShardResponse::FsyncMessages)
+ }
}
}
@@ -958,33 +978,28 @@ impl IggyShard {
}
}
- pub async fn send_request_to_shard_or_recoil(
+ pub async fn send_request_to_shard(
&self,
namespace: &IggyNamespace,
message: ShardMessage,
- ) -> Result<ShardSendRequestResult, IggyError> {
- if let Some(shard) = self.find_shard(namespace) {
- if shard.id == self.id {
- return Ok(ShardSendRequestResult::Recoil(message));
- }
-
- let response = match shard.send_request(message).await {
- Ok(response) => response,
- Err(err) => {
- error!(
- "{COMPONENT} - failed to send request to shard with ID: {}, error: {err}",
- shard.id
- );
- return Err(err);
- }
- };
- Ok(ShardSendRequestResult::Response(response))
- } else {
- Err(IggyError::ShardNotFound(
+ ) -> Result<ShardResponse, IggyError> {
+ let shard = self.find_shard(namespace).ok_or_else(|| {
+ IggyError::ShardNotFound(
namespace.stream_id(),
namespace.topic_id(),
namespace.partition_id(),
- ))
+ )
+ })?;
+
+ match shard.send_request(message).await {
+ Ok(response) => Ok(response),
+ Err(err) => {
+ error!(
+ "{COMPONENT} - failed to send request to shard with ID: {}, error: {err}",
+ shard.id
+ );
+ Err(err)
+ }
}
}
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index 320ce902..da87a1f2 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -21,9 +21,7 @@ use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
use crate::shard::IggyShard;
use crate::shard::namespace::{IggyFullNamespace, IggyNamespace};
use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
- ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
+use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
use crate::shard_trace;
use crate::streaming::polling_consumer::PollingConsumer;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet};
@@ -123,50 +121,12 @@ impl IggyShard {
let payload = ShardRequestPayload::SendMessages { batch };
let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
let message = ShardMessage::Request(request);
- match self
- .send_request_to_shard_or_recoil(&namespace, message)
- .await?
- {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest {
- stream_id,
- topic_id,
- partition_id,
- payload,
- }) = message
- && let ShardRequestPayload::SendMessages { batch } = payload
- {
- let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
- // Encrypt messages if encryptor is enabled in configuration.
- let batch = self.maybe_encrypt_messages(batch)?;
- let messages_count = batch.count();
- self.streams2
- .append_messages(
- self.id,
- &self.config.system,
- &self.task_registry,
- &ns,
- batch,
- )
- .await?;
- self.metrics.increment_messages(messages_count as u64);
- Ok(())
- } else {
- unreachable!(
- "Expected a SendMessages request inside of SendMessages handler, impossible state"
- );
- }
- }
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::SendMessages => Ok(()),
- ShardResponse::ErrorResponse(err) => Err(err),
- _ => unreachable!(
- "Expected a SendMessages response inside of SendMessages handler, impossible state"
- ),
- },
- }?;
- Ok(())
+ match self.send_request_to_shard(&namespace, message).await? {
+ ShardResponse::SendMessages => Ok(()),
+ ShardResponse::ErrorResponse(err) => Err(err),
+ _ => unreachable!("Unexpected response type for SendMessages request"),
+ }
}
pub async fn poll_messages(
@@ -239,101 +199,10 @@ impl IggyShard {
let payload = ShardRequestPayload::PollMessages { consumer, args };
let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
let message = ShardMessage::Request(request);
- let (metadata, batch) = match self
- .send_request_to_shard_or_recoil(&namespace, message)
- .await?
- {
- ShardSendRequestResult::Recoil(message) => {
- if let ShardMessage::Request(ShardRequest {
- partition_id,
- payload,
- ..
- }) = message
- && let ShardRequestPayload::PollMessages { consumer, args } = payload
- {
- let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
- let auto_commit = args.auto_commit;
- let (metadata, batches) =
- self.streams2.poll_messages(&ns, consumer, args).await?;
- let stream_id = ns.stream_id();
- let topic_id = ns.topic_id();
-
- if auto_commit && !batches.is_empty() {
- let offset = batches
- .last_offset()
- .expect("Batch set should have at least one batch");
- trace!(
- "Last offset: {} will be automatically stored for {}, stream: {}, topic: {}, partition: {}",
- offset, consumer, numeric_stream_id, numeric_topic_id, partition_id
- );
- match consumer {
- PollingConsumer::Consumer(consumer_id, _) => {
- self.streams2.with_partition_by_id(
- &stream_id,
- &topic_id,
- partition_id,
- partitions::helpers::store_consumer_offset(
- consumer_id,
- numeric_stream_id,
- numeric_topic_id,
- partition_id,
- offset,
- &self.config.system,
- ),
- );
- self.streams2
- .with_partition_by_id_async(
- &stream_id,
- &topic_id,
- partition_id,
- partitions::helpers::persist_consumer_offset_to_disk(
- self.id,
- consumer_id,
- ),
- )
- .await?;
- }
- PollingConsumer::ConsumerGroup(cg_id, _) => {
- self.streams2.with_partition_by_id(
- &stream_id,
- &topic_id,
- partition_id,
- partitions::helpers::store_consumer_group_member_offset(
- cg_id,
- numeric_stream_id,
- numeric_topic_id,
- partition_id,
- offset,
- &self.config.system,
- ),
- );
- self.streams2.with_partition_by_id_async(
- &stream_id,
- &topic_id,
- partition_id,
- partitions::helpers::persist_consumer_group_member_offset_to_disk(
- self.id,
- cg_id,
- ),
- )
- .await?;
- }
- }
- }
- Ok((metadata, batches))
- } else {
- unreachable!(
- "Expected a PollMessages request inside of PollMessages handler, impossible state"
- );
- }
- }
- ShardSendRequestResult::Response(response) => match response {
- ShardResponse::PollMessages(result) => Ok(result),
- ShardResponse::ErrorResponse(err) => Err(err),
- _ => unreachable!(
- "Expected a SendMessages response inside of SendMessages handler, impossible state"
- ),
- },
+ let (metadata, batch) = match self.send_request_to_shard(&namespace, message).await? {
+ ShardResponse::PollMessages(result) => Ok(result),
+ ShardResponse::ErrorResponse(err) => Err(err),
+ _ => unreachable!("Unexpected response type for PollMessages request"),
}?;
let batch = if let Some(encryptor) = &self.encryptor {
diff --git a/core/server/src/shard/tasks/periodic/message_saver.rs b/core/server/src/shard/tasks/periodic/message_saver.rs
index 9bd8ccc6..ac4b380e 100644
--- a/core/server/src/shard/tasks/periodic/message_saver.rs
+++ b/core/server/src/shard/tasks/periodic/message_saver.rs
@@ -17,6 +17,9 @@
*/
use crate::shard::IggyShard;
+use crate::shard::namespace::IggyNamespace;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
use crate::shard_info;
use iggy_common::{Identifier, IggyError};
use std::rc::Rc;
@@ -55,27 +58,35 @@ async fn save_messages(shard: Rc<IggyShard>) -> Result<(), IggyError> {
let topic_id = Identifier::numeric(ns.topic_id() as u32).unwrap();
let partition_id = ns.partition_id();
- match shard
- .streams2
- .persist_messages(
- shard.id,
- &stream_id,
- &topic_id,
- partition_id,
- REASON,
- &shard.config.system,
- )
- .await
- {
- Ok(batch_count) => {
+ let namespace = IggyNamespace::new(ns.stream_id(), ns.topic_id(), partition_id);
+ let payload = ShardRequestPayload::PersistMessages {
+ reason: REASON.to_string(),
+ };
+ let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
+ let message = ShardMessage::Request(request);
+
+ match shard.send_request_to_shard(&namespace, message).await {
+ Ok(ShardResponse::PersistMessages(batch_count)) => {
total_saved_messages += batch_count;
}
- Err(err) => {
+ Ok(ShardResponse::ErrorResponse(err)) => {
error!(
"Failed to save messages for partition {}: {}",
partition_id, err
);
}
+ Ok(_) => {
+ error!(
+ "Unexpected response type for persist_messages on partition {}",
+ partition_id
+ );
+ }
+ Err(err) => {
+ error!(
+ "Failed to send persist_messages request for partition {}: {}",
+ partition_id, err
+ );
+ }
}
}
@@ -108,23 +119,36 @@ async fn fsync_all_segments_on_shutdown(shard: Rc<IggyShard>, result: Result<(),
let topic_id = Identifier::numeric(ns.topic_id() as u32).unwrap();
let partition_id = ns.partition_id();
- match shard
- .streams2
- .fsync_all_messages(&stream_id, &topic_id, partition_id)
- .await
- {
- Ok(()) => {
+ let namespace = IggyNamespace::new(ns.stream_id(), ns.topic_id(), partition_id);
+ let payload = ShardRequestPayload::FsyncMessages;
+ let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
+ let message = ShardMessage::Request(request);
+
+ match shard.send_request_to_shard(&namespace, message).await {
+ Ok(ShardResponse::FsyncMessages) => {
trace!(
"Successfully fsynced segment for stream: {}, topic: {}, partition: {} during shutdown",
stream_id, topic_id, partition_id
);
}
- Err(err) => {
+ Ok(ShardResponse::ErrorResponse(err)) => {
error!(
"Failed to fsync segment for stream: {}, topic: {}, partition: {} during shutdown: {}",
stream_id, topic_id, partition_id, err
);
}
+ Ok(_) => {
+ error!(
+ "Unexpected response type for fsync_messages on stream: {}, topic: {}, partition: {}",
+ stream_id, topic_id, partition_id
+ );
+ }
+ Err(err) => {
+ error!(
+ "Failed to send fsync_messages request for stream: {}, topic: {}, partition: {} during shutdown: {}",
+ stream_id, topic_id, partition_id, err
+ );
+ }
}
}
}
diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs
index 6539ec4c..f8b05014 100644
--- a/core/server/src/shard/transmission/frame.rs
+++ b/core/server/src/shard/transmission/frame.rs
@@ -27,6 +27,8 @@ use crate::{
pub enum ShardResponse {
PollMessages((IggyPollMetadata, IggyMessagesBatchSet)),
SendMessages,
+ PersistMessages(u32),
+ FsyncMessages,
Event,
ErrorResponse(IggyError),
}
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index 14bf075b..e4e5fbb1 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -25,12 +25,6 @@ use crate::{
};
use iggy_common::Identifier;
-pub enum ShardSendRequestResult {
- // TODO: In the future we can add other variants, for example backpressure from the destination shard,
- Recoil(ShardMessage),
- Response(ShardResponse),
-}
-
#[derive(Debug)]
pub enum ShardMessage {
Request(ShardRequest),
@@ -70,6 +64,10 @@ pub enum ShardRequestPayload {
consumer: PollingConsumer,
args: PollingArgs,
},
+ PersistMessages {
+ reason: String,
+ },
+ FsyncMessages,
}
impl From<ShardRequest> for ShardMessage {
diff --git a/core/server/src/streaming/topics/topic2.rs b/core/server/src/streaming/topics/topic2.rs
index 64d4fc69..7830e102 100644
--- a/core/server/src/streaming/topics/topic2.rs
+++ b/core/server/src/streaming/topics/topic2.rs
@@ -22,8 +22,8 @@ pub struct TopicAuxilary {
impl TopicAuxilary {
pub fn get_next_partition_id(&self, shard_id: u16, upperbound: usize) -> usize {
let mut partition_id = self.current_partition_id.fetch_add(1, Ordering::AcqRel);
- if partition_id > upperbound {
- partition_id = 1;
+ if partition_id >= upperbound {
+ partition_id = 0;
self.current_partition_id
.swap(partition_id + 1, Ordering::Release);
}