You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2025/09/30 11:01:42 UTC

(iggy) 01/01: szpont fix

This is an automated email from the ASF dual-hosted git repository.

hgruszecki pushed a commit to branch io_uring_tpc_szpont_fix
in repository https://gitbox.apache.org/repos/asf/iggy.git

commit 5852d05ffbd4c9d5ca3000a74bd002753d5ac591
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Tue Sep 30 13:00:57 2025 +0200

    szpont fix
---
 core/configs/server.toml                           |   2 +-
 core/server/src/shard/mod.rs                       |  59 +++++---
 core/server/src/shard/system/messages.rs           | 151 ++-------------------
 .../src/shard/tasks/periodic/message_saver.rs      |  66 ++++++---
 core/server/src/shard/transmission/frame.rs        |   2 +
 core/server/src/shard/transmission/message.rs      |  10 +-
 core/server/src/streaming/topics/topic2.rs         |   4 +-
 7 files changed, 101 insertions(+), 193 deletions(-)

diff --git a/core/configs/server.toml b/core/configs/server.toml
index 3d25b0ce..b0312c13 100644
--- a/core/configs/server.toml
+++ b/core/configs/server.toml
@@ -599,4 +599,4 @@ address = "127.0.0.1:8091"
 # - "all": Use all available CPU cores (default)
 # - numeric value (e.g. 4): Use 4 shards (4 threads pinned to cores 0, 1, 2, 3)
 # - range (e.g. "5..8"): Use 3 shards with affinity to cores 5, 6, 7
-cpu_allocation = "all"
+cpu_allocation = "1"
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 035cdaac..af6d88fa 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -35,7 +35,7 @@ use crate::{
         transmission::{
             event::ShardEvent,
             frame::{ShardFrame, ShardResponse},
-            message::{ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult},
+            message::{ShardMessage, ShardRequest, ShardRequestPayload},
         },
     },
     shard_error, shard_info, shard_warn,
@@ -626,6 +626,26 @@ impl IggyShard {
                 }
                 Ok(ShardResponse::PollMessages((metadata, batches)))
             }
+            ShardRequestPayload::PersistMessages { reason } => {
+                let batch_count = self
+                    .streams2
+                    .persist_messages(
+                        self.id,
+                        &stream_id,
+                        &topic_id,
+                        partition_id,
+                        &reason,
+                        &self.config.system,
+                    )
+                    .await?;
+                Ok(ShardResponse::PersistMessages(batch_count))
+            }
+            ShardRequestPayload::FsyncMessages => {
+                self.streams2
+                    .fsync_all_messages(&stream_id, &topic_id, partition_id)
+                    .await?;
+                Ok(ShardResponse::FsyncMessages)
+            }
         }
     }
 
@@ -958,33 +978,28 @@ impl IggyShard {
         }
     }
 
-    pub async fn send_request_to_shard_or_recoil(
+    pub async fn send_request_to_shard(
         &self,
         namespace: &IggyNamespace,
         message: ShardMessage,
-    ) -> Result<ShardSendRequestResult, IggyError> {
-        if let Some(shard) = self.find_shard(namespace) {
-            if shard.id == self.id {
-                return Ok(ShardSendRequestResult::Recoil(message));
-            }
-
-            let response = match shard.send_request(message).await {
-                Ok(response) => response,
-                Err(err) => {
-                    error!(
-                        "{COMPONENT} - failed to send request to shard with ID: {}, error: {err}",
-                        shard.id
-                    );
-                    return Err(err);
-                }
-            };
-            Ok(ShardSendRequestResult::Response(response))
-        } else {
-            Err(IggyError::ShardNotFound(
+    ) -> Result<ShardResponse, IggyError> {
+        let shard = self.find_shard(namespace).ok_or_else(|| {
+            IggyError::ShardNotFound(
                 namespace.stream_id(),
                 namespace.topic_id(),
                 namespace.partition_id(),
-            ))
+            )
+        })?;
+
+        match shard.send_request(message).await {
+            Ok(response) => Ok(response),
+            Err(err) => {
+                error!(
+                    "{COMPONENT} - failed to send request to shard with ID: {}, error: {err}",
+                    shard.id
+                );
+                Err(err)
+            }
         }
     }
 
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index 320ce902..da87a1f2 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -21,9 +21,7 @@ use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
 use crate::shard::IggyShard;
 use crate::shard::namespace::{IggyFullNamespace, IggyNamespace};
 use crate::shard::transmission::frame::ShardResponse;
-use crate::shard::transmission::message::{
-    ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
-};
+use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
 use crate::shard_trace;
 use crate::streaming::polling_consumer::PollingConsumer;
 use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet};
@@ -123,50 +121,12 @@ impl IggyShard {
         let payload = ShardRequestPayload::SendMessages { batch };
         let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
         let message = ShardMessage::Request(request);
-        match self
-            .send_request_to_shard_or_recoil(&namespace, message)
-            .await?
-        {
-            ShardSendRequestResult::Recoil(message) => {
-                if let ShardMessage::Request(ShardRequest {
-                    stream_id,
-                    topic_id,
-                    partition_id,
-                    payload,
-                }) = message
-                    && let ShardRequestPayload::SendMessages { batch } = payload
-                {
-                    let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
-                    // Encrypt messages if encryptor is enabled in configuration.
-                    let batch = self.maybe_encrypt_messages(batch)?;
-                    let messages_count = batch.count();
-                    self.streams2
-                        .append_messages(
-                            self.id,
-                            &self.config.system,
-                            &self.task_registry,
-                            &ns,
-                            batch,
-                        )
-                        .await?;
-                    self.metrics.increment_messages(messages_count as u64);
-                    Ok(())
-                } else {
-                    unreachable!(
-                        "Expected a SendMessages request inside of SendMessages handler, impossible state"
-                    );
-                }
-            }
-            ShardSendRequestResult::Response(response) => match response {
-                ShardResponse::SendMessages => Ok(()),
-                ShardResponse::ErrorResponse(err) => Err(err),
-                _ => unreachable!(
-                    "Expected a SendMessages response inside of SendMessages handler, impossible state"
-                ),
-            },
-        }?;
 
-        Ok(())
+        match self.send_request_to_shard(&namespace, message).await? {
+            ShardResponse::SendMessages => Ok(()),
+            ShardResponse::ErrorResponse(err) => Err(err),
+            _ => unreachable!("Unexpected response type for SendMessages request"),
+        }
     }
 
     pub async fn poll_messages(
@@ -239,101 +199,10 @@ impl IggyShard {
         let payload = ShardRequestPayload::PollMessages { consumer, args };
         let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
         let message = ShardMessage::Request(request);
-        let (metadata, batch) = match self
-            .send_request_to_shard_or_recoil(&namespace, message)
-            .await?
-        {
-            ShardSendRequestResult::Recoil(message) => {
-                if let ShardMessage::Request(ShardRequest {
-                    partition_id,
-                    payload,
-                    ..
-                }) = message
-                    && let ShardRequestPayload::PollMessages { consumer, args } = payload
-                {
-                    let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
-                    let auto_commit = args.auto_commit;
-                    let (metadata, batches) =
-                        self.streams2.poll_messages(&ns, consumer, args).await?;
-                    let stream_id = ns.stream_id();
-                    let topic_id = ns.topic_id();
-
-                    if auto_commit && !batches.is_empty() {
-                        let offset = batches
-                            .last_offset()
-                            .expect("Batch set should have at least one batch");
-                        trace!(
-                            "Last offset: {} will be automatically stored for {}, stream: {}, topic: {}, partition: {}",
-                            offset, consumer, numeric_stream_id, numeric_topic_id, partition_id
-                        );
-                        match consumer {
-                            PollingConsumer::Consumer(consumer_id, _) => {
-                                self.streams2.with_partition_by_id(
-                                    &stream_id,
-                                    &topic_id,
-                                    partition_id,
-                                    partitions::helpers::store_consumer_offset(
-                                        consumer_id,
-                                        numeric_stream_id,
-                                        numeric_topic_id,
-                                        partition_id,
-                                        offset,
-                                        &self.config.system,
-                                    ),
-                                );
-                                self.streams2
-                                    .with_partition_by_id_async(
-                                        &stream_id,
-                                        &topic_id,
-                                        partition_id,
-                                        partitions::helpers::persist_consumer_offset_to_disk(
-                                            self.id,
-                                            consumer_id,
-                                        ),
-                                    )
-                                    .await?;
-                            }
-                            PollingConsumer::ConsumerGroup(cg_id, _) => {
-                                self.streams2.with_partition_by_id(
-                                    &stream_id,
-                                    &topic_id,
-                                    partition_id,
-                                    partitions::helpers::store_consumer_group_member_offset(
-                                        cg_id,
-                                        numeric_stream_id,
-                                        numeric_topic_id,
-                                        partition_id,
-                                        offset,
-                                        &self.config.system,
-                                    ),
-                                );
-                                self.streams2.with_partition_by_id_async(
-                                    &stream_id,
-                                    &topic_id,
-                                    partition_id,
-                                    partitions::helpers::persist_consumer_group_member_offset_to_disk(
-                                        self.id,
-                                        cg_id,
-                                    ),
-                                )
-                                .await?;
-                            }
-                        }
-                    }
-                    Ok((metadata, batches))
-                } else {
-                    unreachable!(
-                        "Expected a PollMessages request inside of PollMessages handler, impossible state"
-                    );
-                }
-            }
-            ShardSendRequestResult::Response(response) => match response {
-                ShardResponse::PollMessages(result) => Ok(result),
-                ShardResponse::ErrorResponse(err) => Err(err),
-                _ => unreachable!(
-                    "Expected a SendMessages response inside of SendMessages handler, impossible state"
-                ),
-            },
+        let (metadata, batch) = match self.send_request_to_shard(&namespace, message).await? {
+            ShardResponse::PollMessages(result) => Ok(result),
+            ShardResponse::ErrorResponse(err) => Err(err),
+            _ => unreachable!("Unexpected response type for PollMessages request"),
         }?;
 
         let batch = if let Some(encryptor) = &self.encryptor {
diff --git a/core/server/src/shard/tasks/periodic/message_saver.rs b/core/server/src/shard/tasks/periodic/message_saver.rs
index 9bd8ccc6..ac4b380e 100644
--- a/core/server/src/shard/tasks/periodic/message_saver.rs
+++ b/core/server/src/shard/tasks/periodic/message_saver.rs
@@ -17,6 +17,9 @@
  */
 
 use crate::shard::IggyShard;
+use crate::shard::namespace::IggyNamespace;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
 use crate::shard_info;
 use iggy_common::{Identifier, IggyError};
 use std::rc::Rc;
@@ -55,27 +58,35 @@ async fn save_messages(shard: Rc<IggyShard>) -> Result<(), IggyError> {
         let topic_id = Identifier::numeric(ns.topic_id() as u32).unwrap();
         let partition_id = ns.partition_id();
 
-        match shard
-            .streams2
-            .persist_messages(
-                shard.id,
-                &stream_id,
-                &topic_id,
-                partition_id,
-                REASON,
-                &shard.config.system,
-            )
-            .await
-        {
-            Ok(batch_count) => {
+        let namespace = IggyNamespace::new(ns.stream_id(), ns.topic_id(), partition_id);
+        let payload = ShardRequestPayload::PersistMessages {
+            reason: REASON.to_string(),
+        };
+        let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
+        let message = ShardMessage::Request(request);
+
+        match shard.send_request_to_shard(&namespace, message).await {
+            Ok(ShardResponse::PersistMessages(batch_count)) => {
                 total_saved_messages += batch_count;
             }
-            Err(err) => {
+            Ok(ShardResponse::ErrorResponse(err)) => {
                 error!(
                     "Failed to save messages for partition {}: {}",
                     partition_id, err
                 );
             }
+            Ok(_) => {
+                error!(
+                    "Unexpected response type for persist_messages on partition {}",
+                    partition_id
+                );
+            }
+            Err(err) => {
+                error!(
+                    "Failed to send persist_messages request for partition {}: {}",
+                    partition_id, err
+                );
+            }
         }
     }
 
@@ -108,23 +119,36 @@ async fn fsync_all_segments_on_shutdown(shard: Rc<IggyShard>, result: Result<(),
         let topic_id = Identifier::numeric(ns.topic_id() as u32).unwrap();
         let partition_id = ns.partition_id();
 
-        match shard
-            .streams2
-            .fsync_all_messages(&stream_id, &topic_id, partition_id)
-            .await
-        {
-            Ok(()) => {
+        let namespace = IggyNamespace::new(ns.stream_id(), ns.topic_id(), partition_id);
+        let payload = ShardRequestPayload::FsyncMessages;
+        let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
+        let message = ShardMessage::Request(request);
+
+        match shard.send_request_to_shard(&namespace, message).await {
+            Ok(ShardResponse::FsyncMessages) => {
                 trace!(
                     "Successfully fsynced segment for stream: {}, topic: {}, partition: {} during shutdown",
                     stream_id, topic_id, partition_id
                 );
             }
-            Err(err) => {
+            Ok(ShardResponse::ErrorResponse(err)) => {
                 error!(
                     "Failed to fsync segment for stream: {}, topic: {}, partition: {} during shutdown: {}",
                     stream_id, topic_id, partition_id, err
                 );
             }
+            Ok(_) => {
+                error!(
+                    "Unexpected response type for fsync_messages on stream: {}, topic: {}, partition: {}",
+                    stream_id, topic_id, partition_id
+                );
+            }
+            Err(err) => {
+                error!(
+                    "Failed to send fsync_messages request for stream: {}, topic: {}, partition: {} during shutdown: {}",
+                    stream_id, topic_id, partition_id, err
+                );
+            }
         }
     }
 }
diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs
index 6539ec4c..f8b05014 100644
--- a/core/server/src/shard/transmission/frame.rs
+++ b/core/server/src/shard/transmission/frame.rs
@@ -27,6 +27,8 @@ use crate::{
 pub enum ShardResponse {
     PollMessages((IggyPollMetadata, IggyMessagesBatchSet)),
     SendMessages,
+    PersistMessages(u32),
+    FsyncMessages,
     Event,
     ErrorResponse(IggyError),
 }
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index 14bf075b..e4e5fbb1 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -25,12 +25,6 @@ use crate::{
 };
 use iggy_common::Identifier;
 
-pub enum ShardSendRequestResult {
-    // TODO: In the future we can add other variants, for example backpressure from the destination shard,
-    Recoil(ShardMessage),
-    Response(ShardResponse),
-}
-
 #[derive(Debug)]
 pub enum ShardMessage {
     Request(ShardRequest),
@@ -70,6 +64,10 @@ pub enum ShardRequestPayload {
         consumer: PollingConsumer,
         args: PollingArgs,
     },
+    PersistMessages {
+        reason: String,
+    },
+    FsyncMessages,
 }
 
 impl From<ShardRequest> for ShardMessage {
diff --git a/core/server/src/streaming/topics/topic2.rs b/core/server/src/streaming/topics/topic2.rs
index 64d4fc69..7830e102 100644
--- a/core/server/src/streaming/topics/topic2.rs
+++ b/core/server/src/streaming/topics/topic2.rs
@@ -22,8 +22,8 @@ pub struct TopicAuxilary {
 impl TopicAuxilary {
     pub fn get_next_partition_id(&self, shard_id: u16, upperbound: usize) -> usize {
         let mut partition_id = self.current_partition_id.fetch_add(1, Ordering::AcqRel);
-        if partition_id > upperbound {
-            partition_id = 1;
+        if partition_id >= upperbound {
+            partition_id = 0;
             self.current_partition_id
                 .swap(partition_id + 1, Ordering::Release);
         }