You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2025/02/26 08:43:20 UTC
(iggy) 01/01: send response vectored
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch zero_copy_schema
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit b92544043192ed2ee0f15332d95d1f58c5514334
Author: numinex <g....@gmail.com>
AuthorDate: Wed Feb 26 09:12:24 2025 +0100
send response vectored
---
.../handlers/messages/poll_messages_handler.rs | 19 ++++++++++++++++++-
server/src/binary/sender.rs | 7 +++++++
server/src/quic/quic_sender.rs | 12 ++++++++++++
server/src/streaming/partitions/messages.rs | 4 +++-
server/src/streaming/segments/fetch_result.rs | 6 +++++-
server/src/streaming/segments/mod.rs | 4 ++--
server/src/streaming/systems/messages.rs | 4 ++--
server/src/streaming/topics/messages.rs | 6 +++---
server/src/tcp/sender.rs | 21 +++++++++++++++++----
server/src/tcp/tcp_sender.rs | 8 ++++++++
server/src/tcp/tcp_tls_sender.rs | 8 ++++++++
11 files changed, 85 insertions(+), 14 deletions(-)
diff --git a/server/src/binary/handlers/messages/poll_messages_handler.rs b/server/src/binary/handlers/messages/poll_messages_handler.rs
index d2342589..8c759462 100644
--- a/server/src/binary/handlers/messages/poll_messages_handler.rs
+++ b/server/src/binary/handlers/messages/poll_messages_handler.rs
@@ -1,3 +1,5 @@
+use std::io::IoSlice;
+
use crate::binary::handlers::messages::COMPONENT;
use crate::binary::mapper;
use crate::binary::sender::SenderKind;
@@ -32,6 +34,21 @@ pub async fn handle(
"{COMPONENT} (error: {error}) - failed to poll messages for consumer: {}, stream_id: {}, topic_id: {}, partition_id: {:?}, session: {}.",
command.consumer, command.stream_id, command.topic_id, command.partition_id, session
))?;
- sender.send_ok_response(&[]).await?;
+ let length = result.slices.iter().map(|s| s.range.len() as u64).sum::<u64>();
+ let length = length.to_le_bytes();
+ // Adding 1 for the header and 2 for the prefix required by `send_ok_response`.
+ let mut slices = Vec::with_capacity(1 + result.slices.len() + 2);
+ let header = result.header.as_bytes();
+ slices.push(IoSlice::new(&header));
+ for slice in result.slices.iter() {
+ // Ergh... it is kinda ugly that we have to index this way, rather than just passing `Range`,
+ // but borrow checker gets really anal about it, since the range is field of `IggyBatchSlice`
+ // that we borrow inside of that loop.
+ // It is fixable by decoupling `Range` from `Bytes`, but w/e.
+ let slice = &slice.bytes[slice.range.start..slice.range.end];
+ slices.push(IoSlice::new(slice));
+ }
+
+ sender.send_ok_response_vectored(&length, slices).await?;
Ok(())
}
diff --git a/server/src/binary/sender.rs b/server/src/binary/sender.rs
index 0c1291e9..965efe71 100644
--- a/server/src/binary/sender.rs
+++ b/server/src/binary/sender.rs
@@ -1,4 +1,5 @@
use std::future::Future;
+use std::io::IoSlice;
use crate::tcp::tcp_sender::TcpSender;
use crate::tcp::tcp_tls_sender::TcpTlsSender;
@@ -35,6 +36,11 @@ pub trait Sender {
&mut self,
payload: &[u8],
) -> impl Future<Output = Result<(), IggyError>> + Send;
+ fn send_ok_response_vectored(
+ &mut self,
+ length: &[u8],
+ slices: Vec<IoSlice<'_>>,
+ ) -> impl Future<Output = Result<(), IggyError>> + Send;
fn send_error_response(
&mut self,
error: IggyError,
@@ -68,6 +74,7 @@ impl SenderKind {
async fn read(&mut self, buffer: &mut [u8]) -> Result<usize, IggyError>;
async fn send_empty_ok_response(&mut self) -> Result<(), IggyError>;
async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError>;
+ async fn send_ok_response_vectored(&mut self, length: &[u8], slices: Vec<IoSlice<'_>>) -> Result<(), IggyError>;
async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError>;
async fn shutdown(&mut self) -> Result<(), ServerError>;
}
diff --git a/server/src/quic/quic_sender.rs b/server/src/quic/quic_sender.rs
index 7e7be69b..7e43965b 100644
--- a/server/src/quic/quic_sender.rs
+++ b/server/src/quic/quic_sender.rs
@@ -1,8 +1,11 @@
+use std::io::IoSlice;
+
use crate::quic::COMPONENT;
use crate::{binary::sender::Sender, server_error::ServerError};
use error_set::ErrContext;
use iggy::error::IggyError;
use quinn::{RecvStream, SendStream};
+use tokio::io::AsyncWriteExt;
use tracing::{debug, error};
const STATUS_OK: &[u8] = &[0; 4];
@@ -39,6 +42,15 @@ impl Sender for QuicSender {
async fn shutdown(&mut self) -> Result<(), ServerError> {
Ok(())
}
+
+ async fn send_ok_response_vectored(
+ &mut self,
+ length: &[u8],
+ slices: Vec<IoSlice<'_>>,
+ ) -> Result<(), IggyError> {
+ //TODO: Fix me, quinn seems to have support for write vectored.
+ todo!()
+ }
}
impl QuicSender {
diff --git a/server/src/streaming/partitions/messages.rs b/server/src/streaming/partitions/messages.rs
index 3fbd226d..65a3d708 100644
--- a/server/src/streaming/partitions/messages.rs
+++ b/server/src/streaming/partitions/messages.rs
@@ -98,7 +98,9 @@ impl Partition {
.await
}
//_ => Self::get_messages_from_segments(segments, start_offset, count).await,
- _ => panic!("todo, the method from above is already implemented, need to flatten the results")
+ _ => panic!(
+ "todo, the method from above is already implemented, need to flatten the results"
+ ),
}
}
diff --git a/server/src/streaming/segments/fetch_result.rs b/server/src/streaming/segments/fetch_result.rs
index 8f9ed869..d2baad29 100644
--- a/server/src/streaming/segments/fetch_result.rs
+++ b/server/src/streaming/segments/fetch_result.rs
@@ -27,6 +27,10 @@ pub struct IggyBatchFetchResult {
impl IggyBatchFetchResult {
pub fn new(slices: Vec<IggyBatchSlice>, msg_count: u32, header: IggyHeader) -> Self {
- Self { slices, msg_count, header }
+ Self {
+ slices,
+ msg_count,
+ header,
+ }
}
}
diff --git a/server/src/streaming/segments/mod.rs b/server/src/streaming/segments/mod.rs
index 06208b3c..324a19f4 100644
--- a/server/src/streaming/segments/mod.rs
+++ b/server/src/streaming/segments/mod.rs
@@ -1,13 +1,13 @@
+mod fetch_result;
mod indexes;
mod logs;
mod reading_messages;
mod segment;
mod writing_messages;
-mod fetch_result;
+pub use fetch_result::*;
pub use indexes::Index;
pub use segment::Segment;
-pub use fetch_result::*;
pub const LOG_EXTENSION: &str = "log";
pub const INDEX_EXTENSION: &str = "index";
diff --git a/server/src/streaming/systems/messages.rs b/server/src/streaming/systems/messages.rs
index beb6a057..e570aaf2 100644
--- a/server/src/streaming/systems/messages.rs
+++ b/server/src/streaming/systems/messages.rs
@@ -66,7 +66,7 @@ impl System {
.await?;
// TODO: Fix me
- /*
+ /*
if polled_messages.messages.is_empty() {
return Ok(polled_messages);
}
@@ -86,7 +86,7 @@ impl System {
if self.encryptor.is_none() {
return Ok(result);
- }
+ }
// TODO: Fix me
/*
let encryptor = self.encryptor.as_ref().unwrap();
diff --git a/server/src/streaming/topics/messages.rs b/server/src/streaming/topics/messages.rs
index 774c9e33..2455e30a 100644
--- a/server/src/streaming/topics/messages.rs
+++ b/server/src/streaming/topics/messages.rs
@@ -59,11 +59,11 @@ impl Topic {
.await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to get messages by timestamp: {value}, count: {count}"))
*/
- todo!()
+ todo!()
}
PollingKind::First => todo!(), //partition.get_first_messages(count).await,
- PollingKind::Last => todo!(), //partition.get_last_messages(count).await,
- PollingKind::Next => todo!(), //partition.get_next_messages(consumer, count).await,
+ PollingKind::Last => todo!(), //partition.get_last_messages(count).await,
+ PollingKind::Next => todo!(), //partition.get_next_messages(consumer, count).await,
}?;
/*
diff --git a/server/src/tcp/sender.rs b/server/src/tcp/sender.rs
index c6ab4f59..68750fe9 100644
--- a/server/src/tcp/sender.rs
+++ b/server/src/tcp/sender.rs
@@ -37,6 +37,17 @@ where
send_response(stream, STATUS_OK, payload).await
}
+pub(crate) async fn send_ok_response_vectored<T>(
+ stream: &mut T,
+ length: &[u8],
+ slices: Vec<IoSlice<'_>>,
+) -> Result<(), IggyError>
+where
+ T: AsyncRead + AsyncWrite + Unpin,
+{
+ send_response_vectored(stream, STATUS_OK, length, slices).await
+}
+
pub(crate) async fn send_error_response<T>(
stream: &mut T,
error: IggyError,
@@ -68,15 +79,17 @@ where
pub(crate) async fn send_response_vectored<T>(
stream: &mut T,
status: &[u8],
- slices: [IoSlice]
+ length: &[u8],
+ mut slices: Vec<IoSlice<'_>>,
) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadExt + AsyncWriteExt + Unpin,
{
debug!("Sending response with status: {:?}...", status);
- let length = (payload.len() as u32).to_le_bytes();
+ let prefix = [IoSlice::new(status), IoSlice::new(length)];
+ slices.splice(0..0, prefix);
stream
- .write_all(&[status, &length, payload].as_slice().concat())
+ .write_vectored(&slices)
.await
.map_err(|_| IggyError::TcpError)?;
debug!("Sent response with status: {:?}", status);
diff --git a/server/src/tcp/tcp_sender.rs b/server/src/tcp/tcp_sender.rs
index a87325eb..66e82da0 100644
--- a/server/src/tcp/tcp_sender.rs
+++ b/server/src/tcp/tcp_sender.rs
@@ -36,4 +36,12 @@ impl Sender for TcpSender {
})
.map_err(ServerError::IoError)
}
+
+ async fn send_ok_response_vectored(
+ &mut self,
+ length: &[u8],
+ slices: Vec<std::io::IoSlice<'_>>,
+ ) -> Result<(), IggyError> {
+ sender::send_ok_response_vectored(&mut self.stream, length, slices).await
+ }
}
diff --git a/server/src/tcp/tcp_tls_sender.rs b/server/src/tcp/tcp_tls_sender.rs
index 34356b8a..4d331f7d 100644
--- a/server/src/tcp/tcp_tls_sender.rs
+++ b/server/src/tcp/tcp_tls_sender.rs
@@ -38,4 +38,12 @@ impl Sender for TcpTlsSender {
})
.map_err(ServerError::IoError)
}
+
+ async fn send_ok_response_vectored(
+ &mut self,
+ length: &[u8],
+ slices: Vec<std::io::IoSlice<'_>>,
+ ) -> Result<(), IggyError> {
+ sender::send_ok_response_vectored(&mut self.stream, length, slices).await
+ }
}