You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/10/26 11:00:44 UTC
(iggy) branch feat/deferred-polling created (now c5ef0d6fd)
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a change to branch feat/deferred-polling
in repository https://gitbox.apache.org/repos/asf/iggy.git
at c5ef0d6fd fix(partitions): admit deferred polls regardless of journal residency
This branch includes the following new commits:
new 5062aa958 feat(rust): make bounded long polling the consumer default
new 097dde82d feat(java): add deferred polling to the TCP and HTTP clients
new c5ef0d6fd fix(partitions): admit deferred polls regardless of journal residency
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iggy) 03/03: fix(partitions): admit deferred polls regardless of journal residency
Posted by hg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch feat/deferred-polling
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit c5ef0d6fde6b7dbd7c1a6efc685182bc8334b310
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Thu Sep 17 17:19:07 2026 +0200
fix(partitions): admit deferred polls regardless of journal residency
A deferred poll was refused with InvalidSizeBytes on any partition holding
more than about 1.3 MiB of resident journal, while an immediate poll on the
same partition at the same instant succeeded. The admission check charged the
whole resident journal against the read budget instead of the offset range the
poll would read, so the refusal did not depend on the request at all. Under
load a partition crossed that line within half a second of production and then
refused every deferred poll from that point on.
The byte budget is already enforced where the bytes exist: the served result is
rejected above the configured max_read_bytes, and the reply is trimmed to the
caller's max_bytes with the trailing batch re-framed. Without the check the
bounded plan and snapshot builders were identical to the plain ones, and the
journal scan behind it, which decoded every resident batch on the shard pump
for every probe, has no caller left.
---
core/partitions/src/iggy_partition.rs | 29 -----------------------------
core/partitions/src/iggy_partitions.rs | 20 --------------------
core/partitions/src/journal.rs | 19 -------------------
core/shard/src/poll/deferred.rs | 16 ++++------------
core/shard/src/poll/deferred_tests.rs | 10 ++++++++++
core/shard/src/poll/test_support.rs | 8 +++++++-
6 files changed, 21 insertions(+), 81 deletions(-)
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs
index 07409ae8c..fc58d86d1 100644
--- a/core/partitions/src/iggy_partition.rs
+++ b/core/partitions/src/iggy_partition.rs
@@ -3820,35 +3820,6 @@ where
}
}
- pub(crate) fn build_bounded_poll_plan(
- &mut self,
- consumer: PollingConsumer,
- args: &PollingArgs,
- validate_checksum: bool,
- max_bytes: usize,
- ) -> Result<PollPlan, IggyError> {
- // Source snapshots, sparse copies and fragment descriptors can coexist.
- const RESIDENT_ALLOCATION_FACTOR: usize = 4;
- // Covers file paths, read handles and fixed IO/selection bookkeeping.
- const READ_METADATA_BYTES: usize = 64 * 1024;
- let resident_bytes = self.log.journal().inner.resident_message_bytes()?;
- let segment_bytes = self
- .log
- .segments()
- .len()
- .saturating_mul(size_of::<DiskSegment>());
- // Selection may copy sparse fragments while the source snapshot remains live.
- if resident_bytes
- .saturating_mul(RESIDENT_ALLOCATION_FACTOR)
- .saturating_add(segment_bytes)
- .saturating_add(READ_METADATA_BYTES)
- > max_bytes
- {
- return Err(IggyError::InvalidSizeBytes);
- }
- Ok(self.build_poll_plan(consumer, args, validate_checksum))
- }
-
/// Snapshot read resources synchronously on the partition owner.
/// Only the owned snapshot crosses a suspension during disk I/O.
#[allow(clippy::too_many_lines)]
diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs
index 75324bd9a..7afe27da3 100644
--- a/core/partitions/src/iggy_partitions.rs
+++ b/core/partitions/src/iggy_partitions.rs
@@ -488,26 +488,6 @@ where
Some(partition.build_poll_plan(consumer, args, validate_checksum))
}
- /// Snapshot a deferred read after checking its resident allocation budget.
- ///
- /// # Errors
- /// Returns a size error when the snapshot exceeds the budget, or a read
- /// error when a resident journal entry cannot be decoded.
- pub fn build_bounded_poll_snapshot(
- &self,
- namespace: &IggyNamespace,
- consumer: PollingConsumer,
- args: &PollingArgs,
- max_bytes: usize,
- ) -> Result<Option<PollPlan>, IggyError> {
- let validate_checksum = self.config.validate_checksum;
- self.get_mut_by_ns(namespace)
- .map(|partition| {
- partition.build_bounded_poll_plan(consumer, args, validate_checksum, max_bytes)
- })
- .transpose()
- }
-
/// Validate and accept a poll synchronously on the owning pump.
/// Attempt the reply synchronously, then immediately await any returned
/// continuation through [`Self::replicate_poll_completion`] on the same pump.
diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs
index e577b57d4..2cc8164e9 100644
--- a/core/partitions/src/journal.rs
+++ b/core/partitions/src/journal.rs
@@ -16,7 +16,6 @@
// under the License.
use iggy_binary_protocol::{Operation, PrepareHeader};
-use iggy_common::IggyError;
use journal::{Journal, Storage};
use server_common::{
iobuf::{Frozen, Owned},
@@ -768,24 +767,6 @@ impl PartitionJournal<PartitionJournalMemStorage> {
entries
}
- /// Conservative snapshot charge without allocating an entry vector.
- pub fn resident_message_bytes(&self) -> Result<usize, IggyError> {
- let offset_to_op = unsafe { &*self.offset_to_op.get() };
- let op_to_storage_offset = unsafe { &*self.op_to_storage_offset.get() };
- let inner = unsafe { &*self.inner.get() };
- offset_to_op.values().try_fold(0usize, |bytes, op| {
- let entry = op_to_storage_offset
- .get(op)
- .and_then(|offset| inner.storage.read_at_sync(*offset))
- .ok_or(IggyError::CannotReadMessage)?;
- decode_prepare_slice_trusted(entry.as_slice())
- .map_err(|_| IggyError::CannotReadMessage)?;
- Ok(bytes
- .saturating_add(entry.allocation_bytes())
- .saturating_add(size_of::<JournalBuffer>()))
- })
- }
-
/// Owned, append-ordered clones of every resident entry above the purge
/// floor, control ops included; one `Frozen` refcount bump each. Linear in
/// the resident journal, so not for the poll path, which takes
diff --git a/core/shard/src/poll/deferred.rs b/core/shard/src/poll/deferred.rs
index 30104ef88..d9c51e75d 100644
--- a/core/shard/src/poll/deferred.rs
+++ b/core/shard/src/poll/deferred.rs
@@ -554,21 +554,13 @@ where
};
// Snapshot, disk selection and index loading can coexist.
let max_bytes = reservation.bytes / READ_BUDGET_PARTS;
- let plan = match self.plane.partitions().build_bounded_poll_snapshot(
+ let Some(plan) = self.plane.partitions().build_poll_snapshot(
&waiter.namespace,
waiter.request.consumer,
&waiter.request.args,
- max_bytes,
- ) {
- Ok(Some(plan)) => plan,
- Ok(None) => {
- self.reject_deferred_poll(id, &waiter, IggyError::TransientNotAccepted);
- return;
- }
- Err(error) => {
- self.reject_deferred_poll(id, &waiter, error);
- return;
- }
+ ) else {
+ self.reject_deferred_poll(id, &waiter, IggyError::TransientNotAccepted);
+ return;
};
waiter.last_visibility = Some(visibility);
if plan.needs_off_pump_io() {
diff --git a/core/shard/src/poll/deferred_tests.rs b/core/shard/src/poll/deferred_tests.rs
index 0d99cba9e..cd5f82481 100644
--- a/core/shard/src/poll/deferred_tests.rs
+++ b/core/shard/src/poll/deferred_tests.rs
@@ -416,3 +416,13 @@ async fn read_crossing_readiness_deadline_finishes_within_request_budget() {
);
}
}
+
+#[compio::test]
+async fn poll_admission_is_independent_of_resident_journal_size() {
+ // 1.5 MiB resident, one message polled: the read budget bounds the reply,
+ // not how much the partition holds.
+ let payload = "x".repeat(768 * 1024);
+ let owner = owner(&[payload.as_str(); 2]).await;
+ let replies = submit(&owner, request(&owner, 1, false, false)).await;
+ assert_eq!(offsets(&replies), [0]);
+}
diff --git a/core/shard/src/poll/test_support.rs b/core/shard/src/poll/test_support.rs
index 6664288ed..d8d13dfc4 100644
--- a/core/shard/src/poll/test_support.rs
+++ b/core/shard/src/poll/test_support.rs
@@ -33,6 +33,8 @@ use server_common::sharding::IggyNamespace;
pub(super) type PollTestMetadata = MuxStateMachine<variadic!(Users, Streams)>;
+const SEGMENT_FLOOR: u64 = 1024 * 1024;
+
/// Commit one batch starting at offset zero and keep it resident. Each call
/// creates an independent partition history, even for the same namespace.
#[allow(clippy::future_not_send)]
@@ -44,7 +46,11 @@ pub(super) async fn partition_with_messages<B: MessageBus + Clone>(
let cluster_id = 1;
let replica_id = 0;
let replica_count = 3;
- let segment_size = IggyByteSize::from(1_048_576_u64);
+ // The batch must fit one segment or it never materialises, and it must stay
+ // under the flush threshold or it stops being resident. Doubling the payload
+ // total covers framing; the floor keeps small fixtures where they were.
+ let payload_bytes: u64 = payloads.iter().map(|payload| payload.len() as u64).sum();
+ let segment_size = IggyByteSize::from(SEGMENT_FLOOR.max(payload_bytes.saturating_mul(2)));
let config = PartitionsConfig {
messages_required_to_save: 100,
size_of_messages_required_to_save: segment_size,
(iggy) 01/03: feat(rust): make bounded long polling the consumer default
Posted by hg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch feat/deferred-polling
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 5062aa958b1256cdbb40fe9d88a1e202ab5631bf
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Thu Sep 17 12:12:48 2026 +0200
feat(rust): make bounded long polling the consumer default
Interval polling repeatedly fetches empty batches and couples batch
size to delivery latency. Fetch-time commits can also acknowledge
messages before application processing.
Separate readiness minimums, response byte limits and the request
deadline. Reuse bounded owner reads, cancellation and session fences,
finishing a terminal read at wait expiry without caching partials.
Carry remaining budgets through binary routing and HTTP forwarding.
Default Rust consumers to Next and manual commits. Track explicit
fetch positions and reserve capacity across in-flight and buffered
replies. Refresh assignments independently and discard revoked data.
Remove normal polling intervals and migrate Rust callers and Bench.
Verify with 2,627 crate tests, 29 SDK doctests and 85 integration
cases.
---
.../bench/src/actors/consumer/client/high_level.rs | 45 +-
core/bench/src/actors/consumer/client/interface.rs | 1 +
core/bench/src/actors/consumer/client/low_level.rs | 3 +-
.../actors/consumer/typed_benchmark_consumer.rs | 2 +
.../typed_benchmark_producing_consumer.rs | 14 +-
core/bench/src/args/common.rs | 44 +-
core/bench/src/benchmarks/common.rs | 4 +
core/binary_protocol/src/codes.rs | 4 +
core/binary_protocol/src/dispatch.rs | 7 +
core/binary_protocol/src/requests/messages/mod.rs | 4 +-
.../src/requests/messages/poll_messages.rs | 147 +++-
.../src/responses/messages/poll_messages.rs | 2 +-
core/common/src/consumer_group_client_state.rs | 56 +-
core/common/src/lib.rs | 3 +-
core/common/src/traits/binary_impls/messages.rs | 52 +-
core/common/src/traits/binary_impls/mod.rs | 2 +-
core/common/src/traits/binary_transport.rs | 16 +
core/common/src/traits/message_client.rs | 57 ++
core/common/src/traits/mod.rs | 2 +-
.../src/types/message/deferred_poll_options.rs | 96 +++
core/common/src/types/message/mod.rs | 3 +
core/common/src/types/message/polled_messages.rs | 28 +-
core/configs/src/server_config/sharding.rs | 33 +
core/connectors/runtime/README.md | 4 +
core/connectors/runtime/src/sink.rs | 17 +-
.../tests/sdk/consumer_group_membership.rs | 1 -
core/integration/tests/sdk/consumer_offset.rs | 2 +-
.../integration/tests/server/poll_semantics_vsr.rs | 737 ++++++++++++++++
...umer_group_auto_commit_reconnection_scenario.rs | 1 -
...er_group_new_messages_after_restart_scenario.rs | 1 -
.../consumer_group_offset_cleanup_scenario.rs | 1 -
.../consumer_timestamp_polling_scenario.rs | 1 -
.../scenarios/reconnect_after_restart_scenario.rs | 1 -
.../stale_client_consumer_group_scenario.rs | 1 -
core/message_bus/src/lib.rs | 22 +-
core/message_bus/src/transports/quic.rs | 44 +-
core/message_bus/src/transports/tcp.rs | 8 +-
core/message_bus/src/transports/tcp_tls.rs | 7 +
core/message_bus/src/transports/ws.rs | 7 +
core/message_bus/src/transports/wss.rs | 7 +
core/partitions/src/iggy_partition.rs | 68 +-
core/partitions/src/iggy_partitions.rs | 26 +-
core/partitions/src/journal.rs | 52 +-
core/partitions/src/poll_plan.rs | 418 +++++++--
core/partitions/src/state_transfer.rs | 2 +
core/partitions/src/types.rs | 7 +-
core/sdk/README.md | 72 +-
.../src/client_wrappers/binary_message_client.rs | 39 +-
core/sdk/src/client_wrappers/client_wrapper.rs | 61 ++
core/sdk/src/clients/binary_message.rs | 27 +-
core/sdk/src/clients/client.rs | 5 -
core/sdk/src/clients/consumer.rs | 957 +++++----------------
core/sdk/src/clients/consumer/deferred.rs | 571 ++++++++++++
core/sdk/src/clients/consumer_builder.rs | 66 +-
core/sdk/src/http/http_client.rs | 266 +++++-
core/sdk/src/http/messages.rs | 48 ++
core/sdk/src/poll_routing.rs | 514 ++++++++++-
core/sdk/src/prelude.rs | 14 +-
core/sdk/src/quic/quic_client.rs | 230 +++--
.../stream_builder/build/build_iggy_consumer.rs | 6 +-
.../stream_builder/config/config_iggy_consumer.rs | 108 +--
.../stream_builder/config/config_iggy_stream.rs | 26 +-
core/sdk/src/tcp/tcp_client.rs | 80 +-
core/sdk/src/websocket/websocket_client.rs | 67 +-
core/server/config.toml | 5 +
core/server/src/boot/recovery.rs | 7 +
core/server/src/dispatch/authz.rs | 13 +-
core/server/src/dispatch/partition.rs | 98 ++-
core/server/src/dispatch/reads.rs | 204 ++++-
core/server/src/http.rs | 10 +-
core/server/src/http/forward.rs | 208 ++++-
core/server/src/http/handlers.rs | 135 ++-
core/server/src/session_manager.rs | 153 +++-
core/server_common/src/consensus_message.rs | 39 +-
core/server_common/src/iobuf.rs | 7 +
core/shard/src/config.rs | 23 +
core/shard/src/lib.rs | 30 +-
core/shard/src/metrics.rs | 60 ++
core/shard/src/poll.rs | 47 +-
core/shard/src/poll/completion.rs | 86 +-
core/shard/src/poll/deferred.rs | 707 +++++++++++++++
core/shard/src/poll/deferred_tests.rs | 418 +++++++++
core/shard/src/poll/timeout_tests.rs | 11 +-
core/shard/src/router.rs | 30 +
core/simulator/src/bus.rs | 9 +
examples/rust/src/multi-tenant/consumer/main.rs | 4 -
examples/rust/src/new-sdk/consumer/main.rs | 11 +-
.../stream-builder/stream-consumer-config/main.rs | 3 +-
88 files changed, 6318 insertions(+), 1217 deletions(-)
diff --git a/core/bench/src/actors/consumer/client/high_level.rs b/core/bench/src/actors/consumer/client/high_level.rs
index 00128ca26..dbcd9e646 100644
--- a/core/bench/src/actors/consumer/client/high_level.rs
+++ b/core/bench/src/actors/consumer/client/high_level.rs
@@ -30,6 +30,8 @@ use std::{sync::Arc, time::Duration};
use tokio::time::{Instant, timeout};
use tracing::{error, warn};
+const PREFETCH_BATCHES: u32 = 16;
+
pub struct HighLevelConsumerClient {
client_factory: Arc<dyn ClientFactory>,
config: BenchmarkConsumerConfig,
@@ -56,6 +58,10 @@ impl HighLevelConsumerClient {
}
impl ConsumerClient for HighLevelConsumerClient {
+ #[expect(
+ clippy::significant_drop_tightening,
+ reason = "The borrowed consumer retains reservations until its buffered messages are drained."
+ )]
async fn consume_batch(&mut self) -> Result<Option<BatchMetrics>, IggyError> {
let consumer = self.consumer.as_mut().expect("Consumer not initialized");
@@ -108,38 +114,47 @@ impl ConsumerClient for HighLevelConsumerClient {
}
impl BenchmarkInit for HighLevelConsumerClient {
+ #[expect(
+ clippy::significant_drop_tightening,
+ reason = "The initialized consumer moves into self and must retain its prefetch reservations."
+ )]
async fn setup(&mut self) -> Result<(), IggyError> {
let topic_id_str = "topic-1";
let client = self.client_factory.create_authenticated_client().await?;
-
- let stream_id_str = self.config.stream_id.clone();
-
- let mut consumer = if let Some(cg_id) = self.config.consumer_group_id {
- let consumer_group_name = format!("cg_{cg_id}");
+ let stream_id_str = &self.config.stream_id;
+ let builder = if let Some(cg_id) = self.config.consumer_group_id {
client
- .consumer_group(&consumer_group_name, &stream_id_str, topic_id_str)?
- .batch_length(self.config.messages_per_batch.get())
+ .consumer_group(&format!("cg_{cg_id}"), stream_id_str, topic_id_str)?
.auto_commit(AutoCommit::When(AutoCommitWhen::PollingMessages))
.create_consumer_group_if_not_exists()
.auto_join_consumer_group()
- .build()
} else {
- // TODO(hubcio): as of now, there is no way to mimic the behavior of
- // PollingKind::Offset, because high level API doesn't provide method
- // to commit local offset manually, only auto-commit on server.
client
.consumer(
&format!("hl_consumer_{}", self.config.consumer_id),
- &stream_id_str,
+ stream_id_str,
topic_id_str,
0,
)?
.polling_strategy(PollingStrategy::offset(0))
- .batch_length(self.config.messages_per_batch.get())
.auto_commit(AutoCommit::Disabled)
- .build()
};
-
+ let mut consumer = builder
+ .batch_length(self.config.messages_per_batch.get())
+ .poll_options(self.config.poll_options)
+ .prefetch_messages(
+ self.config
+ .messages_per_batch
+ .get()
+ .saturating_mul(PREFETCH_BATCHES),
+ )
+ .prefetch_bytes(
+ self.config
+ .poll_options
+ .max_bytes
+ .saturating_mul(PREFETCH_BATCHES),
+ )
+ .build();
consumer.init().await?;
self.consumer = Some(consumer);
self.client = Some(Box::new(client));
diff --git a/core/bench/src/actors/consumer/client/interface.rs b/core/bench/src/actors/consumer/client/interface.rs
index 5b5cc3fdd..d8173b640 100644
--- a/core/bench/src/actors/consumer/client/interface.rs
+++ b/core/bench/src/actors/consumer/client/interface.rs
@@ -28,6 +28,7 @@ pub struct BenchmarkConsumerConfig {
pub messages_per_batch: BenchmarkNumericParameter,
pub warmup_time: IggyDuration,
pub polling_kind: PollingKind,
+ pub poll_options: DeferredPollOptions,
pub origin_timestamp_latency_calculation: bool,
pub pretty: bool,
}
diff --git a/core/bench/src/actors/consumer/client/low_level.rs b/core/bench/src/actors/consumer/client/low_level.rs
index 964231f62..84ef88d11 100644
--- a/core/bench/src/actors/consumer/client/low_level.rs
+++ b/core/bench/src/actors/consumer/client/low_level.rs
@@ -74,7 +74,7 @@ impl ConsumerClient for LowLevelConsumerClient {
};
let before_poll = Instant::now();
let polled = client
- .poll_messages_with_strategy_for(
+ .poll_messages_with_strategy_for_and_options(
&self.stream_id,
&self.topic_id,
self.partition_id,
@@ -82,6 +82,7 @@ impl ConsumerClient for LowLevelConsumerClient {
&strategy_for,
messages_to_receive,
self.auto_commit,
+ Some(self.config.poll_options),
)
.await;
diff --git a/core/bench/src/actors/consumer/typed_benchmark_consumer.rs b/core/bench/src/actors/consumer/typed_benchmark_consumer.rs
index a7aa1fad1..dc16eb372 100644
--- a/core/bench/src/actors/consumer/typed_benchmark_consumer.rs
+++ b/core/bench/src/actors/consumer/typed_benchmark_consumer.rs
@@ -54,6 +54,7 @@ impl TypedBenchmarkConsumer {
sampling_time: IggyDuration,
moving_average_window: u32,
polling_kind: PollingKind,
+ poll_options: DeferredPollOptions,
limit_bytes_per_second: Option<IggyByteSize>,
origin_timestamp_latency_calculation: bool,
pretty: bool,
@@ -65,6 +66,7 @@ impl TypedBenchmarkConsumer {
messages_per_batch,
warmup_time,
polling_kind,
+ poll_options,
origin_timestamp_latency_calculation,
pretty,
};
diff --git a/core/bench/src/actors/producing_consumer/typed_benchmark_producing_consumer.rs b/core/bench/src/actors/producing_consumer/typed_benchmark_producing_consumer.rs
index b193905f1..0f5c1ebb5 100644
--- a/core/bench/src/actors/producing_consumer/typed_benchmark_producing_consumer.rs
+++ b/core/bench/src/actors/producing_consumer/typed_benchmark_producing_consumer.rs
@@ -40,8 +40,8 @@ use bench_report::{
use iggy::prelude::*;
pub enum TypedBenchmarkProducingConsumer {
- High(BenchmarkProducingConsumer<HighLevelProducerClient, HighLevelConsumerClient>),
- Low(BenchmarkProducingConsumer<LowLevelProducerClient, LowLevelConsumerClient>),
+ High(Box<BenchmarkProducingConsumer<HighLevelProducerClient, HighLevelConsumerClient>>),
+ Low(Box<BenchmarkProducingConsumer<LowLevelProducerClient, LowLevelConsumerClient>>),
}
impl TypedBenchmarkProducingConsumer {
#[allow(clippy::too_many_arguments)]
@@ -62,6 +62,7 @@ impl TypedBenchmarkProducingConsumer {
moving_average_window: u32,
limit_bytes_per_second: Option<IggyByteSize>,
polling_kind: PollingKind,
+ poll_options: DeferredPollOptions,
origin_timestamp_latency_calculation: bool,
pretty: bool,
) -> Self {
@@ -82,6 +83,7 @@ impl TypedBenchmarkProducingConsumer {
messages_per_batch,
warmup_time,
polling_kind,
+ poll_options,
origin_timestamp_latency_calculation,
pretty,
};
@@ -90,7 +92,7 @@ impl TypedBenchmarkProducingConsumer {
let producer =
HighLevelProducerClient::new(client_factory.clone(), producer_config.clone());
let consumer = HighLevelConsumerClient::new(client_factory, consumer_config.clone());
- Self::High(BenchmarkProducingConsumer::new(
+ Self::High(Box::new(BenchmarkProducingConsumer::new(
producer,
consumer,
benchmark_kind,
@@ -101,12 +103,12 @@ impl TypedBenchmarkProducingConsumer {
limit_bytes_per_second,
producer_config,
consumer_config,
- ))
+ )))
} else {
let producer =
LowLevelProducerClient::new(client_factory.clone(), producer_config.clone());
let consumer = LowLevelConsumerClient::new(client_factory, consumer_config.clone());
- Self::Low(BenchmarkProducingConsumer::new(
+ Self::Low(Box::new(BenchmarkProducingConsumer::new(
producer,
consumer,
benchmark_kind,
@@ -117,7 +119,7 @@ impl TypedBenchmarkProducingConsumer {
limit_bytes_per_second,
producer_config,
consumer_config,
- ))
+ )))
}
}
diff --git a/core/bench/src/args/common.rs b/core/bench/src/args/common.rs
index da92b2e21..2aedfbb15 100644
--- a/core/bench/src/args/common.rs
+++ b/core/bench/src/args/common.rs
@@ -30,8 +30,8 @@ use bench_report::numeric_parameter::BenchmarkNumericParameter;
use clap::error::ErrorKind;
use clap::{CommandFactory, Parser};
use iggy::prelude::{
- DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, IggyByteSize, IggyDuration, IggyExpiry,
- TransportProtocol,
+ DEFAULT_POLL_MAX_BYTES, DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, DeferredPollOptions,
+ IggyByteSize, IggyDuration, IggyExpiry, TransportProtocol,
};
use std::num::NonZeroU32;
use std::str::FromStr;
@@ -52,6 +52,22 @@ pub struct IggyBenchArgs {
#[arg(long, short = 'P', value_parser = BenchmarkNumericParameter::from_str, default_value_t = BenchmarkNumericParameter::Value(DEFAULT_MESSAGES_PER_BATCH.get()))]
pub messages_per_batch: BenchmarkNumericParameter,
+ /// Maximum deliberate wait for the readiness minimum.
+ #[arg(long, default_value_t = IggyDuration::ONE_SECOND)]
+ pub max_wait: IggyDuration,
+
+ /// Readiness minimum, independent of the maximum messages per batch.
+ #[arg(long, default_value_t = 1)]
+ pub min_count: u32,
+
+ /// Maximum encoded poll reply bytes, including response and batch headers.
+ #[arg(long, default_value_t = DEFAULT_POLL_MAX_BYTES)]
+ pub max_bytes: u32,
+
+ /// Overall budget for routing, readiness, I/O and response handling.
+ #[arg(long, default_value_t = DeferredPollOptions::default().request_timeout)]
+ pub request_timeout: IggyDuration,
+
/// Number of message batches per actor (producer / consumer / producing consumer).
/// This argument is mutually exclusive with `total_messages_size`.
#[arg(long, short = 'b', group = "data_to_process")]
@@ -116,6 +132,15 @@ pub struct IggyBenchArgs {
}
impl IggyBenchArgs {
+ pub const fn poll_options(&self) -> DeferredPollOptions {
+ DeferredPollOptions {
+ max_wait: self.max_wait,
+ min_count: self.min_count,
+ max_bytes: self.max_bytes,
+ request_timeout: self.request_timeout,
+ }
+ }
+
pub fn transport_command(&self) -> &BenchmarkTransportCommand {
self.benchmark_kind.transport_command()
}
@@ -136,6 +161,11 @@ impl IggyBenchArgs {
}
pub fn validate(&mut self) {
+ if let Err(error) = self.poll_options().validate(self.messages_per_batch.min()) {
+ Self::command()
+ .error(ErrorKind::ValueValidation, error.to_string())
+ .exit();
+ }
if self.output_dir().is_none()
&& (self.gitref().is_some()
|| self.identifier().is_some()
@@ -409,6 +439,11 @@ impl IggyBenchArgs {
transport.to_string(),
];
+ parts.push(format!(
+ "wait{}_min{}_bytes{}_timeout{}",
+ self.max_wait, self.min_count, self.max_bytes, self.request_timeout
+ ));
+
if let Some(remark) = &self.remark() {
parts.push(remark.clone());
}
@@ -460,6 +495,11 @@ impl IggyBenchArgs {
self.messages_per_batch(),
);
+ name = format!(
+ "{name}, wait {}, min {}, max {}B, timeout {}",
+ self.max_wait, self.min_count, self.max_bytes, self.request_timeout
+ );
+
if let Some(remark) = &self.remark() {
name = format!("{name} ({remark})");
}
diff --git a/core/bench/src/benchmarks/common.rs b/core/bench/src/benchmarks/common.rs
index a941e3991..2b00e39a0 100644
--- a/core/bench/src/benchmarks/common.rs
+++ b/core/bench/src/benchmarks/common.rs
@@ -167,6 +167,7 @@ pub fn build_consumer_futures(
let moving_average_window = args.moving_average_window();
let kind = args.kind();
let pretty = args.pretty;
+ let poll_options = args.poll_options();
let polling_kind = if cg_count > 0 {
PollingKind::Next
} else {
@@ -235,6 +236,7 @@ pub fn build_consumer_futures(
sampling_time,
moving_average_window,
polling_kind,
+ poll_options,
rate_limit,
origin_timestamp_latency_calculation,
pretty,
@@ -299,6 +301,7 @@ pub fn build_producing_consumers_futures(
args_clone.moving_average_window(),
rate_limit,
polling_kind,
+ args_clone.poll_options(),
origin_timestamp_latency_calculation,
args_clone.pretty,
);
@@ -402,6 +405,7 @@ pub fn build_producing_consumer_groups_futures(
args_clone.moving_average_window(),
rate_limit,
polling_kind,
+ args_clone.poll_options(),
origin_timestamp_latency_calculation,
args_clone.pretty,
);
diff --git a/core/binary_protocol/src/codes.rs b/core/binary_protocol/src/codes.rs
index 3a21e1f6a..106dceefa 100644
--- a/core/binary_protocol/src/codes.rs
+++ b/core/binary_protocol/src/codes.rs
@@ -56,6 +56,8 @@ pub const SEND_MESSAGES_CODE: u32 = 101;
pub const FLUSH_UNSAVED_BUFFER_CODE: u32 = 102;
pub const GET_POLL_ROUTING_CODE: u32 = 103;
pub const POLL_MESSAGES_ON_PRIMARY_CODE: u32 = 104;
+pub const POLL_MESSAGES_DEFERRED_CODE: u32 = 105;
+pub const POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE: u32 = 106;
// -- Consumer Offsets --
pub const GET_CONSUMER_OFFSET_CODE: u32 = 120;
@@ -140,6 +142,8 @@ mod tests {
POLL_MESSAGES_CODE,
GET_POLL_ROUTING_CODE,
POLL_MESSAGES_ON_PRIMARY_CODE,
+ POLL_MESSAGES_DEFERRED_CODE,
+ POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE,
SEND_MESSAGES_CODE,
FLUSH_UNSAVED_BUFFER_CODE,
GET_CONSUMER_OFFSET_CODE,
diff --git a/core/binary_protocol/src/dispatch.rs b/core/binary_protocol/src/dispatch.rs
index ad9e2905c..7bab7c22a 100644
--- a/core/binary_protocol/src/dispatch.rs
+++ b/core/binary_protocol/src/dispatch.rs
@@ -189,6 +189,11 @@ pub const COMMAND_TABLE: &[CommandMeta] = &[
CommandMeta::non_replicated(GET_POLL_ROUTING_CODE, "message.poll_routing"),
CommandMeta::non_replicated(POLL_MESSAGES_ON_PRIMARY_CODE, "message.poll_primary"),
CommandMeta::non_replicated(GET_CONSUMER_OFFSET_ROUTING_CODE, "consumer_offset.routing"),
+ CommandMeta::non_replicated(POLL_MESSAGES_DEFERRED_CODE, "message.poll_deferred"),
+ CommandMeta::non_replicated(
+ POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE,
+ "message.poll_deferred_primary",
+ ),
];
/// Lookup command metadata by command code.
@@ -255,6 +260,8 @@ pub const fn lookup_command(code: u32) -> Option<&'static CommandMeta> {
GET_POLL_ROUTING_CODE => 52,
POLL_MESSAGES_ON_PRIMARY_CODE => 53,
GET_CONSUMER_OFFSET_ROUTING_CODE => 54,
+ POLL_MESSAGES_DEFERRED_CODE => 55,
+ POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE => 56,
_ => return None,
};
Some(&COMMAND_TABLE[idx])
diff --git a/core/binary_protocol/src/requests/messages/mod.rs b/core/binary_protocol/src/requests/messages/mod.rs
index 76584099d..5f2d1141a 100644
--- a/core/binary_protocol/src/requests/messages/mod.rs
+++ b/core/binary_protocol/src/requests/messages/mod.rs
@@ -20,5 +20,7 @@ pub mod poll_messages;
pub mod send_messages;
pub use flush_unsaved_buffer::FlushUnsavedBufferRequest;
-pub use poll_messages::PollMessagesRequest;
+pub use poll_messages::{
+ DeferredPollMessagesRequest, MAX_DEFERRED_POLL_WAIT_US, PollMessagesRequest,
+};
pub use send_messages::{RawMessage, SendMessagesEncoder, SendMessagesHeader};
diff --git a/core/binary_protocol/src/requests/messages/poll_messages.rs b/core/binary_protocol/src/requests/messages/poll_messages.rs
index 4c6413cfc..2b07cce7e 100644
--- a/core/binary_protocol/src/requests/messages/poll_messages.rs
+++ b/core/binary_protocol/src/requests/messages/poll_messages.rs
@@ -17,7 +17,7 @@
use crate::WireError;
use crate::WireIdentifier;
-use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le};
+use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le};
use crate::primitives::consumer::WireConsumer;
use crate::primitives::polling_strategy::WirePollingStrategy;
use bytes::{BufMut, BytesMut};
@@ -49,6 +49,10 @@ const STRATEGY_SIZE: usize = 9;
const COUNT_SIZE: usize = 4;
const AUTO_COMMIT_SIZE: usize = 1;
+/// Finite protocol ceiling; servers may configure a lower admission limit.
+pub const MAX_DEFERRED_POLL_WAIT_US: u64 = 600_000_000;
+pub const MAX_DEFERRED_POLL_TIMEOUT_US: u64 = MAX_DEFERRED_POLL_WAIT_US + 30_000_000;
+
impl WireEncode for PollMessagesRequest {
fn encoded_size(&self) -> usize {
self.consumer.encoded_size()
@@ -120,6 +124,70 @@ impl WireDecode for PollMessagesRequest {
}
}
+/// Readiness-based polling with explicit message and response-byte limits.
+/// New command codes prevent older servers from silently ignoring the wait.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DeferredPollMessagesRequest {
+ pub poll: PollMessagesRequest,
+ pub wait_us: u64,
+ pub min_count: u32,
+ pub max_bytes: u32,
+ pub request_timeout_us: u64,
+}
+
+impl WireEncode for DeferredPollMessagesRequest {
+ fn encoded_size(&self) -> usize {
+ self.poll.encoded_size() + 2 * size_of::<u64>() + 2 * size_of::<u32>()
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.poll.encode(buf);
+ buf.put_u64_le(self.wait_us);
+ buf.put_u32_le(self.min_count);
+ buf.put_u32_le(self.max_bytes);
+ buf.put_u64_le(self.request_timeout_us);
+ }
+}
+
+impl WireDecode for DeferredPollMessagesRequest {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let (poll, consumed) = PollMessagesRequest::decode(buf)?;
+ if buf.len() - consumed != 2 * size_of::<u64>() + 2 * size_of::<u32>() {
+ return Err(WireError::Validation(
+ "deferred poll requires wait, minimum count, byte limit and request timeout".into(),
+ ));
+ }
+ let wait_us = read_u64_le(buf, consumed)?;
+ let min_count = read_u32_le(buf, consumed + size_of::<u64>())?;
+ let max_bytes = read_u32_le(buf, consumed + size_of::<u64>() + size_of::<u32>())?;
+ let request_timeout_us =
+ read_u64_le(buf, consumed + size_of::<u64>() + 2 * size_of::<u32>())?;
+ if wait_us > MAX_DEFERRED_POLL_WAIT_US
+ || min_count == 0
+ || min_count > poll.count
+ || request_timeout_us == 0
+ || request_timeout_us < wait_us
+ || request_timeout_us > MAX_DEFERRED_POLL_TIMEOUT_US
+ || (max_bytes as usize)
+ < crate::responses::messages::poll_messages::POLL_RESPONSE_HEADER_SIZE
+ {
+ return Err(WireError::Validation(
+ "invalid deferred poll wait, message counts or byte limit".into(),
+ ));
+ }
+ Ok((
+ Self {
+ poll,
+ wait_us,
+ min_count,
+ max_bytes,
+ request_timeout_us,
+ },
+ buf.len(),
+ ))
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -217,4 +285,81 @@ mod tests {
);
}
}
+ #[test]
+ fn deferred_preserves_legacy_prefix_and_validates_limits() {
+ let request = DeferredPollMessagesRequest {
+ poll: PollMessagesRequest {
+ consumer: WireConsumer::consumer_group(WireIdentifier::named("group").unwrap()),
+ stream_id: WireIdentifier::named("stream").unwrap(),
+ topic_id: WireIdentifier::numeric(2),
+ partition_id: Some(3),
+ strategy: WirePollingStrategy::offset(42),
+ count: 5,
+ auto_commit: true,
+ },
+ wait_us: MAX_DEFERRED_POLL_WAIT_US,
+ min_count: 1,
+ max_bytes: 1024,
+ request_timeout_us: MAX_DEFERRED_POLL_TIMEOUT_US,
+ };
+ for wait_us in [0, 1, MAX_DEFERRED_POLL_WAIT_US] {
+ let request = DeferredPollMessagesRequest {
+ wait_us,
+ ..request.clone()
+ };
+ let bytes = request.to_bytes();
+ let prefix = request.poll.to_bytes();
+ assert_eq!(&bytes[..prefix.len()], prefix.as_ref());
+ assert_eq!(bytes.len(), prefix.len() + 24);
+ assert_eq!(
+ &bytes[prefix.len()..prefix.len() + 8],
+ &wait_us.to_le_bytes()
+ );
+ assert_eq!(
+ DeferredPollMessagesRequest::decode(&bytes).unwrap(),
+ (request, bytes.len())
+ );
+ for end in 0..bytes.len() {
+ assert!(DeferredPollMessagesRequest::decode(&bytes[..end]).is_err());
+ }
+ let mut extra = bytes.to_vec();
+ extra.push(0);
+ assert!(DeferredPollMessagesRequest::decode(&extra).is_err());
+ }
+ for invalid in [
+ DeferredPollMessagesRequest {
+ wait_us: MAX_DEFERRED_POLL_WAIT_US + 1,
+ ..request.clone()
+ },
+ DeferredPollMessagesRequest {
+ min_count: 0,
+ ..request.clone()
+ },
+ DeferredPollMessagesRequest {
+ min_count: 6,
+ ..request.clone()
+ },
+ DeferredPollMessagesRequest {
+ max_bytes: 15,
+ ..request.clone()
+ },
+ DeferredPollMessagesRequest {
+ request_timeout_us: 0,
+ ..request.clone()
+ },
+ DeferredPollMessagesRequest {
+ request_timeout_us: MAX_DEFERRED_POLL_WAIT_US - 1,
+ ..request.clone()
+ },
+ DeferredPollMessagesRequest {
+ request_timeout_us: MAX_DEFERRED_POLL_TIMEOUT_US + 1,
+ ..request
+ },
+ ] {
+ assert!(
+ DeferredPollMessagesRequest::decode(&invalid.to_bytes()).is_err(),
+ "accepted {invalid:?}"
+ );
+ }
+ }
}
diff --git a/core/binary_protocol/src/responses/messages/poll_messages.rs b/core/binary_protocol/src/responses/messages/poll_messages.rs
index ae7dce74f..ca1a16e70 100644
--- a/core/binary_protocol/src/responses/messages/poll_messages.rs
+++ b/core/binary_protocol/src/responses/messages/poll_messages.rs
@@ -30,7 +30,7 @@ use crate::error::WireError;
use bytes::{BufMut, BytesMut};
/// Size of the `PollMessages` response header: `partition_id(4) + current_offset(8) + count(4)`.
-const POLL_RESPONSE_HEADER_SIZE: usize = 16;
+pub const POLL_RESPONSE_HEADER_SIZE: usize = 16;
/// The 16-byte metadata prefix of a `PollMessages` response.
///
diff --git a/core/common/src/consumer_group_client_state.rs b/core/common/src/consumer_group_client_state.rs
index 0686bc742..45e899a5c 100644
--- a/core/common/src/consumer_group_client_state.rs
+++ b/core/common/src/consumer_group_client_state.rs
@@ -29,6 +29,7 @@
use crate::Identifier;
use std::collections::HashMap;
use std::sync::Mutex;
+use std::sync::atomic::{AtomicU64, Ordering};
#[derive(Debug, Default, Clone)]
struct GroupAssignment {
@@ -43,6 +44,7 @@ struct GroupAssignment {
#[derive(Debug, Default)]
pub struct ConsumerGroupClientState {
assignments: Mutex<HashMap<String, GroupAssignment>>,
+ session_generation: AtomicU64,
balanced_cursors: Mutex<HashMap<String, usize>>,
partition_counts: Mutex<HashMap<String, u32>>,
/// Identifiers of the joined groups, so the heartbeat can rebuild a sync
@@ -52,6 +54,40 @@ pub struct ConsumerGroupClientState {
}
impl ConsumerGroupClientState {
+ /// Snapshot of the session generation, assignment generation and partitions.
+ pub fn assignment_snapshot(&self, key: &str) -> (u64, u64, Vec<u32>) {
+ let assignments = self
+ .assignments
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ let session = self.session_generation.load(Ordering::Acquire);
+ assignments.get(key).map_or_else(
+ || (session, 0, Vec::new()),
+ |assignment| {
+ (
+ session,
+ assignment.generation,
+ assignment.partitions.clone(),
+ )
+ },
+ )
+ }
+
+ /// Validate a buffered batch against the latest locally observed assignment.
+ pub fn is_current_assignment(&self, key: &str, version: (u64, u64), partition: u32) -> bool {
+ if !self.is_registered(key) {
+ return false;
+ }
+ let assignments = self
+ .assignments
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ self.session_generation.load(Ordering::Acquire) == version.0
+ && assignments.get(key).is_some_and(|assignment| {
+ assignment.generation == version.1 && assignment.partitions.contains(&partition)
+ })
+ }
+
#[must_use]
pub fn new() -> Self {
Self::default()
@@ -193,10 +229,14 @@ impl ConsumerGroupClientState {
/// produce round-robin at partition 0 and cost a metadata round trip per
/// topic on every reconnect.
pub fn clear_session_scoped(&self) {
- self.assignments
- .lock()
- .unwrap_or_else(std::sync::PoisonError::into_inner)
- .clear();
+ {
+ let mut assignments = self
+ .assignments
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ assignments.clear();
+ self.session_generation.fetch_add(1, Ordering::Release);
+ }
self.joined_groups
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
@@ -269,7 +309,15 @@ mod tests {
assert_eq!(state.next_balanced_partition("s|t", 3), 0);
state.set_partition_count("s|t".to_owned(), 3);
+ let before = state.assignment_snapshot("s|t|g");
state.clear_session_scoped();
+ state.set_assignment("s|t|g".to_owned(), before.1, before.2.clone());
+ let after = state.assignment_snapshot("s|t|g");
+ assert_ne!(
+ before.0, after.0,
+ "same group generation in a new session must fence queued batches"
+ );
+ state.invalidate_assignment("s|t|g");
assert!(state.registered_groups().is_empty());
assert!(!state.is_registered("s|t|g"));
diff --git a/core/common/src/lib.rs b/core/common/src/lib.rs
index e74eae292..ad233e8e4 100644
--- a/core/common/src/lib.rs
+++ b/core/common/src/lib.rs
@@ -76,6 +76,7 @@ pub use http::streams::*;
pub use http::system::*;
pub use http::topics::*;
pub use http::users::*;
+pub use iggy_binary_protocol::requests::messages::MAX_DEFERRED_POLL_WAIT_US;
pub use iggy_binary_protocol::responses::messages::{
SendMessagesConfirmationResponse, SendMessagesResponse,
};
@@ -86,7 +87,6 @@ pub use traits::client::Client;
pub use traits::cluster_client::ClusterClient;
pub use traits::consumer_group_client::ConsumerGroupClient;
pub use traits::consumer_offset_client::ConsumerOffsetClient;
-pub use traits::decode_send_confirmations;
pub use traits::message_client::MessageClient;
pub use traits::partition_client::PartitionClient;
pub use traits::partitioner::Partitioner;
@@ -98,6 +98,7 @@ pub use traits::system_client::SystemClient;
pub use traits::topic_client::TopicClient;
pub use traits::user_client::UserClient;
pub use traits::validatable::Validatable;
+pub use traits::{decode_send_confirmations, sync_group_assignment};
pub use types::args::*;
pub use types::client::client_info::*;
pub use types::client_state::ClientState;
diff --git a/core/common/src/traits/binary_impls/messages.rs b/core/common/src/traits/binary_impls/messages.rs
index 2dd6679f9..d41891a5e 100644
--- a/core/common/src/traits/binary_impls/messages.rs
+++ b/core/common/src/traits/binary_impls/messages.rs
@@ -50,7 +50,11 @@ fn topic_cache_key(stream_id: &Identifier, topic_id: &Identifier) -> String {
/// Sync the requesting member's assignment from the coordinator into the
/// transport cache. An empty reply means the client is not a member.
-async fn sync_group_assignment<B: BinaryClient>(
+///
+/// # Errors
+/// Propagates transport and assignment-decoding failures.
+#[doc(hidden)]
+pub async fn sync_group_assignment<B: BinaryClient>(
client: &B,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -166,6 +170,7 @@ async fn resolve_partitioning<B: BinaryClient>(
/// (round-robin), ask `strategy_for` where to read it from and send an
/// explicit-partition poll. A coordinator fence rejection (stale assignment
/// after a rebalance) triggers one re-sync + retry.
+#[allow(clippy::too_many_arguments)]
async fn poll_group_messages<B: BinaryClient>(
client: &B,
stream_id: &Identifier,
@@ -174,7 +179,9 @@ async fn poll_group_messages<B: BinaryClient>(
strategy_for: &(dyn Fn(u32) -> PollingStrategy + Send + Sync),
count: u32,
auto_commit: bool,
+ options: Option<crate::DeferredPollOptions>,
) -> Result<PolledMessages, IggyError> {
+ let started = std::time::Instant::now();
let key = group_cache_key(stream_id, topic_id, &consumer.id);
if !client.consumer_group_state().has_assignment(&key) {
sync_group_assignment(client, stream_id, topic_id, &consumer.id).await?;
@@ -209,7 +216,13 @@ async fn poll_group_messages<B: BinaryClient>(
count,
auto_commit,
};
- match client.send_poll_with_response(&request).await {
+ let remaining = options
+ .map(|options| options.remaining(started.elapsed()))
+ .transpose()?;
+ match client
+ .send_poll_with_response_and_options(&request, remaining)
+ .await
+ {
Ok(response) => {
let polled = PolledMessages::from_bytes(response)?;
// The coordinator can't yet signal a generation fence as a typed
@@ -237,6 +250,9 @@ async fn poll_group_messages<B: BinaryClient>(
// empty poll rather than `ConsumerGroupPartitionNotOwned(0, 0)`: the (0, 0)
// ids are fabricated and a normal rebalance must not look like a hard error
// to a CG app that doesn't special-case 5009. The caller just re-polls.
+ if options.is_some() {
+ return Err(IggyError::TransientNotAccepted);
+ }
Ok(PolledMessages::empty())
}
@@ -305,6 +321,33 @@ impl<B: BinaryClient> MessageClient for B {
count: u32,
auto_commit: bool,
) -> Result<PolledMessages, IggyError> {
+ self.poll_messages_with_strategy_for_and_options(
+ stream_id,
+ topic_id,
+ partition_id,
+ consumer,
+ strategy_for,
+ count,
+ auto_commit,
+ None,
+ )
+ .await
+ }
+
+ async fn poll_messages_with_strategy_for_and_options(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: Option<u32>,
+ consumer: &Consumer,
+ strategy_for: &(dyn Fn(u32) -> PollingStrategy + Send + Sync),
+ count: u32,
+ auto_commit: bool,
+ options: Option<crate::DeferredPollOptions>,
+ ) -> Result<PolledMessages, IggyError> {
+ if let Some(options) = options {
+ options.validate(count)?;
+ }
fail_if_not_authenticated(self).await?;
// VSR: a consumer-group poll without an explicit partition is resolved
// client-side from the member's cached assignment (the broker routes
@@ -318,6 +361,7 @@ impl<B: BinaryClient> MessageClient for B {
strategy_for,
count,
auto_commit,
+ options,
)
.await;
}
@@ -331,7 +375,9 @@ impl<B: BinaryClient> MessageClient for B {
count,
auto_commit,
};
- let response = self.send_poll_with_response(&req).await?;
+ let response = self
+ .send_poll_with_response_and_options(&req, options)
+ .await?;
PolledMessages::from_bytes(response)
}
diff --git a/core/common/src/traits/binary_impls/mod.rs b/core/common/src/traits/binary_impls/mod.rs
index 58239f505..3ef3cae0f 100644
--- a/core/common/src/traits/binary_impls/mod.rs
+++ b/core/common/src/traits/binary_impls/mod.rs
@@ -27,7 +27,7 @@ mod system;
mod topics;
mod users;
-pub use messages::decode_send_confirmations;
+pub use messages::{decode_send_confirmations, sync_group_assignment};
use crate::IggyError;
use crate::http::users::defaults::{
diff --git a/core/common/src/traits/binary_transport.rs b/core/common/src/traits/binary_transport.rs
index 1f14f5ac7..74d63e178 100644
--- a/core/common/src/traits/binary_transport.rs
+++ b/core/common/src/traits/binary_transport.rs
@@ -56,6 +56,22 @@ pub trait BinaryTransport {
self.send_raw_with_response(POLL_MESSAGES_CODE, request.to_bytes())
.await
}
+ /// Positive waits require an isolated, bounded connection lease.
+ async fn send_poll_with_response_and_options(
+ &self,
+ request: &PollMessagesRequest,
+ options: Option<crate::DeferredPollOptions>,
+ ) -> Result<Bytes, IggyError>
+ where
+ Self: Sync,
+ {
+ if options.is_none() {
+ self.send_poll_with_response(request).await
+ } else {
+ Err(IggyError::FeatureUnavailable)
+ }
+ }
+
fn get_heartbeat_interval(&self) -> NonZeroIggyDuration;
/// Per-transport consumer-group + partitioning cache used to resolve
diff --git a/core/common/src/traits/message_client.rs b/core/common/src/traits/message_client.rs
index 1bd3ab13e..f2cd5c03d 100644
--- a/core/common/src/traits/message_client.rs
+++ b/core/common/src/traits/message_client.rs
@@ -112,6 +112,63 @@ pub trait MessageClient {
.await
}
+ /// Wait for the readiness minimum and return up to `count` messages.
+ /// Expiry returns an available partial; response bytes are independently bounded.
+ /// A lost response can follow progress admission; see `poll_messages`.
+ #[allow(clippy::too_many_arguments)]
+ async fn poll_messages_deferred(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: Option<u32>,
+ consumer: &Consumer,
+ strategy: &PollingStrategy,
+ count: u32,
+ auto_commit: bool,
+ options: crate::DeferredPollOptions,
+ ) -> Result<PolledMessages, IggyError> {
+ self.poll_messages_with_strategy_for_and_options(
+ stream_id,
+ topic_id,
+ partition_id,
+ consumer,
+ &|_| *strategy,
+ count,
+ auto_commit,
+ Some(options),
+ )
+ .await
+ }
+
+ /// Deferred polling with the strategy resolved after choosing a partition.
+ /// Custom transports must opt in; positive waits never silently downgrade.
+ #[allow(clippy::too_many_arguments)]
+ async fn poll_messages_with_strategy_for_and_options(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: Option<u32>,
+ consumer: &Consumer,
+ strategy_for: &(dyn Fn(u32) -> PollingStrategy + Send + Sync),
+ count: u32,
+ auto_commit: bool,
+ options: Option<crate::DeferredPollOptions>,
+ ) -> Result<PolledMessages, IggyError> {
+ if options.is_some() {
+ return Err(IggyError::FeatureUnavailable);
+ }
+ self.poll_messages_with_strategy_for(
+ stream_id,
+ topic_id,
+ partition_id,
+ consumer,
+ strategy_for,
+ count,
+ auto_commit,
+ )
+ .await
+ }
+
/// Send messages using specified partitioning strategy to the given stream and topic by unique IDs or names.
///
/// Authentication is required, and the permission to send the messages.
diff --git a/core/common/src/traits/mod.rs b/core/common/src/traits/mod.rs
index 2d8056a9b..95db88b38 100644
--- a/core/common/src/traits/mod.rs
+++ b/core/common/src/traits/mod.rs
@@ -35,4 +35,4 @@ pub(crate) mod topic_client;
pub(crate) mod user_client;
pub(crate) mod validatable;
-pub use binary_impls::decode_send_confirmations;
+pub use binary_impls::{decode_send_confirmations, sync_group_assignment};
diff --git a/core/common/src/types/message/deferred_poll_options.rs b/core/common/src/types/message/deferred_poll_options.rs
new file mode 100644
index 000000000..d5074f5cf
--- /dev/null
+++ b/core/common/src/types/message/deferred_poll_options.rs
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use serde::{Deserialize, Serialize};
+use std::time::Duration;
+
+use crate::{IggyDuration, IggyError, MAX_DEFERRED_POLL_WAIT_US};
+use iggy_binary_protocol::requests::messages::DeferredPollMessagesRequest;
+use iggy_binary_protocol::requests::messages::poll_messages::MAX_DEFERRED_POLL_TIMEOUT_US;
+
+pub const DEFAULT_POLL_MAX_BYTES: u32 = 1024 * 1024;
+pub const DEFAULT_POLL_REQUEST_TIMEOUT: Duration = Duration::from_secs(11);
+
+/// Readiness and response limits, independent of the maximum message count.
+/// The wait bounds batching delay; the transport's request timeout also allows
+/// bounded storage and network work. Byte limits include the binary response
+/// metadata and batch framing, including for HTTP's equivalent selection.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(default)]
+pub struct DeferredPollOptions {
+ pub max_wait: IggyDuration,
+ pub min_count: u32,
+ pub max_bytes: u32,
+ pub request_timeout: IggyDuration,
+}
+
+impl Default for DeferredPollOptions {
+ fn default() -> Self {
+ Self {
+ max_wait: IggyDuration::ONE_SECOND,
+ min_count: 1,
+ max_bytes: DEFAULT_POLL_MAX_BYTES,
+ request_timeout: DEFAULT_POLL_REQUEST_TIMEOUT.into(),
+ }
+ }
+}
+
+impl DeferredPollOptions {
+ pub fn validate(&self, count: u32) -> Result<(), IggyError> {
+ if self.min_count == 0 || self.min_count > count {
+ return Err(IggyError::InvalidMessagesCount);
+ }
+ if self.max_wait.as_micros() > MAX_DEFERRED_POLL_WAIT_US
+ || self.request_timeout.is_zero()
+ || self.request_timeout.as_micros() > MAX_DEFERRED_POLL_TIMEOUT_US
+ || self.max_wait.get_duration() > self.request_timeout.get_duration()
+ {
+ return Err(IggyError::InvalidCommand);
+ }
+ if self.max_bytes
+ < iggy_binary_protocol::responses::messages::poll_messages::POLL_RESPONSE_HEADER_SIZE
+ as u32
+ {
+ return Err(IggyError::InvalidSizeBytes);
+ }
+ Ok(())
+ }
+
+ /// Preserve one client budget across assignment refreshes and routing retries.
+ pub fn remaining(self, elapsed: Duration) -> Result<Self, IggyError> {
+ let request_timeout = self.request_timeout.get_duration().saturating_sub(elapsed);
+ if request_timeout.as_micros() == 0 {
+ return Err(IggyError::TransientNotCommitted);
+ }
+ Ok(Self {
+ max_wait: self.max_wait.get_duration().saturating_sub(elapsed).into(),
+ request_timeout: request_timeout.into(),
+ ..self
+ })
+ }
+}
+
+impl From<&DeferredPollMessagesRequest> for DeferredPollOptions {
+ fn from(request: &DeferredPollMessagesRequest) -> Self {
+ Self {
+ max_wait: request.wait_us.into(),
+ min_count: request.min_count,
+ max_bytes: request.max_bytes,
+ request_timeout: request.request_timeout_us.into(),
+ }
+ }
+}
diff --git a/core/common/src/types/message/mod.rs b/core/common/src/types/message/mod.rs
index 86f236054..1ec925041 100644
--- a/core/common/src/types/message/mod.rs
+++ b/core/common/src/types/message/mod.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+mod deferred_poll_options;
mod iggy_message;
mod index;
mod index_view;
@@ -35,6 +36,8 @@ mod user_headers;
pub const INDEX_SIZE: usize = 16;
+pub use deferred_poll_options::{DEFAULT_POLL_MAX_BYTES, DeferredPollOptions};
+
pub use crate::http::messages::poll_messages::PollMessages;
pub use crate::http::messages::send_messages::{
SendMessages, SendMessagesConfirmation, SendMessagesConfirmations,
diff --git a/core/common/src/types/message/polled_messages.rs b/core/common/src/types/message/polled_messages.rs
index d974fa479..10c330ff6 100644
--- a/core/common/src/types/message/polled_messages.rs
+++ b/core/common/src/types/message/polled_messages.rs
@@ -17,7 +17,9 @@
use crate::{IggyMessage, IggyMessageHeader, error::IggyError};
use bytes::Bytes;
-use iggy_binary_protocol::batch::{BATCH_HEADER_SIZE, BatchHeader, BatchMessageHeader};
+use iggy_binary_protocol::batch::{
+ BATCH_HEADER_SIZE, BATCH_MESSAGE_HEADER_SIZE, BatchHeader, BatchMessageHeader,
+};
use serde::{Deserialize, Serialize};
use tracing::error;
@@ -97,6 +99,9 @@ impl PolledMessages {
/// values. Payload and user-header `Bytes` are zero-copy slices of the
/// response buffer.
fn messages_from_batches(buffer: Bytes, count: u32) -> Result<Vec<IggyMessage>, IggyError> {
+ if count as usize > buffer.len() / BATCH_MESSAGE_HEADER_SIZE {
+ return Err(IggyError::InvalidMessagesCount);
+ }
let mut messages = Vec::with_capacity(count as usize);
let mut position = 0usize;
while position < buffer.len() {
@@ -140,6 +145,9 @@ fn messages_from_batches(buffer: Bytes, count: u32) -> Result<Vec<IggyMessage>,
} else {
None
};
+ if messages.len() == count as usize {
+ return Err(IggyError::InvalidMessagesCount);
+ }
messages.push(IggyMessage {
header,
payload,
@@ -150,5 +158,23 @@ fn messages_from_batches(buffer: Bytes, count: u32) -> Result<Vec<IggyMessage>,
position = batch_end;
}
+ if messages.len() != count as usize {
+ return Err(IggyError::InvalidMessagesCount);
+ }
Ok(messages)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn rejects_impossible_count_before_allocating_messages() {
+ let mut prefix = vec![0; 16];
+ prefix[12..16].copy_from_slice(&u32::MAX.to_le_bytes());
+ assert!(matches!(
+ PolledMessages::from_bytes(prefix.into()),
+ Err(IggyError::InvalidMessagesCount)
+ ));
+ }
+}
diff --git a/core/configs/src/server_config/sharding.rs b/core/configs/src/server_config/sharding.rs
index 03fb95f94..76e3e0fdc 100644
--- a/core/configs/src/server_config/sharding.rs
+++ b/core/configs/src/server_config/sharding.rs
@@ -19,6 +19,7 @@
//! defaults read from the embedded `core/server/config.toml`.
use iggy_common::IggyDuration;
+use iggy_common::MAX_DEFERRED_POLL_WAIT_US;
use iggy_common::Validatable;
use serde::{Deserialize, Serialize};
use serde_with::{DisplayFromStr, serde_as};
@@ -101,6 +102,11 @@ pub struct ShardingConfig {
/// rejects new disk polls before I/O. Main and reply inbox traffic uses
/// separate capacity. This counts operations, not retained message bytes.
pub poll_completion_capacity: usize,
+ pub deferred_poll_max_wait_us: u64,
+ pub deferred_poll_max_pending: usize,
+ pub deferred_poll_max_pending_per_session: usize,
+ pub deferred_poll_max_read_bytes: usize,
+ pub deferred_poll_max_inflight_bytes: usize,
/// Wall-clock budget for a single shard's bus drain on shutdown.
/// Drives `IggyMessageBus::shutdown(..)` from the per-shard watchdog
/// and the parallel-join survivor path. Sized larger than typical
@@ -149,6 +155,18 @@ impl Default for ShardingConfig {
inbox_capacity: SERVER_CONFIG.sharding.inbox_capacity as usize,
reply_inbox_capacity: SERVER_CONFIG.sharding.reply_inbox_capacity as usize,
poll_completion_capacity: SERVER_CONFIG.sharding.poll_completion_capacity as usize,
+ deferred_poll_max_wait_us: SERVER_CONFIG.sharding.deferred_poll_max_wait_us as u64,
+ deferred_poll_max_pending: SERVER_CONFIG.sharding.deferred_poll_max_pending as usize,
+ deferred_poll_max_pending_per_session: SERVER_CONFIG
+ .sharding
+ .deferred_poll_max_pending_per_session
+ as usize,
+ deferred_poll_max_read_bytes: SERVER_CONFIG.sharding.deferred_poll_max_read_bytes
+ as usize,
+ deferred_poll_max_inflight_bytes: SERVER_CONFIG
+ .sharding
+ .deferred_poll_max_inflight_bytes
+ as usize,
shutdown_drain_timeout: SERVER_CONFIG
.sharding
.shutdown_drain_timeout
@@ -223,6 +241,21 @@ impl Validatable<ConfigurationError> for ShardingConfig {
return Err(ConfigurationError::InvalidConfigurationValue);
}
+ const MAX_DEFERRED_BYTES: usize = 1024 * 1024 * 1024;
+ if self.deferred_poll_max_wait_us == 0
+ || self.deferred_poll_max_wait_us > MAX_DEFERRED_POLL_WAIT_US
+ || self.deferred_poll_max_pending == 0
+ || self.deferred_poll_max_pending > INBOX_CAPACITY_MAX
+ || self.deferred_poll_max_pending_per_session == 0
+ || self.deferred_poll_max_pending_per_session > self.deferred_poll_max_pending
+ || self.deferred_poll_max_read_bytes == 0
+ || self.deferred_poll_max_read_bytes > self.deferred_poll_max_inflight_bytes
+ || self.deferred_poll_max_inflight_bytes > MAX_DEFERRED_BYTES
+ {
+ eprintln!("Invalid sharding configuration: inconsistent deferred poll limits");
+ return Err(ConfigurationError::InvalidConfigurationValue);
+ }
+
let drain = self.shutdown_drain_timeout.get_duration();
if drain.is_zero() {
eprintln!(
diff --git a/core/connectors/runtime/README.md b/core/connectors/runtime/README.md
index eb907521e..5549f321c 100644
--- a/core/connectors/runtime/README.md
+++ b/core/connectors/runtime/README.md
@@ -49,6 +49,10 @@ Supported scalar fields and indexed list entries use environment variables with
Source destination topics must persist every acknowledged batch before the runtime checkpoints the source or invokes its Ack hook. Missing topics are therefore created with `durability = "persisted"` and `messages_required_to_save = 1`. An existing topic must use `durability = "persisted"`; its save threshold may differ because persisted acknowledgments already wait for durable storage.
+Sink consumers use the Rust SDK's default long polling. A configured stream
+`poll_interval` is deprecated: the runtime logs a warning and ignores it.
+Existing explicit sink auto-commit policies still apply.
+
## State storage
Source plugins can supply optional checkpoint bytes. The runtime stores these opaque bytes using the backend selected by `state.storage`:
diff --git a/core/connectors/runtime/src/sink.rs b/core/connectors/runtime/src/sink.rs
index 994cfc392..37931c750 100644
--- a/core/connectors/runtime/src/sink.rs
+++ b/core/connectors/runtime/src/sink.rs
@@ -28,8 +28,7 @@ use crate::{
use dlopen2::wrapper::Container;
use futures::StreamExt;
use iggy::prelude::{
- AutoCommit, AutoCommitWhen, IggyClient, IggyConsumer, IggyDuration, IggyMessage,
- PollingStrategy,
+ AutoCommit, AutoCommitWhen, IggyClient, IggyConsumer, IggyMessage, PollingStrategy,
};
use iggy_connector_sdk::decoders::avro::{AvroConfig, AvroStreamDecoder};
use iggy_connector_sdk::{
@@ -38,7 +37,6 @@ use iggy_connector_sdk::{
};
use std::{
collections::HashMap,
- str::FromStr,
sync::{Arc, atomic::Ordering},
time::{Duration, Instant},
};
@@ -501,12 +499,12 @@ pub(crate) async fn setup_sink_consumers(
let mut consumers = Vec::new();
for stream in config.streams.iter() {
- let poll_interval = IggyDuration::from_str(
- stream.poll_interval.as_deref().unwrap_or("5ms"),
- )
- .map_err(|error| {
- RuntimeError::InvalidConfiguration(format!("Invalid poll interval: {error}"))
- })?;
+ if stream.poll_interval.is_some() {
+ warn!(
+ "Sink connector: {key}, stream: {} configures deprecated poll_interval; the consumer now uses long polling",
+ stream.stream
+ );
+ }
let default_consumer_group = format!("iggy-connect-sink-{key}");
let consumer_group = stream
.consumer_group
@@ -520,7 +518,6 @@ pub(crate) async fn setup_sink_consumers(
.create_consumer_group_if_not_exists()
.auto_join_consumer_group()
.polling_strategy(PollingStrategy::next())
- .poll_interval(poll_interval)
.batch_length(batch_length)
.build();
consumer.init().await?;
diff --git a/core/integration/tests/sdk/consumer_group_membership.rs b/core/integration/tests/sdk/consumer_group_membership.rs
index a8e7f59e7..5ad71a670 100644
--- a/core/integration/tests/sdk/consumer_group_membership.rs
+++ b/core/integration/tests/sdk/consumer_group_membership.rs
@@ -105,7 +105,6 @@ async fn given_group_member_holds_no_partitions_when_group_deleted_should_surfac
.consumer_group(CONSUMER_GROUP_NAME, STREAM_NAME, TOPIC_NAME)
.unwrap()
.batch_length(1)
- .poll_interval(IggyDuration::new(Duration::from_millis(100)))
.auto_join_consumer_group()
.do_not_create_consumer_group_if_not_exists()
.build();
diff --git a/core/integration/tests/sdk/consumer_offset.rs b/core/integration/tests/sdk/consumer_offset.rs
index c49cd16af..8aa4ad414 100644
--- a/core/integration/tests/sdk/consumer_offset.rs
+++ b/core/integration/tests/sdk/consumer_offset.rs
@@ -143,7 +143,7 @@ async fn given_standalone_stream_consumer_when_creating_topic_should_select_part
.create_topic_if_not_exists(true)
.partitions_count(partitions_count)
.maybe_partition_id(partition_id)
- .polling_interval("1ms".parse().unwrap())
+ .poll_options(iggy::prelude::DeferredPollOptions::default())
.polling_strategy(PollingStrategy::first())
.polling_retry_interval(NonZeroIggyDuration::ONE_SECOND)
.init_retries(0)
diff --git a/core/integration/tests/server/poll_semantics_vsr.rs b/core/integration/tests/server/poll_semantics_vsr.rs
index 243039742..1888fc11c 100644
--- a/core/integration/tests/server/poll_semantics_vsr.rs
+++ b/core/integration/tests/server/poll_semantics_vsr.rs
@@ -24,8 +24,11 @@
//! timestamp poll must be at-or-after, including the message stamped exactly at
//! the queried timestamp (the timestamp replies report per message).
+use futures::StreamExt;
use iggy::prelude::*;
use integration::iggy_harness;
+use std::str::FromStr;
+use std::sync::Arc;
use tokio::time::{Duration, sleep};
#[iggy_harness(
@@ -358,3 +361,737 @@ async fn given_message_at_polled_timestamp_when_polling_should_include_it(harnes
"the message at the queried timestamp is the one returned"
);
}
+
+#[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic, Http])]
+async fn given_deferred_poll_when_messages_arrive_should_fill_count_and_keep_control_available(
+ harness: &TestHarness,
+) {
+ let client = harness.root_client().await.unwrap();
+ client.create_stream("deferred-stream").await.unwrap();
+ let stream = Identifier::from_str_value("deferred-stream").unwrap();
+ client
+ .create_topic(
+ &stream,
+ "deferred-topic",
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ let topic = Identifier::from_str_value("deferred-topic").unwrap();
+ let consumer = Consumer::default();
+ let strategy = PollingStrategy::first();
+ let mut poll = Box::pin(client.poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &consumer,
+ &strategy,
+ 2,
+ false,
+ DeferredPollOptions {
+ max_wait: 2_000_000.into(),
+ min_count: 2,
+ ..Default::default()
+ },
+ ));
+ tokio::select! {
+ result = &mut poll => panic!("empty poll completed before arrivals: {result:?}"),
+ () = sleep(Duration::from_millis(50)) => {}
+ }
+ let mut first = vec![
+ IggyMessage::builder()
+ .payload("first".into())
+ .build()
+ .unwrap(),
+ ];
+ client
+ .send_messages(&stream, &topic, &Partitioning::partition_id(0), &mut first)
+ .await
+ .unwrap();
+ client
+ .ping()
+ .await
+ .expect("control connection remains available");
+ tokio::select! {
+ result = &mut poll => panic!("partial poll completed before count or deadline: {result:?}"),
+ () = sleep(Duration::from_millis(50)) => {}
+ }
+ let mut second = vec![
+ IggyMessage::builder()
+ .payload("second".into())
+ .build()
+ .unwrap(),
+ ];
+ client
+ .send_messages(&stream, &topic, &Partitioning::partition_id(0), &mut second)
+ .await
+ .unwrap();
+ let result = tokio::time::timeout(Duration::from_secs(3), poll)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ result
+ .messages
+ .iter()
+ .map(|message| message.header.offset)
+ .collect::<Vec<_>>(),
+ [0, 1]
+ );
+ let started = tokio::time::Instant::now();
+ let partial = client
+ .poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &consumer,
+ &strategy,
+ 3,
+ false,
+ DeferredPollOptions {
+ max_wait: 200_000.into(),
+ min_count: 3,
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ assert_eq!(partial.messages.len(), 2);
+ assert!(
+ started.elapsed() >= Duration::from_millis(150),
+ "partial result returned before deadline"
+ );
+ let empty = client
+ .poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &consumer,
+ &PollingStrategy::offset(2),
+ 1,
+ false,
+ DeferredPollOptions {
+ max_wait: 50_000.into(),
+ min_count: 1,
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ assert!(empty.messages.is_empty());
+ let immediate = client
+ .poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &consumer,
+ &strategy,
+ 3,
+ false,
+ DeferredPollOptions {
+ max_wait: 0.into(),
+ min_count: 3,
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ assert_eq!(immediate.messages.len(), 2);
+}
+
+#[iggy_harness(
+ test_client_transport = [Tcp, WebSocket, Quic, Http],
+ cluster_nodes = 3,
+ server(
+ http.jwt.encoding_secret = "0123456789abcdef0123456789abcdef",
+ http.jwt.decoding_secret = "0123456789abcdef0123456789abcdef"
+ )
+)]
+async fn given_deferred_auto_commit_when_cancelled_should_preserve_unseen_messages(
+ harness: &TestHarness,
+) {
+ let client = if harness.transport().unwrap() == TransportProtocol::Http {
+ harness
+ .node(1)
+ .http_client()
+ .unwrap()
+ .with_root_login()
+ .connect()
+ .await
+ .unwrap()
+ } else {
+ harness.root_client().await.unwrap()
+ };
+ client.create_stream("cancel-stream").await.unwrap();
+ let stream = Identifier::from_str_value("cancel-stream").unwrap();
+ client
+ .create_topic(
+ &stream,
+ "cancel-topic",
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ let topic = Identifier::from_str_value("cancel-topic").unwrap();
+ let consumer = Consumer::default();
+ let strategy = PollingStrategy::next();
+ let mut messages = vec![
+ IggyMessage::builder()
+ .payload("unseen".into())
+ .build()
+ .unwrap(),
+ ];
+ client
+ .send_messages(
+ &stream,
+ &topic,
+ &Partitioning::partition_id(0),
+ &mut messages,
+ )
+ .await
+ .unwrap();
+ let mut cancelled = Box::pin(client.poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &consumer,
+ &strategy,
+ 2,
+ true,
+ DeferredPollOptions {
+ max_wait: 200_000.into(),
+ min_count: 2,
+ ..Default::default()
+ },
+ ));
+ tokio::select! {
+ result = &mut cancelled => panic!("partial result completed before cancellation: {result:?}"),
+ () = sleep(Duration::from_millis(50)) => {}
+ }
+ drop(cancelled);
+ sleep(Duration::from_millis(250)).await;
+ let next = client
+ .poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &consumer,
+ &strategy,
+ 1,
+ false,
+ DeferredPollOptions {
+ max_wait: 200_000.into(),
+ min_count: 1,
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ next.messages.len(),
+ 1,
+ "cancelled poll must not accept its partial result"
+ );
+ assert_eq!(next.messages[0].header.offset, 0);
+
+ let committed = client
+ .poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &consumer,
+ &strategy,
+ 1,
+ true,
+ DeferredPollOptions {
+ max_wait: 500_000.into(),
+ min_count: 1,
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ assert_eq!(committed.messages[0].header.offset, 0);
+ let next = client
+ .poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &consumer,
+ &strategy,
+ 1,
+ true,
+ DeferredPollOptions {
+ max_wait: 200_000.into(),
+ min_count: 1,
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ assert!(
+ next.messages.is_empty(),
+ "accepted automatic commit advances Next"
+ );
+}
+
+#[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic])]
+async fn given_deferred_group_when_one_partition_is_empty_should_deliver_other_partition(
+ harness: &TestHarness,
+) {
+ let client = harness.root_client().await.unwrap();
+ client.create_stream("group-stream").await.unwrap();
+ let stream = Identifier::from_str_value("group-stream").unwrap();
+ client
+ .create_topic(
+ &stream,
+ "group-topic",
+ &TopicCreateOptions {
+ partitions_count: Some(2),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ let topic = Identifier::from_str_value("group-topic").unwrap();
+ let mut consumer = client
+ .consumer_group("deferred-group", "group-stream", "group-topic")
+ .unwrap()
+ .create_consumer_group_if_not_exists()
+ .auto_join_consumer_group()
+ .auto_commit(AutoCommit::Disabled)
+ .polling_strategy(PollingStrategy::first())
+ .batch_length(1)
+ .poll_options(DeferredPollOptions {
+ max_wait: Duration::from_secs(2).into(),
+ ..Default::default()
+ })
+ .build();
+ consumer.init().await.unwrap();
+ let mut messages = vec![
+ IggyMessage::builder()
+ .payload("ready".into())
+ .build()
+ .unwrap(),
+ ];
+ client
+ .send_messages(
+ &stream,
+ &topic,
+ &Partitioning::partition_id(1),
+ &mut messages,
+ )
+ .await
+ .unwrap();
+ let message = tokio::time::timeout(Duration::from_secs(1), consumer.next())
+ .await
+ .expect("empty partition must not hold up the ready partition")
+ .unwrap()
+ .unwrap();
+ assert_eq!(message.partition_id, 1);
+ consumer.shutdown().await.unwrap();
+}
+
+#[iggy_harness(
+ test_client_transport = [Tcp],
+ server(heartbeat.enabled = true, heartbeat.interval = "500ms")
+)]
+async fn given_deferred_poll_when_wait_exceeds_heartbeat_window_should_keep_sessions_alive(
+ harness: &TestHarness,
+) {
+ let client = TcpClient::create(Arc::new(TcpClientConfig {
+ server_address: harness.server().raw_tcp_addr().unwrap(),
+ heartbeat_interval: NonZeroIggyDuration::from_str("100ms").unwrap(),
+ nodelay: true,
+ ..TcpClientConfig::default()
+ }))
+ .unwrap();
+ Client::connect(&client).await.unwrap();
+ let client = IggyClient::create(ClientWrapper::Tcp(client), None, None);
+ client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ client.create_stream("live-stream").await.unwrap();
+ let stream = Identifier::named("live-stream").unwrap();
+ client
+ .create_topic(
+ &stream,
+ "live-topic",
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ let topic = Identifier::named("live-topic").unwrap();
+ client
+ .create_consumer_group(&stream, &topic, "live-group")
+ .await
+ .unwrap();
+ let group = Identifier::named("live-group").unwrap();
+ client
+ .join_consumer_group(&stream, &topic, &group)
+ .await
+ .unwrap();
+
+ let started = tokio::time::Instant::now();
+ let result = client
+ .poll_messages_deferred(
+ &stream,
+ &topic,
+ None,
+ &Consumer::group(group.clone()),
+ &PollingStrategy::first(),
+ 1,
+ false,
+ DeferredPollOptions {
+ max_wait: 2_000_000.into(),
+ min_count: 1,
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ assert_eq!(result.partition_id, 0);
+ assert!(result.messages.is_empty());
+ assert!(started.elapsed() >= Duration::from_millis(1800));
+ client.ping().await.unwrap();
+ assert_eq!(
+ client
+ .get_consumer_group(&stream, &topic, &group)
+ .await
+ .unwrap()
+ .unwrap()
+ .members_count,
+ 1,
+ "the parent session must remain a group member throughout the deferred wait"
+ );
+}
+
+#[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic, Http])]
+async fn given_large_batch_cap_when_one_message_arrives_should_return_without_waiting_for_batch(
+ harness: &TestHarness,
+) {
+ let client = harness.root_client().await.unwrap();
+ client.create_stream("ready-stream").await.unwrap();
+ let stream = Identifier::named("ready-stream").unwrap();
+ client
+ .create_topic(
+ &stream,
+ "topic",
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ let topic = Identifier::named("topic").unwrap();
+ let consumer = Consumer::default();
+ let strategy = PollingStrategy::offset(0);
+ let mut poll = Box::pin(client.poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &consumer,
+ &strategy,
+ 100,
+ false,
+ DeferredPollOptions {
+ max_wait: Duration::from_secs(5).into(),
+ request_timeout: Duration::from_secs(5).into(),
+ ..Default::default()
+ },
+ ));
+ tokio::select! {
+ result = &mut poll => panic!("completed without an arrival: {result:?}"),
+ () = sleep(Duration::from_millis(50)) => {}
+ }
+ let mut messages = vec![IggyMessage::from_str("ready").unwrap()];
+ client
+ .send_messages(
+ &stream,
+ &topic,
+ &Partitioning::partition_id(0),
+ &mut messages,
+ )
+ .await
+ .unwrap();
+ let polled = tokio::time::timeout(Duration::from_secs(2), poll)
+ .await
+ .expect("minimum is one, not 100")
+ .unwrap();
+ assert_eq!(polled.messages.len(), 1);
+}
+
+#[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic, Http])]
+async fn given_byte_limit_when_batch_is_larger_should_return_prefix_without_skipping_offsets(
+ harness: &TestHarness,
+) {
+ const PAYLOAD: &str = "payload";
+ let client = harness.root_client().await.unwrap();
+ client.create_stream("bytes-stream").await.unwrap();
+ let stream = Identifier::named("bytes-stream").unwrap();
+ client
+ .create_topic(
+ &stream,
+ "topic",
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ let topic = Identifier::named("topic").unwrap();
+ let mut messages = (0..3)
+ .map(|_| IggyMessage::from_str(PAYLOAD).unwrap())
+ .collect::<Vec<_>>();
+ client
+ .send_messages(
+ &stream,
+ &topic,
+ &Partitioning::partition_id(0),
+ &mut messages,
+ )
+ .await
+ .unwrap();
+ let max_bytes =
+ (iggy_binary_protocol::responses::messages::poll_messages::POLL_RESPONSE_HEADER_SIZE
+ + iggy_binary_protocol::batch::BATCH_HEADER_SIZE
+ + iggy_binary_protocol::batch::BATCH_MESSAGE_HEADER_SIZE
+ + PAYLOAD.len()) as u32;
+ let options = DeferredPollOptions {
+ min_count: 3,
+ max_bytes,
+ ..Default::default()
+ };
+ for offset in 0..3 {
+ let polled = tokio::time::timeout(
+ Duration::from_millis(700),
+ client.poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &Consumer::default(),
+ &PollingStrategy::next(),
+ 3,
+ true,
+ options,
+ ),
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(polled.messages.len(), 1);
+ assert_eq!(polled.messages[0].header.offset, offset);
+ }
+ let oversized = client
+ .poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &Consumer::default(),
+ &PollingStrategy::offset(0),
+ 3,
+ false,
+ DeferredPollOptions {
+ max_bytes: max_bytes - 1,
+ ..options
+ },
+ )
+ .await;
+ let error = oversized.expect_err("an oversized first record must fail, never disappear");
+ let code = if let IggyError::HttpResponseError(_, body) = &error {
+ serde_json::from_str::<serde_json::Value>(body).unwrap()["id"]
+ .as_u64()
+ .unwrap()
+ } else {
+ u64::from(error.as_code())
+ };
+ assert_eq!(
+ code,
+ u64::from(IggyError::InvalidSizeBytes.as_code()),
+ "{error:?}"
+ );
+ let replay = client
+ .poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(0),
+ &Consumer::default(),
+ &PollingStrategy::offset(0),
+ 3,
+ false,
+ options,
+ )
+ .await
+ .unwrap();
+ assert_eq!(replay.messages[0].header.offset, 0);
+}
+
+#[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic, Http])]
+async fn given_default_consumer_when_restarted_before_commit_should_replay_unprocessed_messages(
+ harness: &TestHarness,
+) {
+ let client = harness.root_client().await.unwrap();
+ client.create_stream("safe-stream").await.unwrap();
+ let stream = Identifier::named("safe-stream").unwrap();
+ client
+ .create_topic(
+ &stream,
+ "topic",
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ let topic = Identifier::named("topic").unwrap();
+ let mut messages = (0..6)
+ .map(|_| IggyMessage::from_str("uncommitted").unwrap())
+ .collect::<Vec<_>>();
+ client
+ .send_messages(
+ &stream,
+ &topic,
+ &Partitioning::partition_id(0),
+ &mut messages,
+ )
+ .await
+ .unwrap();
+ let mut consumer = client
+ .consumer("safe-reader", "safe-stream", "topic", 0)
+ .unwrap()
+ .batch_length(2)
+ .prefetch_messages(4)
+ .build();
+ consumer.init().await.unwrap();
+ for offset in 0..6 {
+ let received = tokio::time::timeout(Duration::from_secs(2), consumer.next())
+ .await
+ .expect("explicit fetch position must advance without a commit")
+ .unwrap()
+ .unwrap();
+ assert_eq!(received.message.header.offset, offset);
+ }
+ drop(consumer);
+ let identity = Consumer::new(Identifier::named("safe-reader").unwrap());
+ assert!(
+ client
+ .get_consumer_offset(&identity, &stream, &topic, Some(0))
+ .await
+ .unwrap()
+ .is_none()
+ );
+ let mut restarted = client
+ .consumer("safe-reader", "safe-stream", "topic", 0)
+ .unwrap()
+ .batch_length(2)
+ .build();
+ restarted.init().await.unwrap();
+ let first = tokio::time::timeout(Duration::from_secs(2), restarted.next())
+ .await
+ .unwrap()
+ .unwrap()
+ .unwrap();
+ assert_eq!(first.message.header.offset, 0);
+ restarted.store_offset(0, Some(0)).await.unwrap();
+ drop(restarted);
+ let mut committed = client
+ .consumer("safe-reader", "safe-stream", "topic", 0)
+ .unwrap()
+ .build();
+ committed.init().await.unwrap();
+ let next = tokio::time::timeout(Duration::from_secs(2), committed.next())
+ .await
+ .unwrap()
+ .unwrap()
+ .unwrap();
+ assert_eq!(next.message.header.offset, 1);
+ committed.shutdown().await.unwrap();
+}
+
+#[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic])]
+async fn given_buffered_group_batch_when_membership_is_revoked_should_discard_queued_messages(
+ harness: &TestHarness,
+) {
+ let client = harness.root_client().await.unwrap();
+ client.create_stream("revoke-stream").await.unwrap();
+ let stream = Identifier::named("revoke-stream").unwrap();
+ client
+ .create_topic(
+ &stream,
+ "topic",
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ let topic = Identifier::named("topic").unwrap();
+ let group = Identifier::named("readers").unwrap();
+ client
+ .create_consumer_group(&stream, &topic, "readers")
+ .await
+ .unwrap();
+ client
+ .join_consumer_group(&stream, &topic, &group)
+ .await
+ .unwrap();
+ let mut messages = (0..10)
+ .map(|_| IggyMessage::from_str("buffered").unwrap())
+ .collect::<Vec<_>>();
+ client
+ .send_messages(
+ &stream,
+ &topic,
+ &Partitioning::partition_id(0),
+ &mut messages,
+ )
+ .await
+ .unwrap();
+ let mut consumer = client
+ .consumer_group("readers", "revoke-stream", "topic")
+ .unwrap()
+ .do_not_auto_join_consumer_group()
+ .batch_length(10)
+ .prefetch_messages(10)
+ .build();
+ consumer.init().await.unwrap();
+ assert_eq!(
+ consumer
+ .next()
+ .await
+ .unwrap()
+ .unwrap()
+ .message
+ .header
+ .offset,
+ 0
+ );
+ client
+ .leave_consumer_group(&stream, &topic, &group)
+ .await
+ .unwrap();
+ sleep(Duration::from_millis(1200)).await;
+ let received = tokio::time::timeout(Duration::from_secs(2), consumer.next())
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(
+ received.is_err(),
+ "revoked prefetched messages must never reach the application"
+ );
+ consumer.shutdown().await.unwrap();
+}
diff --git a/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs
index 05350f847..277e51304 100644
--- a/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs
+++ b/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs
@@ -135,7 +135,6 @@ async fn create_auto_commit_consumer(client: &IggyClient) -> IggyConsumer {
.consumer_group(CONSUMER_GROUP_NAME, STREAM_NAME, TOPIC_NAME)
.unwrap()
.batch_length(10)
- .poll_interval(IggyDuration::from_str("100ms").expect("Invalid duration"))
.polling_strategy(PollingStrategy::next())
.auto_join_consumer_group()
.auto_commit(AutoCommit::IntervalOrAfter(
diff --git a/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs
index ad4c2012f..ed188c313 100644
--- a/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs
+++ b/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs
@@ -219,7 +219,6 @@ async fn create_consumer(client: &IggyClient) -> IggyConsumer {
.consumer_group(CONSUMER_GROUP_NAME, STREAM_NAME, TOPIC_NAME)
.unwrap()
.batch_length(10)
- .poll_interval(IggyDuration::from_str("100ms").unwrap())
.polling_strategy(PollingStrategy::next())
.auto_join_consumer_group()
.create_consumer_group_if_not_exists()
diff --git a/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs
index f7d1fb08c..bb36fef77 100644
--- a/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs
+++ b/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs
@@ -153,7 +153,6 @@ async fn create_consumer(client: &IggyClient, consumer_group_name: &str) -> Iggy
.consumer_group(consumer_group_name, STREAM_NAME, TOPIC_NAME)
.unwrap()
.batch_length(10)
- .poll_interval(IggyDuration::from_str("10ms").expect("Invalid duration"))
.polling_strategy(PollingStrategy::next())
.auto_join_consumer_group()
.auto_commit(AutoCommit::IntervalOrAfter(
diff --git a/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs b/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs
index 68a37a92c..7e64d064a 100644
--- a/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs
+++ b/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs
@@ -183,7 +183,6 @@ async fn consume_with_strategy(
AutoCommitWhen::ConsumingAllMessages,
))
.polling_strategy(strategy)
- .poll_interval(IggyDuration::from_str("2ms").unwrap())
.batch_length(BATCH_LENGTH)
.build();
diff --git a/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs b/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs
index 5210bde6d..d76c87784 100644
--- a/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs
+++ b/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs
@@ -141,7 +141,6 @@ pub async fn run_consumer(harness: &mut TestHarness) {
.expect("Failed to create consumer builder")
.polling_strategy(PollingStrategy::next())
.batch_length(10)
- .poll_interval(IggyDuration::from_str("100ms").unwrap())
.polling_retry_interval(NonZeroIggyDuration::from_str("500ms").unwrap())
.build();
diff --git a/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs b/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs
index ec130dc66..70fec8427 100644
--- a/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs
+++ b/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs
@@ -279,7 +279,6 @@ async fn should_handle_stale_client_with_auto_reconnection(
.consumer_group(CONSUMER_GROUP_NAME, STREAM_NAME, TOPIC_NAME)
.unwrap()
.batch_length(1)
- .poll_interval(IggyDuration::from_str("100ms").unwrap())
.polling_strategy(PollingStrategy::next())
.auto_join_consumer_group()
.create_consumer_group_if_not_exists()
diff --git a/core/message_bus/src/lib.rs b/core/message_bus/src/lib.rs
index dc2348daa..c56b8e8f1 100644
--- a/core/message_bus/src/lib.rs
+++ b/core/message_bus/src/lib.rs
@@ -109,8 +109,8 @@ use std::array;
use std::cell::{Cell, OnceCell, RefCell};
use std::net::SocketAddr;
use std::rc::Rc;
-use std::sync::Arc;
use std::sync::atomic::{AtomicU16, Ordering};
+use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
/// Maximum number of replicas a single cluster supports. Replica ids are
@@ -481,6 +481,13 @@ pub type ConnectionLostFn = std::rc::Rc<dyn Fn(u8)>;
///
/// A bus impl must preserve this divergence - see each method.
pub trait MessageBus {
+ /// Process-wide monotonic microseconds, on the same timeline as `sleep`.
+ /// Virtual-time implementations must override both methods together.
+ fn monotonic_micros(&self) -> u64 {
+ static ORIGIN: OnceLock<Instant> = OnceLock::new();
+ u64::try_from(ORIGIN.get_or_init(Instant::now).elapsed().as_micros()).unwrap_or(u64::MAX)
+ }
+
/// Queue one frame for `client_id`. Takes anything that converts into a
/// [`BusMessage`]: a single `Frozen<MESSAGE_ALIGN>` buffer or an
/// already fragmented frame.
@@ -1285,6 +1292,10 @@ impl<T: MessageBus + ?Sized> MessageBus for std::rc::Rc<T> {
// spawn queue, and virtual clock) would be silently reverted to the defaults
// through the wrapper. No-op for production, whose `IggyMessageBus` uses those
// defaults anyway.
+ fn monotonic_micros(&self) -> u64 {
+ (**self).monotonic_micros()
+ }
+
fn sleep(&self, duration: Duration) -> impl Future<Output = ()> {
(**self).sleep(duration)
}
@@ -1320,11 +1331,16 @@ impl MessageBus for IggyMessageBus {
// the payload either).
return match self.clients.try_send_or_return(client_id, message) {
ReplyRoute::Delivered(send_result) => send_result.map_err(map_try_send_err),
- ReplyRoute::InProcess(message) => {
+ ReplyRoute::InProcess(mut message) => {
let request = reply_request_id(&message);
+ let receipt = message.take_write_receipt();
self.clients
.fire_in_process(client_id, request, message)
- .map_err(|_| SendError::ClientNotFound(client_id))
+ .map_err(|_| SendError::ClientNotFound(client_id))?;
+ if let Some(receipt) = receipt {
+ receipt.complete();
+ }
+ Ok(())
}
ReplyRoute::NoSlot(_message) => Err(SendError::ClientNotFound(client_id)),
};
diff --git a/core/message_bus/src/transports/quic.rs b/core/message_bus/src/transports/quic.rs
index da9296cb6..dac1c0775 100644
--- a/core/message_bus/src/transports/quic.rs
+++ b/core/message_bus/src/transports/quic.rs
@@ -90,7 +90,11 @@ use compio_quic::{
crypto::rustls::QuicServerConfig,
};
use futures::FutureExt;
-use iggy_binary_protocol::{Command, HEADER_SIZE, ReplyHeader, RequestHeader};
+use iggy_binary_protocol::codes::{
+ POLL_MESSAGES_DEFERRED_CODE, POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE,
+};
+use iggy_binary_protocol::requests::messages::DeferredPollMessagesRequest;
+use iggy_binary_protocol::{Command, HEADER_SIZE, ReplyHeader, RequestHeader, WireDecode};
use std::io;
use std::net::SocketAddr;
use std::sync::Arc;
@@ -436,6 +440,25 @@ impl TransportConn for QuicTransportConn {
.and_then(|bytes| bytemuck::checked::try_from_bytes::<RequestHeader>(bytes).ok())
.map(|header| header.request);
+ let deferred_wait =
+ bytemuck::checked::try_from_bytes::<RequestHeader>(&req.as_slice()[..HEADER_SIZE])
+ .ok()
+ .and_then(|header| {
+ let code = u32::from_le_bytes(
+ header.reserved[..size_of::<u32>()].try_into().ok()?,
+ );
+ if !matches!(
+ code,
+ POLL_MESSAGES_DEFERRED_CODE | POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE
+ ) {
+ return None;
+ }
+ DeferredPollMessagesRequest::decode_from(&req.as_slice()[HEADER_SIZE..])
+ .ok()
+ .map(|request| Duration::from_micros(request.request_timeout_us))
+ })
+ .unwrap_or_default();
+
if in_tx.send(req).await.is_err() {
debug!(%label, %peer, "quic: inbound queue dropped");
break;
@@ -455,10 +478,20 @@ impl TransportConn for QuicTransportConn {
let mut matched: Option<BusMessage> = None;
let mut connection_gone = false;
{
- let mut reply_deadline =
- std::pin::pin!(compio::time::sleep(REPLY_WAIT_BACKSTOP).fuse());
+ let mut reply_deadline = std::pin::pin!(
+ compio::time::sleep(REPLY_WAIT_BACKSTOP.saturating_add(deferred_wait)).fuse()
+ );
loop {
futures::select! {
+ _ = async {
+ if deferred_wait.is_zero() {
+ std::future::pending::<()>().await;
+ }
+ send.stopped().await
+ }.fuse() => {
+ connection_gone = true;
+ break;
+ }
() = shutdown_fut.as_mut() => {
debug!(%label, %peer, "quic: shutdown during reply wait");
connection_gone = true;
@@ -494,7 +527,7 @@ impl TransportConn for QuicTransportConn {
if connection_gone {
break;
}
- let Some(first) = matched else {
+ let Some(mut first) = matched else {
// Unreachable in practice: every no-reply path above sets
// `connection_gone`. Close the send half defensively.
if let Err(e) = send.finish() {
@@ -507,6 +540,7 @@ impl TransportConn for QuicTransportConn {
// quinn-proto flushes pending data + a FIN, signalling the SDK
// that the reply is complete.
let mut write_failed = false;
+ let write_receipt = first.take_write_receipt();
for fragment in first.into_fragments() {
let BufResult(result, _frozen) = send.write_all(fragment).await;
if let Err(e) = result {
@@ -520,6 +554,8 @@ impl TransportConn for QuicTransportConn {
}
if let Err(e) = send.finish() {
debug!(%label, %peer, error = ?e, "quic: send.finish() failed");
+ } else if let Some(receipt) = write_receipt {
+ receipt.complete();
}
// (send, recv) drop here -> stream fully closed.
}
diff --git a/core/message_bus/src/transports/tcp.rs b/core/message_bus/src/transports/tcp.rs
index 53845b41e..6b17d0e00 100644
--- a/core/message_bus/src/transports/tcp.rs
+++ b/core/message_bus/src/transports/tcp.rs
@@ -247,6 +247,7 @@ async fn writer_loop(
let mut iovecs: Vec<Frozen<MESSAGE_ALIGN>> = Vec::with_capacity(max_batch);
// Sized by the first wide batch; most connections never need it.
let mut scratch: Vec<Frozen<MESSAGE_ALIGN>> = Vec::new();
+ let mut write_receipts = Vec::new();
let mut shutdown_fut = Box::pin(shutdown.wait().fuse());
loop {
@@ -278,7 +279,8 @@ async fn writer_loop(
// `drain(..)` keeps the batch allocation for the next round.
#[allow(clippy::iter_with_drain)]
- for frame in batch.drain(..) {
+ for mut frame in batch.drain(..) {
+ write_receipts.extend(frame.take_write_receipt());
iovecs.extend(frame.into_fragments());
}
let result = write_iovecs(&mut write_half, &mut iovecs, &mut scratch).await;
@@ -293,6 +295,10 @@ async fn writer_loop(
);
return;
}
+ #[allow(clippy::iter_with_drain)]
+ for receipt in write_receipts.drain(..) {
+ receipt.complete();
+ }
}
}
diff --git a/core/message_bus/src/transports/tcp_tls.rs b/core/message_bus/src/transports/tcp_tls.rs
index c0155991a..d7d548de4 100644
--- a/core/message_bus/src/transports/tcp_tls.rs
+++ b/core/message_bus/src/transports/tcp_tls.rs
@@ -573,6 +573,10 @@ async fn run_pump(tls: &mut TlsStream<TcpStream>, ctx: ActorContext) {
}
}
let drained = batch.len();
+ let write_receipts: Vec<_> = batch
+ .iter_mut()
+ .filter_map(BusMessage::take_write_receipt)
+ .collect();
// `drain(..)` consumes the batch in FIFO order while
// preserving the Vec's allocation for the next
// iteration; `into_iter()` would move the buffer out.
@@ -602,6 +606,9 @@ async fn run_pump(tls: &mut TlsStream<TcpStream>, ctx: ActorContext) {
);
return;
}
+ for receipt in write_receipts {
+ receipt.complete();
+ }
}
PumpAction::Recv(Ok(Some(frame))) => {
if in_tx.send(frame).await.is_err() {
diff --git a/core/message_bus/src/transports/ws.rs b/core/message_bus/src/transports/ws.rs
index 15b5a963a..91c94d601 100644
--- a/core/message_bus/src/transports/ws.rs
+++ b/core/message_bus/src/transports/ws.rs
@@ -233,6 +233,10 @@ async fn run_pump(ws: &mut WebSocketStream<TcpStream>, ctx: ActorContext) {
}
}
let drained = batch.len();
+ let write_receipts: Vec<_> = batch
+ .iter_mut()
+ .filter_map(BusMessage::take_write_receipt)
+ .collect();
#[allow(clippy::iter_with_drain)]
// One WS frame per message: tungstenite takes a single payload
// buffer, so a multi-fragment frame is joined here (the only
@@ -248,6 +252,9 @@ async fn run_pump(ws: &mut WebSocketStream<TcpStream>, ctx: ActorContext) {
warn!(%label, %peer, error = ?e, batch_len = drained, "ws writer: flush failed");
return;
}
+ for receipt in write_receipts {
+ receipt.complete();
+ }
}
PumpAction::Recv(Ok(msg)) => match msg {
WsMessage::Binary(bytes) => {
diff --git a/core/message_bus/src/transports/wss.rs b/core/message_bus/src/transports/wss.rs
index b28b93ebf..cc00c8d4e 100644
--- a/core/message_bus/src/transports/wss.rs
+++ b/core/message_bus/src/transports/wss.rs
@@ -381,6 +381,10 @@ async fn run_pump(ws: &mut WebSocketStream<TcpStream>, ctx: ActorContext) {
}
}
let drained = batch.len();
+ let write_receipts: Vec<_> = batch
+ .iter_mut()
+ .filter_map(BusMessage::take_write_receipt)
+ .collect();
// `drain(..)` consumes the batch in FIFO order while
// preserving the Vec's allocation for the next
// iteration; `into_iter()` would move the buffer out.
@@ -399,6 +403,9 @@ async fn run_pump(ws: &mut WebSocketStream<TcpStream>, ctx: ActorContext) {
warn!(%label, %peer, error = ?e, batch_len = drained, "wss writer: flush failed");
return;
}
+ for receipt in write_receipts {
+ receipt.complete();
+ }
}
PumpAction::Recv(Ok(msg)) => match msg {
WsMessage::Binary(bytes) => {
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs
index 5e39d0c3e..07409ae8c 100644
--- a/core/partitions/src/iggy_partition.rs
+++ b/core/partitions/src/iggy_partition.rs
@@ -222,6 +222,7 @@ where
pub(crate) pending_consumer_offset_commits: HashMap<u64, PendingConsumerOffsetCommit>,
/// Identity shared with pending polls and replaced when their history retires.
poll_history: PollHistoryId,
+ poll_notifier: Option<Rc<dyn Fn()>>,
/// Committed consumer-offset membership and values. This is deliberately
/// separate from the eager poll maps because follower-local and uncommitted
/// auto-commit progress must never consume a durable slot or enter a state
@@ -614,6 +615,7 @@ where
fatal: None,
pending_consumer_offset_commits: HashMap::new(),
poll_history: PollHistoryId::default(),
+ poll_notifier: None,
durable_consumer_offsets: DurableConsumerOffsets::default(),
consumer_offset_capacity: ConsumerOffsetCapacity::new(
ConsumerKind::Consumer,
@@ -1611,6 +1613,7 @@ where
let raise = !seeded || self.offset.load(Ordering::Acquire) < committed_end;
if raise {
self.offset.store(committed_end, Ordering::Release);
+ self.publish_poll_visibility();
}
raise
});
@@ -1634,6 +1637,7 @@ where
// the pre-first-flush shape, where the committed counter still seeds
// nothing.
self.offset_space.committed_seeded |= frontier > 0;
+ self.publish_poll_visibility();
}
/// Path of the anchor whose lifecycle is this segment's.
@@ -1654,11 +1658,12 @@ where
/// because a segment on disk holds committed messages only). Everything on
/// the live path moves ONE bit -- see [`Self::note_append_live`] and
/// [`Self::note_committed_seeded`].
- pub const fn set_offset_space_used(&mut self, used: bool) {
+ pub fn set_offset_space_used(&mut self, used: bool) {
self.offset_space = OffsetSpace {
append_live: used,
committed_seeded: used,
};
+ self.publish_poll_visibility();
}
/// The append counter is live: an offset has been journaled, so the next
@@ -1679,9 +1684,10 @@ where
/// Implies the append counter is live too -- nothing commits that was not
/// journaled first -- but the reverse does not hold, which is the whole
/// reason the two bits are separate.
- pub const fn note_committed_seeded(&mut self) {
+ pub fn note_committed_seeded(&mut self) {
self.offset_space.append_live = true;
self.offset_space.committed_seeded = true;
+ self.publish_poll_visibility();
}
/// Whether this partition ever stamped an offset, i.e. whether its offset
@@ -1762,6 +1768,7 @@ where
.max(durable)
.max(self.dirty_offset.load(Ordering::Relaxed));
self.offset.store(durable, Ordering::Release);
+ self.publish_poll_visibility();
self.dirty_offset.store(dirty, Ordering::Relaxed);
self.set_offset_space_used(true);
// Everything carried over is already persisted as far as this replica is
@@ -2971,9 +2978,29 @@ where
/// their reads have already completed.
pub(crate) fn invalidate_poll_history(&mut self) {
self.poll_history = PollHistoryId::default();
+ self.publish_poll_visibility();
self.discard_queued_auto_commits();
}
+ pub fn poll_visibility(&self) -> (PollHistoryId, Option<u64>) {
+ (
+ self.poll_history,
+ self.offset_space
+ .committed_seeded
+ .then(|| self.offset.load(Ordering::Acquire)),
+ )
+ }
+
+ pub fn set_poll_notifier(&mut self, notifier: Option<Rc<dyn Fn()>>) {
+ self.poll_notifier = notifier;
+ }
+
+ pub(crate) fn publish_poll_visibility(&self) {
+ if let Some(notify) = &self.poll_notifier {
+ notify();
+ }
+ }
+
fn discard_queued_auto_commits(&self) {
self.consensus.with_pipeline_mut(|pipeline| {
pipeline.retain_requests(|request| request.auto_commit().is_none());
@@ -3793,6 +3820,35 @@ where
}
}
+ pub(crate) fn build_bounded_poll_plan(
+ &mut self,
+ consumer: PollingConsumer,
+ args: &PollingArgs,
+ validate_checksum: bool,
+ max_bytes: usize,
+ ) -> Result<PollPlan, IggyError> {
+ // Source snapshots, sparse copies and fragment descriptors can coexist.
+ const RESIDENT_ALLOCATION_FACTOR: usize = 4;
+ // Covers file paths, read handles and fixed IO/selection bookkeeping.
+ const READ_METADATA_BYTES: usize = 64 * 1024;
+ let resident_bytes = self.log.journal().inner.resident_message_bytes()?;
+ let segment_bytes = self
+ .log
+ .segments()
+ .len()
+ .saturating_mul(size_of::<DiskSegment>());
+ // Selection may copy sparse fragments while the source snapshot remains live.
+ if resident_bytes
+ .saturating_mul(RESIDENT_ALLOCATION_FACTOR)
+ .saturating_add(segment_bytes)
+ .saturating_add(READ_METADATA_BYTES)
+ > max_bytes
+ {
+ return Err(IggyError::InvalidSizeBytes);
+ }
+ Ok(self.build_poll_plan(consumer, args, validate_checksum))
+ }
+
/// Snapshot read resources synchronously on the partition owner.
/// Only the owned snapshot crosses a suspension during disk I/O.
#[allow(clippy::too_many_lines)]
@@ -3886,9 +3942,10 @@ where
if serve_journal_first {
let tier = match self.journal_get_sync(&query) {
- Some((fragments, last_matching_offset)) => PollTier::Resident {
+ Some((fragments, last_matching_offset, message_count)) => PollTier::Resident {
fragments,
last_matching_offset,
+ message_count,
},
None => PollTier::Empty,
};
@@ -6051,6 +6108,7 @@ where
if let Some(durable_offset) = durable_offset {
self.note_committed_seeded();
self.offset.store(durable_offset, Ordering::Release);
+ self.publish_poll_visibility();
self.stats.set_current_offset(durable_offset);
}
Ok(true)
@@ -6541,6 +6599,7 @@ where
.is_none_or(|durable| end_offset > durable)
{
self.offset.store(end_offset, Ordering::Release);
+ self.publish_poll_visibility();
self.stats.set_current_offset(end_offset);
// Advance the aggregate stats with the visible offset. Disk
// persistence is threshold-gated in `commit_messages`, which
@@ -7778,6 +7837,7 @@ where
// reads the counter, and the partition write lock is held across this
// whole body.
self.offset.store(start_offset, Ordering::Release);
+ self.publish_poll_visibility();
self.dirty_offset.store(start_offset, Ordering::Relaxed);
self.set_offset_space_used(false);
@@ -12349,6 +12409,8 @@ mod tests {
fragments,
commit_offset: partition.offsets().commit_offset,
last_matching_offset,
+ message_count: u32::from(last_matching_offset.is_some()),
+ read_error: None,
}
}
diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs
index 3a68203a8..75324bd9a 100644
--- a/core/partitions/src/iggy_partitions.rs
+++ b/core/partitions/src/iggy_partitions.rs
@@ -488,6 +488,26 @@ where
Some(partition.build_poll_plan(consumer, args, validate_checksum))
}
+ /// Snapshot a deferred read after checking its resident allocation budget.
+ ///
+ /// # Errors
+ /// Returns a size error when the snapshot exceeds the budget, or a read
+ /// error when a resident journal entry cannot be decoded.
+ pub fn build_bounded_poll_snapshot(
+ &self,
+ namespace: &IggyNamespace,
+ consumer: PollingConsumer,
+ args: &PollingArgs,
+ max_bytes: usize,
+ ) -> Result<Option<PollPlan>, IggyError> {
+ let validate_checksum = self.config.validate_checksum;
+ self.get_mut_by_ns(namespace)
+ .map(|partition| {
+ partition.build_bounded_poll_plan(consumer, args, validate_checksum, max_bytes)
+ })
+ .transpose()
+ }
+
/// Validate and accept a poll synchronously on the owning pump.
/// Attempt the reply synchronously, then immediately await any returned
/// continuation through [`Self::replicate_poll_completion`] on the same pump.
@@ -947,7 +967,7 @@ mod tests {
// D = 0 (last disk match), so a straddle would continue at offset 1.
// A raw snapshot walk from offset 1 falls through to the next resident
// op (offset 3): exactly the silent skip the gate prevents.
- let (_, leaky_offset) = crate::journal::select_resident(
+ let (_, leaky_offset, _) = crate::journal::select_resident(
&entries,
MessageLookup::Offset {
offset: 1,
@@ -975,7 +995,7 @@ mod tests {
let from_offset = 3u64;
let contiguous = oldest_resident.is_some_and(|oldest| oldest <= from_offset);
assert!(contiguous, "offset 3 is resident; gate must allow it");
- let (_, contiguous_offset) = crate::journal::select_resident(
+ let (_, contiguous_offset, _) = crate::journal::select_resident(
&entries,
MessageLookup::Offset {
offset: from_offset,
@@ -1073,7 +1093,7 @@ mod tests {
}
let entries = partition.log.journal().inner.resident_entries();
- let (_, last) = crate::journal::select_resident(
+ let (_, last, _) = crate::journal::select_resident(
&entries,
MessageLookup::Offset {
offset: 0,
diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs
index 545377cd4..e577b57d4 100644
--- a/core/partitions/src/journal.rs
+++ b/core/partitions/src/journal.rs
@@ -16,6 +16,7 @@
// under the License.
use iggy_binary_protocol::{Operation, PrepareHeader};
+use iggy_common::IggyError;
use journal::{Journal, Storage};
use server_common::{
iobuf::{Frozen, Owned},
@@ -481,7 +482,7 @@ impl PartitionJournal<PartitionJournalMemStorage> {
) -> PollQueryResult<4096> {
let count = query.count();
if count == 0 {
- return (PollFragments::new(), None);
+ return (PollFragments::new(), None, 0);
}
// Disjoint `UnsafeCell`s: this borrows `op_to_storage_offset` while the
@@ -515,7 +516,7 @@ impl PartitionJournal<PartitionJournalMemStorage> {
);
}
- (fragments, last_matching_offset)
+ (fragments, last_matching_offset, matched_messages)
}
/// Drain all accumulated batches, matching the legacy `PartitionJournal` API.
@@ -767,6 +768,24 @@ impl PartitionJournal<PartitionJournalMemStorage> {
entries
}
+ /// Conservative snapshot charge without allocating an entry vector.
+ pub fn resident_message_bytes(&self) -> Result<usize, IggyError> {
+ let offset_to_op = unsafe { &*self.offset_to_op.get() };
+ let op_to_storage_offset = unsafe { &*self.op_to_storage_offset.get() };
+ let inner = unsafe { &*self.inner.get() };
+ offset_to_op.values().try_fold(0usize, |bytes, op| {
+ let entry = op_to_storage_offset
+ .get(op)
+ .and_then(|offset| inner.storage.read_at_sync(*offset))
+ .ok_or(IggyError::CannotReadMessage)?;
+ decode_prepare_slice_trusted(entry.as_slice())
+ .map_err(|_| IggyError::CannotReadMessage)?;
+ Ok(bytes
+ .saturating_add(entry.allocation_bytes())
+ .saturating_add(size_of::<JournalBuffer>()))
+ })
+ }
+
/// Owned, append-ordered clones of every resident entry above the purge
/// floor, control ops included; one `Frozen` refcount bump each. Linear in
/// the resident journal, so not for the poll path, which takes
@@ -1437,7 +1456,7 @@ pub fn select_resident(
);
}
- (!fragments.is_empty()).then_some((fragments, last_matching_offset))
+ (!fragments.is_empty()).then_some((fragments, last_matching_offset, matched_messages))
}
#[cfg(test)]
@@ -1981,8 +2000,9 @@ mod tests {
"fixture must select a sparse fraction of the prepare"
);
- let (fragments, last_matching_offset) =
+ let (fragments, last_matching_offset, matched) =
select_resident(std::slice::from_ref(&prepare), query).expect("one record matches");
+ assert_eq!(matched, 1);
assert_eq!(last_matching_offset, Some(500));
assert_eq!(fragments.len(), 2, "rewritten header plus body slice");
let (header, body) = (&fragments[0], &fragments[1]);
@@ -2004,9 +2024,10 @@ mod tests {
fn resident_whole_batch_selection_keeps_the_zero_copy_slice() {
let prepare = build_resident_prepare(1_000, 300);
- let (fragments, last_matching_offset) =
+ let (fragments, last_matching_offset, matched) =
select_resident(std::slice::from_ref(&prepare), offset_lookup(0, 1_000))
.expect("whole batch matches");
+ assert_eq!(matched, 1_000);
assert_eq!(last_matching_offset, Some(999));
assert_eq!(fragments.len(), 1, "a whole batch ships its original bytes");
assert!(
@@ -2032,7 +2053,7 @@ mod tests {
"fixture must select a sparse fraction that is over the copy cap"
);
- let (fragments, _) =
+ let (fragments, _, _) =
select_resident(std::slice::from_ref(&prepare), query).expect("records match");
assert_eq!(fragments.len(), 2, "rewritten header plus body slice");
assert!(
@@ -2086,4 +2107,23 @@ mod tests {
"dense slice keeps the zero-copy path"
);
}
+ #[test]
+ fn selected_message_count_does_not_include_offset_holes() {
+ let entries = vec![
+ build_message_prepare(1, 0, 1, 8),
+ build_message_prepare(2, 100, 1, 8),
+ ];
+ let (fragments, last, count) = select_resident(
+ &entries,
+ MessageLookup::Offset {
+ offset: 0,
+ count: 3,
+ ceiling: 100,
+ },
+ )
+ .unwrap();
+ assert!(!fragments.is_empty());
+ assert_eq!(last, Some(100));
+ assert_eq!(count, 2);
+ }
}
diff --git a/core/partitions/src/poll_plan.rs b/core/partitions/src/poll_plan.rs
index a4397f371..a65c5b9b2 100644
--- a/core/partitions/src/poll_plan.rs
+++ b/core/partitions/src/poll_plan.rs
@@ -26,13 +26,15 @@ use crate::iggy_index_reader::IggyIndexReader;
use crate::journal::{
MessageLookup, push_selected_batch_fragments, select_batch_slice, unpin_sparse_source,
};
-use crate::{PollFragments, PollingConsumer};
+use crate::{Fragment, PollFragments, PollingConsumer};
use compio::io::AsyncReadAtExt;
+use iggy_binary_protocol::batch::{BATCH_HEADER_SIZE, BatchHeader, BatchRef};
+use iggy_binary_protocol::responses::messages::poll_messages::POLL_RESPONSE_HEADER_SIZE;
use iggy_binary_protocol::{WireError, batch};
use iggy_common::{ConsumerKind, IggyError};
use server_common::iobuf::{Frozen, Owned};
use server_common::poll::PollHistoryId;
-use server_common::send_messages::{BatchIntegrity, COMMAND_HEADER_SIZE};
+use server_common::send_messages::{BatchIntegrity, COMMAND_HEADER_SIZE, frozen_batch_header};
use std::cell::{Cell, RefCell};
use std::rc::Rc;
use tracing::{error, warn};
@@ -188,9 +190,112 @@ pub struct PollReadResult {
pub(crate) commit_offset: u64,
/// Inclusive offset of the last selected message, or `None` for no match.
pub(crate) last_matching_offset: Option<u64>,
+ pub(crate) message_count: u32,
+ pub(crate) read_error: Option<IggyError>,
}
impl PollReadResult {
+ /// Cap the encoded selection before the owner accepts its final offset.
+ /// A sliced batch retains its original offset base and receives a new checksum.
+ ///
+ /// # Errors
+ /// Returns `InvalidSizeBytes` when the first message cannot fit, or
+ /// `CannotReadMessage` if the selected batch framing is invalid.
+ pub fn limit_bytes(mut self, max_bytes: usize) -> Result<(Self, bool), IggyError> {
+ let mut remaining = max_bytes
+ .checked_sub(POLL_RESPONSE_HEADER_SIZE)
+ .ok_or(IggyError::InvalidSizeBytes)?;
+ let mut fragments = std::mem::take(&mut self.fragments).into_iter();
+ self.message_count = 0;
+ self.last_matching_offset = None;
+ let mut limited = false;
+ while let Some(fragment) = fragments.next() {
+ let source = fragment.into_frozen();
+ let mut header =
+ BatchHeader::decode(&source).map_err(|_| IggyError::CannotReadMessage)?;
+ let body = if source.len() == BATCH_HEADER_SIZE {
+ fragments
+ .next()
+ .ok_or(IggyError::CannotReadMessage)?
+ .into_frozen()
+ } else {
+ source.slice(BATCH_HEADER_SIZE..source.len())
+ };
+ if body.len()
+ != header
+ .blob_len()
+ .map_err(|_| IggyError::CannotReadMessage)?
+ {
+ return Err(IggyError::CannotReadMessage);
+ }
+ let batch = BatchRef::new(header, &body);
+ let mut selected = 0;
+ let mut end = 0;
+ for record in batch.iter_with_offsets() {
+ if BATCH_HEADER_SIZE.saturating_add(record.end) > remaining {
+ limited = true;
+ break;
+ }
+ selected += 1;
+ end = record.end;
+ self.last_matching_offset = Some(
+ header
+ .base_offset
+ .checked_add(u64::from(record.message.header.offset_delta))
+ .ok_or(IggyError::CannotReadMessage)?,
+ );
+ }
+ if selected == 0 {
+ if self.message_count == 0 {
+ return Err(IggyError::InvalidSizeBytes);
+ }
+ break;
+ }
+ self.message_count += selected;
+ remaining -= BATCH_HEADER_SIZE + end;
+ limited |= remaining == 0;
+ if selected == header.message_count && source.len() != BATCH_HEADER_SIZE {
+ self.fragments.push(Fragment::whole(source));
+ } else {
+ header.message_count = selected;
+ header.batch_length = (BATCH_HEADER_SIZE + end) as u64;
+ header.batch_checksum = header.checksum_for_blob(&body[..end]);
+ self.fragments
+ .push(Fragment::whole(frozen_batch_header(&header)));
+ self.fragments.push(Fragment::whole(body.slice(..end)));
+ }
+ if limited {
+ break;
+ }
+ }
+ Ok((self, limited))
+ }
+
+ /// Conservative charge: shared allocations count once for each fragment.
+ #[must_use]
+ pub fn retained_bytes(&self) -> usize {
+ self.fragments.iter().fold(
+ self.fragments
+ .capacity()
+ .saturating_mul(size_of::<crate::Fragment>()),
+ |bytes, fragment| bytes.saturating_add(fragment.allocation_bytes()),
+ )
+ }
+
+ #[must_use]
+ pub const fn message_count(&self) -> u32 {
+ self.message_count
+ }
+
+ /// Reject incomplete reads before a deferred poll can cache or accept them.
+ /// Reject a read whose valid prefix was followed by an error.
+ ///
+ /// # Errors
+ /// Returns the underlying read or allocation-limit failure.
+ pub fn checked(mut self) -> Result<Self, IggyError> {
+ self.read_error.take().map_or_else(|| Ok(self), Err)
+ }
+
#[must_use]
pub const fn consumer_kind(&self) -> ConsumerKind {
match self.context.consumer {
@@ -252,49 +357,44 @@ impl PollPlan {
}
/// Read the captured snapshot without changing consumer progress.
- /// The result still requires owner validation, including when it is empty.
+ /// The owner decides whether legacy prefix results or strict errors apply.
pub async fn execute(self) -> PollReadResult {
- let commit_offset = self.commit_offset;
- let (fragments, last_matching_offset) = match self.tier {
- PollTier::Empty => (PollFragments::new(), None),
+ self.execute_with_limit(usize::MAX).await
+ }
+
+ pub async fn execute_with_limit(self, max_bytes: usize) -> PollReadResult {
+ let mut read_error = None;
+ let (fragments, last_matching_offset, message_count) = match self.tier {
+ PollTier::Empty => (PollFragments::new(), None, 0),
PollTier::Resident {
fragments,
last_matching_offset,
- } => (fragments, last_matching_offset),
+ message_count,
+ } => (fragments, last_matching_offset, message_count),
PollTier::Disk {
disk,
query,
resident_tail,
- } => match disk.read_disk(query).await {
- // Disk walked cleanly and matched nothing: the query offset is
- // below disk retention too, so the match (if any) is journal-
- // resident. Serve the journal forward (retention-recovery) from
- // the resident-tail snapshot with the ORIGINAL query (offset or
- // timestamp); no contiguity gate, this is not a straddle.
+ } => match disk.read_disk_with_limit(query, max_bytes).await {
DiskReadOutcome::Empty => {
crate::journal::select_resident(&resident_tail.entries, query)
- .unwrap_or_else(|| (PollFragments::new(), None))
+ .unwrap_or_else(|| (PollFragments::new(), None, 0))
+ }
+ DiskReadOutcome::Limited => {
+ read_error = Some(IggyError::InvalidSizeBytes);
+ (PollFragments::new(), None, 0)
+ }
+ DiskReadOutcome::Faulted => {
+ read_error = Some(IggyError::CannotReadMessage);
+ (PollFragments::new(), None, 0)
}
- // Disk read stopped on a fault. Fail-closed: return an empty poll
- // WITHOUT the journal-forward fallback. Falling forward here would
- // splice the next resident op over the unreadable run and silently
- // skip live data.
- //
- // TODO(partitions): the poll reply has no error channel, so this
- // reaches the consumer as an ordinary empty poll. Fair for a transient
- // IO fault, wrong for a batch that failed its own checksum: data
- // damaged at rest never reads again, so the consumer waits forever.
- // Surfacing it needs a status on the poll reply, an SDK-visible change
- // on every client. Until then the ERROR in `walk_disk_chunk` is the
- // only signal, and it is server-side only.
- DiskReadOutcome::Faulted => (PollFragments::new(), None),
- // Straddle: continue past the last disk match into the resident
- // tail (gate + race argument live on `straddle_continuation`).
DiskReadOutcome::Matched {
mut fragments,
last_matching_offset,
matched,
+ faulted,
} => {
+ read_error = faulted.then_some(IggyError::CannotReadMessage);
let remaining = query.count().saturating_sub(matched);
let continuation = last_matching_offset
.and_then(|last_offset| {
@@ -308,21 +408,26 @@ impl PollPlan {
crate::journal::select_resident(&resident_tail.entries, query)
});
match continuation {
- Some((journal_fragments, journal_last)) => {
+ Some((journal_fragments, journal_last, journal_count)) => {
fragments.extend(journal_fragments);
- (fragments, journal_last.or(last_matching_offset))
+ (
+ fragments,
+ journal_last.or(last_matching_offset),
+ matched + journal_count,
+ )
}
- None => (fragments, last_matching_offset),
+ None => (fragments, last_matching_offset, matched),
}
}
},
};
-
PollReadResult {
context: self.context,
- commit_offset,
+ commit_offset: self.commit_offset,
fragments,
last_matching_offset,
+ message_count,
+ read_error,
}
}
@@ -333,24 +438,24 @@ impl PollPlan {
/// Panics if [`Self::needs_off_pump_io`] is true.
#[must_use]
pub fn execute_resident(self) -> PollReadResult {
- let commit_offset = self.commit_offset;
- let (fragments, last_matching_offset) = match self.tier {
- PollTier::Empty => (PollFragments::new(), None),
+ let (fragments, last_matching_offset, message_count) = match self.tier {
+ PollTier::Empty => (PollFragments::new(), None, 0),
PollTier::Resident {
fragments,
last_matching_offset,
- } => (fragments, last_matching_offset),
- // `needs_off_pump_io` is true for every Disk tier, so the dispatch
- // gate never routes one here.
+ message_count,
+ } => (fragments, last_matching_offset, message_count),
PollTier::Disk { .. } => {
unreachable!("execute_resident on Disk tier; needs_off_pump_io guards this")
}
};
PollReadResult {
context: self.context,
- commit_offset,
+ commit_offset: self.commit_offset,
fragments,
last_matching_offset,
+ message_count,
+ read_error: None,
}
}
}
@@ -360,6 +465,7 @@ pub enum PollTier {
Resident {
fragments: PollFragments<4096>,
last_matching_offset: Option<u64>,
+ message_count: u32,
},
Disk {
disk: DiskReadPlan,
@@ -384,6 +490,7 @@ pub enum DiskReadOutcome {
fragments: PollFragments<4096>,
last_matching_offset: Option<u64>,
matched: u32,
+ faulted: bool,
},
/// Walk completed with no fault and matched nothing. The query offset is
/// below disk retention too, so the caller may serve the journal forward
@@ -393,10 +500,13 @@ pub enum DiskReadOutcome {
/// caller returns an empty poll so the consumer cursor does not advance
/// past data that may still be present-but-unreadable.
Faulted,
+ Limited,
}
/// Ceiling for ordinary disk reads. An incomplete batch may require one
/// larger re-read, without widening subsequent chunks or segments.
+const READ_ALLOCATION_FACTOR: usize = 4;
+
const DISK_POLL_CHUNK_MAX: u64 = 1 << 20;
/// Smallest first read of a disk poll. Below this the syscall and the segment
@@ -416,6 +526,9 @@ enum SegmentWalk {
/// The state one disk walk carries across its segments.
struct DiskWalk {
+ max_bytes: usize,
+ retained_bytes: usize,
+ limited: bool,
/// Byte offset into the segment being walked; reset at each boundary.
position: u64,
/// Messages between the resolved index entry and the requested offset,
@@ -448,6 +561,9 @@ impl DiskWalk {
fragments: PollFragments::new(),
last_matching_offset: None,
batch_read_floor: 0,
+ max_bytes: usize::MAX,
+ retained_bytes: 0,
+ limited: false,
#[cfg(feature = "poll-diagnostics")]
requested_bytes: 0,
#[cfg(feature = "poll-diagnostics")]
@@ -494,7 +610,12 @@ impl DiskReadPlan {
/// the file IO. Walks stamped `[256B BatchHeader][blob]` batches in
/// chunked reads, re-reading a batch split across a chunk boundary in the
/// next chunk.
+ #[cfg(test)]
pub(crate) async fn read_disk(self, query: MessageLookup) -> DiskReadOutcome {
+ self.read_disk_with_limit(query, usize::MAX).await
+ }
+
+ async fn read_disk_with_limit(self, query: MessageLookup, max_bytes: usize) -> DiskReadOutcome {
let count = query.count();
if count == 0 || self.segments.is_empty() {
return DiskReadOutcome::Empty;
@@ -533,7 +654,10 @@ impl DiskReadPlan {
// full-scan fallback). An active first segment keeps its
// resident-index-resolved `start_position` untouched.
let resolved = match self.segments.first() {
- Some(first) => self.resolve_sealed_start(first, query, partition_dir).await,
+ Some(first) => {
+ self.resolve_sealed_start(first, query, partition_dir, max_bytes)
+ .await
+ }
None => None,
};
let position = resolved.map_or(self.start_position, |(position, _)| position);
@@ -551,6 +675,7 @@ impl DiskReadPlan {
_ => 0,
};
let mut walk = DiskWalk::starting_at(position, skipped);
+ walk.max_bytes = max_bytes;
// Set when an open/read retry exhausts. The walk breaks immediately so
// later segments are never read into the result (which would leave a
// gap at the faulted segment). Pre-fault matches are still served.
@@ -602,6 +727,9 @@ impl DiskReadPlan {
"disk poll read accounting"
);
+ if walk.limited {
+ return DiskReadOutcome::Limited;
+ }
if walk.matched > 0 {
// Pre-fault matches are always a contiguous prefix (the walk stops
// at the first fault), so a partial result carries no gap.
@@ -609,6 +737,7 @@ impl DiskReadPlan {
fragments: walk.fragments,
last_matching_offset: walk.last_matching_offset,
matched: walk.matched,
+ faulted,
}
} else if faulted {
DiskReadOutcome::Faulted
@@ -638,6 +767,15 @@ impl DiskReadPlan {
.max(walk.batch_read_floor);
while walk.matched < count && walk.position < persisted {
let len = (persisted - walk.position).min(chunk_len) as usize;
+ // One source buffer plus sparse copies, fragment headers and descriptors.
+ if len
+ .saturating_mul(READ_ALLOCATION_FACTOR)
+ .saturating_add(walk.retained_bytes)
+ > walk.max_bytes
+ {
+ walk.limited = true;
+ return SegmentWalk::Faulted;
+ }
let Some(chunk) = self.read_chunk_with_retry(file, len, walk).await else {
// Chunk read exhausted retries: same fail-closed reason as
// a failed open.
@@ -669,6 +807,16 @@ impl DiskReadPlan {
&chunk,
usize::MAX,
);
+ for fragment in &walk.fragments[fragments_before_chunk..] {
+ walk.retained_bytes = walk
+ .retained_bytes
+ .saturating_add(fragment.allocation_bytes())
+ .saturating_add(size_of::<crate::Fragment>() * 2);
+ }
+ if walk.retained_bytes > walk.max_bytes {
+ walk.limited = true;
+ return SegmentWalk::Faulted;
+ }
if corrupt {
// A batch that does not match its own checksum. Fail closed like
// an IO fault: serving it hands a consumer data provably not what
@@ -750,6 +898,7 @@ impl DiskReadPlan {
segment: &DiskSegment,
query: MessageLookup,
partition_dir: &str,
+ max_bytes: usize,
) -> Option<(u64, u64)> {
// The active segment grows under the reader, so neither the shared
// sparse index nor the offset memo can describe it; its own resident
@@ -792,7 +941,12 @@ impl DiskReadPlan {
return None;
}
};
- if entry_count.saturating_mul(IGGY_INDEX_SIZE as u64) <= SEALED_INDEX_RESIDENT_MAX_BYTES {
+ if entry_count.saturating_mul(IGGY_INDEX_SIZE as u64) <= SEALED_INDEX_RESIDENT_MAX_BYTES
+ && entry_count
+ .saturating_mul(IGGY_INDEX_SIZE as u64)
+ .saturating_mul(2)
+ <= max_bytes as u64
+ {
let index = match reader.load_all().await {
Ok(index) => index,
Err(error) => {
@@ -1028,17 +1182,84 @@ struct ChunkWalk {
mod tests {
use super::*;
use crate::iggy_index::IggyIndex;
- #[cfg(feature = "poll-diagnostics")]
use bytes::Bytes;
use compio::io::AsyncWriteAtExt;
use server_common::iobuf::Owned;
- #[cfg(feature = "poll-diagnostics")]
use server_common::send_messages::{
IggyMessage, IggyMessageHeader, IggyMessages, SendMessagesOwned,
};
- #[cfg(feature = "poll-diagnostics")]
use server_common::sharding::IggyNamespace;
+ #[test]
+ fn byte_limit_rewrites_whole_and_split_batches_with_valid_checksums() {
+ const PAYLOAD: &[u8] = b"payload";
+ let mut messages = IggyMessages::with_capacity(2);
+ for _ in 0..2 {
+ messages.push(IggyMessage {
+ header: IggyMessageHeader {
+ payload_length: u32::try_from(PAYLOAD.len()).unwrap(),
+ ..Default::default()
+ },
+ payload: Bytes::from_static(PAYLOAD),
+ user_headers: None,
+ });
+ }
+ let mut batch =
+ SendMessagesOwned::from_messages(IggyNamespace::new(1, 1, 0), &messages).unwrap();
+ batch.header.base_offset = 42;
+ batch.header.batch_checksum = batch.header.checksum_for_blob(&batch.blob);
+ let mut bytes = vec![0; batch.header.total_size()];
+ batch.header.encode_into(&mut bytes);
+ bytes[BATCH_HEADER_SIZE..].copy_from_slice(&batch.blob);
+ let max_bytes = POLL_RESPONSE_HEADER_SIZE
+ + BATCH_HEADER_SIZE
+ + batch::BATCH_MESSAGE_HEADER_SIZE
+ + PAYLOAD.len();
+ for split in [false, true] {
+ let source: Frozen<4096> = Owned::copy_from_slice(&bytes).into();
+ let mut fragments = PollFragments::new();
+ if split {
+ fragments.push(Fragment::whole(source.slice(..BATCH_HEADER_SIZE)));
+ fragments.push(Fragment::whole(source.slice(BATCH_HEADER_SIZE..)));
+ } else {
+ fragments.push(Fragment::whole(source));
+ }
+ let result = PollPlan {
+ context: PollContext {
+ history: PollHistoryId::default(),
+ consumer: PollingConsumer::Consumer(0, 0),
+ auto_commit: true,
+ },
+ commit_offset: 99,
+ tier: PollTier::Resident {
+ fragments,
+ last_matching_offset: Some(43),
+ message_count: 2,
+ },
+ }
+ .execute_resident();
+ let (result, limited) = result.limit_bytes(max_bytes).unwrap();
+ assert!(limited);
+ assert_eq!(result.message_count(), 1);
+ assert_eq!(result.last_matching_offset, Some(42));
+ assert_eq!(result.commit_offset, 99);
+ let bytes: Vec<u8> = result
+ .fragments
+ .into_iter()
+ .flat_map(|fragment| fragment.into_frozen().to_vec())
+ .collect();
+ assert_eq!(bytes.len() + POLL_RESPONSE_HEADER_SIZE, max_bytes);
+ let header = BatchHeader::decode(&bytes).unwrap();
+ assert_eq!(header.base_offset, 42);
+ assert_eq!(header.message_count, 1);
+ assert_eq!(header.total_size(), bytes.len());
+ assert_eq!(
+ header.batch_checksum,
+ header.checksum_for_blob(&bytes[BATCH_HEADER_SIZE..])
+ );
+ }
+ }
+
/// Write a sealed-segment index file too large to materialize
/// (`entry_count * IGGY_INDEX_SIZE > SEALED_INDEX_RESIDENT_MAX_BYTES`), so
/// `resolve_sealed_start` takes the file-backed lookup path the offset
@@ -1244,7 +1465,6 @@ mod tests {
}
}
- #[cfg(feature = "poll-diagnostics")]
fn disk_batch(payload_length: u32, offset: u64) -> Vec<u8> {
let mut messages = IggyMessages::with_capacity(1);
messages.push(IggyMessage {
@@ -1308,7 +1528,7 @@ mod tests {
// First poll pays the on-file lookup and memoizes entry 2's interval
// [20, 30): offset 25 resolves to entry 2 (offset 20 -> position 200).
let first = plan
- .resolve_sealed_start(&segment, offset_query(25), &partition_dir)
+ .resolve_sealed_start(&segment, offset_query(25), &partition_dir, usize::MAX)
.await;
// The entry offset rides along so the caller can size its first read
// to cover the run between that entry and the requested offset.
@@ -1323,14 +1543,14 @@ mod tests {
// (proof the cursor answered with zero index-file reads)...
std::fs::remove_dir_all(&dir).expect("remove dir");
let in_interval = plan
- .resolve_sealed_start(&segment, offset_query(29), &partition_dir)
+ .resolve_sealed_start(&segment, offset_query(29), &partition_dir, usize::MAX)
.await;
assert_eq!(in_interval, Some((200, 20)));
// ...while an offset past the interval misses the cursor, reaches for
// the (now gone) file, and falls back to the byte-0 scan.
let past_interval = plan
- .resolve_sealed_start(&segment, offset_query(30), &partition_dir)
+ .resolve_sealed_start(&segment, offset_query(30), &partition_dir, usize::MAX)
.await;
assert_eq!(past_interval, None);
}
@@ -1362,6 +1582,7 @@ mod tests {
tier: PollTier::Resident {
fragments: placeholder_fragments(),
last_matching_offset: Some(last_selected_offset),
+ message_count: 1,
},
};
assert!(!resident_plan.needs_off_pump_io());
@@ -1395,4 +1616,101 @@ mod tests {
assert!(read_result.fragments.is_empty());
assert_eq!(read_result.last_matching_offset, None);
}
+ #[compio::test]
+ async fn deferred_disk_result_enforces_byte_limits_and_rejects_corruption() {
+ for corrupt_tail in [false, true] {
+ let directory = tempfile::tempdir().unwrap();
+ let mut bytes = disk_batch(10, 0);
+ let first_batch_bytes = bytes.len();
+ let mut tail = disk_batch(10, 9);
+ if corrupt_tail {
+ *tail.last_mut().unwrap() ^= 1;
+ }
+ bytes.extend(tail);
+ std::fs::write(directory.path().join("00000000000000000000.log"), &bytes).unwrap();
+ let plan = PollPlan {
+ context: PollContext {
+ history: PollHistoryId::default(),
+ consumer: PollingConsumer::Consumer(0, 0),
+ auto_commit: true,
+ },
+ commit_offset: 9,
+ tier: PollTier::Disk {
+ query: MessageLookup::Offset {
+ offset: 0,
+ count: 2,
+ ceiling: 9,
+ },
+ resident_tail: ResidentTailSnapshot {
+ entries: Vec::new(),
+ oldest_resident: None,
+ },
+ disk: DiskReadPlan {
+ partition_dir: PartitionDirResolution::Resolved(
+ directory.path().display().to_string(),
+ ),
+ bytes_per_message: None,
+ widest_batch_bytes: 0,
+ segments: vec![DiskSegment {
+ start_offset: 0,
+ persisted: bytes.len() as u64,
+ read_state: Rc::default(),
+ sealed: false,
+ }],
+ start_position: 0,
+ start_index_offset: None,
+ namespace_raw: 0,
+ validate_checksum: true,
+ },
+ },
+ };
+ let result = plan.execute_with_limit(1 << 20).await;
+ if corrupt_tail {
+ assert_eq!(
+ result.message_count(),
+ 1,
+ "a valid prefix must not hide corruption"
+ );
+ assert!(matches!(
+ result.checked(),
+ Err(IggyError::CannotReadMessage)
+ ));
+ } else {
+ assert_eq!(
+ result.message_count(),
+ 2,
+ "sparse offsets count as two messages"
+ );
+ let (result, limited) = result
+ .checked()
+ .unwrap()
+ .limit_bytes(POLL_RESPONSE_HEADER_SIZE + first_batch_bytes)
+ .unwrap();
+ assert!(limited);
+ assert_eq!(result.message_count(), 1);
+ assert_eq!(result.last_matching_offset, Some(0));
+ assert_eq!(result.commit_offset, 9);
+ }
+ }
+ }
+
+ #[compio::test]
+ async fn deferred_disk_budget_rejects_oversized_batch_before_growing_buffer() {
+ let directory = tempfile::tempdir().unwrap();
+ let bytes = disk_batch(2 << 20, 0);
+ let path = directory.path().join("batch.log");
+ std::fs::write(&path, &bytes).unwrap();
+ let file = compio::fs::File::open(path).await.unwrap();
+ let plan = sizing_plan(Some(1), 0);
+ let mut walk = DiskWalk::starting_at(0, 0);
+ walk.max_bytes = 1 << 20;
+ assert!(matches!(
+ plan.walk_segment(&file, offset_query(0), 1, bytes.len() as u64, &mut walk)
+ .await,
+ SegmentWalk::Faulted
+ ));
+ assert!(walk.limited);
+ assert_eq!(walk.matched, 0);
+ assert!(walk.fragments.is_empty());
+ }
}
diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs
index e4ffae9e2..173f0acaa 100644
--- a/core/partitions/src/state_transfer.rs
+++ b/core/partitions/src/state_transfer.rs
@@ -3328,6 +3328,7 @@ where
// registered handle must keep reading the same cells.
let end = next_offset.saturating_sub(1);
self.offset.store(end, Ordering::Release);
+ self.publish_poll_visibility();
self.dirty_offset.store(end, Ordering::Relaxed);
self.set_offset_space_used(next_offset > 0);
self.recovered_durable_offset = installed_end;
@@ -3637,6 +3638,7 @@ where
// claim goes back to None; the partition is honestly lagging.
let end = minted_next_offset.saturating_sub(1);
self.offset.store(end, Ordering::Release);
+ self.publish_poll_visibility();
self.dirty_offset.store(end, Ordering::Relaxed);
self.set_offset_space_used(minted_next_offset > 0);
self.recovered_durable_offset = None;
diff --git a/core/partitions/src/types.rs b/core/partitions/src/types.rs
index acb6bdfb2..7619552c7 100644
--- a/core/partitions/src/types.rs
+++ b/core/partitions/src/types.rs
@@ -47,6 +47,11 @@ pub struct Fragment<const ALIGN: usize = 4096> {
}
impl<const ALIGN: usize> Fragment<ALIGN> {
+ #[must_use]
+ pub fn allocation_bytes(&self) -> usize {
+ self.source.allocation_bytes()
+ }
+
#[must_use]
pub fn whole(source: Frozen<ALIGN>) -> Self {
let end = source.len();
@@ -108,7 +113,7 @@ pub struct PollingArgs {
}
pub type PollFragments<const ALIGN: usize = 4096> = SmallVec<[Fragment<ALIGN>; 4]>;
-pub type PollQueryResult<const ALIGN: usize = 4096> = (PollFragments<ALIGN>, Option<u64>);
+pub type PollQueryResult<const ALIGN: usize = 4096> = (PollFragments<ALIGN>, Option<u64>, u32);
impl PollingArgs {
#[must_use]
diff --git a/core/sdk/README.md b/core/sdk/README.md
index 826d93b25..486863c18 100644
--- a/core/sdk/README.md
+++ b/core/sdk/README.md
@@ -108,14 +108,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
let mut consumer = client
.consumer_group(CONSUMER_GROUP, STREAM, TOPIC)?
- .auto_commit(AutoCommit::IntervalOrWhen(
- NonZeroIggyDuration::from_str("1s")?,
- AutoCommitWhen::ConsumingAllMessages,
- ))
.create_consumer_group_if_not_exists()
.auto_join_consumer_group()
.polling_strategy(PollingStrategy::next())
- .poll_interval(IggyDuration::from_str("1ms")?)
.batch_length(1000)
.build();
consumer.init().await?;
@@ -136,6 +131,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
println!(" header {key} = {value:?}");
}
}
+ consumer
+ .store_offset(message.message.header.offset, Some(message.partition_id))
+ .await?;
}
Err(error) => eprintln!("poll error: {error}"),
}
@@ -148,6 +146,70 @@ For lower-level control over individual commands (login, stream/topic management
For `IggyConsumerConfig`, `partitions_count` controls topic creation only. An ordinary consumer uses partition `0` unless the builder's `partition_id` or the config's `with_partition_id` selects another partition. Code that previously used `partitions_count` to select an existing partition must set `partition_id` explicitly. Consumer-group assignment ignores `partition_id`.
+## Deferred polling
+
+High-level consumers use long polling by default. `DeferredPollOptions` separates
+readiness from batch limits: `min_count = 1`, `max_wait = 1s`, `max_bytes = 1 MiB`
+and `request_timeout = 11s`. `batch_length` remains the maximum message count.
+Use `.poll_options(options)` on either consumer builder, or pass options to the
+low-level `poll_messages_deferred` method. A zero `max_wait` skips readiness
+waiting while preserving the same byte limits and request budget.
+
+The server responds when the minimum is readable, the byte cap is reached, or the
+readiness wait expires. Expiry allows a final bounded read; the separate request
+timeout bounds routing, waiting, I/O and response handling. It returns partial or
+empty data when appropriate. Read failures and exhausted request budgets return
+errors. `max_bytes` covers the binary body, including response metadata and batch
+framing; HTTP applies the same selection limit before JSON encoding. A first
+record that cannot fit returns `InvalidSizeBytes`, without advancing its offset.
+
+Both builders default to `Next` and manual commits. Process messages in partition order,
+then call `store_offset(offset, Some(partition_id))`. Fetch positions advance
+independently of stored offsets; an initial `Next` is resolved once per partition
+and assignment. Restarting before a commit can replay work. Explicit auto-commit
+policies remain available, but `PollingMessages` can commit prefetched messages
+before application delivery or processing.
+
+Prefetch reserves both byte and message capacity before each request and holds
+it through queued and partially consumed batches. Defaults are 16 MiB of encoded
+response bytes and 16,000 messages; use `.prefetch_bytes(...)` and
+`.prefetch_messages(...)` to change them. Each must fit at least one maximum
+response. Decoded message metadata and bounded HTTP JSON overhead are additional;
+these limits are not process RSS limits. Memory handed to application code is
+outside the consumer budget. Assignment refresh, reconnection and cancellation
+continue while the application stalls. Revoked generations are discarded before
+delivery. The normal `poll_interval` setting has been removed; error backoff is
+still configurable.
+
+Binary data requests lease separate connections, preserving control traffic.
+At most 16 single-partition polls run concurrently, rotating through larger
+assignments. This does not watch every partition simultaneously when there are
+more than 16. Cancellation discards the leased connection. An ambiguous reply
+failure is not replayed inside the transport; the high-level worker can retry its
+explicit position. With opt-in server auto-commit, cancellation may race an
+accepted offset update.
+
+HTTP uses `GET /streams/{stream}/topics/{topic}/messages/deferred` with the ordinary
+poll fields plus `wait_us`, `min_count`, `max_bytes` and `request_timeout_us`.
+Omitted options use the SDK defaults. Binary commands 105 and 106 carry the same
+contract. Upgrade servers before SDKs; unsupported servers reject these requests.
+Retries and forwarding deduct from the original budgets. Proxy timeouts should
+exceed the configured request timeout. The low-level immediate poll API retains
+its existing command and route.
+
+The server defaults to a 30-second maximum readiness wait, 1024 pending polls per
+shard, 64 per logical session (or HTTP user), 16 MiB per read reservation and
+64 MiB of in-flight read reservations. Conservative snapshot and disk allocation
+bounds can reject a small selection backed by a large allocation. Configure
+`sharding.deferred_poll_*` accordingly. The response cap is applied to the selected
+result; it does not cap all storage work. Capacity refusal returns
+`TransientNotAccepted`. Metrics expose pending polls, read reservations and
+completion outcomes.
+
+Bench exposes `--max-wait`, `--min-count`, `--max-bytes` and `--request-timeout`
+for both consumer APIs and records them in report names. Intentional batching
+waits contribute to measured latency.
+
## Versioning
Stable releases follow semver (`x.y.z`). Edge releases (`x.y.z-edge.N`) are cut from `master` between stable versions and may include unreleased fixes; pin to a stable version for production.
diff --git a/core/sdk/src/client_wrappers/binary_message_client.rs b/core/sdk/src/client_wrappers/binary_message_client.rs
index 3fb72563a..1ea9d303f 100644
--- a/core/sdk/src/client_wrappers/binary_message_client.rs
+++ b/core/sdk/src/client_wrappers/binary_message_client.rs
@@ -56,11 +56,35 @@ impl MessageClient for ClientWrapper {
strategy_for: &(dyn Fn(u32) -> PollingStrategy + Send + Sync),
count: u32,
auto_commit: bool,
+ ) -> Result<PolledMessages, IggyError> {
+ self.poll_messages_with_strategy_for_and_options(
+ stream_id,
+ topic_id,
+ partition_id,
+ consumer,
+ strategy_for,
+ count,
+ auto_commit,
+ None,
+ )
+ .await
+ }
+
+ async fn poll_messages_with_strategy_for_and_options(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: Option<u32>,
+ consumer: &Consumer,
+ strategy_for: &(dyn Fn(u32) -> PollingStrategy + Send + Sync),
+ count: u32,
+ auto_commit: bool,
+ options: Option<iggy_common::DeferredPollOptions>,
) -> Result<PolledMessages, IggyError> {
match self {
ClientWrapper::Iggy(client) => {
client
- .poll_messages_with_strategy_for(
+ .poll_messages_with_strategy_for_and_options(
stream_id,
topic_id,
partition_id,
@@ -68,12 +92,13 @@ impl MessageClient for ClientWrapper {
strategy_for,
count,
auto_commit,
+ options,
)
.await
}
ClientWrapper::Http(client) => {
client
- .poll_messages_with_strategy_for(
+ .poll_messages_with_strategy_for_and_options(
stream_id,
topic_id,
partition_id,
@@ -81,12 +106,13 @@ impl MessageClient for ClientWrapper {
strategy_for,
count,
auto_commit,
+ options,
)
.await
}
ClientWrapper::Tcp(client) => {
client
- .poll_messages_with_strategy_for(
+ .poll_messages_with_strategy_for_and_options(
stream_id,
topic_id,
partition_id,
@@ -94,12 +120,13 @@ impl MessageClient for ClientWrapper {
strategy_for,
count,
auto_commit,
+ options,
)
.await
}
ClientWrapper::Quic(client) => {
client
- .poll_messages_with_strategy_for(
+ .poll_messages_with_strategy_for_and_options(
stream_id,
topic_id,
partition_id,
@@ -107,12 +134,13 @@ impl MessageClient for ClientWrapper {
strategy_for,
count,
auto_commit,
+ options,
)
.await
}
ClientWrapper::WebSocket(client) => {
client
- .poll_messages_with_strategy_for(
+ .poll_messages_with_strategy_for_and_options(
stream_id,
topic_id,
partition_id,
@@ -120,6 +148,7 @@ impl MessageClient for ClientWrapper {
strategy_for,
count,
auto_commit,
+ options,
)
.await
}
diff --git a/core/sdk/src/client_wrappers/client_wrapper.rs b/core/sdk/src/client_wrappers/client_wrapper.rs
index 4b347ab0c..239d5e433 100644
--- a/core/sdk/src/client_wrappers/client_wrapper.rs
+++ b/core/sdk/src/client_wrappers/client_wrapper.rs
@@ -20,6 +20,8 @@ use crate::http::http_client::HttpClient;
use crate::quic::quic_client::QuicClient;
use crate::tcp::tcp_client::TcpClient;
use crate::websocket::websocket_client::WebSocketClient;
+use iggy_common::locking::IggyRwLockFn;
+use iggy_common::{BinaryTransport, Identifier, IggyError, sync_group_assignment};
#[allow(clippy::large_enum_variant)]
#[derive(Debug)]
@@ -30,3 +32,62 @@ pub enum ClientWrapper {
Quic(QuicClient),
WebSocket(WebSocketClient),
}
+
+impl ClientWrapper {
+ pub(crate) async fn deferred_group_state(
+ &self,
+ ) -> Result<std::sync::Arc<iggy_common::ConsumerGroupClientState>, IggyError> {
+ match self {
+ Self::Tcp(client) => Ok(client.consumer_group_state()),
+ Self::Quic(client) => Ok(client.consumer_group_state()),
+ Self::WebSocket(client) => Ok(client.consumer_group_state()),
+ Self::Http(_) => Err(IggyError::FeatureUnavailable),
+ Self::Iggy(client) => Box::pin(client.client.read().await.deferred_group_state()).await,
+ }
+ }
+
+ pub(crate) async fn deferred_poll_partitions(
+ &self,
+ stream: &Identifier,
+ topic: &Identifier,
+ group: &Identifier,
+ refresh: bool,
+ ) -> Result<(u64, u64, Vec<u32>), IggyError> {
+ if let Self::Iggy(client) = self {
+ return Box::pin(
+ client
+ .client
+ .read()
+ .await
+ .deferred_poll_partitions(stream, topic, group, refresh),
+ )
+ .await;
+ }
+ let state = match self {
+ Self::Tcp(client) => client.consumer_group_state(),
+ Self::Quic(client) => client.consumer_group_state(),
+ Self::WebSocket(client) => client.consumer_group_state(),
+ Self::Http(_) => return Err(IggyError::FeatureUnavailable),
+ Self::Iggy(_) => unreachable!("nested clients handled above"),
+ };
+ let key = format!("{stream}|{topic}|{group}");
+ if refresh || !state.is_registered(&key) {
+ match self {
+ Self::Tcp(client) => sync_group_assignment(client, stream, topic, group).await?,
+ Self::Quic(client) => sync_group_assignment(client, stream, topic, group).await?,
+ Self::WebSocket(client) => {
+ sync_group_assignment(client, stream, topic, group).await?
+ }
+ Self::Http(_) | Self::Iggy(_) => unreachable!("binary transport selected above"),
+ }
+ }
+ if !state.is_registered(&key) {
+ return Err(IggyError::ConsumerGroupMemberNotFound(
+ 0,
+ group.clone(),
+ topic.clone(),
+ ));
+ }
+ Ok(state.assignment_snapshot(&key))
+ }
+}
diff --git a/core/sdk/src/clients/binary_message.rs b/core/sdk/src/clients/binary_message.rs
index e83321bbd..882422906 100644
--- a/core/sdk/src/clients/binary_message.rs
+++ b/core/sdk/src/clients/binary_message.rs
@@ -58,6 +58,30 @@ impl MessageClient for IggyClient {
strategy_for: &(dyn Fn(u32) -> PollingStrategy + Send + Sync),
count: u32,
auto_commit: bool,
+ ) -> Result<PolledMessages, IggyError> {
+ self.poll_messages_with_strategy_for_and_options(
+ stream_id,
+ topic_id,
+ partition_id,
+ consumer,
+ strategy_for,
+ count,
+ auto_commit,
+ None,
+ )
+ .await
+ }
+
+ async fn poll_messages_with_strategy_for_and_options(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: Option<u32>,
+ consumer: &Consumer,
+ strategy_for: &(dyn Fn(u32) -> PollingStrategy + Send + Sync),
+ count: u32,
+ auto_commit: bool,
+ options: Option<iggy_common::DeferredPollOptions>,
) -> Result<PolledMessages, IggyError> {
if count == 0 {
return Err(IggyError::InvalidMessagesCount);
@@ -67,7 +91,7 @@ impl MessageClient for IggyClient {
.client
.read()
.await
- .poll_messages_with_strategy_for(
+ .poll_messages_with_strategy_for_and_options(
stream_id,
topic_id,
partition_id,
@@ -75,6 +99,7 @@ impl MessageClient for IggyClient {
strategy_for,
count,
auto_commit,
+ options,
)
.await?;
diff --git a/core/sdk/src/clients/client.rs b/core/sdk/src/clients/client.rs
index 09b8beff4..a35f25b20 100644
--- a/core/sdk/src/clients/client.rs
+++ b/core/sdk/src/clients/client.rs
@@ -173,7 +173,6 @@ const SESSION_CONTROL_CODES: [u32; 5] = [
/// .consumer("consumer_name", "stream_name", "topic_name", 1)?
/// .auto_commit(AutoCommit::When(AutoCommitWhen::PollingMessages))
/// .polling_strategy(PollingStrategy::next())
-/// .poll_interval(IggyDuration::ONE_SECOND)
/// .batch_length(1000)
/// .build();
/// consumer.init().await?;
@@ -625,7 +624,6 @@ impl IggyClient {
/// .consumer("consumer_name", "stream_name", "topic_name", 1)? // returns IggyConsumerBuilder from IggyClient
/// .auto_commit(AutoCommit::When(AutoCommitWhen::PollingMessages))
/// .polling_strategy(PollingStrategy::next())
- /// .poll_interval(IggyDuration::ONE_SECOND)
/// .batch_length(1000)
/// .build(); // returns IggyConsumer from IggyConsumerBuilder
/// # let _ = consumer;
@@ -655,7 +653,6 @@ impl IggyClient {
topic.try_into()?,
Some(partition),
self.encryptor.clone(),
- None,
))
}
@@ -697,7 +694,6 @@ impl IggyClient {
/// .consumer_group("group_name", "stream_name", "topic_name")? // returns IggyConsumerBuilder from IggyClient
/// .auto_commit(AutoCommit::When(AutoCommitWhen::PollingMessages))
/// .polling_strategy(PollingStrategy::next())
- /// .poll_interval(IggyDuration::ONE_SECOND)
/// .batch_length(1000)
/// .build(); // returns IggyConsumer from IggyConsumerBuilder
/// # let _ = consumer;
@@ -726,7 +722,6 @@ impl IggyClient {
topic.try_into()?,
None,
self.encryptor.clone(),
- None,
))
}
diff --git a/core/sdk/src/clients/consumer.rs b/core/sdk/src/clients/consumer.rs
index 52a953c4d..1791287d4 100644
--- a/core/sdk/src/clients/consumer.rs
+++ b/core/sdk/src/clients/consumer.rs
@@ -16,35 +16,27 @@
// under the License.
use crate::client_wrappers::client_wrapper::ClientWrapper;
-use bytes::Bytes;
use dashmap::DashMap;
use futures::Stream;
-use futures_util::{FutureExt, StreamExt};
+use futures_util::StreamExt;
use iggy_common::locking::{IggyRwLock, IggyRwLockFn};
+use iggy_common::{Client, ConsumerGroupClient, ConsumerOffsetClient, StreamClient, TopicClient};
use iggy_common::{
- Client, ConsumerGroupClient, ConsumerOffsetClient, MessageClient, StreamClient, TopicClient,
-};
-use iggy_common::{
- Consumer, ConsumerKind, DiagnosticEvent, EncryptorKind, IdKind, Identifier, IggyDuration,
- IggyError, IggyMessage, IggyTimestamp, NO_ASSIGNED_PARTITION, NonZeroIggyDuration,
- PolledMessages, PollingKind, PollingStrategy,
+ Consumer, ConsumerKind, DeferredPollOptions, DiagnosticEvent, EncryptorKind, IdKind,
+ Identifier, IggyDuration, IggyError, IggyMessage, NonZeroIggyDuration, PollingStrategy,
};
use std::collections::VecDeque;
use std::fmt::{self, Debug, Formatter};
-use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64};
use std::task::{Context, Poll};
-use std::time::Duration;
use tokio::sync::Notify;
use tokio::task::JoinHandle;
use tokio::time;
-use tokio::time::sleep;
use tracing::{debug, error, info, trace, warn};
const ORDERING: std::sync::atomic::Ordering = std::sync::atomic::Ordering::SeqCst;
-type PollMessagesFuture = Pin<Box<dyn Future<Output = Result<PolledMessages, IggyError>> + Send>>;
/// The auto-commit configuration for storing the offset on the server.
#[derive(Debug, PartialEq, Copy, Clone)]
@@ -261,112 +253,31 @@ impl IggyConsumerState {
}
}
-// SAFETY: IggyConsumer is Sync because:
-// 1. The only non-Sync field is `poll_future: Option<PollMessagesFuture>`
-// 2. `poll_future` is only accessed through `poll_next()` which requires `Pin<&mut Self>`
-// (exclusive mutable access), so concurrent access to `poll_future` is impossible
-// 3. All other fields are inherently Sync (Arc<AtomicX>, Arc<DashMap>, etc.) or
-// only accessed through `&mut self` methods
-// 4. All `&self` methods only access Sync-safe fields
-unsafe impl Sync for IggyConsumer {}
-
-/// Reads messages from the partitions of one topic and yields them one at a time.
-///
-/// A topic is split into partitions, and a partition is an ordered log that producers append to.
-/// Every message sits at an *offset*, its position in that log. Reading is therefore always the
-/// same three decisions: which partition to read, where in it to start, and how to keep track of
-/// how far you got so the next run can continue there.
+/// Reads a topic as a stream of messages, using long polling and bounded prefetch.
///
-/// `IggyConsumer` handles all three. It fetches batches of messages from the server, keeps them in
-/// an in-memory buffer, decrypts them when it has an encryptor, and records how far it has read.
-/// **It implements [`Stream`], so consuming is a loop over [`StreamExt::next`].**
+/// Call [`Self::init`] before reading and [`Self::shutdown`] before dropping the consumer.
+/// A standalone consumer reads one partition. A group member reads its assignment on
+/// independent data connections, while control traffic and assignment refresh continue
+/// even when the application stops reading. At most 16 partition fetches run concurrently.
///
-/// You can use a consumer as a worker draining a topic, a reader that replays a
-/// partition from a chosen point, and a pool of consumers sharing a workload through a consumer
-/// group.
+/// A poll returns when at least `min_count` messages are available (one by default),
+/// its byte limit is reached, or `max_wait` expires. `batch_length` is the maximum
+/// message count. The separate request timeout covers routing, waiting and reading.
+/// There is no normal polling interval; errors and reconnects retain retry backoff.
///
-/// # Creating a consumer
+/// # Positions and commits
///
-/// Easiest way is to use the [`IggyClient`] with a configured connection. Then:
-/// - [`IggyClient::consumer()`] builds a standalone consumer, bound to the one partition passed
-/// in.
-/// - [`IggyClient::consumer_group()`] builds a member of a consumer group. The server gives every
-/// partition to exactly one member, so several consumers using the same group name split the
-/// topic between them and share one set of offsets.
+/// The initial polling strategy is resolved separately for each partition. `Next`
+/// starts after the stored offset, or at zero when no offset exists. Subsequent
+/// fetches use explicit offsets, so a lost reply can be retried without advancing
+/// past unseen messages. Reassignment discards prefetched data and resolves the
+/// starting position again. Messages already delivered are filtered unless replay
+/// is enabled. Delivery and durable processing progress are separate.
///
-/// Note, building never talks to the server. [`init()`](Self::init) must be awaited once before the
-/// first message is read.
-///
-/// # Examples
-///
-/// A standalone consumer reading partition 1 with the defaults:
-///
-/// ```rust,no_run
-/// use futures_util::StreamExt;
-/// use iggy::prelude::*;
-///
-/// # async fn example() -> Result<(), IggyError> {
-/// let client = IggyClient::from_connection_string("iggy://iggy:iggy@localhost:8090")?;
-/// client.connect().await?;
-///
-/// let mut consumer = client
-/// .consumer("my-consumer", "my-stream", "my-topic", 1)?
-/// .batch_length(100)
-/// .poll_interval(IggyDuration::new_from_secs(1))
-/// .build();
-/// consumer.init().await?;
-///
-/// while let Some(received) = consumer.next().await {
-/// match received {
-/// Ok(received) => println!("Offset: {}", received.message.header.offset),
-/// Err(error) => eprintln!("Failed to read a message: {error}"),
-/// }
-/// }
-/// # Ok(())
-/// # }
-/// ```
-///
-/// A group member that queues a commit for every message just before handing it over, and shuts
-/// down cleanly:
-///
-/// ```rust,no_run
-/// use futures_util::StreamExt;
-/// use iggy::prelude::*;
-///
-/// # async fn handle(message: &IggyMessage) {}
-/// # async fn example() -> Result<(), IggyError> {
-/// let client = IggyClient::from_connection_string("iggy://iggy:iggy@localhost:8090")?;
-/// client.connect().await?;
-///
-/// let mut consumer = client
-/// .consumer_group("order-workers", "my-stream", "my-topic")?
-/// .auto_commit(AutoCommit::When(AutoCommitWhen::ConsumingEachMessage))
-/// .polling_strategy(PollingStrategy::next())
-/// .build();
-/// consumer.init().await?;
-///
-/// let mut consumed = 0;
-/// while let Some(received) = consumer.next().await {
-/// match received {
-/// Ok(received) => {
-/// handle(&received.message).await;
-/// consumed += 1;
-/// }
-/// Err(error) => eprintln!("Failed to read a message: {error}"),
-/// }
-/// if consumed == 100 {
-/// break;
-/// }
-/// }
-///
-/// consumer.shutdown().await?;
-/// # Ok(())
-/// # }
-/// ```
-///
-/// Committing by hand, so that a message the handler could not process comes back on the next
-/// run. Every commit is one round trip, and no auto-commit setting substitutes, since each of them
-/// also commits a message whose handler failed:
+/// The default is [`AutoCommit::Disabled`]. Store offsets only after successfully
+/// processing messages, in partition order. Fetching ahead never commits under this
+/// default. Opting into auto-commit can acknowledge messages before processing;
+/// `PollingMessages` also acknowledges batches still in the prefetch buffer.
///
/// ```rust,no_run
/// use futures_util::StreamExt;
@@ -376,265 +287,43 @@ unsafe impl Sync for IggyConsumer {}
/// # async fn example() -> Result<(), IggyError> {
/// let client = IggyClient::from_connection_string("iggy://iggy:iggy@localhost:8090")?;
/// client.connect().await?;
-///
-/// let mut consumer = client
-/// .consumer("my-consumer", "my-stream", "my-topic", 1)?
-/// .auto_commit(AutoCommit::Disabled)
-/// .polling_strategy(PollingStrategy::next())
-/// .build();
+/// let mut consumer = client.consumer("reader", "stream", "topic", 0)?.build();
/// consumer.init().await?;
-///
/// while let Some(received) = consumer.next().await {
-/// let received = match received {
-/// Ok(received) => received,
-/// Err(error) => {
-/// eprintln!("Failed to read a message: {error}");
-/// continue;
-/// }
-/// };
-/// // Leaving a failed message uncommitted is what brings it back on the next run.
-/// if handle(&received.message).await.is_err() {
-/// break;
-/// }
-/// consumer
-/// .store_offset(received.message.header.offset, Some(received.partition_id))
-/// .await?;
+/// let received = received?;
+/// handle(&received.message).await?;
+/// consumer.store_offset(received.message.header.offset, Some(received.partition_id)).await?;
/// }
-///
/// consumer.shutdown().await?;
/// # Ok(())
/// # }
/// ```
///
-/// # Which partitions are read
-///
-/// A **standalone consumer** reads exactly one partition, the one passed to
-/// [`IggyClient::consumer()`]. Covering a whole topic with several partitions
-/// means running one consumer per partition and dividing the work yourself.
-///
-/// A **consumer group member** does not choose. The server hands every partition of the topic to
-/// exactly one member, so consumers sharing a group name split the topic between them without
-/// coordinating. [`ReceivedMessage::partition_id`] tells where a message came from.
-///
-/// What to know when working with consumer groups:
-/// - With [`auto_join_consumer_group()`] (the default) a member joins during [`init()`](Self::init),
-/// creating the group first if [`create_consumer_group_if_not_exists()`] is set (the default).
-/// It rejoins on its own after a reconnect and whenever the server reports that its membership
-/// is gone.
-/// - Such a member does not poll until it is in the group. A join that fails is yielded as
-/// `Some(Err(..))` after [`polling_retry_interval()`], and the next call tries again.
-/// - Partitions are redistributed whenever members join or leave, so a member reads different
-/// partitions over time and messages from several partitions interleave in its stream.
-/// - More members than partitions leaves the surplus members without partitions. Such a member
-/// keeps asking the server for an assignment, parking for [`polling_retry_interval()`] between
-/// attempts. The partition count of the topic is the ceiling on how far one group can be
-/// scaled out.
-/// - The group shares one set of stored offsets, kept under the group name. Thus,
-/// a partition taken over by another member continues where the previous one
-/// committed.
-///
-/// # How messages are read
+/// # Buffer bounds
///
-/// Reading is done by polling. One request fetches up to [`batch_length()`] messages. The consumer
-/// passes the first one to the caller and buffers the rest. The next request is sent once that buffer
-/// is empty.
+/// Both builders reserve a full possible batch in bytes and messages before sending
+/// each request. The reservation covers in-flight, completed and buffered replies
+/// until the batch is drained or discarded. The byte budget measures encoded poll
+/// replies; decoded message metadata is additionally bounded by the message budget.
+/// Messages retained by application code are outside these SDK buffer limits.
+/// Assignment changes invalidate buffered generations before the next delivery.
///
-/// [`poll_interval()`] sets the smallest gap between two requests, measured from one send to the
-/// next. Without it the next request goes out as soon as the previous one is answered, which is
-/// the fastest option but keeps a busy loop running against an idle topic.
-///
-/// [`polling_strategy()`] decides **where** in the partition reading begins:
-///
-/// | Strategy | Starts at |
-/// | --- | --- |
-/// | [`PollingStrategy::next()`] (default) | the message after the offset stored on the server, or the first message when nothing is stored yet |
-/// | [`PollingStrategy::first()`] | the oldest message in the partition |
-/// | [`PollingStrategy::last()`] | the end of the partition (returns up to [`batch_length()`] of the most recent messages) |
-/// | [`PollingStrategy::offset()`] | a custom offset |
-/// | [`PollingStrategy::timestamp()`] | the first message at or after a given point in time |
-///
-/// Only [`PollingStrategy::next()`] consults the offset stored on the server.
-/// Use this if you want to resume where a previous run stopped. The other four are the starting
-/// point for the first request to each partition. From then on the consumer asks that partition
-/// for whatever follows the last message it handed over from it, and it keeps that position per
-/// partition. A partition that moves to another member and back therefore continues from this
-/// consumer's own position, not from where the other member got to, so a rebalance can repeat
-/// messages under these strategies, which ignore the group's stored offsets by definition.
-///
-/// [`StreamExt::next`] yields `None` once [`shutdown()`](Self::shutdown) has been called, and never
-/// otherwise: not when the topic is empty and not while the client is disconnected. A request that
-/// comes back empty is not an error and not the end of the stream, it just means nothing new has
-/// arrived yet.
-///
-/// A failed poll request waits [`polling_retry_interval()`] before yielding `Some(Err(..))` and leaves
-/// the consumer usable for the next call. This delay also applies to terminal server errors and when
-/// the ordinary poll interval is disabled. Connection and authentication failures are yielded at once
-/// and pause polling until the client has reconnected and signed in again, which the consumer handles
-/// automatically; the next call parks for [`polling_retry_interval()`] while polling is paused. Hence,
-/// deciding when to give up on repeated errors is up to you.
-///
-/// With [`AutoCommitWhen::PollingMessages`], a missing response can leave the server's
-/// cursor ahead of messages this consumer received. Continuing with
-/// [`PollingStrategy::next()`] can skip those messages. See the
-/// [poll recovery contract](MessageClient::poll_messages) for recovery from explicit
-/// checkpoints for each partition.
-///
-/// For a boilerplate implementation of such a loop Iggy provides [`IggyConsumerMessageExt::consume_messages`].
-///
-/// # Tracking what has been read
-///
-/// An offset is the index tracking what has been already read from a partition by the consumer.
-/// Managing the offset has implications on where consumers resume reading messages.
-///
-/// There are two positions (offsets) tracked in two different places:
-/// - The **reading position** is held by the consumer, one per partition, and is the offset of the
-/// last message handed over
-/// ([`get_last_consumed_offset()`](Self::get_last_consumed_offset)). It dies with the process.
-/// - The **stored offset** is committed to the server under the consumer or group name.
-/// Its restart guarantees depend on the topic's consumer-offset durability policy.
-///
-/// Committing matters because [`PollingStrategy::next()`] resumes from the stored offset. A
-/// consumer that never commits starts over from the same place on every run. Within a run it
-/// stalls instead: the server serves the same messages again on every request, messages at or
-/// below the reading position are dropped (see [Guarantees](#guarantees)), so the stream goes
-/// quiet once the reading position is [`batch_length()`] or more ahead of the stored offset. Under
-/// [`PollingStrategy::next()`], keep commits within [`batch_length()`] of the reading position.
-///
-/// [`auto_commit()`] decides when the consumer commits by itself:
-///
-/// | Setting | Commits |
-/// | --- | --- |
-/// | [`AutoCommit::Disabled`] | never, not even on [`shutdown()`](Self::shutdown). Commit with [`store_offset()`](Self::store_offset) |
-/// | [`AutoCommit::Interval`] | on every tick, the reading position of every partition read so far |
-/// | [`AutoCommitWhen::PollingMessages`] | sends the commit with the poll request itself, before your code sees the batch |
-/// | [`AutoCommitWhen::ConsumingEachMessage`] | queued just before every message is handed over to the calling code. Commits queued faster than they are sent collapse into the latest one per partition |
-/// | [`AutoCommitWhen::ConsumingEveryNthMessage`] | queued just before a message whose offset divides by `n` is handed over |
-/// | [`AutoCommitWhen::ConsumingAllMessages`] | queued when the buffer of the current batch runs empty |
-/// | [`AutoCommitAfter`] variants | once the handler returned, `Ok` or `Err`, and only under [`IggyConsumerMessageExt::consume_messages`], see below |
-///
-/// [`AutoCommit::IntervalOrWhen`] and [`AutoCommit::IntervalOrAfter`] combine an interval with a
-/// message trigger. The default is [`AutoCommit::IntervalOrWhen`] with one second and
-/// [`AutoCommitWhen::PollingMessages`].
-/// Important implications of these settings:
-/// - [`AutoCommitWhen::PollingMessages`] marks a batch as consumed while it is being delivered,
-/// before your code has seen any of it. For a crash-safe option configure with [`AutoCommit::Disabled`]
-/// and manually store the offset with [`Self::store_offset()`].
-/// - [`AutoCommitWhen::ConsumingEveryNthMessage`] tests the offset of a message, not a counter of
-/// messages this process handled, so it commits at every `n`-th offset of the partition. With
-/// `n = 0` the trigger never fires, so without an interval only [`shutdown()`](Self::shutdown)
-/// commits.
-/// - [`AutoCommitAfter::ConsumingAllMessages`] fires for the message whose offset equals the
-/// partition head seen by the poll ([`ReceivedMessage::current_offset`]), not when the buffer
-/// runs empty, so a consumer that lags behind commits nothing until it has caught up. Every
-/// [`AutoCommitAfter`] variant commits after a handler that returned `Err` as well.
-///
-/// ## Guarantees
-///
-/// - **Each message is handed over once per consumer.** Messages whose offset is not greater than
-/// the reading position of their partition are dropped before they reach the stream. Re-reading
-/// a partition, or seeing a failed message again within the same consumer, needs
-/// [`allow_replay()`], which turns that filter off. A new consumer starts with an empty filter.
-/// - **Delivering at-least-once.** If you cannot tolerate missing any messages, use [`AutoCommit::Disabled`]
-/// and store the offset using [`Self::store_offset()`] after handling a message. Every other
-/// setting except the plain [`AutoCommit::After`] variants can commit a message before your
-/// handler is done with it, so a crash in the handler loses it. [`AutoCommit::IntervalOrAfter`]
-/// still commits on its interval tick, and [`shutdown()`](Self::shutdown) commits the reading
-/// position under every setting but [`AutoCommit::Disabled`], a failed message included.
-///
-/// # Options and defaults
-///
-/// Everything is configured on the [`IggyConsumerBuilder`] before [`build()`] and is fixed
-/// afterwards.
-///
-/// | Option | Default | Controls |
-/// | --- | --- | --- |
-/// | [`stream()`], [`topic()`], [`partition()`] | the values passed to the entry point | what is read. [`partition()`] is for standalone consumers, a group member ignores it with a warning and reads the server's assignment |
-/// | [`batch_length()`] | 1000 | messages fetched per request |
-/// | [`poll_interval()`] | none | smallest gap between two requests |
-/// | [`polling_strategy()`] | [`PollingStrategy::next()`] | where reading each partition starts |
-/// | [`auto_commit()`] | [`AutoCommit::IntervalOrWhen`], one second, [`AutoCommitWhen::PollingMessages`] | when offsets are committed |
-/// | [`allow_replay()`] | off | whether a message can be handed over again |
-/// | [`auto_join_consumer_group()`] | on | joining the group during [`init()`](Self::init) and again whenever the membership is lost. With [`do_not_auto_join_consumer_group()`] joining is up to the caller, and a poll without a membership fails with [`IggyError::ConsumerGroupMemberNotFound`] |
-/// | [`create_consumer_group_if_not_exists()`] | on | creating the group when it is missing |
-/// | [`polling_retry_interval()`] | one second | delay before yielding poll errors other than connection and authentication failures, and between attempts while polling is blocked or the member holds no partitions |
-/// | [`init_retries()`] | none, one second apart | retries when the stream or topic is missing at [`init()`](Self::init) |
-/// | [`offset_drain_timeout()`] | five seconds | how long [`shutdown()`](Self::shutdown) waits for pending commits |
-/// | [`encryptor()`] | inherited from the client | decrypting payloads and user headers, see [Encryption](#encryption) |
-///
-/// The switches have inverse setters as well, such as [`without_poll_interval()`],
-/// [`without_encryptor()`], [`do_not_auto_join_consumer_group()`] and
-/// [`do_not_create_consumer_group_if_not_exists()`].
-///
-/// # Encryption
-///
-/// A consumer with an encryptor, inherited from the [`IggyClient`] or set with [`encryptor()`],
-/// decrypts payloads and user headers before a message is yielded. That only works if the producer
-/// encrypted them with the same key, which an [`IggyProducer`] and an `IggyConsumer` from the same
-/// client share unless one of them overrides it on its builder. Without an encryptor the consumer
-/// yields payloads as stored, encrypted or not.
-///
-/// A message that cannot be decrypted is yielded as an `Err` and the whole batch is dropped. The
-/// next request fetches the same batch and fails the same way until
-/// [`store_offset()`](Self::store_offset) moves the offset past it. Under
-/// [`AutoCommitWhen::PollingMessages`] the server would have committed the batch with the poll and
-/// skipped it for good, so [`init()`](Self::init) rejects that setting, the default included, with
-/// [`IggyError::InvalidConfiguration`] when the consumer has an encryptor.
-///
-/// # Concurrency
-///
-/// `IggyConsumer` is `Send` and `Sync` but not `Clone`. Driving the stream
-/// ([`StreamExt::next`]) and [`shutdown()`](Self::shutdown) take `&mut self`, so one task owns and
-/// drives a consumer end to end. To read offsets or commit from another task, take an
-/// [`IggyConsumerState`] via [`state()`](Self::state): it is a cheap clone of the shared
-/// bookkeeping and needs no lock.
-///
-/// Besides the stream, a consumer runs background tasks: one watching the connection lifecycle,
-/// one sending queued commits, and an interval commit task for the [`AutoCommit`] variants that
-/// carry an interval. See [`init()`](Self::init).
-///
-/// # Shutting down
-///
-/// Call [`shutdown()`](Self::shutdown) once done consuming. It drains the commit tasks, commits
-/// the reading position of every partition unless [`auto_commit()`] is [`AutoCommit::Disabled`],
-/// leaves the consumer group and stops the background tasks. Dropping an `IggyConsumer` instead
-/// skips the final commit and the group leave, so the server reassigns the member's partitions
-/// only once the connection is gone. Commits already queued are still sent and the background
-/// tasks still stop.
-///
-/// [`IggyClient`]: crate::prelude::IggyClient
-/// [`IggyClient::consumer()`]: crate::prelude::IggyClient::consumer
-/// [`IggyClient::consumer_group()`]: crate::prelude::IggyClient::consumer_group
-/// [`IggyProducer`]: crate::prelude::IggyProducer
-/// [`IggyConsumerBuilder`]: crate::prelude::IggyConsumerBuilder
-/// [`IggyConsumerMessageExt::consume_messages`]: crate::prelude::IggyConsumerMessageExt::consume_messages
-/// [`allow_replay()`]: crate::prelude::IggyConsumerBuilder::allow_replay
-/// [`auto_commit()`]: crate::prelude::IggyConsumerBuilder::auto_commit
-/// [`auto_join_consumer_group()`]: crate::prelude::IggyConsumerBuilder::auto_join_consumer_group
-/// [`batch_length()`]: crate::prelude::IggyConsumerBuilder::batch_length
-/// [`build()`]: crate::prelude::IggyConsumerBuilder::build
-/// [`create_consumer_group_if_not_exists()`]: crate::prelude::IggyConsumerBuilder::create_consumer_group_if_not_exists
-/// [`do_not_auto_join_consumer_group()`]: crate::prelude::IggyConsumerBuilder::do_not_auto_join_consumer_group
-/// [`do_not_create_consumer_group_if_not_exists()`]: crate::prelude::IggyConsumerBuilder::do_not_create_consumer_group_if_not_exists
-/// [`encryptor()`]: crate::prelude::IggyConsumerBuilder::encryptor
-/// [`init_retries()`]: crate::prelude::IggyConsumerBuilder::init_retries
-/// [`offset_drain_timeout()`]: crate::prelude::IggyConsumerBuilder::offset_drain_timeout
-/// [`partition()`]: crate::prelude::IggyConsumerBuilder::partition
-/// [`poll_interval()`]: crate::prelude::IggyConsumerBuilder::poll_interval
-/// [`polling_retry_interval()`]: crate::prelude::IggyConsumerBuilder::polling_retry_interval
-/// [`polling_strategy()`]: crate::prelude::IggyConsumerBuilder::polling_strategy
-/// [`stream()`]: crate::prelude::IggyConsumerBuilder::stream
-/// [`topic()`]: crate::prelude::IggyConsumerBuilder::topic
-/// [`without_encryptor()`]: crate::prelude::IggyConsumerBuilder::without_encryptor
-/// [`without_poll_interval()`]: crate::prelude::IggyConsumerBuilder::without_poll_interval
-///
-/// A server-side auto-commit poll can fail with `TooManyConsumerOffsets` when
-/// its consumer needs a new offset key at the partition's configured limit.
-/// The rejected poll returns no messages. Other auto-commit modes store the
-/// same key through a client request. Background and shutdown stores log a
-/// capacity failure but do not yield it through this stream. Only
-/// [`AutoCommit::Disabled`] avoids automatic key allocation. Existing keys
-/// remain usable.
+/// Encryption failures are returned without advancing the fetch position. Polling
+/// auto-commit with an encryptor is rejected because the server would commit before
+/// decryption. Other auto-commit policies retain their explicit semantics, including
+/// interval commits of delivered messages and `AutoCommitAfter` commits after a
+/// handler returns either success or failure.
pub struct IggyConsumer {
+ poll_options: DeferredPollOptions,
+ prefetch_bytes: u32,
+ prefetch_messages: u32,
+ deferred_polls: Option<deferred::DeferredPolls>,
+ fetch_generation: Arc<AtomicU64>,
+ fetch_notify: Arc<Notify>,
+ buffered_generation: u64,
+ buffered_assignment: Option<(u64, u64)>,
+ assignment_state: Option<(Arc<iggy_common::ConsumerGroupClientState>, String)>,
+ buffered_capacity: Option<deferred::FetchCapacity>,
initialized: bool,
shutdown: Arc<AtomicBool>,
can_poll: Arc<AtomicBool>,
@@ -647,10 +336,6 @@ pub struct IggyConsumer {
topic_id: Arc<Identifier>,
partition_id: Option<u32>,
polling_strategy: PollingStrategy,
- /// The next offset to ask each partition for. Empty under [`PollingStrategy::next()`], which
- /// leaves the continuation to the offset stored on the server.
- next_offsets: Arc<DashMap<u32, u64>>,
- poll_interval_micros: u64,
batch_length: u32,
auto_commit: AutoCommit,
auto_commit_after_polling: bool,
@@ -658,7 +343,6 @@ pub struct IggyConsumer {
create_consumer_group_if_not_exists: bool,
state: IggyConsumerState,
current_offsets: Arc<DashMap<u32, AtomicU64>>,
- poll_future: Option<PollMessagesFuture>,
buffered_messages: VecDeque<IggyMessage>,
encryptor: Option<Arc<EncryptorKind>>,
/// The latest offset each message trigger asked to commit, per partition. The store task
@@ -672,7 +356,6 @@ pub struct IggyConsumer {
store_offset_after_each_message: bool,
store_offset_after_all_messages: bool,
store_after_every_nth_message: u64,
- last_polled_at: Arc<AtomicU64>,
reconnection_retry_interval: NonZeroIggyDuration,
init_retries: Option<u32>,
init_retry_interval: NonZeroIggyDuration,
@@ -689,7 +372,9 @@ impl IggyConsumer {
stream_id: Identifier,
topic_id: Identifier,
partition_id: Option<u32>,
- polling_interval: Option<IggyDuration>,
+ poll_options: DeferredPollOptions,
+ prefetch_bytes: u32,
+ prefetch_messages: u32,
polling_strategy: PollingStrategy,
batch_length: u32,
auto_commit: AutoCommit,
@@ -723,6 +408,16 @@ impl IggyConsumer {
allow_replay,
);
Self {
+ poll_options,
+ prefetch_bytes,
+ prefetch_messages,
+ deferred_polls: None,
+ fetch_generation: Arc::new(AtomicU64::new(0)),
+ fetch_notify: Arc::new(Notify::new()),
+ buffered_generation: 0,
+ buffered_assignment: None,
+ assignment_state: None,
+ buffered_capacity: None,
initialized: false,
shutdown: Arc::new(AtomicBool::new(false)),
is_consumer_group,
@@ -735,11 +430,8 @@ impl IggyConsumer {
topic_id,
partition_id,
polling_strategy,
- next_offsets: Arc::new(DashMap::new()),
- poll_interval_micros: polling_interval.map_or(0, |interval| interval.as_micros()),
state,
current_offsets: Arc::new(DashMap::new()),
- poll_future: None,
batch_length,
auto_commit,
auto_commit_after_polling: matches!(
@@ -774,7 +466,6 @@ impl IggyConsumer {
}
_ => 0,
},
- last_polled_at: Arc::new(AtomicU64::new(0)),
reconnection_retry_interval,
init_retries,
init_retry_interval,
@@ -876,12 +567,11 @@ impl IggyConsumer {
}
/// Returns the offset this consumer last stored on the server for the given partition, or
- /// `None` while it has neither polled nor stored for that partition. The first poll or store
- /// seeds the entry, so `Some(0)` also covers "seen, nothing stored yet".
+ /// `None` until this consumer observes a successful commit for the partition.
///
/// The value is this consumer's own record of what it committed, kept in memory rather than
/// read back from the server.
- /// Under auto-commit-on-poll (the default) this can trail the server by up to one batch.
+ /// With auto-commit-on-poll, the server can be ahead by prefetched batches.
pub fn get_last_stored_offset(&self, partition_id: u32) -> Option<u64> {
self.state.get_last_stored_offset(partition_id)
}
@@ -923,7 +613,7 @@ impl IggyConsumer {
///
/// When the consumer commits is decided by
/// [`auto_commit()`](crate::prelude::IggyConsumerBuilder::auto_commit), see
- /// [Tracking what has been read](IggyConsumer#tracking-what-has-been-read). `init()` spawns the
+ /// [Positions and commits](IggyConsumer#positions-and-commits). `init()` spawns the
/// tasks behind it:
/// - An interval task, only for the variants that carry an interval ([`AutoCommit::Interval`],
/// [`AutoCommit::IntervalOrWhen`], [`AutoCommit::IntervalOrAfter`]). Every tick it stores the
@@ -934,16 +624,15 @@ impl IggyConsumer {
///
/// Both skip an offset that is not ahead of this consumer's own record of what it stored
/// ([`get_last_stored_offset()`](Self::get_last_stored_offset)). Only offset `0` is always sent.
- /// Under auto-commit-on-poll (the default) that record trails the server by one batch, so every
- /// tick re-sends the reading position and the server, which takes an explicit store as is,
- /// moves its offset back to it until the next poll.
+ /// Combining interval commits with auto-commit-on-poll can move the server offset
+ /// back to the delivered position while later prefetched batches are already committed.
///
/// # Errors
///
/// - [`IggyError::InvalidConfiguration`] when the consumer has an encryptor and
/// [`auto_commit()`](crate::prelude::IggyConsumerBuilder::auto_commit) is
- /// [`AutoCommitWhen::PollingMessages`], checked before anything is sent. See
- /// [Encryption](IggyConsumer#encryption).
+ /// [`AutoCommitWhen::PollingMessages`], checked before anything is sent because
+ /// the server would commit before decryption succeeds.
/// - [`IggyError::StreamNameNotFound`] or [`IggyError::TopicNameNotFound`] when the
/// stream or the topic still does not exist once the retries are exhausted.
/// - [`IggyError::ConsumerGroupNameNotFound`] when the consumer group does not exist
@@ -952,6 +641,10 @@ impl IggyConsumer {
/// while creating or joining the consumer group. Such an error ends initialization
/// immediately instead of consuming a retry.
pub async fn init(&mut self) -> Result<(), IggyError> {
+ self.poll_options.validate(self.batch_length)?;
+ if self.prefetch_slots() == 0 {
+ return Err(IggyError::InvalidConfiguration);
+ }
if self.initialized {
return Ok(());
}
@@ -1043,6 +736,12 @@ impl IggyConsumer {
previous.abort();
}
self.init_consumer_group().await?;
+ if self.is_consumer_group {
+ self.assignment_state = Some((
+ self.client.read().await.deferred_group_state().await?,
+ format!("{}|{}|{}", self.stream_id, self.topic_id, self.consumer.id),
+ ));
+ }
match self.auto_commit {
AutoCommit::Interval(interval)
@@ -1056,6 +755,7 @@ impl IggyConsumer {
self.store_offset_task = Some(self.store_pending_commits_in_background());
self.initialized = true;
+ self.deferred_polls = Some(deferred::DeferredPolls::start(self));
info!(
"Consumer: {consumer_name} has been initialized for stream: {}, topic: {}.",
self.stream_id, self.topic_id
@@ -1072,7 +772,7 @@ impl IggyConsumer {
// Wait for the task until either the interval has passed or
// the task is explicitly notified, which happens when shutdown() is called.
tokio::select! {
- _ = sleep(interval.get_duration()) => {}
+ _ = time::sleep(interval.get_duration()) => {}
_ = notify.notified() => {}
}
@@ -1176,6 +876,8 @@ impl IggyConsumer {
let is_consumer_group = self.is_consumer_group;
let can_poll = self.can_poll.clone();
let joined_consumer_group = self.joined_consumer_group.clone();
+ let generation = Arc::clone(&self.fetch_generation);
+ let notify = Arc::clone(&self.fetch_notify);
tokio::spawn(async move {
while let Some(event) = receiver.next().await {
@@ -1185,6 +887,8 @@ impl IggyConsumer {
warn!("Consumer has been shutdown");
joined_consumer_group.store(false, ORDERING);
can_poll.store(false, ORDERING);
+ generation.fetch_add(1, ORDERING);
+ notify.notify_one();
break;
}
DiagnosticEvent::Connected => {
@@ -1207,214 +911,12 @@ impl IggyConsumer {
can_poll.store(false, ORDERING);
}
}
+ generation.fetch_add(1, ORDERING);
+ notify.notify_one();
}
})
}
- fn create_poll_messages_future(
- &self,
- ) -> impl Future<Output = Result<PolledMessages, IggyError>> + use<> {
- let stream_id = self.stream_id.clone();
- let topic_id = self.topic_id.clone();
- let partition_id = self.partition_id;
- let consumer = self.consumer.clone();
- let polling_strategy = self.polling_strategy;
- let next_offsets = self.next_offsets.clone();
- let client = self.client.clone();
- let count = self.batch_length;
- let auto_commit_after_polling = self.auto_commit_after_polling;
- let interval = self.poll_interval_micros;
- let last_polled_at = self.last_polled_at.clone();
- let can_poll = self.can_poll.clone();
- let retry_interval = self.reconnection_retry_interval;
- let last_stored_offset = self.state.last_stored_offsets.clone();
- let last_consumed_offset = self.state.last_consumed_offsets.clone();
- let allow_replay = self.allow_replay;
- let is_consumer_group = self.is_consumer_group;
- let auto_join_consumer_group = self.auto_join_consumer_group;
- let create_consumer_group_if_not_exists = self.create_consumer_group_if_not_exists;
- let joined_consumer_group = self.joined_consumer_group.clone();
- let consumer_name = self.consumer_name.clone();
-
- async move {
- if interval > 0 {
- Self::wait_before_polling(interval, last_polled_at.load(ORDERING)).await;
- }
-
- while !can_poll.load(ORDERING) {
- trace!("Cannot poll yet, waiting {retry_interval}...");
- sleep(retry_interval.get_duration()).await;
- }
-
- // A member that joins on its own is in the group before it polls. One built with
- // `do_not_auto_join_consumer_group()` polls right away and gets a missing
- // membership reported as a poll error.
- if is_consumer_group
- && auto_join_consumer_group
- && !joined_consumer_group.load(ORDERING)
- && let Err(error) = Self::initialize_consumer_group(
- client.clone(),
- create_consumer_group_if_not_exists,
- stream_id.clone(),
- topic_id.clone(),
- consumer.clone(),
- &consumer_name,
- joined_consumer_group.clone(),
- )
- .await
- {
- error!(
- "Failed to join consumer group: {consumer_name} for stream: {stream_id}, topic: {topic_id}. {error}"
- );
- sleep(retry_interval.get_duration()).await;
- return Err(error);
- }
-
- trace!("Sending poll messages request");
- last_polled_at.store(IggyTimestamp::now().into(), ORDERING);
- // The map guard is dropped inside `map_or`, and the only writer is `poll_next`,
- // which runs after this future has returned, so the lookup cannot block.
- let strategy_for = |partition: u32| {
- next_offsets
- .get(&partition)
- .map_or(polling_strategy, |offset| PollingStrategy::offset(*offset))
- };
- let polled_messages = client
- .read()
- .await
- .poll_messages_with_strategy_for(
- &stream_id,
- &topic_id,
- partition_id,
- &consumer,
- &strategy_for,
- count,
- auto_commit_after_polling,
- )
- .await;
-
- if let Ok(polled) = &polled_messages
- && polled.partition_id == NO_ASSIGNED_PARTITION
- {
- trace!(
- "No partition assigned to consumer: {consumer_name}, waiting {retry_interval}..."
- );
- sleep(retry_interval.get_duration()).await;
- }
-
- if let Ok(mut polled_messages) = polled_messages {
- if polled_messages.messages.is_empty() {
- return Ok(polled_messages);
- }
-
- let partition_id = polled_messages.partition_id;
- let consumed_offset;
- let has_consumed_offset;
- if let Some(offset_entry) = last_consumed_offset.get(&partition_id) {
- has_consumed_offset = true;
- consumed_offset = offset_entry.load(ORDERING);
- } else {
- consumed_offset = 0;
- has_consumed_offset = false;
- last_consumed_offset.insert(partition_id, AtomicU64::new(0));
- }
-
- if !allow_replay && has_consumed_offset {
- polled_messages
- .messages
- .retain(|message| message.header.offset > consumed_offset);
- polled_messages.count = polled_messages.messages.len() as u32;
- if polled_messages.messages.is_empty() {
- return Ok(polled_messages);
- }
- }
-
- let stored_offset;
- if let Some(stored_offset_entry) = last_stored_offset.get(&partition_id) {
- if auto_commit_after_polling {
- stored_offset_entry.store(consumed_offset, ORDERING);
- stored_offset = consumed_offset;
- } else {
- stored_offset = stored_offset_entry.load(ORDERING);
- }
- } else {
- if auto_commit_after_polling {
- stored_offset = consumed_offset;
- } else {
- stored_offset = 0;
- }
- last_stored_offset.insert(partition_id, AtomicU64::new(stored_offset));
- }
-
- trace!(
- "Last consumed offset: {consumed_offset}, current offset: {}, stored offset: {stored_offset}, in partition ID: {partition_id}, topic: {topic_id}, stream: {stream_id}, consumer: {consumer}",
- polled_messages.current_offset
- );
- return Ok(polled_messages);
- }
-
- let error = polled_messages.unwrap_err();
- error!("Failed to poll messages: {error}");
-
- if is_consumer_group
- && auto_join_consumer_group
- && matches!(&error, IggyError::ConsumerGroupMemberNotFound(..))
- {
- info!(
- "Consumer group membership was revoked for consumer: {consumer_name}, stream: {stream_id}, topic: {topic_id}. Rejoining on the next poll..."
- );
- joined_consumer_group.store(false, ORDERING);
- return Ok(PolledMessages::empty());
- }
-
- // Connection and auth errors: disable polling until the event task
- // re-enables it after reconnection and rejoin complete. Yielded at
- // once: the next poll already parks on `can_poll`, so a retry sleep
- // here would only delay the caller's view of an error it does not
- // act on.
- if matches!(
- error,
- IggyError::Disconnected | IggyError::Unauthenticated | IggyError::StaleClient
- ) {
- can_poll.store(false, ORDERING);
- if is_consumer_group {
- joined_consumer_group.store(false, ORDERING);
- }
- return Err(error);
- }
- trace!("Retrying to poll messages in {retry_interval}...");
- sleep(retry_interval.get_duration()).await;
- Err(error)
- }
- }
-
- async fn wait_before_polling(interval: u64, last_sent_at: u64) {
- if interval == 0 {
- return;
- }
-
- let now: u64 = IggyTimestamp::now().into();
- if now < last_sent_at {
- warn!(
- "Returned monotonic time went backwards, now < last_sent_at: ({now} < {last_sent_at})"
- );
- sleep(Duration::from_micros(interval)).await;
- return;
- }
-
- let elapsed = now - last_sent_at;
- if elapsed >= interval {
- trace!("No need to wait before polling messages. {now} - {last_sent_at} = {elapsed}");
- return;
- }
-
- let remaining = interval - elapsed;
- trace!(
- "Waiting for {remaining} microseconds before polling messages... {interval} - {elapsed} = {remaining}"
- );
- sleep(Duration::from_micros(remaining)).await;
- }
-
async fn initialize_consumer_group(
client: IggyRwLock<ClientWrapper>,
create_consumer_group_if_not_exists: bool,
@@ -1490,6 +992,15 @@ impl IggyConsumer {
);
Ok(())
}
+
+ fn prefetch_slots(&self) -> usize {
+ if self.batch_length == 0 || self.poll_options.max_bytes == 0 {
+ return 0;
+ }
+ (self.prefetch_messages / self.batch_length)
+ .min(self.prefetch_bytes / self.poll_options.max_bytes)
+ .min(crate::poll_routing::MAX_DEFERRED_CONNECTIONS as u32) as usize
+ }
}
/// A single message handed over by an [`IggyConsumer`].
@@ -1534,171 +1045,107 @@ impl Stream for IggyConsumer {
if self.shutdown.load(ORDERING) {
return Poll::Ready(None);
}
-
- let partition_id = self.state.partition_id();
- if let Some(message) = self.buffered_messages.pop_front() {
+ if !self.initialized {
+ return Poll::Ready(Some(Err(IggyError::InvalidConfiguration)));
+ }
+ loop {
+ let assignment_valid = self.assignment_state.as_ref().is_none_or(|(state, key)| {
+ self.buffered_assignment.is_some_and(|assignment| {
+ state.is_current_assignment(key, assignment, self.state.partition_id())
+ })
+ });
+ if self.buffered_generation != self.fetch_generation.load(ORDERING) || !assignment_valid
{
- // Since a consumer can be standalone or a member of a consumer group, in which case
- // it can be reassigned to another partition, either update the offset of a partition
- // the consumer already worked with or add a new record, if it got reassigned.
- if let Some(last_consumed_offset_entry) =
- self.state.last_consumed_offsets.get(&partition_id)
+ self.buffered_messages.clear();
+ self.buffered_capacity = None;
+ }
+ let partition_id = self.state.partition_id();
+ while let Some(message) = self.buffered_messages.pop_front() {
+ let offset = message.header.offset;
+ if !self.allow_replay
+ && self
+ .state
+ .get_last_consumed_offset(partition_id)
+ .is_some_and(|consumed| offset <= consumed)
{
- last_consumed_offset_entry.store(message.header.offset, ORDERING);
- } else {
- self.state
- .last_consumed_offsets
- .insert(partition_id, AtomicU64::new(message.header.offset));
+ continue;
}
-
- if (self.store_after_every_nth_message > 0
- && message.header.offset % self.store_after_every_nth_message == 0)
- || self.store_offset_after_each_message
+ self.state
+ .last_consumed_offsets
+ .entry(partition_id)
+ .or_insert_with(|| AtomicU64::new(offset))
+ .store(offset, ORDERING);
+ let last = self.buffered_messages.is_empty();
+ if self.store_offset_after_each_message
+ || (self.store_after_every_nth_message > 0
+ && offset % self.store_after_every_nth_message == 0)
+ || (self.store_offset_after_all_messages && last)
{
- self.send_store_offset(partition_id, message.header.offset);
- }
- }
-
- // Popping above may have left the buffer empty, so the next turn polls the server.
- // `polling_strategy` is only where reading a partition starts; from then on each
- // poll continues after the last message handed over from that partition.
- if self.buffered_messages.is_empty() {
- if self.polling_strategy.kind != PollingKind::Next {
- self.next_offsets
- .insert(partition_id, message.header.offset + 1);
+ self.send_store_offset(partition_id, offset);
}
-
- if self.store_offset_after_all_messages {
- self.send_store_offset(partition_id, message.header.offset);
+ if last {
+ self.buffered_capacity = None;
}
+ let current_offset = self
+ .current_offsets
+ .get(&partition_id)
+ .map_or(0, |offset| offset.load(ORDERING));
+ return Poll::Ready(Some(Ok(ReceivedMessage::new(
+ message,
+ current_offset,
+ partition_id,
+ ))));
}
-
- // Not the position of this message but the newest offset the partition had when the
- // batch was polled. So every message of a batch reports the same value.
- let current_offset;
- if let Some(current_offset_entry) = self.current_offsets.get(&partition_id) {
- current_offset = current_offset_entry.load(ORDERING);
- } else {
- current_offset = 0;
+ self.buffered_capacity = None;
+ if self.deferred_polls.is_none() {
+ self.deferred_polls = Some(deferred::DeferredPolls::start(&self));
}
-
- return Poll::Ready(Some(Ok(ReceivedMessage::new(
- message,
- current_offset,
- partition_id,
- ))));
- }
-
- // A used (and therefore invalid) future was dropped, thus create a fresh one.
- if self.poll_future.is_none() {
- let future = self.create_poll_messages_future();
- self.poll_future = Some(Box::pin(future));
- }
-
- while let Some(future) = self.poll_future.as_mut() {
- match future.poll_unpin(cx) {
- Poll::Ready(Ok(polled_messages)) => {
- let PolledMessages {
- partition_id,
- current_offset,
- messages,
- ..
- } = polled_messages;
- let mut messages = VecDeque::from(messages);
- let Some(mut first) = messages.pop_front() else {
- self.poll_future = Some(Box::pin(self.create_poll_messages_future()));
+ let received = self
+ .deferred_polls
+ .as_mut()
+ .expect("fetch worker initialized")
+ .poll_recv(cx);
+ match received {
+ Poll::Pending => return Poll::Pending,
+ Poll::Ready(None) => {
+ self.deferred_polls = None;
+ return Poll::Ready(Some(Err(IggyError::ClientShutdown)));
+ }
+ Poll::Ready(Some(batch)) => {
+ if batch.generation != self.fetch_generation.load(ORDERING) {
continue;
- };
-
- // Only a response that carries messages names a partition; an empty one can
- // carry a sentinel instead of a real id.
- self.state
- .current_partition_id
- .store(partition_id, ORDERING);
-
- if let Some(ref encryptor) = self.encryptor {
- for message in std::iter::once(&mut first).chain(messages.iter_mut()) {
- let offset = message.header.offset;
- let payload = encryptor.decrypt(&message.payload);
- if let Err(error) = payload {
- self.poll_future = None;
- error!(
- "Failed to decrypt the message payload at offset: {offset}, partition ID: {partition_id}",
- );
- return Poll::Ready(Some(Err(error)));
+ }
+ match batch.result {
+ Err(error) => return Poll::Ready(Some(Err(error))),
+ Ok(polled) => {
+ if polled.messages.is_empty() {
+ continue;
}
-
- let payload = payload.unwrap();
- message.payload = Bytes::from(payload);
- message.header.payload_length = message.payload.len() as u32;
-
- if let Some(ref user_headers) = message.user_headers {
- let decrypted_headers = encryptor.decrypt(user_headers);
- if let Err(error) = decrypted_headers {
- self.poll_future = None;
- error!(
- "Failed to decrypt the message user headers at offset: {offset}, partition ID: {partition_id}",
- );
- return Poll::Ready(Some(Err(error)));
- }
- let decrypted_headers = decrypted_headers.unwrap();
- message.header.user_headers_length = decrypted_headers.len() as u32;
- message.user_headers = Some(Bytes::from(decrypted_headers));
+ self.state
+ .current_partition_id
+ .store(polled.partition_id, ORDERING);
+ self.current_offsets
+ .entry(polled.partition_id)
+ .or_insert_with(|| AtomicU64::new(polled.current_offset))
+ .store(polled.current_offset, ORDERING);
+ if self.auto_commit_after_polling
+ && let Some(last) = polled.messages.last()
+ {
+ self.state
+ .last_stored_offsets
+ .entry(polled.partition_id)
+ .or_insert_with(|| AtomicU64::new(last.header.offset))
+ .store(last.header.offset, ORDERING);
}
+ self.buffered_messages = polled.messages.into();
+ self.buffered_generation = batch.generation;
+ self.buffered_assignment = batch.assignment;
+ self.buffered_capacity = Some(batch.capacity);
}
}
-
- if let Some(current_offset_entry) = self.current_offsets.get(&partition_id) {
- current_offset_entry.store(current_offset, ORDERING);
- } else {
- self.current_offsets
- .insert(partition_id, AtomicU64::new(current_offset));
- }
-
- // A poll is only sent once the buffer has run empty, so nothing is overwritten.
- self.buffered_messages = messages;
-
- if self.polling_strategy.kind != PollingKind::Next {
- self.next_offsets
- .insert(partition_id, first.header.offset + 1);
- }
-
- if let Some(last_consumed_offset_entry) =
- self.state.last_consumed_offsets.get(&partition_id)
- {
- last_consumed_offset_entry.store(first.header.offset, ORDERING);
- } else {
- self.state
- .last_consumed_offsets
- .insert(partition_id, AtomicU64::new(first.header.offset));
- }
-
- if (self.store_after_every_nth_message > 0
- && first.header.offset % self.store_after_every_nth_message == 0)
- || self.store_offset_after_each_message
- || (self.store_offset_after_all_messages
- && self.buffered_messages.is_empty())
- {
- self.send_store_offset(partition_id, first.header.offset);
- }
-
- // Drop future since it is [invalid after being ready](https://doc.rust-lang.org/std/future/trait.Future.html#panics)
- self.poll_future = None;
- return Poll::Ready(Some(Ok(ReceivedMessage::new(
- first,
- current_offset,
- partition_id,
- ))));
}
- Poll::Ready(Err(err)) => {
- self.poll_future = None;
- return Poll::Ready(Some(Err(err)));
- }
- Poll::Pending => return Poll::Pending,
}
}
-
- Poll::Pending
}
}
@@ -1712,7 +1159,7 @@ impl IggyConsumer {
/// forcing it to abort.
/// - commit the reading position of every partition where it is ahead of this consumer's own
/// record of what it stored, unless [`auto_commit()`] is [`AutoCommit::Disabled`]. Under
- /// auto-commit-on-poll (the default) the poll already committed the whole batch, so this
+ /// explicit auto-commit-on-poll the poll already committed the whole batch, so this
/// store moves the server offset back to the last message handed over, and the next run
/// resumes right after it instead of after the last batch fetched.
/// - leave the consumer group, if this consumer is a group member. This lets the server give its partitions to
@@ -1732,6 +1179,10 @@ impl IggyConsumer {
return Ok(());
}
+ self.deferred_polls = None;
+ self.buffered_messages.clear();
+ self.buffered_capacity = None;
+
info!("Shutting down consumer: {}...", self.consumer_name);
// Drain the background commit tasks while still a group member, before
@@ -1836,6 +1287,8 @@ impl Drop for IggyConsumer {
}
}
+pub(crate) mod deferred;
+
#[cfg(test)]
mod tests {
use super::*;
@@ -1846,6 +1299,7 @@ mod tests {
use iggy_common::locking::IggyRwLockFn;
use std::str::FromStr;
use std::task::Waker;
+ use std::time::Duration;
use tokio::time::timeout;
const POLL_RETRY_INTERVAL: Duration = Duration::from_millis(10);
@@ -1860,7 +1314,6 @@ mod tests {
Identifier::numeric(1).unwrap(),
None,
None,
- None,
)
}
@@ -1874,6 +1327,7 @@ mod tests {
.batch_length(1)
.auto_commit(AutoCommit::Disabled)
.build();
+ consumer.initialized = true;
consumer.buffered_messages.extend([
IggyMessage::from_str("a").unwrap(),
IggyMessage::from_str("b").unwrap(),
@@ -1887,7 +1341,7 @@ mod tests {
consumer.shutdown().await.unwrap();
- assert_eq!(consumer.buffered_messages.len(), 1);
+ assert!(consumer.buffered_messages.is_empty());
assert!(matches!(
Pin::new(&mut consumer).poll_next(&mut context),
Poll::Ready(None)
@@ -1907,7 +1361,7 @@ mod tests {
}
#[tokio::test]
- async fn consumer_group_should_not_create_poll_future_after_shutdown() {
+ async fn consumer_group_should_not_create_fetch_worker_after_shutdown() {
let mut consumer = builder_for(Consumer::group(Identifier::numeric(1).unwrap()))
.auto_commit(AutoCommit::Disabled)
.build();
@@ -1919,7 +1373,7 @@ mod tests {
Pin::new(&mut consumer).poll_next(&mut context),
Poll::Ready(None)
));
- assert!(consumer.poll_future.is_none());
+ assert!(consumer.deferred_polls.is_none());
}
#[test]
@@ -1946,6 +1400,7 @@ mod tests {
/// Hands over `messages` as one buffered batch read from `partition_id`.
fn hand_over_batch(consumer: &mut IggyConsumer, partition_id: u32, messages: Vec<IggyMessage>) {
+ consumer.initialized = true;
consumer
.state
.current_partition_id
@@ -1960,38 +1415,33 @@ mod tests {
}
}
- fn next_offset(consumer: &IggyConsumer, partition_id: u32) -> Option<u64> {
- consumer
- .next_offsets
- .get(&partition_id)
- .map(|offset| *offset)
- }
-
#[test]
- fn group_member_should_continue_each_partition_after_its_last_message() {
+ fn group_member_tracks_delivered_positions_separately_per_partition() {
let mut consumer = builder_for(Consumer::group(Identifier::numeric(1).unwrap()))
.polling_strategy(PollingStrategy::first())
.auto_commit(AutoCommit::Disabled)
.build();
hand_over_batch(&mut consumer, 3, vec![message_at(10), message_at(11)]);
- assert_eq!(next_offset(&consumer, 3), Some(12));
+ assert_eq!(consumer.get_last_consumed_offset(3), Some(11));
assert_eq!(consumer.polling_strategy, PollingStrategy::first());
hand_over_batch(&mut consumer, 4, vec![message_at(7)]);
- assert_eq!(next_offset(&consumer, 4), Some(8));
- assert_eq!(next_offset(&consumer, 3), Some(12));
+ assert_eq!(consumer.get_last_consumed_offset(4), Some(7));
+ assert_eq!(consumer.get_last_consumed_offset(3), Some(11));
}
#[test]
- fn next_strategy_should_leave_the_continuation_to_the_server() {
+ fn default_consumer_does_not_commit_delivered_messages() {
let mut consumer = builder_for(Consumer::group(Identifier::numeric(1).unwrap()))
.auto_commit(AutoCommit::Disabled)
.build();
hand_over_batch(&mut consumer, 3, vec![message_at(10), message_at(11)]);
- assert!(consumer.next_offsets.is_empty());
+ assert_eq!(consumer.auto_commit, AutoCommit::Disabled);
+ assert!(consumer.pending_commits.is_empty());
+ assert_eq!(consumer.get_last_stored_offset(3), None);
}
/// Polls once as a group member on a client that is not connected. The outcome must be an
@@ -2002,6 +1452,7 @@ mod tests {
let mut consumer = builder
.polling_retry_interval(NonZeroIggyDuration::new(POLL_RETRY_INTERVAL).unwrap())
.build();
+ consumer.initialized = true;
timeout(POLL_TIMEOUT, consumer.next())
.await
.expect("a group member must poll or report an error instead of waiting for a join")
diff --git a/core/sdk/src/clients/consumer/deferred.rs b/core/sdk/src/clients/consumer/deferred.rs
new file mode 100644
index 000000000..67428a54c
--- /dev/null
+++ b/core/sdk/src/clients/consumer/deferred.rs
@@ -0,0 +1,571 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::sync::Arc;
+use std::sync::atomic::AtomicU64;
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+use bytes::Bytes;
+use iggy_common::locking::IggyRwLockFn;
+use iggy_common::{
+ ConsumerOffsetClient, IggyError, MessageClient, PolledMessages, PollingKind, PollingStrategy,
+ TopicClient,
+};
+use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, mpsc};
+use tokio::task::{JoinHandle, JoinSet};
+use tokio::time::{Instant, MissedTickBehavior};
+
+use super::{IggyConsumer, ORDERING};
+use crate::poll_routing::MAX_DEFERRED_CONNECTIONS;
+
+const ASSIGNMENT_REFRESH_INTERVAL: Duration = Duration::from_secs(1);
+pub(crate) const DEFAULT_PREFETCH_BYTES: u32 = 16 * 1024 * 1024;
+pub(crate) const DEFAULT_PREFETCH_MESSAGES: u32 = 16_000;
+
+/// A reservation covers the whole possible reply, including work still in flight.
+/// Keep it until the batch is drained: message slices can share its backing allocation.
+pub(super) struct FetchCapacity {
+ permit: Option<OwnedSemaphorePermit>,
+ wake: Arc<Notify>,
+}
+
+impl Drop for FetchCapacity {
+ fn drop(&mut self) {
+ drop(self.permit.take());
+ self.wake.notify_one();
+ }
+}
+
+pub(super) struct PrefetchedBatch {
+ pub generation: u64,
+ pub assignment: Option<(u64, u64)>,
+ pub result: Result<PolledMessages, IggyError>,
+ pub capacity: FetchCapacity,
+}
+
+pub(super) struct DeferredPolls {
+ receiver: mpsc::Receiver<PrefetchedBatch>,
+ task: JoinHandle<()>,
+}
+
+impl DeferredPolls {
+ pub(super) fn start(consumer: &IggyConsumer) -> Self {
+ let slots = consumer.prefetch_slots();
+ let (sender, receiver) = mpsc::channel(slots);
+ let worker = Worker::new(consumer, sender);
+ Self {
+ receiver,
+ task: tokio::spawn(worker.run()),
+ }
+ }
+
+ pub(super) fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll<Option<PrefetchedBatch>> {
+ self.receiver.poll_recv(cx)
+ }
+}
+
+impl Drop for DeferredPolls {
+ fn drop(&mut self) {
+ self.task.abort();
+ }
+}
+
+struct CompletedPoll {
+ partition: u32,
+ strategy: Option<PollingStrategy>,
+ result: Result<PolledMessages, IggyError>,
+ capacity: FetchCapacity,
+}
+
+struct Worker {
+ client: iggy_common::locking::IggyRwLock<crate::client_wrappers::client_wrapper::ClientWrapper>,
+ stream: Arc<iggy_common::Identifier>,
+ topic: Arc<iggy_common::Identifier>,
+ consumer: Arc<iggy_common::Consumer>,
+ name: String,
+ standalone_partition: u32,
+ is_group: bool,
+ auto_join: bool,
+ create_group: bool,
+ joined: Arc<std::sync::atomic::AtomicBool>,
+ can_poll: Arc<std::sync::atomic::AtomicBool>,
+ generation: Arc<AtomicU64>,
+ wake: Arc<Notify>,
+ strategy: PollingStrategy,
+ options: iggy_common::DeferredPollOptions,
+ count: u32,
+ auto_commit: bool,
+ encryptor: Option<Arc<iggy_common::EncryptorKind>>,
+ retry_interval: Duration,
+ sender: mpsc::Sender<PrefetchedBatch>,
+ capacity: Arc<Semaphore>,
+ tasks: JoinSet<CompletedPoll>,
+ active: BTreeSet<u32>,
+ positions: BTreeMap<u32, PollingStrategy>,
+ retry_at: BTreeMap<u32, Instant>,
+ partitions: Vec<u32>,
+ assignment: Option<(u64, u64)>,
+ pending_error: Option<IggyError>,
+ epoch: u64,
+ cursor: Option<u32>,
+}
+
+impl Worker {
+ fn new(consumer: &IggyConsumer, sender: mpsc::Sender<PrefetchedBatch>) -> Self {
+ let slots = consumer.prefetch_slots();
+ Self {
+ client: consumer.client.clone(),
+ stream: Arc::clone(&consumer.stream_id),
+ topic: Arc::clone(&consumer.topic_id),
+ consumer: Arc::clone(&consumer.consumer),
+ name: consumer.consumer_name.clone(),
+ standalone_partition: consumer.partition_id.unwrap_or_default(),
+ is_group: consumer.is_consumer_group,
+ auto_join: consumer.auto_join_consumer_group,
+ create_group: consumer.create_consumer_group_if_not_exists,
+ joined: Arc::clone(&consumer.joined_consumer_group),
+ can_poll: Arc::clone(&consumer.can_poll),
+ generation: Arc::clone(&consumer.fetch_generation),
+ wake: Arc::clone(&consumer.fetch_notify),
+ strategy: consumer.polling_strategy,
+ options: consumer.poll_options,
+ count: consumer.batch_length,
+ auto_commit: consumer.auto_commit_after_polling,
+ encryptor: consumer.encryptor.clone(),
+ retry_interval: consumer.reconnection_retry_interval.get_duration(),
+ sender,
+ capacity: Arc::new(Semaphore::new(slots)),
+ tasks: JoinSet::new(),
+ active: BTreeSet::new(),
+ positions: BTreeMap::new(),
+ retry_at: BTreeMap::new(),
+ partitions: Vec::new(),
+ assignment: None,
+ pending_error: None,
+ epoch: consumer.fetch_generation.load(ORDERING),
+ cursor: None,
+ }
+ }
+
+ async fn run(mut self) {
+ let mut refresh = tokio::time::interval(ASSIGNMENT_REFRESH_INTERVAL);
+ refresh.set_missed_tick_behavior(MissedTickBehavior::Delay);
+ loop {
+ if self.sender.is_closed() {
+ return;
+ }
+ if self.epoch != self.generation.load(ORDERING) {
+ self.clear();
+ self.epoch = self.generation.load(ORDERING);
+ refresh.reset_immediately();
+ }
+ self.flush_error();
+ self.schedule();
+ let retry_at = self
+ .retry_at
+ .values()
+ .min()
+ .copied()
+ .unwrap_or_else(|| Instant::now() + ASSIGNMENT_REFRESH_INTERVAL);
+ tokio::select! {
+ _ = self.sender.closed() => return,
+ _ = self.wake.notified() => {},
+ _ = tokio::time::sleep_until(retry_at), if !self.retry_at.is_empty() => {
+ self.retry_at.retain(|_, deadline| *deadline > Instant::now());
+ },
+ _ = refresh.tick() => {
+ if !self.can_poll.load(ORDERING) { continue; }
+ let result = tokio::time::timeout(self.options.request_timeout.get_duration(), self.refresh()).await
+ .unwrap_or(Err(IggyError::TransientNotCommitted));
+ if let Err(error) = result {
+ self.invalidate();
+ self.report(error);
+ refresh.reset_after(self.retry_interval);
+ } else {
+ refresh.reset();
+ }
+ },
+ Some(completed) = self.tasks.join_next(), if !self.tasks.is_empty() => {
+ match completed {
+ Ok(completed) => {
+ self.complete(completed);
+ if self.partitions.is_empty() {
+ refresh.reset_after(self.retry_interval);
+ }
+ },
+ Err(_) => {
+ self.invalidate();
+ self.report(IggyError::ClientShutdown);
+ return;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ fn clear(&mut self) {
+ self.tasks = JoinSet::new();
+ self.active.clear();
+ self.positions.clear();
+ self.retry_at.clear();
+ self.partitions.clear();
+ self.assignment = None;
+ self.pending_error = None;
+ }
+
+ fn invalidate(&mut self) {
+ self.epoch = self.generation.fetch_add(1, ORDERING) + 1;
+ self.clear();
+ }
+
+ async fn refresh(&mut self) -> Result<(), IggyError> {
+ if self.is_group && self.auto_join && !self.joined.load(ORDERING) {
+ IggyConsumer::initialize_consumer_group(
+ self.client.clone(),
+ self.create_group,
+ Arc::clone(&self.stream),
+ Arc::clone(&self.topic),
+ Arc::clone(&self.consumer),
+ &self.name,
+ Arc::clone(&self.joined),
+ )
+ .await?;
+ }
+ let (session, generation, mut partitions) = if self.is_group {
+ self.client
+ .read()
+ .await
+ .deferred_poll_partitions(&self.stream, &self.topic, &self.consumer.id, true)
+ .await?
+ } else {
+ (0, 0, vec![self.standalone_partition])
+ };
+ partitions.sort_unstable();
+ if self.assignment != Some((session, generation)) || self.partitions != partitions {
+ self.invalidate();
+ self.partitions = partitions;
+ self.assignment = Some((session, generation));
+ }
+ Ok(())
+ }
+
+ fn reserve(&self) -> Option<FetchCapacity> {
+ Arc::clone(&self.capacity)
+ .try_acquire_owned()
+ .ok()
+ .map(|permit| FetchCapacity {
+ permit: Some(permit),
+ wake: Arc::clone(&self.wake),
+ })
+ }
+
+ fn report(&mut self, error: IggyError) {
+ if matches!(error, IggyError::ConsumerGroupMemberNotFound(..)) {
+ self.joined.store(false, ORDERING);
+ if self.auto_join {
+ return;
+ }
+ }
+ if matches!(
+ error,
+ IggyError::Disconnected | IggyError::Unauthenticated | IggyError::StaleClient
+ ) {
+ self.can_poll.store(false, ORDERING);
+ self.joined.store(false, ORDERING);
+ }
+ self.pending_error = Some(error);
+ self.flush_error();
+ }
+
+ fn flush_error(&mut self) {
+ let Some(error) = self.pending_error.take() else {
+ return;
+ };
+ let Some(capacity) = self.reserve() else {
+ self.pending_error = Some(error);
+ return;
+ };
+ let _ = self.sender.try_send(PrefetchedBatch {
+ generation: self.epoch,
+ assignment: self.assignment,
+ result: Err(error),
+ capacity,
+ });
+ }
+
+ fn schedule(&mut self) {
+ if !self.can_poll.load(ORDERING) || self.partitions.is_empty() {
+ return;
+ }
+ let start = self.cursor.map_or(0, |cursor| {
+ self.partitions.partition_point(|id| *id <= cursor)
+ });
+ for index in 0..self.partitions.len() {
+ if self.tasks.len() >= MAX_DEFERRED_CONNECTIONS {
+ break;
+ }
+ let partition = self.partitions[(start + index) % self.partitions.len()];
+ if self.active.contains(&partition) || self.retry_at.contains_key(&partition) {
+ continue;
+ }
+ let Some(capacity) = self.reserve() else {
+ break;
+ };
+ let client = self.client.clone();
+ let stream = Arc::clone(&self.stream);
+ let topic = Arc::clone(&self.topic);
+ let consumer = Arc::clone(&self.consumer);
+ let initial = self.strategy;
+ let mut strategy = self.positions.get(&partition).copied();
+ let count = self.count;
+ let auto_commit = self.auto_commit;
+ let options = self.options;
+ let encryptor = self.encryptor.clone();
+ self.tasks.spawn(async move {
+ let started = Instant::now();
+ let result = tokio::time::timeout(options.request_timeout.get_duration(), async {
+ let client = client.read().await;
+ let resolved = match strategy {
+ Some(strategy) => strategy,
+ None => match initial.kind {
+ PollingKind::Next => {
+ let stored = client
+ .get_consumer_offset(
+ &consumer,
+ &stream,
+ &topic,
+ Some(partition),
+ )
+ .await?;
+ let offset = stored
+ .map(|offset| {
+ offset
+ .stored_offset
+ .checked_add(1)
+ .ok_or(IggyError::InvalidOffset(offset.stored_offset))
+ })
+ .transpose()?
+ .unwrap_or_default();
+ PollingStrategy::offset(offset)
+ }
+ PollingKind::First => PollingStrategy::offset(0),
+ PollingKind::Last => {
+ let details = client.get_topic(&stream, &topic).await?;
+ let head = details
+ .as_ref()
+ .and_then(|topic| {
+ topic.partitions.iter().find(|entry| entry.id == partition)
+ })
+ .ok_or_else(|| {
+ IggyError::PartitionNotFound(
+ partition as usize,
+ (*topic).clone(),
+ (*stream).clone(),
+ )
+ })?;
+ PollingStrategy::offset(
+ head.current_offset.saturating_sub(u64::from(count) - 1),
+ )
+ }
+ _ => initial,
+ },
+ };
+ strategy = Some(resolved);
+ let mut result = client
+ .poll_messages_deferred(
+ &stream,
+ &topic,
+ Some(partition),
+ &consumer,
+ &resolved,
+ count,
+ auto_commit,
+ options.remaining(started.elapsed())?,
+ )
+ .await?;
+ if result.count > count || result.messages.len() > count as usize {
+ return Err(IggyError::InvalidMessagesCount);
+ }
+ if let Some(encryptor) = encryptor {
+ for message in &mut result.messages {
+ message.payload = Bytes::from(encryptor.decrypt(&message.payload)?);
+ message.header.payload_length = message.payload.len() as u32;
+ if let Some(headers) = &message.user_headers {
+ let headers = encryptor.decrypt(headers)?;
+ message.header.user_headers_length = headers.len() as u32;
+ message.user_headers = Some(Bytes::from(headers));
+ }
+ }
+ }
+ Ok(result)
+ })
+ .await
+ .unwrap_or(Err(IggyError::TransientNotCommitted));
+ CompletedPoll {
+ partition,
+ strategy,
+ result,
+ capacity,
+ }
+ });
+ self.active.insert(partition);
+ self.cursor = Some(partition);
+ }
+ }
+
+ fn complete(&mut self, mut completed: CompletedPoll) {
+ self.active.remove(&completed.partition);
+ if self.epoch != self.generation.load(ORDERING) {
+ return;
+ }
+ if let Some(strategy) = completed.strategy {
+ self.positions.insert(completed.partition, strategy);
+ }
+ if let Ok(result) = &completed.result {
+ if result.partition_id == iggy_common::RESYNC_REQUIRED_PARTITION_SENTINEL {
+ self.invalidate();
+ self.wake.notify_one();
+ return;
+ }
+ if let Some(last) = result.messages.last() {
+ match last.header.offset.checked_add(1) {
+ Some(next) => {
+ self.positions
+ .insert(completed.partition, PollingStrategy::offset(next));
+ }
+ None => {
+ completed.result = Err(IggyError::InvalidOffset(last.header.offset));
+ }
+ }
+ } else {
+ return;
+ }
+ }
+ if let Err(error) = completed.result {
+ if matches!(
+ error,
+ IggyError::ConsumerGroupMemberNotFound(..)
+ | IggyError::Disconnected
+ | IggyError::Unauthenticated
+ | IggyError::StaleClient
+ ) {
+ self.invalidate();
+ self.report(error);
+ return;
+ }
+ self.retry_at
+ .insert(completed.partition, Instant::now() + self.retry_interval);
+ completed.result = Err(error);
+ }
+ let _ = self.sender.try_send(PrefetchedBatch {
+ generation: self.epoch,
+ assignment: self.assignment,
+ result: completed.result,
+ capacity: completed.capacity,
+ });
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::client_wrappers::client_wrapper::ClientWrapper;
+ use crate::clients::consumer_builder::IggyConsumerBuilder;
+ use crate::tcp::tcp_client::TcpClient;
+ use iggy_common::locking::{IggyRwLock, IggyRwLockFn};
+ use iggy_common::{Consumer, DeferredPollOptions, Identifier};
+
+ #[tokio::test(start_paused = true)]
+ async fn byte_and_message_reservations_cover_inflight_and_queued_replies() {
+ const BATCH_COUNT: u32 = 10;
+ const REPLY_BYTES: u32 = 1024;
+ for (bytes, messages) in [
+ (REPLY_BYTES * 2, BATCH_COUNT * 10),
+ (REPLY_BYTES * 10, BATCH_COUNT * 2),
+ ] {
+ let consumer = IggyConsumerBuilder::new(
+ IggyRwLock::new(ClientWrapper::Tcp(TcpClient::default())),
+ "reader".into(),
+ Consumer::default(),
+ Identifier::numeric(1).unwrap(),
+ Identifier::numeric(1).unwrap(),
+ None,
+ None,
+ )
+ .batch_length(BATCH_COUNT)
+ .poll_options(DeferredPollOptions {
+ max_bytes: REPLY_BYTES,
+ ..Default::default()
+ })
+ .prefetch_bytes(bytes)
+ .prefetch_messages(messages)
+ .build();
+ let client = consumer.client.clone();
+ let _blocked_connection = client.write().await;
+ let (sender, mut receiver) = mpsc::channel(consumer.prefetch_slots());
+ let mut worker = Worker::new(&consumer, sender);
+ worker.partitions = vec![0, 1, 2, 3];
+ worker.schedule();
+ assert_eq!(worker.tasks.len(), 2, "reserve before sending");
+ assert_eq!(worker.capacity.available_permits(), 0);
+ while let Some(completed) = worker.tasks.join_next().await {
+ let completed = completed.unwrap();
+ assert!(matches!(
+ completed.result,
+ Err(IggyError::TransientNotCommitted)
+ ));
+ worker.complete(completed);
+ }
+ assert_eq!(
+ receiver.len(),
+ 2,
+ "completed failures retain their bounded queue reservations"
+ );
+ worker.schedule();
+ assert!(
+ worker.tasks.is_empty(),
+ "a stalled application stops new fetches"
+ );
+ drop(receiver.recv().await.unwrap());
+ worker.retry_at.clear();
+ worker.schedule();
+ assert_eq!(
+ worker.tasks.len(),
+ 1,
+ "draining releases exactly one full-batch reservation"
+ );
+ while let Some(completed) = worker.tasks.join_next().await {
+ worker.complete(completed.unwrap());
+ }
+ worker.invalidate();
+ worker.report(IggyError::Disconnected);
+ assert!(
+ worker.pending_error.is_some(),
+ "retain control errors under backpressure"
+ );
+ drop(receiver.recv().await.unwrap());
+ worker.flush_error();
+ assert_ne!(receiver.recv().await.unwrap().generation, worker.epoch);
+ let error = receiver.recv().await.unwrap();
+ assert_eq!(error.generation, worker.epoch);
+ assert!(matches!(error.result, Err(IggyError::Disconnected)));
+ }
+ }
+}
diff --git a/core/sdk/src/clients/consumer_builder.rs b/core/sdk/src/clients/consumer_builder.rs
index 91cb9ac83..b1fb0beee 100644
--- a/core/sdk/src/clients/consumer_builder.rs
+++ b/core/sdk/src/clients/consumer_builder.rs
@@ -16,10 +16,11 @@
// under the License.
use crate::client_wrappers::client_wrapper::ClientWrapper;
-use crate::prelude::{AutoCommit, AutoCommitWhen, IggyConsumer};
+use crate::prelude::{AutoCommit, IggyConsumer};
use iggy_common::locking::IggyRwLock;
use iggy_common::{
- Consumer, EncryptorKind, Identifier, IggyDuration, NonZeroIggyDuration, PollingStrategy,
+ Consumer, DeferredPollOptions, EncryptorKind, Identifier, IggyDuration, NonZeroIggyDuration,
+ PollingStrategy,
};
use std::sync::Arc;
@@ -32,8 +33,10 @@ pub struct IggyConsumerBuilder {
topic: Identifier,
partition: Option<u32>,
polling_strategy: PollingStrategy,
- polling_interval: Option<IggyDuration>,
batch_length: u32,
+ poll_options: DeferredPollOptions,
+ prefetch_bytes: u32,
+ prefetch_messages: u32,
auto_commit: AutoCommit,
auto_join_consumer_group: bool,
create_consumer_group_if_not_exists: bool,
@@ -55,7 +58,6 @@ impl IggyConsumerBuilder {
topic_id: Identifier,
partition_id: Option<u32>,
encryptor: Option<Arc<EncryptorKind>>,
- polling_interval: Option<IggyDuration>,
) -> Self {
Self {
client,
@@ -66,14 +68,13 @@ impl IggyConsumerBuilder {
partition: partition_id,
polling_strategy: PollingStrategy::next(),
batch_length: 1000,
- auto_commit: AutoCommit::IntervalOrWhen(
- NonZeroIggyDuration::ONE_SECOND,
- AutoCommitWhen::PollingMessages,
- ),
+ poll_options: DeferredPollOptions::default(),
+ prefetch_bytes: super::consumer::deferred::DEFAULT_PREFETCH_BYTES,
+ prefetch_messages: super::consumer::deferred::DEFAULT_PREFETCH_MESSAGES,
+ auto_commit: AutoCommit::Disabled,
auto_join_consumer_group: true,
create_consumer_group_if_not_exists: true,
encryptor,
- polling_interval,
polling_retry_interval: NonZeroIggyDuration::ONE_SECOND,
init_retries: None,
init_retry_interval: NonZeroIggyDuration::ONE_SECOND,
@@ -115,6 +116,33 @@ impl IggyConsumerBuilder {
}
}
+ /// Sets readiness, response-byte and overall request limits. Defaults to one
+ /// message, a 1 MiB reply, a one-second wait and an eleven-second request timeout.
+ pub fn poll_options(self, poll_options: DeferredPollOptions) -> Self {
+ Self {
+ poll_options,
+ ..self
+ }
+ }
+
+ /// Caps encoded bytes reserved for in-flight and buffered replies. Default: 16 MiB.
+ /// Must fit at least one complete reply configured by `poll_options`.
+ pub fn prefetch_bytes(self, prefetch_bytes: u32) -> Self {
+ Self {
+ prefetch_bytes,
+ ..self
+ }
+ }
+
+ /// Caps messages reserved for in-flight and buffered replies. Default: 16,000.
+ /// Must fit at least `batch_length` messages.
+ pub fn prefetch_messages(self, prefetch_messages: u32) -> Self {
+ Self {
+ prefetch_messages,
+ ..self
+ }
+ }
+
/// Sets the auto-commit configuration for storing the offset on the server.
pub fn auto_commit(self, auto_commit: AutoCommit) -> Self {
Self {
@@ -158,22 +186,6 @@ impl IggyConsumerBuilder {
}
}
- /// Sets the polling interval for messages.
- pub fn poll_interval(self, interval: IggyDuration) -> Self {
- Self {
- polling_interval: Some(interval),
- ..self
- }
- }
-
- /// Clears the polling interval for messages.
- pub fn without_poll_interval(self) -> Self {
- Self {
- polling_interval: None,
- ..self
- }
- }
-
/// Sets the encryptor for decrypting the messages' payloads.
pub fn encryptor(self, encryptor: Arc<EncryptorKind>) -> Self {
Self {
@@ -239,7 +251,9 @@ impl IggyConsumerBuilder {
self.stream,
self.topic,
self.partition,
- self.polling_interval,
+ self.poll_options,
+ self.prefetch_bytes,
+ self.prefetch_messages,
self.polling_strategy,
self.batch_length,
self.auto_commit,
diff --git a/core/sdk/src/http/http_client.rs b/core/sdk/src/http/http_client.rs
index f3b03d120..30e1dae9a 100644
--- a/core/sdk/src/http/http_client.rs
+++ b/core/sdk/src/http/http_client.rs
@@ -16,6 +16,7 @@
// under the License.
use crate::http::http_transport::HttpTransport;
+use crate::poll_routing::{ROUTING_RETRY_INTERVAL, ROUTING_RETRY_MAX_INTERVAL};
use crate::prelude::{Client, HttpClientConfig, IggyError, NonZeroIggyDuration};
use async_broadcast::{Receiver, Sender, broadcast};
use async_trait::async_trait;
@@ -29,7 +30,7 @@ use reqwest::{Method, Response, StatusCode, Url};
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff};
use reqwest_tracing::{SpanBackendWithUrl, TracingMiddleware};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
use std::ops::Deref;
use std::str::FromStr;
use std::sync::Arc;
@@ -50,6 +51,7 @@ pub struct HttpClient {
pub api_url: Url,
pub(crate) heartbeat_interval: NonZeroIggyDuration,
client: ClientWithMiddleware,
+ deferred_leases: tokio::sync::Semaphore,
access_token: IggyRwLock<String>,
events: (Sender<DiagnosticEvent>, Receiver<DiagnosticEvent>),
}
@@ -256,6 +258,86 @@ impl HttpTransport for HttpClient {
}
impl HttpClient {
+ pub(super) async fn get_deferred_poll(
+ &self,
+ path: &str,
+ poll: &iggy_common::PollMessages,
+ options: iggy_common::DeferredPollOptions,
+ ) -> Result<iggy_common::PolledMessages, IggyError> {
+ options.validate(poll.count)?;
+ let started = tokio::time::Instant::now();
+ let deadline = started + options.request_timeout.get_duration();
+ let _permit = tokio::time::timeout_at(deadline, self.deferred_leases.acquire())
+ .await
+ .map_err(|_| IggyError::TransientNotAccepted)?
+ .map_err(|_| IggyError::ClientShutdown)?;
+ let url = self.get_url(path)?;
+ self.fail_if_not_authenticated(path).await?;
+ let mut retry_interval = ROUTING_RETRY_INTERVAL;
+ loop {
+ let request = {
+ let token = self.access_token.read().await;
+ let remaining = options.remaining(started.elapsed())?;
+ // Deferred auto-commits cannot replay an ambiguous transport failure.
+ self.client
+ .as_ref()
+ .get(url.clone())
+ .bearer_auth(token.deref())
+ .query(poll)
+ .query(&[
+ ("wait_us", remaining.max_wait.as_micros()),
+ ("min_count", u64::from(remaining.min_count)),
+ ("max_bytes", u64::from(remaining.max_bytes)),
+ ("request_timeout_us", remaining.request_timeout.as_micros()),
+ ])
+ .timeout(remaining.request_timeout.get_duration())
+ };
+ let mut response = request
+ .send()
+ .await
+ .map_err(|_| IggyError::InvalidHttpRequest)?;
+ let status = response.status();
+ // Bound JSON expansion and error bodies under the same request budget.
+ const MAX_JSON_EXPANSION: usize = 16;
+ let limit = (options.max_bytes as usize).saturating_mul(MAX_JSON_EXPANSION);
+ if response
+ .content_length()
+ .is_some_and(|length| length > limit as u64)
+ {
+ return Err(IggyError::InvalidSizeBytes);
+ }
+ let mut body = Vec::new();
+ while let Some(chunk) = tokio::time::timeout_at(deadline, response.chunk())
+ .await
+ .map_err(|_| IggyError::TransientNotCommitted)?
+ .map_err(|_| IggyError::InvalidHttpRequest)?
+ {
+ if chunk.len() > limit.saturating_sub(body.len()) {
+ return Err(IggyError::InvalidSizeBytes);
+ }
+ body.extend_from_slice(&chunk);
+ }
+ if status.is_success() {
+ let messages: iggy_common::PolledMessages =
+ serde_json::from_slice(&body).map_err(|_| IggyError::InvalidJsonResponse)?;
+ if messages.count > poll.count || messages.messages.len() != messages.count as usize
+ {
+ return Err(IggyError::InvalidMessagesCount);
+ }
+ return Ok(messages);
+ }
+ let error = Self::response_error(status, String::from_utf8_lossy(&body).into_owned());
+ if !HttpRejection::is_not_accepted(&error) {
+ return Err(error);
+ }
+ if tokio::time::Instant::now() + retry_interval >= deadline {
+ return Err(IggyError::TransientNotAccepted);
+ }
+ tokio::time::sleep(retry_interval).await;
+ retry_interval = (retry_interval * 2).min(ROUTING_RETRY_MAX_INTERVAL);
+ }
+ }
+
/// Create a new HTTP client for interacting with the Iggy API using the provided API URL.
pub fn new(api_url: &str) -> Result<Self, IggyError> {
Self::create(Arc::new(HttpClientConfig {
@@ -279,6 +361,9 @@ impl HttpClient {
Ok(Self {
api_url,
client,
+ deferred_leases: tokio::sync::Semaphore::new(
+ crate::poll_routing::MAX_DEFERRED_CONNECTIONS,
+ ),
heartbeat_interval: config.heartbeat_interval,
access_token: IggyRwLock::new(access_token),
events: broadcast(1000),
@@ -342,12 +427,7 @@ impl HttpClient {
true => Ok(response),
false => {
let reason = response.text().await.unwrap_or("error".to_string());
- match status {
- StatusCode::UNAUTHORIZED => Err(IggyError::Unauthenticated),
- StatusCode::FORBIDDEN => Err(IggyError::Unauthorized),
- StatusCode::NOT_FOUND => Err(IggyError::ResourceNotFound(reason)),
- _ => Err(IggyError::HttpResponseError(status.as_u16(), reason)),
- }
+ Err(Self::response_error(status, reason))
}
}
}
@@ -369,6 +449,15 @@ impl HttpClient {
async fn disconnect(&self) -> Result<(), IggyError> {
Ok(())
}
+
+ fn response_error(status: StatusCode, reason: String) -> IggyError {
+ match status {
+ StatusCode::UNAUTHORIZED => IggyError::Unauthenticated,
+ StatusCode::FORBIDDEN => IggyError::Unauthorized,
+ StatusCode::NOT_FOUND => IggyError::ResourceNotFound(reason),
+ _ => IggyError::HttpResponseError(status.as_u16(), reason),
+ }
+ }
}
#[derive(Debug, Serialize)]
@@ -376,11 +465,174 @@ struct RefreshToken {
token: String,
}
+#[derive(Deserialize)]
+struct HttpRejection {
+ id: u32,
+}
+
+impl HttpRejection {
+ fn is_not_accepted(error: &IggyError) -> bool {
+ matches!(error, IggyError::HttpResponseError(status, body)
+ if *status == StatusCode::SERVICE_UNAVAILABLE.as_u16()
+ && serde_json::from_str::<Self>(body)
+ .is_ok_and(|error| error.id == IggyError::TransientNotAccepted.as_code()))
+ }
+}
+
/// Unit tests for HttpClient.
/// TODO: Add complete unit tests for HttpClient.
#[cfg(test)]
mod tests {
use super::*;
+ use iggy_common::{Consumer, Identifier, PollMessages, PollingStrategy};
+ use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
+ use tokio::net::TcpListener;
+ use tokio::task::JoinHandle;
+
+ #[tokio::test]
+ async fn deferred_poll_retries_only_nonadmission_with_remaining_wait() {
+ let (client, requests) = deferred_response_server(
+ [
+ IggyError::TransientNotAccepted.as_code(),
+ IggyError::TransientNotCommitted.as_code(),
+ ]
+ .into_iter()
+ .map(|id| (503, format!(r#"{{"id":{id}}}"#)))
+ .collect(),
+ )
+ .await;
+ let result = client
+ .get_deferred_poll(
+ "messages/deferred",
+ &deferred_request(),
+ iggy_common::DeferredPollOptions::default(),
+ )
+ .await;
+ assert!(
+ matches!(result, Err(IggyError::HttpResponseError(503, body))
+ if serde_json::from_str::<HttpRejection>(&body).unwrap().id
+ == IggyError::TransientNotCommitted.as_code())
+ );
+ let waits = requests.await.unwrap();
+ assert_eq!(waits.len(), 2);
+ assert!(waits[0] <= 1_000_000);
+ assert!(waits[1] > 0 && waits[1] < waits[0], "waits: {waits:?}");
+ }
+
+ #[tokio::test]
+ async fn deferred_poll_exhausted_retry_budget_does_not_send_zero_wait() {
+ let (client, requests) = deferred_response_server(vec![(
+ 503,
+ format!(r#"{{"id":{}}}"#, IggyError::TransientNotAccepted.as_code()),
+ )])
+ .await;
+ let result = client
+ .get_deferred_poll(
+ "messages/deferred",
+ &deferred_request(),
+ iggy_common::DeferredPollOptions {
+ max_wait: 50_000.into(),
+ request_timeout: 50_000.into(),
+ ..Default::default()
+ },
+ )
+ .await;
+ assert!(matches!(result, Err(IggyError::TransientNotAccepted)));
+ assert_eq!(
+ tokio::time::timeout(std::time::Duration::from_secs(5), requests)
+ .await
+ .unwrap()
+ .unwrap()
+ .len(),
+ 1
+ );
+ }
+
+ #[tokio::test]
+ async fn deferred_poll_bounds_success_and_error_bodies() {
+ const MAX_BYTES: u32 = 16;
+ for status in [200, 503] {
+ let (client, requests) =
+ deferred_response_server(vec![(status, " ".repeat(16 * MAX_BYTES as usize + 1))])
+ .await;
+ let result = client
+ .get_deferred_poll(
+ "messages/deferred",
+ &deferred_request(),
+ iggy_common::DeferredPollOptions {
+ max_bytes: MAX_BYTES,
+ ..Default::default()
+ },
+ )
+ .await;
+ assert!(
+ matches!(result, Err(IggyError::InvalidSizeBytes)),
+ "status {status}: {result:?}"
+ );
+ assert_eq!(requests.await.unwrap().len(), 1);
+ }
+ }
+
+ fn deferred_request() -> PollMessages {
+ PollMessages {
+ consumer: Consumer::default(),
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: Some(0),
+ strategy: PollingStrategy::default(),
+ count: 1,
+ auto_commit: true,
+ }
+ }
+
+ async fn deferred_response_server(
+ responses: Vec<(u16, String)>,
+ ) -> (HttpClient, JoinHandle<Vec<u64>>) {
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let client = HttpClient::create(Arc::new(HttpClientConfig {
+ api_url: format!("http://{}", listener.local_addr().unwrap()),
+ jwt: Some("test-token".to_owned()),
+ ..HttpClientConfig::default()
+ }))
+ .unwrap();
+ let requests = tokio::spawn(async move {
+ let mut waits = Vec::new();
+ for (status, body) in responses {
+ let (stream, _) = listener.accept().await.unwrap();
+ let mut stream = BufReader::new(stream);
+ let mut line = String::new();
+ stream.read_line(&mut line).await.unwrap();
+ let path = line.split_whitespace().nth(1).unwrap();
+ let url = Url::parse(&format!("http://localhost{path}")).unwrap();
+ let wait = url
+ .query_pairs()
+ .find(|(name, _)| name == "wait_us")
+ .unwrap()
+ .1
+ .parse::<u64>()
+ .unwrap();
+ waits.push(wait);
+ loop {
+ line.clear();
+ assert_ne!(stream.read_line(&mut line).await.unwrap(), 0);
+ if line == "\r\n" {
+ break;
+ }
+ }
+ let response = format!(
+ "HTTP/1.1 {status} Test\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ );
+ stream
+ .get_mut()
+ .write_all(response.as_bytes())
+ .await
+ .unwrap();
+ }
+ waits
+ });
+ (client, requests)
+ }
#[test]
fn should_fail_with_empty_connection_string() {
diff --git a/core/sdk/src/http/messages.rs b/core/sdk/src/http/messages.rs
index d12f76b24..6e6e6bce7 100644
--- a/core/sdk/src/http/messages.rs
+++ b/core/sdk/src/http/messages.rs
@@ -29,6 +29,54 @@ use iggy_common::flush_unsaved_buffer::FlushUnsavedBuffer;
#[async_trait]
impl MessageClient for HttpClient {
+ async fn poll_messages_with_strategy_for_and_options(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: Option<u32>,
+ consumer: &Consumer,
+ strategy_for: &(dyn Fn(u32) -> PollingStrategy + Send + Sync),
+ count: u32,
+ auto_commit: bool,
+ options: Option<iggy_common::DeferredPollOptions>,
+ ) -> Result<PolledMessages, IggyError> {
+ let Some(options) = options else {
+ return self
+ .poll_messages_with_strategy_for(
+ stream_id,
+ topic_id,
+ partition_id,
+ consumer,
+ strategy_for,
+ count,
+ auto_commit,
+ )
+ .await;
+ };
+ options.validate(count)?;
+ if consumer.kind == iggy_common::ConsumerKind::ConsumerGroup && partition_id.is_none() {
+ return Err(IggyError::FeatureUnavailable);
+ }
+ let path = format!(
+ "{}/deferred",
+ get_path(&stream_id.as_cow_str(), &topic_id.as_cow_str())
+ );
+ self.get_deferred_poll(
+ &path,
+ &PollMessages {
+ stream_id: stream_id.clone(),
+ topic_id: topic_id.clone(),
+ partition_id,
+ consumer: consumer.clone(),
+ strategy: strategy_for(partition_id.unwrap_or(0)),
+ count,
+ auto_commit,
+ },
+ options,
+ )
+ .await
+ }
+
async fn poll_messages(
&self,
stream_id: &Identifier,
diff --git a/core/sdk/src/poll_routing.rs b/core/sdk/src/poll_routing.rs
index 5056e775e..98a7c004d 100644
--- a/core/sdk/src/poll_routing.rs
+++ b/core/sdk/src/poll_routing.rs
@@ -28,8 +28,11 @@ use iggy_binary_protocol::codes::{
ATTACH_CONSUMER_SESSION_CODE, GET_CLUSTER_METADATA_CODE, GET_CONSUMER_OFFSET_ROUTING_CODE,
GET_POLL_ROUTING_CODE, PING_CODE, POLL_MESSAGES_CODE, POLL_MESSAGES_ON_PRIMARY_CODE,
};
+use iggy_binary_protocol::codes::{
+ POLL_MESSAGES_DEFERRED_CODE, POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE,
+};
use iggy_binary_protocol::requests::consumer_offsets::GetConsumerOffsetRequest;
-use iggy_binary_protocol::requests::messages::PollMessagesRequest;
+use iggy_binary_protocol::requests::messages::{DeferredPollMessagesRequest, PollMessagesRequest};
use iggy_binary_protocol::requests::system::AttachConsumerSessionRequest;
use iggy_binary_protocol::responses::messages::PollRoutingResponse;
use iggy_binary_protocol::responses::system::get_cluster_metadata::ClusterMetadataResponse;
@@ -42,15 +45,16 @@ use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
-use tokio::sync::Mutex as AsyncMutex;
+use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard, Semaphore};
use tokio::time::{Instant, sleep, timeout, timeout_at};
use tracing::error;
const MAX_CACHED_ROUTES: usize = 4096;
const MAX_DATA_CONNECTIONS: usize = 256;
+pub(crate) const MAX_DEFERRED_CONNECTIONS: usize = 16;
const POLL_TIMEOUT: Duration = Duration::from_secs(30);
-const ROUTING_RETRY_INTERVAL: Duration = Duration::from_millis(50);
-const ROUTING_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(1);
+pub(crate) const ROUTING_RETRY_INTERVAL: Duration = Duration::from_millis(50);
+pub(crate) const ROUTING_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(1);
pub(crate) const ROSTER_READ_TIMEOUT: Duration = Duration::from_secs(5);
pub(crate) const fn is_poll_routing_code(code: u32) -> bool {
@@ -59,6 +63,8 @@ pub(crate) const fn is_poll_routing_code(code: u32) -> bool {
ATTACH_CONSUMER_SESSION_CODE
| GET_POLL_ROUTING_CODE
| POLL_MESSAGES_ON_PRIMARY_CODE
+ | POLL_MESSAGES_DEFERRED_CODE
+ | POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE
| GET_CONSUMER_OFFSET_ROUTING_CODE
)
}
@@ -69,6 +75,12 @@ pub(crate) trait PollTransport: BinaryClient + Send + Sync + Sized {
async fn connect_poll_client(&self, endpoint: &str) -> Result<Self, IggyError>;
+ async fn local_poll_session(
+ &self,
+ ) -> Result<(String, AttachConsumerSessionRequest), IggyError> {
+ Err(IggyError::FeatureUnavailable)
+ }
+
/// One exchange on this connection, with no node movement or automatic
/// replay of an ambiguous outcome, including replicated offset writes.
async fn send_poll_request(&self, code: u32, payload: Bytes) -> Result<Bytes, IggyError>;
@@ -108,6 +120,8 @@ pub(crate) struct PollRouter<T> {
session_generation: AtomicU64,
routes: Mutex<HashMap<RouteKey, Arc<PollRoute>>>,
connections: Mutex<HashMap<String, ConnectionSlot<T>>>,
+ deferred_connections: Mutex<Vec<(String, ConnectionSlot<T>)>>,
+ deferred_leases: Semaphore,
credentials: Mutex<Option<(Credentials, u32)>>,
next_heartbeat: Mutex<Option<Instant>>,
}
@@ -120,6 +134,8 @@ impl<T> Default for PollRouter<T> {
session_generation: AtomicU64::new(0),
routes: Mutex::default(),
connections: Mutex::default(),
+ deferred_connections: Mutex::default(),
+ deferred_leases: Semaphore::new(MAX_DEFERRED_CONNECTIONS),
credentials: Mutex::default(),
next_heartbeat: Mutex::default(),
}
@@ -144,6 +160,10 @@ impl<T> PollRouter<T> {
self.session_generation.fetch_add(1, Ordering::AcqRel);
routes.clear();
connections.clear();
+ self.deferred_connections
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .clear();
}
pub(crate) fn remember_credentials(&self, credentials: Credentials, user_id: u32) {
@@ -256,6 +276,234 @@ impl<T: PollTransport> PollRouter<T> {
Ok(self.roster_size.load(Ordering::Acquire) > 1)
}
+ pub(crate) async fn poll_deferred(
+ &self,
+ coordinator: &T,
+ request: &PollMessagesRequest,
+ options: iggy_common::DeferredPollOptions,
+ ) -> Result<Bytes, IggyError> {
+ options.validate(request.count)?;
+ let started = Instant::now();
+ let deadline = started + options.request_timeout.get_duration();
+ let wait_deadline = started + options.max_wait.get_duration();
+ let operation = async {
+ let primary = request.auto_commit && self.is_clustered(coordinator).await?;
+ let payload = request.to_bytes();
+ let prefix_len = request.consumer.encoded_size()
+ + request.stream_id.encoded_size()
+ + request.topic_id.encoded_size()
+ + size_of::<u8>()
+ + size_of::<u32>();
+ let key = (GET_POLL_ROUTING_CODE, payload.slice(..prefix_len));
+ let mut retry_interval = ROUTING_RETRY_INTERVAL;
+ loop {
+ if Instant::now() >= deadline {
+ return Err(IggyError::TransientNotAccepted);
+ }
+ let result = self
+ .deferred_attempt(
+ coordinator,
+ request,
+ primary,
+ &key,
+ &payload,
+ deadline,
+ wait_deadline,
+ options,
+ )
+ .await;
+ if !matches!(result, Err(IggyError::TransientNotAccepted)) {
+ return result;
+ }
+ self.routes
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .remove(&key);
+ if Instant::now() + retry_interval >= deadline {
+ return result;
+ }
+ sleep(retry_interval).await;
+ retry_interval = (retry_interval * 2).min(ROUTING_RETRY_MAX_INTERVAL);
+ }
+ };
+ tokio::select! {
+ result = timeout_at(deadline, operation) => result.unwrap_or(Err(IggyError::TransientNotCommitted)),
+ _ = self.maintain_poll_parent(coordinator) => Err(IggyError::TransientNotCommitted),
+ }
+ }
+
+ async fn maintain_poll_parent(&self, coordinator: &T) {
+ let interval = coordinator.get_heartbeat_interval().get_duration();
+ loop {
+ sleep((interval / 2).max(Duration::from_nanos(1))).await;
+ let now = Instant::now();
+ let due = {
+ let mut next = self
+ .next_heartbeat
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ if next.is_none_or(|next| now >= next) {
+ *next = Some(now + interval);
+ true
+ } else {
+ false
+ }
+ };
+ if due
+ && coordinator
+ .send_poll_control(PING_CODE, Bytes::new())
+ .await
+ .is_err()
+ {
+ return;
+ }
+ }
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ async fn deferred_attempt(
+ &self,
+ coordinator: &T,
+ request: &PollMessagesRequest,
+ primary: bool,
+ key: &RouteKey,
+ payload: &Bytes,
+ deadline: Instant,
+ wait_deadline: Instant,
+ options: iggy_common::DeferredPollOptions,
+ ) -> Result<Bytes, IggyError> {
+ let route = if primary {
+ self.route(coordinator, key, payload).await?
+ } else {
+ let generation = self.session_generation.load(Ordering::Acquire);
+ let (endpoint, consumer_session) = coordinator.local_poll_session().await?;
+ Arc::new(PollRoute {
+ generation,
+ endpoint,
+ consumer_session,
+ })
+ };
+ let _permit = timeout_at(deadline, self.deferred_leases.acquire())
+ .await
+ .map_err(|_| IggyError::TransientNotAccepted)?
+ .map_err(|_| IggyError::ClientShutdown)?;
+ let mut lease = DeferredLease(self.lease_deferred_connection(&route)?);
+ if lease
+ .0
+ .as_ref()
+ .is_some_and(|connection| !connection.usable)
+ {
+ lease.0.take();
+ }
+ if lease.0.is_none() {
+ let client = timeout_at(deadline, coordinator.connect_poll_client(&route.endpoint))
+ .await
+ .map_err(|_| IggyError::TransientNotAccepted)?
+ .map_err(unaccepted_data_error)?;
+ self.validate_route(&route)?;
+ *lease.0 = Some(PollConnection {
+ client,
+ consumer_session: None,
+ usable: true,
+ });
+ }
+ let connection = lease.0.as_mut().ok_or(IggyError::NotConnected)?;
+ connection.usable = false;
+ if connection.consumer_session != Some(route.consumer_session) {
+ timeout_at(
+ deadline,
+ connection.client.send_poll_request(
+ ATTACH_CONSUMER_SESSION_CODE,
+ route.consumer_session.to_bytes(),
+ ),
+ )
+ .await
+ .map_err(|_| IggyError::TransientNotAccepted)?
+ .map_err(unaccepted_data_error)?;
+ connection.consumer_session = Some(route.consumer_session);
+ }
+ self.validate_route(&route)?;
+ let now = Instant::now();
+ let wait_us = u64::try_from(wait_deadline.saturating_duration_since(now).as_micros())
+ .map_err(|_| IggyError::InvalidCommand)?;
+ let request_timeout_us = u64::try_from(deadline.saturating_duration_since(now).as_micros())
+ .map_err(|_| IggyError::InvalidCommand)?;
+ if request_timeout_us == 0 {
+ return Err(IggyError::TransientNotAccepted);
+ }
+ let code = if primary {
+ POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE
+ } else {
+ POLL_MESSAGES_DEFERRED_CODE
+ };
+ let wire = DeferredPollMessagesRequest {
+ poll: request.clone(),
+ wait_us,
+ min_count: options.min_count,
+ max_bytes: options.max_bytes,
+ request_timeout_us,
+ };
+ let result = connection
+ .client
+ .send_poll_request(code, wire.to_bytes())
+ .await;
+ connection.usable = !result.as_ref().is_err_and(poll_connection_failed);
+ if result
+ .as_ref()
+ .is_ok_and(|bytes| bytes.len() > options.max_bytes as usize)
+ {
+ connection.usable = false;
+ return Err(IggyError::InvalidSizeBytes);
+ }
+ if route.generation != self.session_generation.load(Ordering::Acquire) {
+ return Err(IggyError::StaleClient);
+ }
+ if matches!(result, Err(IggyError::TransientNotAccepted)) {
+ connection.consumer_session = None;
+ }
+ result.map_err(|error| {
+ if poll_connection_failed(&error) {
+ IggyError::TransientNotCommitted
+ } else {
+ error
+ }
+ })
+ }
+
+ fn lease_deferred_connection(
+ &self,
+ route: &PollRoute,
+ ) -> Result<OwnedMutexGuard<Option<PollConnection<T>>>, IggyError> {
+ let mut pool = self
+ .deferred_connections
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ self.validate_route(route)?;
+ for (endpoint, slot) in pool.iter() {
+ if endpoint == &route.endpoint
+ && let Ok(guard) = Arc::clone(slot).try_lock_owned()
+ {
+ return Ok(guard);
+ }
+ }
+ if pool.len() < MAX_DEFERRED_CONNECTIONS {
+ let slot = Arc::new(AsyncMutex::new(None));
+ let guard = Arc::clone(&slot)
+ .try_lock_owned()
+ .map_err(|_| IggyError::TransientNotAccepted)?;
+ pool.push((route.endpoint.clone(), slot));
+ return Ok(guard);
+ }
+ for (endpoint, slot) in pool.iter_mut() {
+ if let Ok(mut guard) = Arc::clone(slot).try_lock_owned() {
+ guard.take();
+ *endpoint = route.endpoint.clone();
+ return Ok(guard);
+ }
+ }
+ Err(IggyError::TransientNotAccepted)
+ }
+
async fn send_routed(
&self,
coordinator: &T,
@@ -510,6 +758,60 @@ fn unaccepted_data_error(error: IggyError) -> IggyError {
}
}
+struct DeferredLease<T>(OwnedMutexGuard<Option<PollConnection<T>>>);
+
+impl<T> Drop for DeferredLease<T> {
+ fn drop(&mut self) {
+ if self.0.as_ref().is_some_and(|connection| !connection.usable) {
+ self.0.take();
+ }
+ }
+}
+
+pub(crate) fn deferred_response_limit(code: u32, payload: &[u8]) -> Result<usize, IggyError> {
+ if matches!(
+ code,
+ POLL_MESSAGES_DEFERRED_CODE | POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE
+ ) {
+ let request = DeferredPollMessagesRequest::decode_from(payload)
+ .map_err(|_| IggyError::InvalidCommand)?;
+ Ok(request.max_bytes as usize)
+ } else {
+ Ok(usize::MAX)
+ }
+}
+
+pub(crate) fn deferred_exchange_timeout(code: u32, payload: &[u8]) -> Result<Duration, IggyError> {
+ if matches!(
+ code,
+ POLL_MESSAGES_DEFERRED_CODE | POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE
+ ) {
+ let request = DeferredPollMessagesRequest::decode_from(payload)
+ .map_err(|_| IggyError::InvalidCommand)?;
+ Ok(Duration::from_micros(request.request_timeout_us))
+ } else {
+ Ok(Duration::ZERO)
+ }
+}
+
+/// Deferred exchanges own a disposable socket. Cancelling them must also
+/// cancel the detached reader so a parked owner request sees disconnection.
+pub(crate) struct DeferredExchangeTask(Option<tokio::task::AbortHandle>);
+
+impl DeferredExchangeTask {
+ pub(crate) fn new<T>(wait: Duration, task: &tokio::task::JoinHandle<T>) -> Self {
+ Self((!wait.is_zero()).then(|| task.abort_handle()))
+ }
+}
+
+impl Drop for DeferredExchangeTask {
+ fn drop(&mut self) {
+ if let Some(task) = &self.0 {
+ task.abort();
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -559,6 +861,7 @@ mod tests {
exchanges: Mutex<VecDeque<Exchange>>,
route_queries: AtomicUsize,
attachments: Mutex<Vec<AttachConsumerSessionRequest>>,
+ deferred_waits: Mutex<Vec<u64>>,
connections: AtomicUsize,
pause: Mutex<Option<Arc<Pause>>>,
}
@@ -607,6 +910,16 @@ mod tests {
) {
self.script.route_queries.fetch_add(1, Ordering::Relaxed);
}
+ if matches!(
+ code,
+ POLL_MESSAGES_DEFERRED_CODE | POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE
+ ) {
+ self.script.deferred_waits.lock().unwrap().push(
+ DeferredPollMessagesRequest::decode_from(payload)
+ .unwrap()
+ .wait_us,
+ );
+ }
if code == ATTACH_CONSUMER_SESSION_CODE {
self.script
.attachments
@@ -675,6 +988,18 @@ mod tests {
#[async_trait]
impl PollTransport for Transport {
const PROTOCOL: TransportProtocol = TransportProtocol::Tcp;
+ async fn local_poll_session(
+ &self,
+ ) -> Result<(String, AttachConsumerSessionRequest), IggyError> {
+ Ok((
+ "127.0.0.1:8090".to_string(),
+ AttachConsumerSessionRequest {
+ client_id: 7,
+ session: 1,
+ metadata_watermark: 1,
+ },
+ ))
+ }
async fn connect_poll_client(&self, _endpoint: &str) -> Result<Self, IggyError> {
self.script.connections.fetch_add(1, Ordering::Relaxed);
self.script.pause_at(PausePoint::Connect).await;
@@ -1233,6 +1558,187 @@ mod tests {
router.write_offset(coordinator, code, payload).await
}
+ #[tokio::test]
+ async fn deferred_manual_polls_lease_separate_connections_and_leave_control_available() {
+ let (router, coordinator, mut request) = fixture([
+ (
+ Channel::Data,
+ ATTACH_CONSUMER_SESSION_CODE,
+ Ok(Bytes::new()),
+ ),
+ (
+ Channel::Data,
+ POLL_MESSAGES_DEFERRED_CODE,
+ Ok(Bytes::from_static(b"first")),
+ ),
+ (Channel::Coordinator, PING_CODE, Ok(Bytes::new())),
+ (
+ Channel::Data,
+ ATTACH_CONSUMER_SESSION_CODE,
+ Ok(Bytes::new()),
+ ),
+ (
+ Channel::Data,
+ POLL_MESSAGES_DEFERRED_CODE,
+ Ok(Bytes::from_static(b"second")),
+ ),
+ ]);
+ request.auto_commit = false;
+ let pause = Arc::new(Pause {
+ point: PausePoint::Reply(POLL_MESSAGES_DEFERRED_CODE),
+ reached: Notify::new(),
+ resume: Notify::new(),
+ });
+ *coordinator.script.pause.lock().unwrap() = Some(Arc::clone(&pause));
+ let mut first = Box::pin(router.poll_deferred(
+ &coordinator,
+ &request,
+ iggy_common::DeferredPollOptions::default(),
+ ));
+ tokio::select! {
+ _ = pause.reached.notified() => {}
+ _ = &mut first => panic!("first poll must remain parked"),
+ }
+ coordinator
+ .send_poll_request(PING_CODE, Bytes::new())
+ .await
+ .unwrap();
+ assert_eq!(
+ router
+ .poll_deferred(
+ &coordinator,
+ &request,
+ iggy_common::DeferredPollOptions::default()
+ )
+ .await
+ .unwrap(),
+ Bytes::from_static(b"second")
+ );
+ assert_eq!(coordinator.script.connections.load(Ordering::Relaxed), 2);
+ pause.resume.notify_one();
+ assert_eq!(first.await.unwrap(), Bytes::from_static(b"first"));
+ assert!(coordinator.script.exchanges.lock().unwrap().is_empty());
+ }
+
+ #[tokio::test]
+ async fn cancelling_deferred_poll_discards_its_connection() {
+ let (router, coordinator, mut request) = fixture([
+ (
+ Channel::Data,
+ ATTACH_CONSUMER_SESSION_CODE,
+ Ok(Bytes::new()),
+ ),
+ (Channel::Data, POLL_MESSAGES_DEFERRED_CODE, Ok(Bytes::new())),
+ (
+ Channel::Data,
+ ATTACH_CONSUMER_SESSION_CODE,
+ Ok(Bytes::new()),
+ ),
+ (Channel::Data, POLL_MESSAGES_DEFERRED_CODE, Ok(Bytes::new())),
+ ]);
+ request.auto_commit = false;
+ let pause = Arc::new(Pause {
+ point: PausePoint::Reply(POLL_MESSAGES_DEFERRED_CODE),
+ reached: Notify::new(),
+ resume: Notify::new(),
+ });
+ *coordinator.script.pause.lock().unwrap() = Some(Arc::clone(&pause));
+ let mut first = Box::pin(router.poll_deferred(
+ &coordinator,
+ &request,
+ iggy_common::DeferredPollOptions::default(),
+ ));
+ tokio::select! {
+ _ = pause.reached.notified() => {}
+ _ = &mut first => panic!("first poll must remain parked"),
+ }
+ drop(first);
+ router
+ .poll_deferred(
+ &coordinator,
+ &request,
+ iggy_common::DeferredPollOptions::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(coordinator.script.connections.load(Ordering::Relaxed), 2);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn deferred_retries_only_nonadmission_and_keeps_one_deadline() {
+ for failure in [
+ IggyError::TransientNotAccepted,
+ IggyError::CannotReadMessage,
+ ] {
+ let mut exchanges = vec![
+ (
+ Channel::Data,
+ ATTACH_CONSUMER_SESSION_CODE,
+ Ok(Bytes::new()),
+ ),
+ (
+ Channel::Data,
+ POLL_MESSAGES_DEFERRED_CODE,
+ Err(failure.clone()),
+ ),
+ ];
+ if matches!(failure, IggyError::TransientNotAccepted) {
+ exchanges.extend([
+ (
+ Channel::Data,
+ ATTACH_CONSUMER_SESSION_CODE,
+ Ok(Bytes::new()),
+ ),
+ (Channel::Data, POLL_MESSAGES_DEFERRED_CODE, Ok(Bytes::new())),
+ ]);
+ }
+ let (router, coordinator, mut request) = fixture(exchanges);
+ request.auto_commit = false;
+ let result = router
+ .poll_deferred(
+ &coordinator,
+ &request,
+ iggy_common::DeferredPollOptions::default(),
+ )
+ .await;
+ let waits = coordinator.script.deferred_waits.lock().unwrap();
+ if matches!(failure, IggyError::TransientNotAccepted) {
+ assert!(result.is_ok());
+ assert_eq!(waits.len(), 2);
+ assert!(waits[1] < waits[0]);
+ } else {
+ assert!(matches!(result, Err(IggyError::CannotReadMessage)));
+ assert_eq!(waits.len(), 1);
+ }
+ }
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn deferred_pool_is_bounded_and_lease_wait_uses_request_deadline() {
+ let (router, coordinator, mut request) = fixture([]);
+ request.auto_commit = false;
+ let _leases = router
+ .deferred_leases
+ .acquire_many(u32::try_from(MAX_DEFERRED_CONNECTIONS).unwrap())
+ .await
+ .unwrap();
+ assert!(matches!(
+ router
+ .poll_deferred(
+ &coordinator,
+ &request,
+ iggy_common::DeferredPollOptions {
+ max_wait: 1_000.into(),
+ request_timeout: 1_000.into(),
+ ..Default::default()
+ }
+ )
+ .await,
+ Err(IggyError::TransientNotAccepted)
+ ));
+ assert_eq!(coordinator.script.connections.load(Ordering::Relaxed), 0);
+ }
+
fn fixture(
exchanges: impl IntoIterator<Item = Exchange>,
) -> (PollRouter<Transport>, Transport, PollMessagesRequest) {
diff --git a/core/sdk/src/prelude.rs b/core/sdk/src/prelude.rs
index c16c643a1..6a5a2511f 100644
--- a/core/sdk/src/prelude.rs
+++ b/core/sdk/src/prelude.rs
@@ -53,11 +53,11 @@ pub use iggy_common::{
Aes256GcmEncryptor, Args, ArgsOptional, AutoLogin, CacheMetrics, CacheMetricsKey, ClientError,
ClientInfoDetails, ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus,
CompressionAlgorithm, Consumer, ConsumerGroup, ConsumerGroupDetails, ConsumerGroupMember,
- ConsumerKind, Credentials, Durability, EncryptorKind, GlobalPermissions, HeaderField,
- HeaderKey, HeaderKind, HeaderValue, HttpClientConfig, HttpClientConfigBuilder, HttpMethod,
- IdKind, Identifier, IdentityInfo, IggyByteSize, IggyDuration, IggyError, IggyExpiry,
- IggyIndexView, IggyMessage, IggyMessageHeader, IggyMessageHeaderView, IggyMessageView,
- IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, NonZeroDurationError,
+ ConsumerKind, Credentials, DeferredPollOptions, Durability, EncryptorKind, GlobalPermissions,
+ HeaderField, HeaderKey, HeaderKind, HeaderValue, HttpClientConfig, HttpClientConfigBuilder,
+ HttpMethod, IdKind, Identifier, IdentityInfo, IggyByteSize, IggyDuration, IggyError,
+ IggyExpiry, IggyIndexView, IggyMessage, IggyMessageHeader, IggyMessageHeaderView,
+ IggyMessageView, IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, NonZeroDurationError,
NonZeroIggyDuration, OptionSpec, OptionValue, OptionsScope, Partition, Partitioner,
Partitioning, Permissions, PersonalAccessTokenExpiry, PollMessages, PolledMessages,
PollingKind, PollingStrategy, QuicClientConfig, QuicClientConfigBuilder,
@@ -76,11 +76,11 @@ pub use iggy_common::{
TopicClient, UserClient,
};
pub use iggy_common::{
- IGGY_MESSAGE_CHECKSUM_OFFSET_RANGE, IGGY_MESSAGE_HEADER_SIZE,
+ DEFAULT_POLL_MAX_BYTES, IGGY_MESSAGE_CHECKSUM_OFFSET_RANGE, IGGY_MESSAGE_HEADER_SIZE,
IGGY_MESSAGE_HEADERS_LENGTH_OFFSET_RANGE, IGGY_MESSAGE_ID_OFFSET_RANGE,
IGGY_MESSAGE_OFFSET_OFFSET_RANGE, IGGY_MESSAGE_ORIGIN_TIMESTAMP_OFFSET_RANGE,
IGGY_MESSAGE_PAYLOAD_LENGTH_OFFSET_RANGE, IGGY_MESSAGE_TIMESTAMP_OFFSET_RANGE, INDEX_SIZE,
- MAX_PAYLOAD_SIZE, MAX_USER_HEADERS_SIZE, NO_ASSIGNED_PARTITION,
+ MAX_DEFERRED_POLL_WAIT_US, MAX_PAYLOAD_SIZE, MAX_USER_HEADERS_SIZE, NO_ASSIGNED_PARTITION,
RESYNC_REQUIRED_PARTITION_SENTINEL, SEC_IN_MICRO,
defaults::{DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USER_ID, DEFAULT_ROOT_USERNAME},
};
diff --git a/core/sdk/src/quic/quic_client.rs b/core/sdk/src/quic/quic_client.rs
index 1110e7291..89d243e4a 100644
--- a/core/sdk/src/quic/quic_client.rs
+++ b/core/sdk/src/quic/quic_client.rs
@@ -151,6 +151,17 @@ impl BinaryTransport for QuicClient {
) -> Result<Bytes, IggyError> {
self.poll_router.poll(self, request).await
}
+
+ async fn send_poll_with_response_and_options(
+ &self,
+ request: &iggy_binary_protocol::requests::messages::PollMessagesRequest,
+ options: Option<iggy_common::DeferredPollOptions>,
+ ) -> Result<Bytes, IggyError> {
+ match options {
+ Some(options) => self.poll_router.poll_deferred(self, request, options).await,
+ None => self.poll_router.poll(self, request).await,
+ }
+ }
async fn get_state(&self) -> ClientState {
*self.state.lock().await
}
@@ -497,6 +508,33 @@ impl BinaryClient for QuicClient {}
impl PollTransport for QuicClient {
const PROTOCOL: TransportProtocol = TransportProtocol::Quic;
+ async fn local_poll_session(
+ &self,
+ ) -> Result<
+ (
+ String,
+ iggy_binary_protocol::requests::system::AttachConsumerSessionRequest,
+ ),
+ IggyError,
+ > {
+ let endpoint = self.current_server_address.lock().await.clone();
+ let session = self
+ .consensus_session
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ Ok((
+ endpoint,
+ iggy_binary_protocol::requests::system::AttachConsumerSessionRequest {
+ client_id: session.client_id(),
+ session: session.session().ok_or(IggyError::Unauthenticated)?,
+ metadata_watermark: self
+ .poll_router
+ .metadata_watermark
+ .load(std::sync::atomic::Ordering::Acquire),
+ },
+ ))
+ }
+
async fn connect_poll_client(&self, endpoint: &str) -> Result<Self, IggyError> {
let mut config = (*self.config).clone();
config.server_address = endpoint.to_owned();
@@ -1024,8 +1062,11 @@ impl QuicClient {
let response_buffer_size = self.config.response_buffer_size;
let consensus_session = self.consensus_session.clone();
let metadata_watermark = Arc::clone(&self.poll_router.metadata_watermark);
+ let deferred_timeout = crate::poll_routing::deferred_exchange_timeout(code, &payload)?;
+ let response_limit = crate::poll_routing::deferred_response_limit(code, &payload)?
+ .saturating_add(iggy_binary_protocol::HEADER_SIZE);
// SAFETY: we run code holding the `connection` lock in a task so we can't be cancelled while holding the lock.
- tokio::spawn(async move {
+ let task = tokio::spawn(async move {
let connection = connection.lock().await;
let Some(connection) = connection.as_ref() else {
error!("Cannot send data. Client is not connected.");
@@ -1033,106 +1074,105 @@ impl QuicClient {
};
let (request_header, request_size) = {
- let mut consensus_session = consensus_session
- .lock()
- .expect("consensus session mutex poisoned");
- crate::vsr::encode_request_header(&mut consensus_session, code, &payload)?
- };
- trace!("Sending a QUIC VSR request of size {request_size} with code: {code}");
- // Same-connection transient resend, gated on the EXPLICIT
- // `TransientNotCommitted` frame only. The server answers every
- // transient submit with that frame (it is pre-commit by
- // construction, so replaying the SAME request header on a fresh
- // bidi cannot double-commit), and it no longer abandons a bidi
- // whose op is still committing. Silence therefore is NOT a
- // retry signal: the partition plane has no dedup or reply
- // cache, so resending a silently-unanswered request whose
- // first attempt was buffered and later commits would commit it
- // twice (duplicate `SendMessages`, or a succeeded delete coming
- // back as terminal `ConsumerOffsetNotFound`). A silent deadline
- // expiry surfaces `Disconnected` and takes the
- // reconnect path in `send_raw_with_response`, same as TCP.
- let header_bytes = bytemuck::bytes_of(&request_header);
- let deadline = tokio::time::Instant::now() + RESPONSE_READ_TIMEOUT;
- // `TransientNotAccepted` gets a short same-connection window
- // only: past it the refusal is a verdict about who leads, not
- // load, and the caller runs a leader recheck or roster walk.
- // Login/register keeps the full budget on this connection: the
- // connect flow owns its leader settlement.
- let not_accepted_deadline = if is_login_register_code(code) {
- deadline
- } else {
- deadline.min(tokio::time::Instant::now() + TRANSIENT_FAILOVER_CHECK_INTERVAL)
- };
- loop {
- let (mut send, mut recv) = connection.open_bi().await.map_err(|error| {
- error!("Failed to open a bidirectional stream: {error}");
- IggyError::QuicError
- })?;
- send.write_all(header_bytes).await.map_err(|error| {
- error!("Failed to write VSR request header: {error}");
- IggyError::QuicError
- })?;
- if !payload.is_empty() {
- send.write_all(&payload).await.map_err(|error| {
- error!("Failed to write VSR request payload: {error}");
- IggyError::QuicError
- })?;
- }
- send.finish().map_err(|error| {
- error!("Failed to finish VSR request stream: {error}");
+ let mut consensus_session = consensus_session
+ .lock()
+ .expect("consensus session mutex poisoned");
+ crate::vsr::encode_request_header(&mut consensus_session, code, &payload)?
+ };
+ trace!("Sending a QUIC VSR request of size {request_size} with code: {code}");
+ // Same-connection transient resend, gated on the EXPLICIT
+ // `TransientNotCommitted` frame only. The server answers every
+ // transient submit with that frame (it is pre-commit by
+ // construction, so replaying the SAME request header on a fresh
+ // bidi cannot double-commit), and it no longer abandons a bidi
+ // whose op is still committing. Silence therefore is NOT a
+ // retry signal: the partition plane has no dedup or reply
+ // cache, so resending a silently-unanswered request whose
+ // first attempt was buffered and later commits would commit it
+ // twice (duplicate `SendMessages`, or a succeeded delete coming
+ // back as terminal `ConsumerOffsetNotFound`). A silent deadline
+ // expiry surfaces `Disconnected` and takes the
+ // reconnect path in `send_raw_with_response`, same as TCP.
+ let header_bytes = bytemuck::bytes_of(&request_header);
+ let deadline = tokio::time::Instant::now() + RESPONSE_READ_TIMEOUT + deferred_timeout;
+ // `TransientNotAccepted` gets a short same-connection window
+ // only: past it the refusal is a verdict about who leads, not
+ // load, and the caller runs a leader recheck or roster walk.
+ // Login/register keeps the full budget on this connection: the
+ // connect flow owns its leader settlement.
+ let not_accepted_deadline = if is_login_register_code(code) {
+ deadline
+ } else {
+ deadline.min(tokio::time::Instant::now() + TRANSIENT_FAILOVER_CHECK_INTERVAL)
+ };
+ loop {
+ let (mut send, mut recv) = connection.open_bi().await.map_err(|error| {
+ error!("Failed to open a bidirectional stream: {error}");
+ IggyError::QuicError
+ })?;
+ send.write_all(header_bytes).await.map_err(|error| {
+ error!("Failed to write VSR request header: {error}");
+ IggyError::QuicError
+ })?;
+ if !payload.is_empty() {
+ send.write_all(&payload).await.map_err(|error| {
+ error!("Failed to write VSR request payload: {error}");
IggyError::QuicError
})?;
- let remaining =
- deadline.saturating_duration_since(tokio::time::Instant::now());
- if remaining.is_zero() {
- return Err(IggyError::Disconnected);
+ }
+ send.finish().map_err(|error| {
+ error!("Failed to finish VSR request stream: {error}");
+ IggyError::QuicError
+ })?;
+ let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
+ if remaining.is_zero() {
+ return Err(IggyError::Disconnected);
+ }
+ match QuicClient::handle_response(
+ &mut recv,
+ (response_buffer_size as usize).min(response_limit),
+ remaining,
+ &metadata_watermark,
+ )
+ .await
+ {
+ Ok(reply) => return Ok(reply),
+ Err(error) if !retry_transient => return Err(error),
+ // `TransientNotCommitted` = the server replied with an
+ // explicit retry frame with an outcome that may still
+ // be resolving (not-caught-up / in-flight /
+ // pipeline-full / view-change cancel). Replaying the
+ // same request id on the same session is safe because
+ // metadata dedup returns the committed reply if needed.
+ // Anything else, including a silent read timeout, is
+ // terminal here and handled by the caller.
+ Err(IggyError::TransientNotAccepted)
+ if tokio::time::Instant::now() >= not_accepted_deadline =>
+ {
+ // Never admitted, so re-issuable anywhere: hand it
+ // back for a leader recheck or a roster walk
+ // instead of replaying into the same refusal for
+ // the whole request budget.
+ return Err(IggyError::TransientNotAccepted);
}
- match QuicClient::handle_response(
- &mut recv,
- response_buffer_size as usize,
- remaining,
- &metadata_watermark,
- )
- .await
+ Err(IggyError::TransientNotCommitted | IggyError::TransientNotAccepted)
+ if tokio::time::Instant::now() < deadline =>
{
- Ok(reply) => return Ok(reply),
- Err(error) if !retry_transient => return Err(error),
- // `TransientNotCommitted` = the server replied with an
- // explicit retry frame with an outcome that may still
- // be resolving (not-caught-up / in-flight /
- // pipeline-full / view-change cancel). Replaying the
- // same request id on the same session is safe because
- // metadata dedup returns the committed reply if needed.
- // Anything else, including a silent read timeout, is
- // terminal here and handled by the caller.
- Err(IggyError::TransientNotAccepted)
- if tokio::time::Instant::now() >= not_accepted_deadline =>
- {
- // Never admitted, so re-issuable anywhere: hand it
- // back for a leader recheck or a roster walk
- // instead of replaying into the same refusal for
- // the whole request budget.
- return Err(IggyError::TransientNotAccepted);
- }
- Err(IggyError::TransientNotCommitted | IggyError::TransientNotAccepted)
- if tokio::time::Instant::now() < deadline =>
- {
- // The explicit frame returns promptly (no read
- // timeout elapsed), so pace the replay.
- let remaining =
- deadline.saturating_duration_since(tokio::time::Instant::now());
- tokio::time::sleep(NOT_READY_RETRY_INTERVAL.min(remaining)).await;
- warn!(
- "QUIC request code {code} not committed (transient); resending on a new stream"
- );
- }
- Err(error) => return Err(error),
+ // The explicit frame returns promptly (no read
+ // timeout elapsed), so pace the replay.
+ let remaining =
+ deadline.saturating_duration_since(tokio::time::Instant::now());
+ tokio::time::sleep(NOT_READY_RETRY_INTERVAL.min(remaining)).await;
+ warn!(
+ "QUIC request code {code} not committed (transient); resending on a new stream"
+ );
}
+ Err(error) => return Err(error),
}
- })
- .await
- .map_err(|e| {
+ }
+ });
+ let _cancellation = crate::poll_routing::DeferredExchangeTask::new(deferred_timeout, &task);
+ task.await.map_err(|e| {
error!("Task execution failed during QUIC request: {}", e);
IggyError::QuicError
})?
diff --git a/core/sdk/src/stream_builder/build/build_iggy_consumer.rs b/core/sdk/src/stream_builder/build/build_iggy_consumer.rs
index c5257dc38..f64abd97f 100644
--- a/core/sdk/src/stream_builder/build/build_iggy_consumer.rs
+++ b/core/sdk/src/stream_builder/build/build_iggy_consumer.rs
@@ -49,7 +49,7 @@ pub(crate) async fn build_iggy_consumer(
let consumer_kind = config.consumer_kind();
let consumer_name = config.consumer_name();
let batch_length = config.batch_length();
- let polling_interval = config.polling_interval();
+ let poll_options = config.poll_options();
let polling_strategy = config.polling_strategy();
let partition = config.partition_id();
let polling_retry_interval = config.polling_retry_interval();
@@ -63,7 +63,9 @@ pub(crate) async fn build_iggy_consumer(
.create_consumer_group_if_not_exists()
.auto_join_consumer_group()
.polling_strategy(polling_strategy)
- .poll_interval(polling_interval)
+ .poll_options(poll_options)
+ .prefetch_bytes(config.prefetch_bytes())
+ .prefetch_messages(config.prefetch_messages())
.batch_length(batch_length)
.polling_retry_interval(polling_retry_interval);
diff --git a/core/sdk/src/stream_builder/config/config_iggy_consumer.rs b/core/sdk/src/stream_builder/config/config_iggy_consumer.rs
index 01c76849e..679ee5d60 100644
--- a/core/sdk/src/stream_builder/config/config_iggy_consumer.rs
+++ b/core/sdk/src/stream_builder/config/config_iggy_consumer.rs
@@ -15,9 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use crate::clients::consumer::{AutoCommit, AutoCommitWhen};
+use crate::clients::consumer::AutoCommit;
+use crate::clients::consumer::deferred::{DEFAULT_PREFETCH_BYTES, DEFAULT_PREFETCH_MESSAGES};
use crate::prelude::{
- ConsumerKind, EncryptorKind, Identifier, IggyDuration, IggyError, NonZeroIggyDuration,
+ ConsumerKind, DeferredPollOptions, EncryptorKind, Identifier, IggyError, NonZeroIggyDuration,
PollingStrategy,
};
use bon::Builder;
@@ -38,10 +39,17 @@ pub struct IggyConsumerConfig {
/// Name of the topic. Must be unique.
topic_name: String,
/// The auto-commit configuration for storing the message offset on the server. See `AutoCommit` for details.
+ #[builder(default = AutoCommit::Disabled)]
auto_commit: AutoCommit,
/// The max number of messages to poll in a batch. The greater the batch length, the higher the throughput for bulk data.
/// Note, there is a tradeoff between batch size and latency, so you want to benchmark your setup.
batch_length: u32,
+ /// Maximum encoded bytes reserved for buffered and in-flight batches.
+ #[builder(default = DEFAULT_PREFETCH_BYTES)]
+ prefetch_bytes: u32,
+ /// Maximum messages reserved for buffered and in-flight batches.
+ #[builder(default = DEFAULT_PREFETCH_MESSAGES)]
+ prefetch_messages: u32,
/// Create the stream if it doesn't exist.
create_stream_if_not_exists: bool,
/// Create the topic if it doesn't exist.
@@ -55,9 +63,11 @@ pub struct IggyConsumerConfig {
/// Partition ID for an ordinary consumer. Defaults to 0 and is ignored by consumer groups.
#[builder(default = DEFAULT_PARTITION_ID)]
partition_id: u32,
- /// The polling interval for messages.
- polling_interval: IggyDuration,
+ /// Readiness, response size and request timeout limits.
+ #[builder(default)]
+ poll_options: DeferredPollOptions,
/// `PollingStrategy` specifies from where to start polling messages. See `PollingStrategy` for details.
+ #[builder(default = PollingStrategy::next())]
polling_strategy: PollingStrategy,
/// Sets the polling retry interval in case of server disconnection.
polling_retry_interval: NonZeroIggyDuration,
@@ -80,14 +90,16 @@ impl Default for IggyConsumerConfig {
stream_name: "test_stream".to_string(),
topic_id,
topic_name: "test_topic".to_string(),
- auto_commit: AutoCommit::When(AutoCommitWhen::PollingMessages),
+ auto_commit: AutoCommit::Disabled,
batch_length: 100,
+ prefetch_bytes: DEFAULT_PREFETCH_BYTES,
+ prefetch_messages: DEFAULT_PREFETCH_MESSAGES,
create_stream_if_not_exists: false,
create_topic_if_not_exists: false,
consumer_name: "test_consumer".to_string(),
consumer_kind: ConsumerKind::ConsumerGroup,
- polling_interval: IggyDuration::from_str("5ms").unwrap(),
- polling_strategy: PollingStrategy::last(),
+ poll_options: DeferredPollOptions::default(),
+ polling_strategy: PollingStrategy::next(),
partitions_count: 1,
partition_id: DEFAULT_PARTITION_ID,
encryptor: None,
@@ -113,7 +125,7 @@ impl IggyConsumerConfig {
/// * `create_topic_if_not_exists` - Whether to create the topic if it does not exists.
/// * `consumer_name` - The consumer name.
/// * `consumer_kind` - The consumer kind.
- /// * `polling_interval` - The interval between polling for new messages.
+ /// * `poll_options` - Readiness, response size and request timeout limits.
/// * `polling_strategy` - The polling strategy.
/// * `partitions_count` - Topic creation count.
/// * `encryptor` - The encryptor.
@@ -139,7 +151,7 @@ impl IggyConsumerConfig {
create_topic_if_not_exists: bool,
consumer_name: String,
consumer_kind: ConsumerKind,
- polling_interval: IggyDuration,
+ poll_options: DeferredPollOptions,
polling_strategy: PollingStrategy,
partitions_count: u32,
encryptor: Option<Arc<EncryptorKind>>,
@@ -154,11 +166,13 @@ impl IggyConsumerConfig {
topic_name,
auto_commit,
batch_length,
+ prefetch_bytes: DEFAULT_PREFETCH_BYTES,
+ prefetch_messages: DEFAULT_PREFETCH_MESSAGES,
create_stream_if_not_exists,
create_topic_if_not_exists,
consumer_name,
consumer_kind,
- polling_interval,
+ poll_options,
polling_strategy,
partitions_count,
partition_id: DEFAULT_PARTITION_ID,
@@ -176,7 +190,7 @@ impl IggyConsumerConfig {
/// * `stream` - The stream name.
/// * `topic` - The topic name.
/// * `batch_length` - The max number of messages to poll in a batch.
- /// * `polling_interval` - The interval between polling for new messages.
+ /// * `poll_options` - Readiness, response size and request timeout limits.
///
/// Returns:
/// A new `IggyConsumerConfig`.
@@ -185,7 +199,7 @@ impl IggyConsumerConfig {
stream: &str,
topic: &str,
batch_length: u32,
- polling_interval: IggyDuration,
+ poll_options: DeferredPollOptions,
) -> Result<Self, IggyError> {
let stream_id = Identifier::from_str_value(stream)?;
let topic_id = Identifier::from_str_value(topic)?;
@@ -195,14 +209,16 @@ impl IggyConsumerConfig {
stream_name: stream.to_string(),
topic_id,
topic_name: topic.to_string(),
- auto_commit: AutoCommit::When(AutoCommitWhen::PollingMessages),
+ auto_commit: AutoCommit::Disabled,
batch_length,
+ prefetch_bytes: DEFAULT_PREFETCH_BYTES,
+ prefetch_messages: DEFAULT_PREFETCH_MESSAGES,
create_stream_if_not_exists: false,
create_topic_if_not_exists: false,
consumer_name: format!("consumer-{stream}-{topic}"),
consumer_kind: ConsumerKind::ConsumerGroup,
- polling_interval,
- polling_strategy: PollingStrategy::last(),
+ poll_options,
+ polling_strategy: PollingStrategy::next(),
partitions_count: 1,
partition_id: DEFAULT_PARTITION_ID,
encryptor: None,
@@ -240,6 +256,14 @@ impl IggyConsumerConfig {
self.auto_commit
}
+ pub fn prefetch_bytes(&self) -> u32 {
+ self.prefetch_bytes
+ }
+
+ pub fn prefetch_messages(&self) -> u32 {
+ self.prefetch_messages
+ }
+
pub fn batch_length(&self) -> u32 {
self.batch_length
}
@@ -259,8 +283,8 @@ impl IggyConsumerConfig {
self.consumer_kind
}
- pub fn polling_interval(&self) -> IggyDuration {
- self.polling_interval
+ pub fn poll_options(&self) -> DeferredPollOptions {
+ self.poll_options
}
pub fn polling_strategy(&self) -> PollingStrategy {
@@ -307,14 +331,11 @@ mod tests {
.stream_name("test_stream".to_string())
.topic_id(topic_id)
.topic_name("test_topic".to_string())
- .auto_commit(AutoCommit::When(AutoCommitWhen::PollingMessages))
.batch_length(100)
.create_stream_if_not_exists(true)
.create_topic_if_not_exists(true)
.consumer_name("test_consumer".to_string())
.consumer_kind(ConsumerKind::ConsumerGroup)
- .polling_interval(IggyDuration::from_str("5ms").unwrap())
- .polling_strategy(PollingStrategy::last())
.polling_retry_interval(NonZeroIggyDuration::ONE_SECOND)
.partitions_count(1)
.init_retries(3)
@@ -331,20 +352,14 @@ mod tests {
&Identifier::from_str_value("test_topic").unwrap()
);
assert_eq!(config.topic_name(), "test_topic");
- assert_eq!(
- config.auto_commit(),
- AutoCommit::When(AutoCommitWhen::PollingMessages)
- );
+ assert_eq!(config.auto_commit(), AutoCommit::Disabled);
assert_eq!(config.batch_length(), 100);
assert!(config.create_stream_if_not_exists());
assert!(config.create_topic_if_not_exists());
assert_eq!(config.consumer_name(), "test_consumer");
assert_eq!(config.consumer_kind(), ConsumerKind::ConsumerGroup);
- assert_eq!(
- config.polling_interval(),
- IggyDuration::from_str("5ms").unwrap()
- );
- assert_eq!(config.polling_strategy(), PollingStrategy::last());
+ assert_eq!(config.poll_options(), DeferredPollOptions::default());
+ assert_eq!(config.polling_strategy(), PollingStrategy::next());
assert_eq!(config.partitions_count(), 1);
assert_eq!(
@@ -369,20 +384,14 @@ mod tests {
assert_eq!(config.stream_name(), "test_stream");
assert_eq!(config.topic_id(), &topic_id);
assert_eq!(config.topic_name(), "test_topic");
- assert_eq!(
- config.auto_commit(),
- AutoCommit::When(AutoCommitWhen::PollingMessages)
- );
+ assert_eq!(config.auto_commit(), AutoCommit::Disabled);
assert_eq!(config.batch_length(), 100);
assert!(!config.create_stream_if_not_exists());
assert!(!config.create_topic_if_not_exists());
assert_eq!(config.consumer_name(), "test_consumer");
assert_eq!(config.consumer_kind(), ConsumerKind::ConsumerGroup);
- assert_eq!(
- config.polling_interval(),
- IggyDuration::from_str("5ms").unwrap()
- );
- assert_eq!(config.polling_strategy(), PollingStrategy::last());
+ assert_eq!(config.poll_options(), DeferredPollOptions::default());
+ assert_eq!(config.polling_strategy(), PollingStrategy::next());
assert_eq!(config.partitions_count(), 1);
assert_eq!(
@@ -403,13 +412,13 @@ mod tests {
"test_stream".to_string(),
Identifier::from_str_value("test_topic").unwrap(),
"test_topic".to_string(),
- AutoCommit::When(AutoCommitWhen::PollingMessages),
+ AutoCommit::Disabled,
100,
false,
false,
"test_consumer".to_string(),
ConsumerKind::ConsumerGroup,
- IggyDuration::from_str("5ms").unwrap(),
+ DeferredPollOptions::default(),
PollingStrategy::last(),
1,
None,
@@ -427,19 +436,13 @@ mod tests {
&Identifier::from_str_value("test_topic").unwrap()
);
assert_eq!(config.topic_name(), "test_topic");
- assert_eq!(
- config.auto_commit(),
- AutoCommit::When(AutoCommitWhen::PollingMessages)
- );
+ assert_eq!(config.auto_commit(), AutoCommit::Disabled);
assert_eq!(config.batch_length(), 100);
assert!(!config.create_stream_if_not_exists());
assert!(!config.create_topic_if_not_exists());
assert_eq!(config.consumer_name(), "test_consumer");
assert_eq!(config.consumer_kind(), ConsumerKind::ConsumerGroup);
- assert_eq!(
- config.polling_interval(),
- IggyDuration::from_str("5ms").unwrap()
- );
+ assert_eq!(config.poll_options(), DeferredPollOptions::default());
assert_eq!(config.polling_strategy(), PollingStrategy::last());
assert_eq!(config.partitions_count(), 1);
@@ -460,7 +463,7 @@ mod tests {
"test_stream",
"test_topic",
100,
- IggyDuration::from_str("5ms").unwrap(),
+ DeferredPollOptions::default(),
);
assert!(res.is_ok());
let config = res.unwrap();
@@ -472,11 +475,8 @@ mod tests {
assert!(!config.create_topic_if_not_exists());
assert_eq!(config.consumer_name(), "consumer-test_stream-test_topic");
assert_eq!(config.consumer_kind(), ConsumerKind::ConsumerGroup);
- assert_eq!(
- config.polling_interval(),
- IggyDuration::from_str("5ms").unwrap()
- );
- assert_eq!(config.polling_strategy(), PollingStrategy::last());
+ assert_eq!(config.poll_options(), DeferredPollOptions::default());
+ assert_eq!(config.polling_strategy(), PollingStrategy::next());
assert_eq!(config.partitions_count(), 1);
}
}
diff --git a/core/sdk/src/stream_builder/config/config_iggy_stream.rs b/core/sdk/src/stream_builder/config/config_iggy_stream.rs
index 45792e2ba..7b0cd4d39 100644
--- a/core/sdk/src/stream_builder/config/config_iggy_stream.rs
+++ b/core/sdk/src/stream_builder/config/config_iggy_stream.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::prelude::{Identifier, IggyDuration, IggyError};
+use crate::prelude::{DeferredPollOptions, Identifier, IggyDuration, IggyError};
use crate::stream_builder::{IggyConsumerConfig, IggyProducerConfig};
use bon::Builder;
@@ -44,7 +44,7 @@ impl IggyStreamConfig {
}
/// Creates a new `IggyStreamConfig` from the given stream and topic names, along with the max
- /// batch size, the send interval and the polling interval.
+ /// batch size, send interval and poll limits.
///
/// # Args
///
@@ -52,7 +52,7 @@ impl IggyStreamConfig {
/// * `topic` - The topic name.
/// * `batch_length` - The max number of messages to send in a batch.
/// * `linger_time` - The interval between messages sent.
- /// * `polling_interval` - The interval between polling for new messages.
+ /// * `poll_options` - Readiness, response size and request timeout limits.
///
/// Returns:
/// A new `IggyStreamConfig`.
@@ -62,10 +62,10 @@ impl IggyStreamConfig {
topic: &str,
batch_length: u32,
linger_time: IggyDuration,
- polling_interval: IggyDuration,
+ poll_options: DeferredPollOptions,
) -> Result<Self, IggyError> {
let consumer_config =
- IggyConsumerConfig::from_stream_topic(stream, topic, batch_length, polling_interval)?;
+ IggyConsumerConfig::from_stream_topic(stream, topic, batch_length, poll_options)?;
let producer_config =
IggyProducerConfig::from_stream_topic(stream, topic, batch_length, linger_time)?;
@@ -114,7 +114,7 @@ mod tests {
"test_stream",
"test_topic",
100,
- IggyDuration::from_str("5ms").unwrap(),
+ DeferredPollOptions::default(),
)
.unwrap();
@@ -131,8 +131,8 @@ mod tests {
assert_eq!(config.consumer_config().batch_length(), 100);
assert_eq!(config.producer_config().batch_length(), 100);
assert_eq!(
- config.consumer_config().polling_interval(),
- IggyDuration::from_str("5ms").unwrap()
+ config.consumer_config().poll_options(),
+ DeferredPollOptions::default()
);
assert_eq!(
config.producer_config().linger_time(),
@@ -148,8 +148,8 @@ mod tests {
assert_eq!(config.consumer_config().batch_length(), 100);
assert_eq!(config.producer_config().batch_length(), 100);
assert_eq!(
- config.consumer_config().polling_interval(),
- IggyDuration::from_str("5ms").unwrap()
+ config.consumer_config().poll_options(),
+ DeferredPollOptions::default()
);
assert_eq!(
config.producer_config().linger_time(),
@@ -164,7 +164,7 @@ mod tests {
"test_topic",
100,
IggyDuration::from_str("5ms").unwrap(),
- IggyDuration::from_str("5ms").unwrap(),
+ DeferredPollOptions::default(),
);
assert!(res.is_ok());
@@ -175,8 +175,8 @@ mod tests {
assert_eq!(config.consumer_config().batch_length(), 100);
assert_eq!(config.producer_config().batch_length(), 100);
assert_eq!(
- config.consumer_config().polling_interval(),
- IggyDuration::from_str("5ms").unwrap()
+ config.consumer_config().poll_options(),
+ DeferredPollOptions::default()
);
assert_eq!(
config.producer_config().linger_time(),
diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs
index 8cf7154bf..1d3c9dfd1 100644
--- a/core/sdk/src/tcp/tcp_client.rs
+++ b/core/sdk/src/tcp/tcp_client.rs
@@ -205,6 +205,17 @@ impl BinaryTransport for TcpClient {
) -> Result<Bytes, IggyError> {
self.poll_router.poll(self, request).await
}
+
+ async fn send_poll_with_response_and_options(
+ &self,
+ request: &iggy_binary_protocol::requests::messages::PollMessagesRequest,
+ options: Option<iggy_common::DeferredPollOptions>,
+ ) -> Result<Bytes, IggyError> {
+ match options {
+ Some(options) => self.poll_router.poll_deferred(self, request, options).await,
+ None => self.poll_router.poll(self, request).await,
+ }
+ }
async fn get_state(&self) -> ClientState {
*self.state.lock().await
}
@@ -493,6 +504,33 @@ impl BinaryClient for TcpClient {}
impl PollTransport for TcpClient {
const PROTOCOL: TransportProtocol = TransportProtocol::Tcp;
+ async fn local_poll_session(
+ &self,
+ ) -> Result<
+ (
+ String,
+ iggy_binary_protocol::requests::system::AttachConsumerSessionRequest,
+ ),
+ IggyError,
+ > {
+ let endpoint = self.current_server_address.lock().await.clone();
+ let session = self
+ .consensus_session
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ Ok((
+ endpoint,
+ iggy_binary_protocol::requests::system::AttachConsumerSessionRequest {
+ client_id: session.client_id(),
+ session: session.session().ok_or(IggyError::Unauthenticated)?,
+ metadata_watermark: self
+ .poll_router
+ .metadata_watermark
+ .load(std::sync::atomic::Ordering::Acquire),
+ },
+ ))
+ }
+
async fn connect_poll_client(&self, endpoint: &str) -> Result<Self, IggyError> {
let mut config = (*self.config).clone();
config.server_address = endpoint.to_owned();
@@ -508,9 +546,17 @@ impl PollTransport for TcpClient {
}
async fn send_poll_request(&self, code: u32, payload: Bytes) -> Result<Bytes, IggyError> {
+ let wait = crate::poll_routing::deferred_exchange_timeout(code, &payload)?;
let now = tokio::time::Instant::now();
let (_, result) = self
- .send_raw_vsr_attempt(code, payload, None, now, now + RESPONSE_READ_TIMEOUT, false)
+ .send_raw_vsr_attempt(
+ code,
+ payload,
+ None,
+ now,
+ now + RESPONSE_READ_TIMEOUT + wait,
+ false,
+ )
.await;
if matches!(result, Err(IggyError::Disconnected | IggyError::TcpError)) {
self.set_state(ClientState::Disconnected).await;
@@ -1505,12 +1551,31 @@ impl TcpClient {
Option<iggy_binary_protocol::consensus::RequestHeader>,
Result<Bytes, IggyError>,
) {
+ let deferred_timeout = match crate::poll_routing::deferred_exchange_timeout(code, &payload)
+ {
+ Ok(wait) => wait,
+ Err(error) => return (None, Err(error)),
+ };
+ let response_limit = match crate::poll_routing::deferred_response_limit(code, &payload) {
+ Ok(limit) => limit,
+ Err(error) => return (None, Err(error)),
+ };
let stream = self.stream.clone();
let consensus_session = self.consensus_session.clone();
let metadata_watermark = Arc::clone(&self.poll_router.metadata_watermark);
// SAFETY: we run code holding the `stream` lock in a task so we can't be cancelled while holding the lock.
- let joined = tokio::spawn(async move {
- let mut stream_guard = stream.lock().await;
+ let task = tokio::spawn(async move {
+ let mut held_stream = stream.lock().await;
+ let mut disposable = if deferred_timeout.is_zero() {
+ None
+ } else {
+ held_stream.take()
+ };
+ let stream_guard = if deferred_timeout.is_zero() {
+ &mut *held_stream
+ } else {
+ &mut disposable
+ };
let Some(stream) = stream_guard.as_mut() else {
error!("Cannot send data. Client is not connected.");
return (None, Err(IggyError::NotConnected));
@@ -1589,6 +1654,7 @@ impl TcpClient {
let response_size = crate::vsr::response_size(&response_header)?;
let body_size = response_size - iggy_binary_protocol::HEADER_SIZE;
+ if body_size > response_limit { return Err(IggyError::InvalidSizeBytes); }
let body = if body_size > 0 {
let mut body = BytesMut::with_capacity(body_size);
let body_read = tokio::time::timeout_at(
@@ -1652,9 +1718,13 @@ impl TcpClient {
if !frame_complete {
stream_guard.take();
}
+ if !deferred_timeout.is_zero() && frame_complete {
+ *held_stream = disposable;
+ }
(Some(request_header), outcome)
- })
- .await;
+ });
+ let _cancellation = crate::poll_routing::DeferredExchangeTask::new(deferred_timeout, &task);
+ let joined = task.await;
match joined {
Ok(result) => result,
Err(e) => {
diff --git a/core/sdk/src/websocket/websocket_client.rs b/core/sdk/src/websocket/websocket_client.rs
index f18a4ad53..e9dc48c4b 100644
--- a/core/sdk/src/websocket/websocket_client.rs
+++ b/core/sdk/src/websocket/websocket_client.rs
@@ -145,6 +145,17 @@ impl BinaryTransport for WebSocketClient {
) -> Result<Bytes, IggyError> {
self.poll_router.poll(self, request).await
}
+
+ async fn send_poll_with_response_and_options(
+ &self,
+ request: &iggy_binary_protocol::requests::messages::PollMessagesRequest,
+ options: Option<iggy_common::DeferredPollOptions>,
+ ) -> Result<Bytes, IggyError> {
+ match options {
+ Some(options) => self.poll_router.poll_deferred(self, request, options).await,
+ None => self.poll_router.poll(self, request).await,
+ }
+ }
async fn get_state(&self) -> ClientState {
*self.state.lock().await
}
@@ -493,6 +504,33 @@ impl BinaryClient for WebSocketClient {}
impl PollTransport for WebSocketClient {
const PROTOCOL: TransportProtocol = TransportProtocol::WebSocket;
+ async fn local_poll_session(
+ &self,
+ ) -> Result<
+ (
+ String,
+ iggy_binary_protocol::requests::system::AttachConsumerSessionRequest,
+ ),
+ IggyError,
+ > {
+ let endpoint = self.current_server_address.lock().await.clone();
+ let session = self
+ .consensus_session
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ Ok((
+ endpoint,
+ iggy_binary_protocol::requests::system::AttachConsumerSessionRequest {
+ client_id: session.client_id(),
+ session: session.session().ok_or(IggyError::Unauthenticated)?,
+ metadata_watermark: self
+ .poll_router
+ .metadata_watermark
+ .load(std::sync::atomic::Ordering::Acquire),
+ },
+ ))
+ }
+
async fn connect_poll_client(&self, endpoint: &str) -> Result<Self, IggyError> {
let mut config = (*self.config).clone();
config.server_address = endpoint.to_owned();
@@ -1094,11 +1132,23 @@ impl WebSocketClient {
let stream = self.stream.clone();
let consensus_session = self.consensus_session.clone();
let metadata_watermark = Arc::clone(&self.poll_router.metadata_watermark);
+ let deferred_timeout = crate::poll_routing::deferred_exchange_timeout(code, &payload)?;
+ let response_limit = crate::poll_routing::deferred_response_limit(code, &payload)?;
// The spawned task owns the lockstep exchange to completion. Cancelling
// the caller after a partial WebSocket frame or response header must
// not release the stream lock while leaving that connection reusable.
- tokio::spawn(async move {
- let mut stream_guard = stream.lock().await;
+ let task = tokio::spawn(async move {
+ let mut held_stream = stream.lock().await;
+ let mut disposable = if deferred_timeout.is_zero() {
+ None
+ } else {
+ held_stream.take()
+ };
+ let stream_guard = if deferred_timeout.is_zero() {
+ &mut *held_stream
+ } else {
+ &mut disposable
+ };
if stream_guard.is_none() {
trace!("Cannot send data. Client is not connected.");
return Err(IggyError::NotConnected);
@@ -1122,7 +1172,8 @@ impl WebSocketClient {
request.len()
);
// One deadline bounds the whole request including transient replays.
- let retry_deadline = tokio::time::Instant::now() + RESPONSE_READ_TIMEOUT;
+ let retry_deadline =
+ tokio::time::Instant::now() + RESPONSE_READ_TIMEOUT + deferred_timeout;
// `TransientNotAccepted` gets a short same-connection window only:
// past it the refusal is a verdict about who leads, not load, and
// the caller runs a leader recheck or roster walk. Login/register
@@ -1160,6 +1211,7 @@ impl WebSocketClient {
let response_size = crate::vsr::response_size(&response_header)?;
let body_size = response_size - iggy_binary_protocol::HEADER_SIZE;
+ if body_size > response_limit { return Err(IggyError::InvalidSizeBytes); }
let body = if body_size > 0 {
let mut body = vec![0u8; body_size];
let body_read =
@@ -1209,10 +1261,13 @@ impl WebSocketClient {
if !frame_complete {
stream_guard.take();
}
+ if !deferred_timeout.is_zero() && frame_complete {
+ *held_stream = disposable;
+ }
outcome
- })
- .await
- .map_err(|error| {
+ });
+ let _cancellation = crate::poll_routing::DeferredExchangeTask::new(deferred_timeout, &task);
+ task.await.map_err(|error| {
error!("Task execution failed during {NAME} request: {error}");
IggyError::WebSocketSendError
})?
diff --git a/core/server/config.toml b/core/server/config.toml
index b32988b0f..b32584774 100644
--- a/core/server/config.toml
+++ b/core/server/config.toml
@@ -737,6 +737,11 @@ reply_inbox_capacity = 1024
# new disk polls are rejected before I/O. Main and reply inbox traffic uses
# separate capacity. This limits operation count, not retained message bytes.
poll_completion_capacity = 1024
+deferred_poll_max_wait_us = 30000000
+deferred_poll_max_pending = 1024
+deferred_poll_max_pending_per_session = 64
+deferred_poll_max_read_bytes = 16777216
+deferred_poll_max_inflight_bytes = 67108864
# Wall-clock budget for a single shard's bus drain on shutdown. Drives
# the per-shard watchdog and the parallel-join survivor path; sized
diff --git a/core/server/src/boot/recovery.rs b/core/server/src/boot/recovery.rs
index 9f3dd9628..87c6639a7 100644
--- a/core/server/src/boot/recovery.rs
+++ b/core/server/src/boot/recovery.rs
@@ -303,6 +303,13 @@ pub(in crate::boot) async fn build_shard_for_thread(
.map_err(ServerError::ShardConstruction)?;
let shard = Rc::new(built.shard);
+ shard.set_deferred_poll_config(shard::config::DeferredPollConfig {
+ max_wait_us: config.sharding.deferred_poll_max_wait_us,
+ max_pending: config.sharding.deferred_poll_max_pending,
+ max_pending_per_session: config.sharding.deferred_poll_max_pending_per_session,
+ max_read_bytes: config.sharding.deferred_poll_max_read_bytes,
+ max_inflight_bytes: config.sharding.deferred_poll_max_inflight_bytes,
+ });
// Repair pacing is shared by both planes' repair loops, so it is a
// per-shard tunable set once here rather than per consensus group.
shard.set_repair_retry_ticks(repair_retry_ticks(config));
diff --git a/core/server/src/dispatch/authz.rs b/core/server/src/dispatch/authz.rs
index 80ed05b7c..1d3be7311 100644
--- a/core/server/src/dispatch/authz.rs
+++ b/core/server/src/dispatch/authz.rs
@@ -423,7 +423,8 @@ mod tests {
GET_CONSUMER_OFFSET_CODE, GET_CONSUMER_OFFSET_ROUTING_CODE, GET_ME_CODE,
GET_POLL_ROUTING_CODE, GET_SNAPSHOT_FILE_CODE, LOGIN_REGISTER_CODE,
LOGIN_REGISTER_WITH_PAT_CODE, LOGIN_USER_CODE, LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE,
- LOGOUT_USER_CODE, PING_CODE, POLL_MESSAGES_CODE, POLL_MESSAGES_ON_PRIMARY_CODE,
+ LOGOUT_USER_CODE, PING_CODE, POLL_MESSAGES_CODE, POLL_MESSAGES_DEFERRED_CODE,
+ POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE, POLL_MESSAGES_ON_PRIMARY_CODE,
SYNC_CONSUMER_GROUP_CODE,
};
use iggy_common::defaults::DEFAULT_ROOT_USER_ID;
@@ -493,6 +494,16 @@ mod tests {
invalid_command,
),
(POLL_MESSAGES_CODE, invalid_command, invalid_command),
+ (
+ POLL_MESSAGES_DEFERRED_CODE,
+ invalid_command,
+ invalid_command,
+ ),
+ (
+ POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE,
+ invalid_command,
+ invalid_command,
+ ),
(
ATTACH_CONSUMER_SESSION_CODE,
invalid_command,
diff --git a/core/server/src/dispatch/partition.rs b/core/server/src/dispatch/partition.rs
index 3ebd37ed4..0ec4fe2cb 100644
--- a/core/server/src/dispatch/partition.rs
+++ b/core/server/src/dispatch/partition.rs
@@ -43,7 +43,7 @@ use iggy_binary_protocol::PrepareHeader;
use iggy_binary_protocol::primitives::consumer::WireConsumer;
use iggy_binary_protocol::primitives::polling_strategy::WirePollingStrategy;
use iggy_binary_protocol::requests::consumer_offsets::GetConsumerOffsetRequest;
-use iggy_binary_protocol::requests::messages::PollMessagesRequest;
+use iggy_binary_protocol::requests::messages::{DeferredPollMessagesRequest, PollMessagesRequest};
use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest;
use iggy_binary_protocol::{KIND_CONSUMER_GROUP, Operation, RoutedRequestHeader, WireDecode};
use iggy_common::{ConsumerKind, IggyError, PollingStrategy, RESYNC_REQUIRED_PARTITION_SENTINEL};
@@ -58,7 +58,10 @@ use partitions::{PollingArgs, PollingConsumer};
use server_common::Message;
use server_common::sharding::IggyNamespace;
use shard::shards_table::ShardsTable;
-use shard::{ConsumerAttachment, PartitionRead, PartitionReadReply};
+use shard::{
+ ConsumerAttachment, DeferredPollContext, DeferredPollRequest, PartitionRead,
+ PartitionReadReply, PollQuotaIdentity,
+};
use std::rc::Rc;
use tracing::{debug, warn};
@@ -519,6 +522,95 @@ pub(in crate::dispatch) async fn handle_poll_messages<B, MJ, S, SB>(
.await;
}
+pub(super) fn resolve_deferred_poll<B, MJ, S, SB>(
+ shard: &Rc<ShellShard<B, MJ, S, SB>>,
+ wire: &DeferredPollMessagesRequest,
+ context: (u128, SessionAttachment, u32, bool),
+) -> Result<(IggyNamespace, u32, PartitionRead), IggyError>
+where
+ B: ShellBus,
+ MJ: JournalHandle + 'static,
+ MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>,
+ S: 'static,
+ SB: SuperblockStore + 'static,
+{
+ let (consumer_client_id, session, user_id, primary) = context;
+ shard.validate_deferred_poll(wire.poll.count, wire.into())?;
+ if let Some(status) = authorize_partition_read(
+ shard,
+ &wire.poll.stream_id,
+ &wire.poll.topic_id,
+ Some(user_id),
+ |permissioner, user_id, stream_id, topic_id| {
+ permissioner.poll_messages(user_id, stream_id, topic_id)
+ },
+ ) {
+ return Err(IggyError::from_code(status));
+ }
+ let (namespace, partition_id, consumer, args) =
+ resolve_poll_request(shard, &wire.poll, consumer_client_id)?;
+ let group_id = match consumer {
+ PollingConsumer::ConsumerGroup(group, _) => Some(group as u64),
+ PollingConsumer::Consumer(..) => None,
+ };
+ let metadata = shard
+ .plane
+ .metadata()
+ .mux_stm
+ .streams()
+ .poll_metadata(namespace, group_id, consumer_client_id)
+ .ok_or(IggyError::TransientNotAccepted)?;
+ let request = DeferredPollRequest::new(
+ consumer,
+ args,
+ DeferredPollContext {
+ session: Some(session),
+ metadata,
+ user_id,
+ quota: PollQuotaIdentity::Session(consumer_client_id),
+ primary,
+ },
+ wire.into(),
+ );
+ Ok((
+ namespace,
+ partition_id,
+ PartitionRead::DeferredPoll(Box::new(request)),
+ ))
+}
+
+#[allow(clippy::future_not_send)]
+pub(super) async fn read_deferred_poll<B, MJ, S, SB>(
+ shard: &Rc<ShellShard<B, MJ, S, SB>>,
+ request: &Message<RoutedRequestHeader>,
+ (namespace, partition_id, read): (IggyNamespace, u32, PartitionRead),
+) -> Result<BusMessage, IggyError>
+where
+ B: ShellBus,
+ MJ: JournalHandle + 'static,
+ MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>,
+ S: 'static,
+ SB: SuperblockStore + 'static,
+{
+ match shard.partition_read(namespace, read).await {
+ Some(PartitionReadReply::Poll {
+ fragments,
+ current_offset,
+ }) => build_polled_messages_reply(
+ request.header(),
+ current_metadata_commit(shard),
+ partition_id,
+ current_offset,
+ fragments,
+ shard.plane.partitions().config().encryptor.as_deref(),
+ )
+ .map_err(|_| IggyError::CannotReadMessage),
+ Some(PartitionReadReply::Rejected(error)) => Err(error),
+ Some(PartitionReadReply::NotFound) => Err(IggyError::TransientNotAccepted),
+ _ => Err(IggyError::ShardCommunicationError),
+ }
+}
+
/// Run the resolved poll on the owning shard and re-encode the stored
/// batches into the wire `PolledMessages` reply. Owner rejections preserve
/// their error; a missing reply reports a communication error without claiming
@@ -622,7 +714,7 @@ enum ReadPolledMessagesError {
/// 16-byte empty poll for `partition_id`, riding the re-sync sentinel
/// channel when the id is the sentinel and the empty-frame channel
/// otherwise.
-fn empty_poll_fallback(partition_id: u32) -> (Bytes, FrameChannel) {
+pub(super) fn empty_poll_fallback(partition_id: u32) -> (Bytes, FrameChannel) {
let channel = if partition_id == RESYNC_REQUIRED_PARTITION_SENTINEL {
FrameChannel::ResyncSentinel
} else {
diff --git a/core/server/src/dispatch/reads.rs b/core/server/src/dispatch/reads.rs
index eec0d9c30..1bc91f73a 100644
--- a/core/server/src/dispatch/reads.rs
+++ b/core/server/src/dispatch/reads.rs
@@ -31,7 +31,8 @@ use crate::dispatch::failure::{
FrameChannel, send_host_frame, send_non_replicated_bytes, send_non_replicated_deny,
};
use crate::dispatch::partition::{
- handle_get_consumer_offset, handle_poll_messages, resolve_poll_request,
+ empty_poll_fallback, handle_get_consumer_offset, handle_poll_messages, read_deferred_poll,
+ resolve_deferred_poll, resolve_poll_request,
};
use crate::responses::{
build_empty_reply, build_get_me_response, build_get_personal_access_tokens_response,
@@ -45,19 +46,21 @@ use crate::wire::request_body;
use bytes::Bytes;
use configs::server::ServerConfig;
use consensus::MetadataHandle;
-use futures::future::{Either, select};
+use consensus::client_table::SessionAttachment;
+use futures::future::{Abortable, Either, select};
use iggy_binary_protocol::PrepareHeader;
use iggy_binary_protocol::codes::{
ATTACH_CONSUMER_SESSION_CODE, DESCRIBE_OPTIONS_CODE, GET_CLIENT_CODE, GET_CLIENTS_CODE,
GET_CLUSTER_METADATA_CODE, GET_CONSUMER_OFFSET_CODE, GET_CONSUMER_OFFSET_ROUTING_CODE,
GET_ME_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, GET_POLL_ROUTING_CODE, GET_SNAPSHOT_FILE_CODE,
- GET_STATS_CODE, PING_CODE, POLL_MESSAGES_CODE, POLL_MESSAGES_ON_PRIMARY_CODE,
+ GET_STATS_CODE, PING_CODE, POLL_MESSAGES_CODE, POLL_MESSAGES_DEFERRED_CODE,
+ POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE, POLL_MESSAGES_ON_PRIMARY_CODE,
SYNC_CONSUMER_GROUP_CODE,
};
use iggy_binary_protocol::dispatch::lookup_command;
use iggy_binary_protocol::requests::consumer_groups::SyncConsumerGroupRequest;
use iggy_binary_protocol::requests::consumer_offsets::GetConsumerOffsetRequest;
-use iggy_binary_protocol::requests::messages::PollMessagesRequest;
+use iggy_binary_protocol::requests::messages::{DeferredPollMessagesRequest, PollMessagesRequest};
use iggy_binary_protocol::requests::system::AttachConsumerSessionRequest;
use iggy_binary_protocol::requests::system::get_client::GetClientRequest;
use iggy_binary_protocol::requests::system::get_snapshot::GetSnapshotRequest;
@@ -68,7 +71,10 @@ use iggy_binary_protocol::responses::consumer_groups::SyncConsumerGroupResponse;
use iggy_binary_protocol::responses::messages::PollRoutingResponse;
use iggy_binary_protocol::responses::system::get_snapshot::GetSnapshotResponse;
use iggy_binary_protocol::{HEADER_SIZE, RoutedRequestHeader, WireDecode, WireEncode};
-use iggy_common::{ClusterNodeRole, IggyError, SnapshotCompression, SystemSnapshotType};
+use iggy_common::{
+ ClusterNodeRole, IggyError, RESYNC_REQUIRED_PARTITION_SENTINEL, SnapshotCompression,
+ SystemSnapshotType,
+};
use journal::superblock::SuperblockStore;
use journal::{Journal, JournalHandle};
use message_bus::framing::MAX_MESSAGE_SIZE;
@@ -77,7 +83,7 @@ use metadata::impls::metadata::StreamsFrontend;
use metadata::permissioner::Permissioner;
use server_common::Message;
use shard::{PartitionRead, PartitionReadReply};
-use std::cell::RefCell;
+use std::cell::{Cell, RefCell};
use std::future::Future;
use std::net::{IpAddr, SocketAddr};
use std::pin::pin;
@@ -203,6 +209,8 @@ pub const fn read_needs_metadata_frontier(code: u32) -> bool {
| GET_CLUSTER_METADATA_CODE
| POLL_MESSAGES_CODE
| POLL_MESSAGES_ON_PRIMARY_CODE
+ | POLL_MESSAGES_DEFERRED_CODE
+ | POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE
| GET_CONSUMER_OFFSET_CODE
| GET_SNAPSHOT_FILE_CODE
) && lookup_command(code).is_some()
@@ -535,6 +543,17 @@ pub(in crate::dispatch) async fn handle_non_replicated_request<B, MJ, S, SB>(
}
}
}
+ POLL_MESSAGES_DEFERRED_CODE | POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE => {
+ handle_deferred_poll(
+ shard,
+ sessions,
+ transport_client_id,
+ &request,
+ user_id,
+ code == POLL_MESSAGES_DEFERRED_ON_PRIMARY_CODE,
+ )
+ .await;
+ }
POLL_MESSAGES_CODE | POLL_MESSAGES_ON_PRIMARY_CODE => {
let consumer_client = if code == POLL_MESSAGES_ON_PRIMARY_CODE {
sessions
@@ -622,6 +641,30 @@ where
let wire = AttachConsumerSessionRequest::decode_from(request_body(request))
.map_err(|_| IggyError::InvalidCommand)?;
let watermark = wire.metadata_watermark.max(wire.session);
+ let attachment = attach_poll_session(shard, &wire, user_id).await?;
+ sessions.borrow_mut().attach_consumer_session(
+ transport_client_id,
+ wire.client_id,
+ attachment,
+ watermark,
+ )?;
+ Ok(Bytes::new())
+}
+
+#[allow(clippy::future_not_send)]
+async fn attach_poll_session<B, MJ, S, SB>(
+ shard: &Rc<ShellShard<B, MJ, S, SB>>,
+ wire: &AttachConsumerSessionRequest,
+ user_id: u32,
+) -> Result<SessionAttachment, IggyError>
+where
+ B: ShellBus,
+ MJ: JournalHandle + 'static,
+ MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>,
+ S: 'static,
+ SB: SuperblockStore + 'static,
+{
+ let watermark = wire.metadata_watermark.max(wire.session);
await_metadata_read_frontier(shard, watermark).await?;
let attachment = if shard.id == 0 {
shard
@@ -650,13 +693,148 @@ where
Either::Right(_) => return Err(IggyError::TransientNotAccepted),
}
};
- sessions.borrow_mut().attach_consumer_session(
- transport_client_id,
- wire.client_id,
- attachment,
- watermark,
- )?;
- Ok(Bytes::new())
+ Ok(attachment)
+}
+
+#[allow(
+ clippy::future_not_send,
+ clippy::too_many_arguments,
+ clippy::too_many_lines
+)]
+async fn handle_deferred_poll<B, MJ, S, SB>(
+ shard: &Rc<ShellShard<B, MJ, S, SB>>,
+ sessions: &Rc<RefCell<SessionManager>>,
+ transport_client_id: u128,
+ request: &Message<RoutedRequestHeader>,
+ user_id: Option<u32>,
+ primary: bool,
+) where
+ B: ShellBus,
+ MJ: JournalHandle + 'static,
+ MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>,
+ S: 'static,
+ SB: SuperblockStore + 'static,
+{
+ let started = shard.bus.monotonic_micros();
+ let Ok(mut wire) = DeferredPollMessagesRequest::decode_from(request_body(request)) else {
+ send_non_replicated_deny(
+ shard,
+ request,
+ transport_client_id,
+ IggyError::InvalidCommand.as_code(),
+ )
+ .await;
+ return;
+ };
+ let setup_timeout = Duration::from_micros(wire.request_timeout_us);
+ let setup = async {
+ let user_id = user_id.ok_or(IggyError::Unauthenticated)?;
+ shard.validate_deferred_poll(wire.poll.count, (&wire).into())?;
+ let attached = sessions
+ .borrow()
+ .attached_consumer_session(transport_client_id)?;
+ let (consumer_client_id, attachment) = if let Some(attached) = attached {
+ attached
+ } else {
+ let (client_id, session) = sessions
+ .borrow()
+ .get_session(transport_client_id)
+ .ok_or(IggyError::Unauthenticated)?;
+ let metadata_watermark = sessions.borrow().metadata_watermark(transport_client_id);
+ let attachment = attach_poll_session(
+ shard,
+ &AttachConsumerSessionRequest {
+ client_id,
+ session,
+ metadata_watermark,
+ },
+ user_id,
+ )
+ .await?;
+ (client_id, attachment)
+ };
+ let remaining = iggy_common::DeferredPollOptions::from(&wire).remaining(
+ Duration::from_micros(shard.bus.monotonic_micros().saturating_sub(started)),
+ )?;
+ wire.wait_us = remaining.max_wait.as_micros();
+ wire.request_timeout_us = remaining.request_timeout.as_micros();
+ let watchdog = remaining.request_timeout.get_duration();
+ let resolved = resolve_deferred_poll(
+ shard,
+ &wire,
+ (consumer_client_id, attachment, user_id, primary),
+ )?;
+ Ok::<_, IggyError>((resolved, watchdog))
+ };
+ let setup = match select(pin!(setup), pin!(shard.bus.sleep(setup_timeout))).await {
+ Either::Left((result, _)) => result,
+ Either::Right(_) => Err(IggyError::TransientNotCommitted),
+ };
+ let (resolved, watchdog) = match setup {
+ Ok(setup) => setup,
+ Err(IggyError::ConsumerGroupPartitionNotOwned(..)) => {
+ let (body, channel) = empty_poll_fallback(RESYNC_REQUIRED_PARTITION_SENTINEL);
+ send_non_replicated_bytes(
+ shard,
+ request,
+ transport_client_id,
+ body,
+ channel,
+ "poll_deferred_resync",
+ )
+ .await;
+ return;
+ }
+ Err(error) => {
+ send_non_replicated_deny(shard, request, transport_client_id, error.as_code()).await;
+ return;
+ }
+ };
+ let (guard, cancellation) =
+ match SessionManager::begin_deferred_request(sessions, transport_client_id, watchdog) {
+ Ok(active) => active,
+ Err(error) => {
+ send_non_replicated_deny(shard, request, transport_client_id, error.as_code())
+ .await;
+ return;
+ }
+ };
+ let response_queued = Cell::new(false);
+ let outcome = {
+ let operation = Abortable::new(
+ async {
+ let mut reply = read_deferred_poll(shard, request, resolved).await?;
+ let written = reply.track_write();
+ shard
+ .bus
+ .send_to_client(transport_client_id, reply)
+ .await
+ .map_err(|_| IggyError::ShardCommunicationError)?;
+ response_queued.set(true);
+ written
+ .await
+ .map_err(|_| IggyError::ShardCommunicationError)
+ },
+ cancellation,
+ );
+ let operation = pin!(operation);
+ let timeout = pin!(shard.bus.sleep(watchdog));
+ match select(operation, timeout).await {
+ Either::Left((Ok(result), _)) => Some(result),
+ Either::Left((Err(_), _)) => None,
+ Either::Right(_) => Some(Err(IggyError::ShardCommunicationError)),
+ }
+ };
+ if matches!(outcome, Some(Ok(()))) {
+ guard.written();
+ }
+ drop(guard);
+ // A queued reply can still reach the peer after the write watchdog expires.
+ if let Some(Err(error)) = outcome
+ && !response_queued.get()
+ {
+ send_non_replicated_deny(shard, request, transport_client_id, error.as_code()).await;
+ }
}
#[allow(clippy::future_not_send, clippy::too_many_arguments)]
diff --git a/core/server/src/http.rs b/core/server/src/http.rs
index 9d04339f5..1618cb876 100644
--- a/core/server/src/http.rs
+++ b/core/server/src/http.rs
@@ -72,9 +72,9 @@ use crate::http::handlers::{
delete_stream, delete_topic, delete_user, describe_options, get_cg, get_cgs, get_client,
get_clients, get_cluster_metadata, get_consumer_offset, get_pats, get_snapshot, get_stats,
get_stream, get_streams, get_topic, get_topics, get_user, get_users, login_user,
- login_with_personal_access_token, logout_user, ping, poll_messages, purge_stream, purge_topic,
- refresh_token, send_messages, store_consumer_offset, update_permissions, update_stream,
- update_topic, update_user,
+ login_with_personal_access_token, logout_user, ping, poll_messages, poll_messages_deferred,
+ purge_stream, purge_topic, refresh_token, send_messages, store_consumer_offset,
+ update_permissions, update_stream, update_topic, update_user,
};
use crate::http::jwt::JwtManager;
use crate::http::session::RegistrationBarrier;
@@ -427,6 +427,10 @@ fn partition_write_routes(state: HttpState) -> Router<HttpState> {
"/streams/{stream_id}/topics/{topic_id}/messages",
post(send_messages).get(poll_messages),
)
+ .route(
+ "/streams/{stream_id}/topics/{topic_id}/messages/deferred",
+ get(poll_messages_deferred),
+ )
.route(
"/streams/{stream_id}/topics/{topic_id}/consumer-offsets",
put(store_consumer_offset),
diff --git a/core/server/src/http/forward.rs b/core/server/src/http/forward.rs
index b39074909..65c0e7486 100644
--- a/core/server/src/http/forward.rs
+++ b/core/server/src/http/forward.rs
@@ -83,7 +83,7 @@ use crate::http::error::{
CustomError, error_response, gateway_timeout_response, primary_http_socket, with_retry_after,
};
use crate::http::extractor::{bearer_token, resolve_credential};
-use crate::http::handlers::DURABILITY_HEADER;
+use crate::http::handlers::{DURABILITY_HEADER, DeferredPollQuery};
use crate::http::state::{APPLIED_OP_HEADER, ForwardState, HttpInner, VIEW_HEADER};
use crate::server_error::ServerError;
@@ -344,6 +344,7 @@ async fn forward_partition_or_pass(state: HttpState, request: Request, next: Nex
if request.method() == Method::GET && !wants_auto_commit(request.uri().query()) {
return next.run(request).await;
}
+ let deferred_deadline = deferred_poll_deadline(&request);
let bearer = match bearer_token(request.headers()) {
Ok(bearer) => bearer,
Err(error) => return CustomError::from(error).into_response(),
@@ -388,7 +389,10 @@ async fn forward_partition_or_pass(state: HttpState, request: Request, next: Nex
.consensus
.as_ref()
.map(consensus::VsrConsensus::replica);
- let deadline = Instant::now() + FORWARD_RETRY_DEADLINE;
+ let deadline = deferred_deadline.map_or_else(
+ || Instant::now() + FORWARD_RETRY_DEADLINE,
+ |deadline| deadline.request,
+ );
let mut skip_node = self_id;
loop {
for socket in partition_http_sockets(&state.roster, skip_node) {
@@ -396,10 +400,26 @@ async fn forward_partition_or_pass(state: HttpState, request: Request, next: Nex
break;
}
let url = format!("{}://{socket}{path_and_query}", state.forward.scheme);
- match attempt(&state, &method, &request_headers, &body, &url, false).await {
+ match attempt(
+ &state,
+ &method,
+ &request_headers,
+ &body,
+ &url,
+ false,
+ deferred_deadline,
+ )
+ .await
+ {
AttemptOutcome::Relay(response) => {
return if method == Method::GET && response.status().is_success() {
- retain_forward_guard(response, guard, FORWARD_ATTEMPT_TIMEOUT)
+ retain_forward_guard(
+ response,
+ guard,
+ deferred_deadline.map_or(FORWARD_ATTEMPT_TIMEOUT, |deadline| {
+ deadline.request.saturating_duration_since(Instant::now())
+ }),
+ )
} else {
response
};
@@ -446,7 +466,16 @@ async fn forward(state: &HttpInner, request: Request) -> Response {
None => AttemptOutcome::Retry,
Some(socket) => {
let url = format!("{}://{socket}{path_and_query}", state.forward.scheme);
- attempt(state, &parts.method, &parts.headers, &body, &url, true).await
+ attempt(
+ state,
+ &parts.method,
+ &parts.headers,
+ &body,
+ &url,
+ true,
+ None,
+ )
+ .await
}
};
match outcome {
@@ -486,6 +515,7 @@ async fn attempt(
body: &Bytes,
url: &str,
retry_redirect: bool,
+ deferred_deadline: Option<DeferredPollDeadline>,
) -> AttemptOutcome {
let builder = match state.forward.client.request(method.clone(), url) {
Ok(builder) => builder,
@@ -494,18 +524,33 @@ async fn attempt(
return AttemptOutcome::Relay(bad_gateway());
}
};
- let request = builder
+ let mut request = builder
.headers(forwarded_headers(request_headers))
.body(body.clone())
.build();
+ let timeout = match deferred_deadline {
+ Some(deadline) => {
+ let Some(remaining) = reduce_deferred_wait(&mut request, deadline) else {
+ return AttemptOutcome::Retry;
+ };
+ remaining
+ }
+ None => FORWARD_ATTEMPT_TIMEOUT,
+ };
let attempt = async {
let response = match state.forward.client.execute(request).await {
Ok(response) => response,
Err(error) => return classify_transport_error(&error),
};
- classify_forwarded_reply(response, method, retry_redirect).await
+ classify_forwarded_reply(
+ response,
+ method,
+ retry_redirect,
+ deferred_deadline.map(|deadline| deadline.request),
+ )
+ .await
};
- match compio::time::timeout(FORWARD_ATTEMPT_TIMEOUT, attempt).await {
+ match compio::time::timeout(timeout, attempt).await {
// Elapsed: the request may be mid-commit on the primary. Outcome
// unknown, so never retried - 504, same contract as a local commit
// wait that timed out.
@@ -517,10 +562,57 @@ async fn attempt(
}
}
+#[derive(Clone, Copy)]
+struct DeferredPollDeadline {
+ wait: Instant,
+ request: Instant,
+}
+
+fn deferred_poll_deadline(request: &Request) -> Option<DeferredPollDeadline> {
+ if request.method() != Method::GET || !request.uri().path().ends_with("/messages/deferred") {
+ return None;
+ }
+ let Query(query) = Query::<DeferredPollQuery>::try_from_uri(request.uri()).ok()?;
+ let options = iggy_common::DeferredPollOptions::from(query);
+ options.validate(options.min_count).ok()?;
+ let now = Instant::now();
+ Some(DeferredPollDeadline {
+ wait: now.checked_add(options.max_wait.get_duration())?,
+ request: now.checked_add(options.request_timeout.get_duration())?,
+ })
+}
+
+fn reduce_deferred_wait(
+ request: &mut cyper::Request,
+ deadline: DeferredPollDeadline,
+) -> Option<Duration> {
+ let now = Instant::now();
+ let remaining = deadline.request.checked_duration_since(now)?;
+ let wait_us = deadline.wait.saturating_duration_since(now).as_micros();
+ if remaining.as_micros() == 0 {
+ return None;
+ }
+ let query: Vec<_> = request
+ .url()
+ .query_pairs()
+ .filter(|(key, _)| key != "wait_us" && key != "request_timeout_us")
+ .map(|(key, value)| (key.into_owned(), value.into_owned()))
+ .collect();
+ request
+ .url_mut()
+ .query_pairs_mut()
+ .clear()
+ .extend_pairs(query)
+ .append_pair("wait_us", &wait_us.to_string())
+ .append_pair("request_timeout_us", &remaining.as_micros().to_string());
+ Some(remaining)
+}
+
async fn classify_forwarded_reply(
response: cyper::Response,
method: &Method,
retry_redirect: bool,
+ deadline: Option<Instant>,
) -> AttemptOutcome {
let status = response.status();
// Only the relayed subset survives; the response is consumed by the
@@ -533,7 +625,7 @@ async fn classify_forwarded_reply(
})
.collect();
if method == Method::GET && !retry_redirect && status.is_success() {
- let mut response = Response::new(stream_poll_body(response.bytes_stream()));
+ let mut response = Response::new(stream_poll_body(response.bytes_stream(), deadline));
*response.status_mut() = status;
for (name, value) in relayed_headers {
response.headers_mut().insert(name, value);
@@ -577,9 +669,15 @@ async fn classify_forwarded_reply(
classify_reply(status, relayed_headers, Bytes::from(body), retry_redirect)
}
-fn stream_poll_body(stream: impl Stream<Item = Result<Bytes, cyper::Error>> + 'static) -> Body {
- let stream = futures::stream::try_unfold(Box::pin(stream), |mut stream| async move {
- match compio::time::timeout(FORWARD_ATTEMPT_TIMEOUT, stream.next()).await {
+fn stream_poll_body(
+ stream: impl Stream<Item = Result<Bytes, cyper::Error>> + 'static,
+ deadline: Option<Instant>,
+) -> Body {
+ let stream = futures::stream::try_unfold(Box::pin(stream), move |mut stream| async move {
+ let timeout = deadline.map_or(FORWARD_ATTEMPT_TIMEOUT, |deadline| {
+ deadline.saturating_duration_since(Instant::now())
+ });
+ match compio::time::timeout(timeout, stream.next()).await {
Ok(Some(Ok(bytes))) => Ok(Some((bytes, stream))),
Ok(None) => Ok(None),
Ok(Some(Err(error))) => Err(std::io::Error::other(error.to_string())),
@@ -930,6 +1028,79 @@ mod tests {
assert!(!wants_auto_commit(Some("%61uto_commit=%66alse")));
}
+ #[test]
+ fn deferred_forward_keeps_one_wait_budget_and_preserves_other_query_fields() {
+ let mut request = cyper::Request::new(
+ Method::GET,
+ "http://localhost/streams/0/topics/0/messages/deferred?%77ait_us=2000000&auto_commit=true&consistency=linearizable"
+ .parse()
+ .unwrap(),
+ );
+ let remaining = reduce_deferred_wait(
+ &mut request,
+ DeferredPollDeadline {
+ wait: Instant::now() + Duration::from_secs(1),
+ request: Instant::now() + Duration::from_secs(1),
+ },
+ )
+ .unwrap();
+ let query: Vec<_> = request.url().query_pairs().collect();
+ let waits: Vec<_> = query.iter().filter(|(key, _)| key == "wait_us").collect();
+ assert_eq!(waits.len(), 1);
+ assert!(waits[0].1.parse::<u128>().unwrap() <= remaining.as_micros());
+ assert!(remaining.as_micros() > 0 && remaining <= Duration::from_secs(1));
+ assert!(
+ query
+ .iter()
+ .any(|(key, value)| key == "auto_commit" && value == "true")
+ );
+ assert!(
+ query
+ .iter()
+ .any(|(key, value)| key == "consistency" && value == "linearizable")
+ );
+ let previous = request.url().clone();
+ assert!(
+ reduce_deferred_wait(
+ &mut request,
+ DeferredPollDeadline {
+ wait: Instant::now(),
+ request: Instant::now()
+ }
+ )
+ .is_none()
+ );
+ assert_eq!(
+ request.url(),
+ &previous,
+ "request timeout must stop forwarding"
+ );
+ }
+
+ #[test]
+ fn forwarding_extends_only_valid_deferred_poll_requests() {
+ for (path, deferred) in [
+ (
+ "/streams/0/topics/0/messages/deferred?wait_us=1000000",
+ true,
+ ),
+ ("/streams/0/topics/0/messages?wait_us=1000000", false),
+ ("/streams/0/topics/0/messages/deferred", true),
+ ("/streams/0/topics/0/messages/deferred?wait_us=0", true),
+ (
+ "/streams/0/topics/0/messages/deferred?wait_us=600000001",
+ false,
+ ),
+ ] {
+ let request = Request::builder().uri(path).body(Body::empty()).unwrap();
+ assert_eq!(
+ deferred_poll_deadline(&request).is_some(),
+ deferred,
+ "{path}"
+ );
+ }
+ }
+
#[test]
fn transient_not_accepted_body_matches_only_its_code() {
let accepted = format!(
@@ -1025,7 +1196,7 @@ mod tests {
Ok::<_, cyper::Error>(chunk.clone())
}));
let response = retain_forward_guard(
- Response::new(stream_poll_body(stream)),
+ Response::new(stream_poll_body(stream, None)),
guard,
FORWARD_ATTEMPT_TIMEOUT,
);
@@ -1046,7 +1217,7 @@ mod tests {
let in_flight = Rc::new(Cell::new(0));
let guard = ForwardGuard::admit(&in_flight).expect("forward admitted");
let response = retain_forward_guard(
- Response::new(stream_poll_body(futures::stream::pending())),
+ Response::new(stream_poll_body(futures::stream::pending(), None)),
guard,
FORWARD_ATTEMPT_TIMEOUT,
);
@@ -1066,8 +1237,11 @@ mod tests {
let stream = futures::stream::unfold(held, |held| async move {
Some((Ok::<_, cyper::Error>(Bytes::from_static(b"chunk")), held))
});
- let response =
- retain_forward_guard(Response::new(stream_poll_body(stream)), guard, BODY_TIMEOUT);
+ let response = retain_forward_guard(
+ Response::new(stream_poll_body(stream, None)),
+ guard,
+ BODY_TIMEOUT,
+ );
compio::time::sleep(BODY_TIMEOUT * 3).await;
assert_eq!(in_flight.get(), 0, "unread bodies must release admission");
assert_eq!(Rc::strong_count(&upstream), 1, "upstream must be dropped");
@@ -1112,7 +1286,7 @@ mod tests {
.expect("poll response");
assert_eq!(response.content_length(), Some(65 * 1024 * 1024));
let AttemptOutcome::Relay(response) =
- classify_forwarded_reply(response, &Method::GET, false).await
+ classify_forwarded_reply(response, &Method::GET, false, None).await
else {
panic!("successful poll must not be retried")
};
diff --git a/core/server/src/http/handlers.rs b/core/server/src/http/handlers.rs
index cbb505a7f..10e1fda55 100644
--- a/core/server/src/http/handlers.rs
+++ b/core/server/src/http/handlers.rs
@@ -117,7 +117,9 @@ use metadata::permissioner::Permissioner;
use secrecy::ExposeSecret;
use send_wrapper::SendWrapper;
use serde::Deserialize;
-use shard::{PartitionRead, PartitionReadReply};
+use shard::{
+ DeferredPollContext, DeferredPollRequest, PartitionRead, PartitionReadReply, PollQuotaIdentity,
+};
use crate::dispatch::partition::{resolve_consumer_offset_request, resolve_poll_request};
use crate::dispatch::session_ops::{verify_login_credentials, verify_pat_credentials};
@@ -1258,6 +1260,85 @@ pub(in crate::http) async fn poll_messages(
Query(query): Query<PollMessages>,
Query(consistency): Query<ConsistencyQuery>,
) -> Result<Json<PolledMessages>, ReadError> {
+ poll_messages_impl(
+ state,
+ identity,
+ stream_id,
+ topic_id,
+ query,
+ consistency,
+ None,
+ )
+ .await
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(default)]
+pub(in crate::http) struct DeferredPollQuery {
+ pub wait_us: u64,
+ pub min_count: u32,
+ pub max_bytes: u32,
+ pub request_timeout_us: u64,
+}
+
+impl Default for DeferredPollQuery {
+ fn default() -> Self {
+ let options = iggy_common::DeferredPollOptions::default();
+ Self {
+ wait_us: options.max_wait.as_micros(),
+ min_count: options.min_count,
+ max_bytes: options.max_bytes,
+ request_timeout_us: options.request_timeout.as_micros(),
+ }
+ }
+}
+
+impl From<DeferredPollQuery> for iggy_common::DeferredPollOptions {
+ fn from(query: DeferredPollQuery) -> Self {
+ Self {
+ max_wait: query.wait_us.into(),
+ min_count: query.min_count,
+ max_bytes: query.max_bytes,
+ request_timeout: query.request_timeout_us.into(),
+ }
+ }
+}
+
+pub(in crate::http) async fn poll_messages_deferred(
+ State(state): State<HttpState>,
+ identity: Identity,
+ Path((stream_id, topic_id)): Path<(String, String)>,
+ Query(query): Query<PollMessages>,
+ Query(wait): Query<DeferredPollQuery>,
+ Query(consistency): Query<ConsistencyQuery>,
+) -> Result<Json<PolledMessages>, ReadError> {
+ let options = iggy_common::DeferredPollOptions::from(wait);
+ state
+ .shard
+ .validate_deferred_poll(query.count, options)
+ .map_err(ReadError::Rejected)?;
+ poll_messages_impl(
+ state,
+ identity,
+ stream_id,
+ topic_id,
+ query,
+ consistency,
+ Some(options),
+ )
+ .await
+}
+
+async fn poll_messages_impl(
+ state: HttpState,
+ identity: Identity,
+ stream_id: String,
+ topic_id: String,
+ query: PollMessages,
+ consistency: ConsistencyQuery,
+ options: Option<iggy_common::DeferredPollOptions>,
+) -> Result<Json<PolledMessages>, ReadError> {
+ let started = std::time::Instant::now();
let stream_id = Identifier::from_str_value(&stream_id).map_err(ReadError::Rejected)?;
let topic_id = Identifier::from_str_value(&topic_id).map_err(ReadError::Rejected)?;
SendWrapper::new(gate_local_read(
@@ -1294,12 +1375,33 @@ pub(in crate::http) async fn poll_messages(
// as the legacy 404 body.
Err(_) => return Err(ReadError::NotFound),
};
- let reply = SendWrapper::new(
- state
+ let read = if let Some(options) = options {
+ let metadata = state
.shard
- .partition_read(namespace, PartitionRead::Poll { consumer, args }),
- )
- .await;
+ .plane
+ .metadata()
+ .mux_stm
+ .streams()
+ .poll_metadata(namespace, None, HTTP_READ_CLIENT_ID)
+ .ok_or(ReadError::Rejected(IggyError::TransientNotAccepted))?;
+ PartitionRead::DeferredPoll(Box::new(DeferredPollRequest::new(
+ consumer,
+ args,
+ DeferredPollContext {
+ session: None,
+ metadata,
+ user_id: identity.user_id,
+ quota: PollQuotaIdentity::User(identity.user_id),
+ primary: query.auto_commit,
+ },
+ options
+ .remaining(started.elapsed())
+ .map_err(ReadError::Rejected)?,
+ )))
+ } else {
+ PartitionRead::Poll { consumer, args }
+ };
+ let reply = SendWrapper::new(state.shard.partition_read(namespace, read)).await;
match reply {
Some(PartitionReadReply::Poll {
fragments,
@@ -1925,4 +2027,25 @@ mod tests {
.expect("confirmations serialize");
assert_eq!(json, r#"{"confirmations":[]}"#);
}
+ #[test]
+ fn deferred_query_uses_ordinary_numeric_fields_and_default_options() {
+ let uri = "/messages/deferred?consumer_id=0&consumer_kind=consumer&kind=offset&value=0&partition_id=0&count=2&auto_commit=false&wait_us=1000".parse().unwrap();
+ let Query(poll) = Query::<PollMessages>::try_from_uri(&uri).unwrap();
+ let Query(wait) = Query::<DeferredPollQuery>::try_from_uri(&uri).unwrap();
+ assert_eq!(poll.partition_id, Some(0));
+ assert_eq!(poll.count, 2);
+ assert_eq!(wait.wait_us, 1000);
+ let Query(defaults) = Query::<DeferredPollQuery>::try_from_uri(
+ &"/messages/deferred?count=2".parse().unwrap(),
+ )
+ .unwrap();
+ let options = iggy_common::DeferredPollOptions::default();
+ assert_eq!(defaults.wait_us, options.max_wait.as_micros());
+ assert_eq!(defaults.min_count, options.min_count);
+ assert_eq!(defaults.max_bytes, options.max_bytes);
+ assert_eq!(
+ defaults.request_timeout_us,
+ options.request_timeout.as_micros()
+ );
+ }
}
diff --git a/core/server/src/session_manager.rs b/core/server/src/session_manager.rs
index dfc14a6f4..aecdc9e52 100644
--- a/core/server/src/session_manager.rs
+++ b/core/server/src/session_manager.rs
@@ -28,9 +28,11 @@
use crate::cluster_meta::ClusterRoster;
use ahash::AHashMap;
use consensus::client_table::SessionAttachment;
+use futures::future::{AbortHandle, AbortRegistration};
use iggy_common::IggyError;
use message_bus::installer::conn_info::ClientTransportKind;
use shard::ConnectedClientInfo;
+use std::cell::RefCell;
use std::net::SocketAddr;
use std::rc::Rc;
use std::time::{Duration, Instant};
@@ -117,6 +119,7 @@ pub struct Connection {
/// session it binds.
pub metadata_watermark: u64,
consumer_session: Option<(u128, SessionAttachment)>,
+ deferred_request: Option<(Instant, AbortHandle)>,
}
/// Bridges transport connections to consensus sessions.
@@ -185,6 +188,7 @@ impl SessionManager {
sdk: None,
metadata_watermark: 0,
consumer_session: None,
+ deferred_request: None,
});
}
@@ -224,6 +228,11 @@ impl SessionManager {
self.connections
.iter()
.filter(|(_, conn)| !matches!(conn.state, ConnectionState::Connected))
+ .filter(|(_, conn)| {
+ conn.deferred_request
+ .as_ref()
+ .is_none_or(|(deadline, _)| now >= *deadline)
+ })
.filter(|(_, conn)| now.duration_since(conn.last_heartbeat) > max_age)
.map(|(&id, _)| id)
.collect()
@@ -255,17 +264,50 @@ impl SessionManager {
/// one, so the caller can submit a session-matched `Logout` (the committed
/// apply releases the client-table slot cluster-wide).
pub fn remove_connection(&mut self, connection_id: u128) -> Option<(u128, u64)> {
- if let Some(conn) = self.connections.remove(&connection_id)
- && let ConnectionState::Bound {
+ if let Some(conn) = self.connections.remove(&connection_id) {
+ if let Some((_, abort)) = conn.deferred_request {
+ abort.abort();
+ }
+ if let ConnectionState::Bound {
client_id, session, ..
} = conn.state
- {
- self.client_to_connection.remove(&client_id);
- return Some((client_id, session));
+ {
+ self.client_to_connection.remove(&client_id);
+ return Some((client_id, session));
+ }
}
None
}
+ /// # Errors
+ /// Rejects a missing connection, another active request or an overflowing watchdog.
+ pub fn begin_deferred_request(
+ sessions: &Rc<RefCell<Self>>,
+ connection_id: u128,
+ watchdog: Duration,
+ ) -> Result<(DeferredRequestGuard, AbortRegistration), IggyError> {
+ let deadline = Instant::now()
+ .checked_add(watchdog)
+ .ok_or(IggyError::InvalidCommand)?;
+ let (abort, registration) = AbortHandle::new_pair();
+ let mut manager = sessions.borrow_mut();
+ let connection = manager
+ .connections
+ .get_mut(&connection_id)
+ .ok_or(IggyError::StaleClient)?;
+ if connection.deferred_request.is_some() {
+ return Err(IggyError::TransientNotAccepted);
+ }
+ connection.deferred_request = Some((deadline, abort));
+ Ok((
+ DeferredRequestGuard {
+ sessions: Rc::clone(sessions),
+ connection_id,
+ },
+ registration,
+ ))
+ }
+
/// Transition to `Authenticated` after successful login.
///
/// # Errors
@@ -499,6 +541,41 @@ impl SessionManager {
}
}
+pub struct DeferredRequestGuard {
+ sessions: Rc<RefCell<SessionManager>>,
+ connection_id: u128,
+}
+
+impl DeferredRequestGuard {
+ pub fn written(&self) {
+ if let Some(connection) = self
+ .sessions
+ .borrow_mut()
+ .connections
+ .get_mut(&self.connection_id)
+ && connection
+ .deferred_request
+ .as_ref()
+ .is_some_and(|(deadline, _)| Instant::now() < *deadline)
+ {
+ connection.last_heartbeat = Instant::now();
+ }
+ }
+}
+
+impl Drop for DeferredRequestGuard {
+ fn drop(&mut self) {
+ if let Some(connection) = self
+ .sessions
+ .borrow_mut()
+ .connections
+ .get_mut(&self.connection_id)
+ {
+ connection.deferred_request = None;
+ }
+ }
+}
+
impl Default for SessionManager {
fn default() -> Self {
Self::new()
@@ -567,6 +644,8 @@ const fn state_name(state: &ConnectionState) -> &'static str {
#[cfg(test)]
mod tests {
+ use futures::FutureExt;
+
use super::*;
use crate::responses::build_empty_reply;
use consensus::ClientTable;
@@ -871,4 +950,68 @@ mod tests {
mgr.record_metadata_watermark(9, 5);
assert_eq!(mgr.metadata_watermark(9), 0);
}
+ #[test]
+ fn deferred_guard_is_connection_local_finite_and_refreshes_only_after_write() {
+ let sessions = Rc::new(RefCell::new(SessionManager::new()));
+ let old = Instant::now().checked_sub(Duration::from_secs(30)).unwrap();
+ for connection in [1, 2] {
+ let mut manager = sessions.borrow_mut();
+ manager.ensure_connection(connection, addr(5000), ClientTransportKind::Tcp);
+ manager.login(connection, 0).unwrap();
+ manager
+ .connections
+ .get_mut(&connection)
+ .unwrap()
+ .last_heartbeat = old;
+ }
+ let (guard, _) =
+ SessionManager::begin_deferred_request(&sessions, 2, Duration::from_secs(60)).unwrap();
+ let now = Instant::now();
+ assert_eq!(
+ sessions
+ .borrow()
+ .collect_stale(Duration::from_secs(10), now),
+ [1]
+ );
+ assert_eq!(sessions.borrow().connections[&1].last_heartbeat, old);
+ assert_eq!(
+ sessions
+ .borrow()
+ .collect_stale(Duration::from_secs(10), now + Duration::from_secs(61))
+ .len(),
+ 2
+ );
+ assert!(
+ SessionManager::begin_deferred_request(&sessions, 2, Duration::from_secs(60)).is_err()
+ );
+ guard.written();
+ drop(guard);
+ assert_eq!(
+ sessions
+ .borrow()
+ .collect_stale(Duration::from_secs(10), Instant::now()),
+ [1]
+ );
+ assert_eq!(sessions.borrow().connections[&1].last_heartbeat, old);
+ let (failed, registration) =
+ SessionManager::begin_deferred_request(&sessions, 1, Duration::from_secs(60)).unwrap();
+ sessions.borrow_mut().remove_connection(1);
+ let cancelled = futures::future::Abortable::new(std::future::pending::<()>(), registration);
+ assert!(cancelled.now_or_never().unwrap().is_err());
+ drop(failed);
+ }
+
+ #[test]
+ fn dropped_deferred_guard_does_not_extend_heartbeat() {
+ let sessions = Rc::new(RefCell::new(SessionManager::new()));
+ sessions
+ .borrow_mut()
+ .ensure_connection(1, addr(5000), ClientTransportKind::Tcp);
+ let before = sessions.borrow().connections[&1].last_heartbeat;
+ let (guard, _) =
+ SessionManager::begin_deferred_request(&sessions, 1, Duration::from_secs(1)).unwrap();
+ drop(guard);
+ assert_eq!(sessions.borrow().connections[&1].last_heartbeat, before);
+ assert!(sessions.borrow().connections[&1].deferred_request.is_none());
+ }
}
diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs
index 0e4af1e11..17c630b3f 100644
--- a/core/server_common/src/consensus_message.rs
+++ b/core/server_common/src/consensus_message.rs
@@ -18,6 +18,7 @@
use crate::iobuf::{Frozen, Owned};
use crate::sharding::METADATA_GROUP;
use aligned_vec::{AVec, ConstAlign};
+use futures::channel::oneshot;
use iggy_binary_protocol::{
Command, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader,
ForwardLogoutHeader, ForwardLogoutResultHeader, ForwardRegisterHeader,
@@ -29,6 +30,7 @@ use iggy_binary_protocol::{
prepare_identity_checksum_bytes,
};
use smallvec::SmallVec;
+use std::sync::{Arc, Mutex};
use std::{
marker::PhantomData,
mem::{offset_of, size_of},
@@ -86,12 +88,41 @@ pub struct RequestBacking {
#[derive(Debug, Clone)]
pub struct ResponseBacking {
fragments: ResponseFragments,
+ write_receipt: Option<WriteReceipt>,
+}
+
+/// Completed by the transport only after the whole frame is written and flushed.
+/// Dropping every copy without completion cancels the observer.
+#[derive(Debug, Clone)]
+pub struct WriteReceipt(Arc<Mutex<Option<oneshot::Sender<()>>>>);
+
+impl WriteReceipt {
+ pub fn complete(self) {
+ if let Some(sender) = self
+ .0
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .take()
+ {
+ let _ = sender.send(());
+ }
+ }
}
impl RequestBackingKind for RequestBacking {}
impl ResponseBackingKind for ResponseBacking {}
impl ResponseBacking {
+ pub fn track_write(&mut self) -> oneshot::Receiver<()> {
+ let (sender, receiver) = oneshot::channel();
+ self.write_receipt = Some(WriteReceipt(Arc::new(Mutex::new(Some(sender)))));
+ receiver
+ }
+
+ pub fn take_write_receipt(&mut self) -> Option<WriteReceipt> {
+ self.write_receipt.take()
+ }
+
/// The fragment carrying the frame header.
#[must_use]
pub fn first(&self) -> &Frozen<MESSAGE_ALIGN> {
@@ -138,6 +169,7 @@ impl From<Frozen<MESSAGE_ALIGN>> for ResponseBacking {
fn from(frozen: Frozen<MESSAGE_ALIGN>) -> Self {
Self {
fragments: smallvec::smallvec![frozen],
+ write_receipt: None,
}
}
}
@@ -606,7 +638,12 @@ where
});
}
- Ok(unsafe { Self::from_backing_unchecked(ResponseBacking { fragments }) })
+ Ok(unsafe {
+ Self::from_backing_unchecked(ResponseBacking {
+ fragments,
+ write_receipt: None,
+ })
+ })
}
}
diff --git a/core/server_common/src/iobuf.rs b/core/server_common/src/iobuf.rs
index 8f97b86a6..53a597be5 100644
--- a/core/server_common/src/iobuf.rs
+++ b/core/server_common/src/iobuf.rs
@@ -286,6 +286,13 @@ impl<const ALIGN: usize> From<Owned<ALIGN>> for Frozen<ALIGN> {
}
impl<const ALIGN: usize> Frozen<ALIGN> {
+ /// Bytes pinned by this view, including its allocation's control block.
+ pub fn allocation_bytes(&self) -> usize {
+ // SAFETY: this live extent retains the immutable capacity in its control block.
+ let capacity = unsafe { self.inner.ctrlb.as_ref().capacity };
+ capacity.saturating_add(size_of::<ControlBlock>())
+ }
+
pub fn as_slice(&self) -> &[u8] {
self.inner.as_slice()
}
diff --git a/core/shard/src/config.rs b/core/shard/src/config.rs
index 0406210b1..3fbae5463 100644
--- a/core/shard/src/config.rs
+++ b/core/shard/src/config.rs
@@ -46,3 +46,26 @@ impl Default for CoordinatorConfig {
}
}
}
+
+/// Owner-local deferred poll limits. Running reads retain their byte reservation
+/// until I/O and completion delivery end, even if their caller disconnects.
+#[derive(Debug, Clone, Copy)]
+pub struct DeferredPollConfig {
+ pub max_wait_us: u64,
+ pub max_pending: usize,
+ pub max_pending_per_session: usize,
+ pub max_read_bytes: usize,
+ pub max_inflight_bytes: usize,
+}
+
+impl Default for DeferredPollConfig {
+ fn default() -> Self {
+ Self {
+ max_wait_us: 30_000_000,
+ max_pending: 1024,
+ max_pending_per_session: 64,
+ max_read_bytes: 16 * 1024 * 1024,
+ max_inflight_bytes: 64 * 1024 * 1024,
+ }
+ }
+}
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index 803547fba..e005b4a3f 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -24,6 +24,7 @@ mod router;
pub mod shards_table;
pub use config::CoordinatorConfig;
+pub use poll::deferred::{DeferredPollContext, DeferredPollRequest, PollQuotaIdentity};
pub use poll::{ConsumerAttachment, PollCompleted};
pub use router::CONSENSUS_TICK_INTERVAL;
@@ -333,6 +334,7 @@ const LIST_CLIENTS_GATHER_TIMEOUT: std::time::Duration = std::time::Duration::fr
#[derive(Debug)]
pub enum PartitionRead {
Primary,
+ DeferredPoll(Box<DeferredPollRequest>),
PollOnPrimary {
consumer: PollingConsumer,
args: PollingArgs,
@@ -1488,6 +1490,7 @@ where
/// Disk reads reserve this lane before I/O so ordinary frames cannot
/// displace their results. Only the owner pump validates completions.
poll_completions: poll::completion::PollCompletionLane,
+ deferred_polls: poll::deferred::DeferredPolls,
/// Partition namespace -> owning shard lookup.
shards_table: T,
@@ -1792,6 +1795,7 @@ where
inbox,
reply_inbox,
poll_completions,
+ deferred_polls: poll::deferred::DeferredPolls::new(metrics.deferred_polls.clone()),
shards_table,
partition_consensus,
coordinator,
@@ -2045,8 +2049,29 @@ where
pub async fn partition_read(
&self,
namespace: IggyNamespace,
- read: PartitionRead,
+ mut read: PartitionRead,
) -> Option<PartitionReadReply> {
+ let timeout = if let PartitionRead::DeferredPoll(request) = &mut read {
+ let Some(deadline) = self
+ .bus
+ .monotonic_micros()
+ .checked_add(request.options.max_wait.as_micros())
+ else {
+ return Some(PartitionReadReply::Rejected(IggyError::InvalidCommand));
+ };
+ request.deadline = deadline;
+ let Some(request_deadline) = self
+ .bus
+ .monotonic_micros()
+ .checked_add(request.options.request_timeout.as_micros())
+ else {
+ return Some(PartitionReadReply::Rejected(IggyError::InvalidCommand));
+ };
+ request.request_deadline = request_deadline;
+ request.options.request_timeout.get_duration()
+ } else {
+ PARTITION_READ_TIMEOUT
+ };
let Some(target) = self.shards_table.shard_for(namespace) else {
tracing::warn!(
shard = self.id,
@@ -2078,7 +2103,7 @@ where
IggyError::TransientNotAccepted,
));
}
- match bus_timeout(&self.bus, PARTITION_READ_TIMEOUT, reply_rx.recv()).await {
+ match bus_timeout(&self.bus, timeout, reply_rx.recv()).await {
Some(Ok(reply)) => Some(reply),
Some(Err(_)) => {
tracing::warn!(
@@ -2288,6 +2313,7 @@ where
shard_count: 1,
inbox,
reply_inbox,
+ deferred_polls: poll::deferred::DeferredPolls::new(metrics.deferred_polls.clone()),
poll_completions: poll::completion::PollCompletionLane::new(
POLL_COMPLETION_CAPACITY,
&metrics,
diff --git a/core/shard/src/metrics.rs b/core/shard/src/metrics.rs
index 4caf701f2..d43a61156 100644
--- a/core/shard/src/metrics.rs
+++ b/core/shard/src/metrics.rs
@@ -202,6 +202,63 @@ const fn consumer_kind_index(kind: ConsumerKind) -> usize {
}
}
+#[derive(Clone, Default)]
+pub(crate) struct DeferredPollMetrics {
+ pub pending: Gauge,
+ pub inflight_bytes: Gauge,
+ pub rejected: Counter,
+ pub failed: Counter,
+ pub cancelled: Counter,
+ pub expired: Counter,
+ pub filled: Counter,
+ pub late: Counter,
+}
+
+impl DeferredPollMetrics {
+ fn register(&self, registry: &mut Registry) {
+ registry.register(
+ "deferred_poll_pending",
+ "pending deferred polls",
+ self.pending.clone(),
+ );
+ registry.register(
+ "deferred_poll_inflight_bytes",
+ "reserved bytes for running or queued deferred reads",
+ self.inflight_bytes.clone(),
+ );
+ registry.register(
+ "deferred_poll_rejected",
+ "deferred polls rejected before admission",
+ self.rejected.clone(),
+ );
+ registry.register(
+ "deferred_poll_failed",
+ "admitted deferred polls rejected by a read error or fence",
+ self.failed.clone(),
+ );
+ registry.register(
+ "deferred_poll_cancelled",
+ "deferred polls cancelled by their caller",
+ self.cancelled.clone(),
+ );
+ registry.register(
+ "deferred_poll_expired",
+ "deferred polls completed at their deadline",
+ self.expired.clone(),
+ );
+ registry.register(
+ "deferred_poll_filled",
+ "deferred polls that filled their requested count",
+ self.filled.clone(),
+ );
+ registry.register(
+ "deferred_poll_late",
+ "deferred read completions discarded after expiry or cancellation",
+ self.late.clone(),
+ );
+ }
+}
+
/// Per-shard metric handles.
///
/// Cheap to clone (`Arc` of a `Family` under the hood). Each shard owns
@@ -220,6 +277,7 @@ const fn consumer_kind_index(kind: ConsumerKind) -> usize {
/// resolved at scrape time via the per-shard registry, not as a label.
#[derive(Clone)]
pub struct ShardMetrics {
+ pub(crate) deferred_polls: DeferredPollMetrics,
partition_wal_disk_bytes: Gauge,
partition_wal_retained_bytes: Gauge,
partition_wal_queued_bytes: Gauge,
@@ -291,6 +349,7 @@ impl ShardMetrics {
.clone();
let consumer_offset_stranded_gauges = [consumer_stranded, group_stranded];
Self {
+ deferred_polls: DeferredPollMetrics::default(),
partition_wal_disk_bytes: Gauge::default(),
partition_wal_retained_bytes: Gauge::default(),
partition_wal_queued_bytes: Gauge::default(),
@@ -768,6 +827,7 @@ impl ShardMetrics {
/// counters.
pub fn register(&self, registry: &mut Registry) {
self.register_persistence(registry);
+ self.deferred_polls.register(registry);
registry.register(
"frame_drops",
"frames shed instead of delivered, by frame class and refusal reason",
diff --git a/core/shard/src/poll.rs b/core/shard/src/poll.rs
index 1e13b2c21..5d6fec724 100644
--- a/core/shard/src/poll.rs
+++ b/core/shard/src/poll.rs
@@ -45,6 +45,7 @@ use server_common::sharding::IggyNamespace;
pub mod completion;
#[cfg(test)]
mod completion_tests;
+pub mod deferred;
#[cfg(test)]
mod test_support;
#[cfg(test)]
@@ -71,14 +72,23 @@ pub struct PollCompleted {
namespace: IggyNamespace,
/// Snapshot facts that have not yet been accepted as consumer progress.
result: PollReadResult,
- /// Return path for the accepted read or a rejection.
- reply: Sender<PartitionReadReply>,
- attachment: Option<ConsumerAttachment>,
+ target: PollTarget,
/// Inbox enqueue time for disk diagnostics, or `None` for resident completion.
#[cfg(feature = "poll-diagnostics")]
queued_at: Option<std::time::Instant>,
}
+pub enum PollTarget {
+ Immediate {
+ reply: Sender<PartitionReadReply>,
+ attachment: Option<ConsumerAttachment>,
+ },
+ Deferred {
+ request_id: u64,
+ _reservation: deferred::ReadReservation,
+ },
+}
+
impl<B, MJ, S, M, T, SB> IggyShard<B, MJ, S, M, T, SB>
where
B: MessageBus + 'static,
@@ -138,6 +148,10 @@ where
read: PartitionRead,
reply: Sender<PartitionReadReply>,
) {
+ if let PartitionRead::DeferredPoll(request) = read {
+ self.admit_deferred_poll(namespace, *request, reply).await;
+ return;
+ }
let partitions = self.plane.partitions();
let rejected = partitions
.with_partition(&namespace, |partition| {
@@ -172,6 +186,7 @@ where
read => (read, None),
};
let result = match read {
+ PartitionRead::DeferredPoll(_) => unreachable!("deferred admission handled above"),
PartitionRead::Primary => partitions
.with_partition(&namespace, |partition| {
let consensus = partition.consensus();
@@ -232,8 +247,7 @@ where
self.on_poll_completed(PollCompleted {
namespace,
result: plan.execute_resident(),
- reply,
- attachment,
+ target: PollTarget::Immediate { reply, attachment },
#[cfg(feature = "poll-diagnostics")]
queued_at: None,
})
@@ -283,8 +297,7 @@ where
let PollCompleted {
namespace,
result,
- reply,
- attachment,
+ target,
#[cfg(feature = "poll-diagnostics")]
queued_at,
} = completion;
@@ -299,6 +312,16 @@ where
"partition poll completion"
);
}
+ let (reply, attachment) = match target {
+ PollTarget::Immediate { reply, attachment } => (reply, attachment),
+ PollTarget::Deferred {
+ request_id,
+ _reservation,
+ } => {
+ self.on_deferred_poll_completed(request_id, result).await;
+ return;
+ }
+ };
if reply.is_disconnected() {
return;
}
@@ -313,6 +336,16 @@ where
));
return;
}
+ self.accept_poll_result(namespace, result, reply).await;
+ }
+
+ #[allow(clippy::future_not_send)]
+ async fn accept_poll_result(
+ &self,
+ namespace: IggyNamespace,
+ result: PollReadResult,
+ reply: Sender<PartitionReadReply>,
+ ) {
let partitions = self.plane.partitions();
let consumer_kind = result.consumer_kind();
match partitions.complete_poll(&namespace, result) {
diff --git a/core/shard/src/poll/completion.rs b/core/shard/src/poll/completion.rs
index 9aa83e7f9..a7d4c49ac 100644
--- a/core/shard/src/poll/completion.rs
+++ b/core/shard/src/poll/completion.rs
@@ -37,7 +37,7 @@ use iggy_common::IggyError;
use partitions::PollReadResult;
use server_common::sharding::IggyNamespace;
-use super::{ConsumerAttachment, PollCompleted};
+use super::{ConsumerAttachment, PollCompleted, PollTarget};
use crate::coordinator::classify_try_send_err;
use crate::metrics::{FrameDropMetrics, ShardMetrics, frame_drop_reason, frame_drop_variant};
use crate::{PartitionReadReply, Receiver, Sender, channel};
@@ -79,29 +79,56 @@ impl PollCompletionLane {
reply: Sender<PartitionReadReply>,
attachment: Option<ConsumerAttachment>,
) -> Option<PollCompletionSender> {
+ let slot = match self.reserve_slot() {
+ Ok(slot) => slot,
+ Err(reason) => {
+ reject(&reply, &self.state.metrics, reason);
+ return None;
+ }
+ };
+ Some(PollCompletionSender {
+ inbox: self.sender.clone(),
+ slot,
+ namespace,
+ target: PollTarget::Immediate { reply, attachment },
+ })
+ }
+
+ pub(crate) fn try_reserve_deferred(
+ &self,
+ namespace: IggyNamespace,
+ request_id: u64,
+ reservation: super::deferred::ReadReservation,
+ ) -> Result<PollCompletionSender, IggyError> {
+ let slot = self.reserve_slot().map_err(|reason| {
+ self.state
+ .metrics
+ .record(frame_drop_variant::PARTITION_POLL_COMPLETION, reason);
+ IggyError::TransientNotAccepted
+ })?;
+ Ok(PollCompletionSender {
+ inbox: self.sender.clone(),
+ slot,
+ namespace,
+ target: PollTarget::Deferred {
+ request_id,
+ _reservation: reservation,
+ },
+ })
+ }
+
+ fn reserve_slot(&self) -> Result<CompletionSlot, &'static str> {
if self.state.closed.load(Ordering::Relaxed) || self.sender.is_disconnected() {
- reject(&reply, &self.state.metrics, frame_drop_reason::DISCONNECTED);
- return None;
+ return Err(frame_drop_reason::DISCONNECTED);
}
- if self
- .state
+ self.state
.reserved
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |reserved| {
(reserved < self.state.capacity).then_some(reserved + 1)
})
- .is_err()
- {
- reject(&reply, &self.state.metrics, frame_drop_reason::FULL);
- return None;
- }
- Some(PollCompletionSender {
- inbox: self.sender.clone(),
- slot: CompletionSlot {
- state: self.state.clone(),
- },
- namespace,
- reply,
- attachment,
+ .map_err(|_| frame_drop_reason::FULL)?;
+ Ok(CompletionSlot {
+ state: self.state.clone(),
})
}
@@ -153,8 +180,7 @@ pub struct PollCompletionSender {
inbox: Sender<QueuedCompletion>,
slot: CompletionSlot,
namespace: IggyNamespace,
- reply: Sender<PartitionReadReply>,
- attachment: Option<ConsumerAttachment>,
+ target: PollTarget,
}
impl PollCompletionSender {
@@ -163,8 +189,8 @@ impl PollCompletionSender {
/// owner rejects the result and releases the reservation.
pub(crate) fn complete(self, result: PollReadResult) {
if self.slot.state.closed.load(Ordering::Relaxed) || self.inbox.is_disconnected() {
- reject(
- &self.reply,
+ reject_target(
+ &self.target,
&self.slot.state.metrics,
frame_drop_reason::DISCONNECTED,
);
@@ -174,8 +200,7 @@ impl PollCompletionSender {
result: Box::new(PollCompleted {
namespace: self.namespace,
result,
- reply: self.reply,
- attachment: self.attachment,
+ target: self.target,
#[cfg(feature = "poll-diagnostics")]
queued_at: Some(std::time::Instant::now()),
}),
@@ -190,8 +215,8 @@ impl PollCompletionSender {
}
TrySendError::Disconnected(completion) => completion,
};
- reject(
- &completion.result.reply,
+ reject_target(
+ &completion.result.target,
&completion.slot.state.metrics,
reason,
);
@@ -226,6 +251,15 @@ pub(super) fn reject(
));
}
+fn reject_target(target: &PollTarget, metrics: &FrameDropMetrics, reason: &'static str) {
+ match target {
+ PollTarget::Immediate { reply, .. } => reject(reply, metrics, reason),
+ PollTarget::Deferred { .. } => {
+ metrics.record(frame_drop_variant::PARTITION_POLL_COMPLETION, reason);
+ }
+ }
+}
+
struct LaneState {
capacity: usize,
/// Includes slots held by running reads and by results awaiting dequeue.
diff --git a/core/shard/src/poll/deferred.rs b/core/shard/src/poll/deferred.rs
new file mode 100644
index 000000000..30104ef88
--- /dev/null
+++ b/core/shard/src/poll/deferred.rs
@@ -0,0 +1,707 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Deferred reads remain provisional until the owner selects one result.
+//! One registry owns readiness deadlines and quotas. Detached readers
+//! carry only a request identity and a reservation that survives cancellation.
+
+use std::cell::RefCell;
+use std::collections::{BTreeMap, BTreeSet, HashSet, VecDeque};
+use std::ops::Bound::{Excluded, Unbounded};
+use std::rc::Rc;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::task::{Context, Poll, Waker};
+
+use consensus::client_table::SessionAttachment;
+use consensus::{Consensus, MetadataHandle, PartitionsHandle};
+use iggy_common::{DeferredPollOptions, IggyError, PollingKind, PollingStrategy};
+use journal::superblock::SuperblockStore;
+use message_bus::MessageBus;
+use metadata::impls::metadata::StreamsFrontend;
+use metadata::stm::stream::PollMetadata;
+use partitions::{Partition, PollReadResult, PollingArgs, PollingConsumer};
+use server_common::poll::PollHistoryId;
+use server_common::sharding::IggyNamespace;
+
+use crate::config::DeferredPollConfig;
+use crate::metrics::DeferredPollMetrics;
+use crate::shards_table::ShardsTable;
+use crate::{IggyShard, PartitionReadReply, Sender};
+use prometheus_client::metrics::gauge::Gauge;
+
+const SERVICE_BUDGET: usize = 8;
+const READ_BUDGET_PARTS: usize = 3;
+const MAINTENANCE_BUDGET: usize = 16;
+
+/// Auxiliary connections are charged to their parent session. HTTP requests
+/// without a binary session are charged to their authenticated user.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum PollQuotaIdentity {
+ Session(u128),
+ User(u32),
+}
+
+#[derive(Debug)]
+pub struct DeferredPollContext {
+ pub session: Option<SessionAttachment>,
+ pub metadata: PollMetadata,
+ pub user_id: u32,
+ pub quota: PollQuotaIdentity,
+ pub primary: bool,
+}
+
+#[derive(Debug)]
+pub struct DeferredPollRequest {
+ pub consumer: PollingConsumer,
+ pub args: PollingArgs,
+ pub context: DeferredPollContext,
+ pub options: DeferredPollOptions,
+ pub(crate) deadline: u64,
+ pub(crate) request_deadline: u64,
+}
+
+impl DeferredPollRequest {
+ #[must_use]
+ pub const fn new(
+ consumer: PollingConsumer,
+ args: PollingArgs,
+ context: DeferredPollContext,
+ options: DeferredPollOptions,
+ ) -> Self {
+ Self {
+ consumer,
+ args,
+ context,
+ options,
+ deadline: 0,
+ request_deadline: 0,
+ }
+ }
+}
+
+struct Waiter {
+ namespace: IggyNamespace,
+ request: DeferredPollRequest,
+ reply: Sender<PartitionReadReply>,
+ history: PollHistoryId,
+ view: u32,
+ last_visibility: Option<(PollHistoryId, Option<u64>)>,
+ running: bool,
+ terminal: bool,
+}
+
+#[derive(Default)]
+struct Signals {
+ active: HashSet<IggyNamespace>,
+ queued: HashSet<IggyNamespace>,
+ dirty: VecDeque<IggyNamespace>,
+ waker: Option<Waker>,
+}
+
+impl Signals {
+ fn mark(&mut self, namespace: IggyNamespace) {
+ if self.active.contains(&namespace) && self.queued.insert(namespace) {
+ self.dirty.push_back(namespace);
+ if let Some(waker) = self.waker.take() {
+ waker.wake();
+ }
+ }
+ }
+
+ fn remove(&mut self, namespace: IggyNamespace) {
+ self.active.remove(&namespace);
+ self.queued.remove(&namespace);
+ self.dirty.retain(|queued| *queued != namespace);
+ }
+}
+
+#[derive(Default)]
+struct State {
+ next_id: u64,
+ waiters: BTreeMap<u64, Waiter>,
+ partitions: BTreeMap<IggyNamespace, BTreeSet<u64>>,
+ deadlines: BTreeSet<(u64, u64)>,
+ sessions: BTreeMap<PollQuotaIdentity, usize>,
+ ready: VecDeque<u64>,
+ queued: BTreeSet<u64>,
+ maintenance_cursor: u64,
+ closed: bool,
+}
+
+impl State {
+ fn enqueue(&mut self, id: u64) {
+ if self.waiters.contains_key(&id) && self.queued.insert(id) {
+ self.ready.push_back(id);
+ }
+ }
+}
+
+pub struct DeferredPolls {
+ metrics: DeferredPollMetrics,
+ config: RefCell<DeferredPollConfig>,
+ state: RefCell<State>,
+ signals: Rc<RefCell<Signals>>,
+ inflight_bytes: Arc<AtomicUsize>,
+}
+
+impl DeferredPolls {
+ pub(crate) fn new(metrics: DeferredPollMetrics) -> Self {
+ Self {
+ metrics,
+ config: RefCell::new(DeferredPollConfig::default()),
+ state: RefCell::new(State::default()),
+ signals: Rc::new(RefCell::new(Signals::default())),
+ inflight_bytes: Arc::new(AtomicUsize::new(0)),
+ }
+ }
+}
+
+impl DeferredPolls {
+ pub(crate) fn earliest_deadline(&self) -> Option<u64> {
+ self.state
+ .borrow()
+ .deadlines
+ .first()
+ .map(|(deadline, _)| *deadline)
+ }
+
+ pub(crate) fn poll_ready(&self, context: &Context<'_>) -> Poll<()> {
+ let mut signals = self.signals.borrow_mut();
+ if !signals.dirty.is_empty() || !self.state.borrow().ready.is_empty() {
+ Poll::Ready(())
+ } else {
+ signals.waker = Some(context.waker().clone());
+ Poll::Pending
+ }
+ }
+
+ pub(crate) fn maintenance(&self) {
+ let mut state = self.state.borrow_mut();
+ let mut ids: Vec<_> = state
+ .waiters
+ .range((Excluded(state.maintenance_cursor), Unbounded))
+ .take(MAINTENANCE_BUDGET)
+ .map(|(id, _)| *id)
+ .collect();
+ if ids.is_empty() {
+ ids.extend(state.waiters.keys().take(MAINTENANCE_BUDGET).copied());
+ }
+ if let Some(last) = ids.last() {
+ state.maintenance_cursor = *last;
+ }
+ for id in ids {
+ state.enqueue(id);
+ }
+ }
+
+ pub(crate) const fn pump_guard(&self) -> PumpGuard<'_> {
+ PumpGuard(self)
+ }
+
+ pub(crate) fn close(&self) {
+ let mut state = self.state.borrow_mut();
+ state.closed = true;
+ for waiter in state.waiters.values() {
+ let _ = waiter.reply.try_send(PartitionReadReply::Rejected(
+ IggyError::TransientNotAccepted,
+ ));
+ }
+ state.waiters.clear();
+ state.partitions.clear();
+ state.deadlines.clear();
+ state.sessions.clear();
+ state.ready.clear();
+ state.queued.clear();
+ self.metrics.pending.set(0);
+ *self.signals.borrow_mut() = Signals::default();
+ }
+
+ fn reserve_read(&self) -> Result<ReadReservation, IggyError> {
+ let config = *self.config.borrow();
+ self.inflight_bytes
+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| {
+ used.checked_add(config.max_read_bytes)
+ .filter(|next| *next <= config.max_inflight_bytes)
+ })
+ .map_err(|_| IggyError::TransientNotAccepted)?;
+ self.metrics
+ .inflight_bytes
+ .inc_by(i64::try_from(config.max_read_bytes).unwrap_or(i64::MAX));
+ Ok(ReadReservation {
+ gauge: self.metrics.inflight_bytes.clone(),
+ bytes: config.max_read_bytes,
+ used: Arc::clone(&self.inflight_bytes),
+ })
+ }
+
+ fn remove(&self, id: u64, waiter: &Waiter) -> bool {
+ let mut state = self.state.borrow_mut();
+ state.deadlines.remove(&(waiter.request.deadline, id));
+ state
+ .deadlines
+ .remove(&(waiter.request.request_deadline, id));
+ state.queued.remove(&id);
+ state.ready.retain(|queued| *queued != id);
+ self.metrics.pending.dec();
+ if let Some(count) = state.sessions.get_mut(&waiter.request.context.quota) {
+ *count -= 1;
+ if *count == 0 {
+ state.sessions.remove(&waiter.request.context.quota);
+ }
+ }
+ let empty = state
+ .partitions
+ .get_mut(&waiter.namespace)
+ .is_some_and(|ids| {
+ ids.remove(&id);
+ ids.is_empty()
+ });
+ if empty {
+ state.partitions.remove(&waiter.namespace);
+ self.signals.borrow_mut().remove(waiter.namespace);
+ }
+ empty
+ }
+}
+
+pub struct PumpGuard<'a>(&'a DeferredPolls);
+
+impl Drop for PumpGuard<'_> {
+ fn drop(&mut self) {
+ self.0.close();
+ }
+}
+
+/// May cross the completion channel; it accounts for running and queued reads.
+pub struct ReadReservation {
+ gauge: Gauge,
+ bytes: usize,
+ used: Arc<AtomicUsize>,
+}
+
+impl Drop for ReadReservation {
+ fn drop(&mut self) {
+ self.used.fetch_sub(self.bytes, Ordering::Relaxed);
+ self.gauge
+ .dec_by(i64::try_from(self.bytes).unwrap_or(i64::MAX));
+ }
+}
+
+impl<B, MJ, S, M, T, SB> IggyShard<B, MJ, S, M, T, SB>
+where
+ B: MessageBus + 'static,
+ T: ShardsTable,
+ M: StreamsFrontend,
+ SB: SuperblockStore,
+{
+ pub fn set_deferred_poll_config(&self, config: DeferredPollConfig) {
+ *self.deferred_polls.config.borrow_mut() = config;
+ }
+
+ /// # Errors
+ /// Rejects invalid readiness limits and waits outside the admission limit.
+ pub fn validate_deferred_poll(
+ &self,
+ count: u32,
+ options: DeferredPollOptions,
+ ) -> Result<(), IggyError> {
+ options.validate(count)?;
+ if options.max_wait.as_micros() > self.deferred_polls.config.borrow().max_wait_us {
+ return Err(IggyError::InvalidCommand);
+ }
+ Ok(())
+ }
+
+ fn validate_deferred_context(
+ &self,
+ namespace: IggyNamespace,
+ context: &DeferredPollContext,
+ ) -> Result<(PollHistoryId, Option<u64>, u32), IggyError> {
+ if context
+ .session
+ .as_ref()
+ .is_some_and(|session| !session.is_valid())
+ {
+ return Err(IggyError::StaleClient);
+ }
+ let metadata = &self.plane.metadata().mux_stm;
+ metadata.users().authorize(|permissioner| {
+ permissioner.poll_messages(context.user_id, namespace.stream_id(), namespace.topic_id())
+ })?;
+ if !context.metadata.is_valid(metadata.streams(), namespace) {
+ return Err(IggyError::TransientNotAccepted);
+ }
+ self.plane
... 1008 lines suppressed ...
(iggy) 02/03: feat(java): add deferred polling to the TCP and HTTP clients
Posted by hg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch feat/deferred-polling
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 097dde82de77d4ed70a9d708a2b8d74faa023b4f
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Thu Sep 17 15:56:01 2026 +0200
feat(java): add deferred polling to the TCP and HTTP clients
A consumer could only read what the server already held, so an
empty topic meant sleeping and asking again, with the client
guessing an interval that costs either latency or requests.
pollMessagesDeferred sends a readiness wait, a readiness target,
a byte cap and a total budget as a 24-byte trailer on the
unchanged poll body, under commands 105 and 106.
On TCP each deferred exchange takes one of 16 exclusive leases
on a data connection of its own. The coordinator drains in
order, so a held reply would stall every later request and
expire the heartbeat queued behind it. HTTP gets its own pool,
with automatic retries off and the body bounded before it is
read.
A poll has no deduplication key, so only an explicit
TRANSIENT_NOT_ACCEPTED permits a replay; a lost reply or a
disconnect may already have been served and auto-committed, and
surfaces as it happened. One monotonic deadline spans lease
acquisition, routing, login, attachment and decoding, and a
retry inherits what is left of it.
---
examples/java/README.md | 4 +-
.../apache/iggy/examples/async/AsyncConsumer.java | 62 ++-
.../apache/iggy/examples/ConsumerOffsetTest.java | 155 ++++--
foreign/java/README.md | 30 ++
.../apache/iggy/client/async/MessagesClient.java | 79 +++
.../iggy/client/async/tcp/AsyncTcpConnection.java | 74 ++-
.../iggy/client/async/tcp/MessagesTcpClient.java | 218 ++++++--
.../apache/iggy/client/async/tcp/PollRouter.java | 597 ++++++++++++++++++++-
.../client/async/tcp/vsr/ConsensusSession.java | 29 +
.../iggy/client/async/tcp/vsr/VsrFrameDecoder.java | 23 +-
.../iggy/client/blocking/MessagesClient.java | 75 +++
.../client/blocking/http/InternalHttpClient.java | 162 +++++-
.../client/blocking/http/MessagesHttpClient.java | 115 ++++
.../iggy/client/blocking/tcp/FutureUtil.java | 25 +
.../client/blocking/tcp/MessagesTcpClient.java | 16 +
.../apache/iggy/message/DeferredPollOptions.java | 163 ++++++
.../java/org/apache/iggy/serde/CommandCode.java | 4 +-
.../iggy/client/DeferredPollCompatibilityTest.java | 170 ++++++
.../AsyncIggyTcpClientTransientFailoverTest.java | 239 +--------
.../iggy/client/async/tcp/PollRouterTest.java | 560 +++++++++++++++++++
.../iggy/client/async/tcp/VsrLoopbackPeer.java | 325 +++++++++++
.../client/async/tcp/vsr/VsrFrameDecoderTest.java | 42 ++
.../client/blocking/MessagesClientBaseTest.java | 130 +++++
.../blocking/http/MessagesHttpDeferredTest.java | 250 +++++++++
.../client/blocking/tcp/TlsConnectionTest.java | 53 ++
.../iggy/message/DeferredPollOptionsTest.java | 120 +++++
26 files changed, 3330 insertions(+), 390 deletions(-)
diff --git a/examples/java/README.md b/examples/java/README.md
index 407741a3e..4b628eb53 100644
--- a/examples/java/README.md
+++ b/examples/java/README.md
@@ -132,10 +132,10 @@ Non-blocking async consumption with advanced patterns:
Shows:
-- Backpressure management (don't poll faster than you can process)
+- Deferred polling, so the server holds the request until messages are ready
+- An idle limit that stops the consumer after a quiet period
- Error recovery with exponential backoff
- Thread pool separation (Netty I/O threads vs. processing threads)
-- Offset-based polling with CompletableFuture
**CRITICAL ASYNC PATTERN - Thread Pool Management:**
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncConsumer.java b/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncConsumer.java
index 948aeb221..361a1b31e 100644
--- a/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncConsumer.java
+++ b/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncConsumer.java
@@ -24,6 +24,7 @@ import org.apache.iggy.client.async.tcp.AsyncIggyTcpClient;
import org.apache.iggy.consumergroup.Consumer;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
import org.apache.iggy.message.Message;
import org.apache.iggy.message.PolledMessages;
import org.apache.iggy.message.PollingStrategy;
@@ -32,12 +33,14 @@ import org.slf4j.LoggerFactory;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -45,7 +48,7 @@ import java.util.concurrent.atomic.AtomicReference;
*
* <p>Demonstrates advanced async message consumption patterns including:
* <ul>
- * <li>Non-blocking continuous polling</li>
+ * <li>Deferred polling: the server holds each poll until data is ready</li>
* <li>Backpressure management (don't poll faster than you can process)</li>
* <li>Error recovery with exponential backoff</li>
* <li>Offloading CPU-intensive work from Netty threads</li>
@@ -87,9 +90,12 @@ public final class AsyncConsumer {
// Polling configuration
private static final int POLL_BATCH_SIZE = 100;
- private static final int POLL_INTERVAL_MS = 1000;
private static final int BATCHES_LIMIT = 5; // Exit after receiving this many batches
- private static final int MAX_EMPTY_POLLS = 5; // Exit if no messages after consecutive empty polls
+ // The server holds each poll for the readiness wait, so the client never
+ // sleeps between polls. Idle time is measured directly instead of counting
+ // empty polls, whose meaning changes with the wait.
+ private static final Duration IDLE_LIMIT = Duration.ofSeconds(5);
+ private static final DeferredPollOptions POLL_OPTIONS = DeferredPollOptions.defaults();
// Error recovery configuration
private static final int MAX_RETRY_ATTEMPTS = 5;
@@ -179,7 +185,7 @@ public final class AsyncConsumer {
log.info("Starting async polling loop (limit: {} batches)...", BATCHES_LIMIT);
AtomicInteger totalReceived = new AtomicInteger(0);
- AtomicInteger emptyPolls = new AtomicInteger(0);
+ AtomicLong lastProgressNanos = new AtomicLong(System.nanoTime());
AtomicInteger consumedBatches = new AtomicInteger(0);
AtomicReference<BigInteger> offset = new AtomicReference<>(BigInteger.ZERO);
@@ -189,15 +195,16 @@ public final class AsyncConsumer {
// until we've finished processing the current batch.
CompletableFuture<Void> pollingLoop = new CompletableFuture<>();
- pollBatch(client, processingPool, totalReceived, emptyPolls, consumedBatches, offset, 0, pollingLoop);
+ pollBatch(client, processingPool, totalReceived, lastProgressNanos, consumedBatches, offset, 0, pollingLoop);
return pollingLoop;
}
+ @SuppressWarnings("checkstyle:ParameterNumber")
private static void pollBatch(
AsyncIggyTcpClient client,
ExecutorService processingPool,
AtomicInteger totalReceived,
- AtomicInteger emptyPolls,
+ AtomicLong lastProgressNanos,
AtomicInteger consumedBatches,
AtomicReference<BigInteger> offset,
int retryAttempt,
@@ -216,14 +223,15 @@ public final class AsyncConsumer {
Consumer consumer = Consumer.of(CONSUMER_ID);
client.messages()
- .pollMessages(
+ .pollMessagesDeferred(
streamId,
topicId,
Optional.of(PARTITION_ID),
consumer,
PollingStrategy.offset(offset.get()),
(long) POLL_BATCH_SIZE,
- false)
+ false,
+ POLL_OPTIONS)
.thenComposeAsync(
polled -> {
// OFFLOAD TO PROCESSING POOL:
@@ -241,33 +249,31 @@ public final class AsyncConsumer {
.offset()
.add(BigInteger.ONE));
consumedBatches.incrementAndGet();
- emptyPolls.set(0);
+ lastProgressNanos.set(System.nanoTime());
});
+ }
+ // An empty reply means the server waited and the topic
+ // stayed quiet, so the next poll goes out immediately.
+ if (System.nanoTime() - lastProgressNanos.get() >= IDLE_LIMIT.toNanos()) {
+ log.info("No new messages for {}, finishing.", IDLE_LIMIT);
+ running = false;
} else {
- int empty = emptyPolls.incrementAndGet();
- if (empty >= MAX_EMPTY_POLLS) {
- log.info("No more messages after {} empty polls, finishing.", MAX_EMPTY_POLLS);
- running = false;
- return CompletableFuture.completedFuture(null);
- }
- log.info("Caught up - no new messages. Waiting...");
- // Sleep without blocking Netty threads
- return CompletableFuture.runAsync(
- () -> {
- try {
- Thread.sleep(POLL_INTERVAL_MS);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- },
- processingPool);
+ log.info("Caught up - no new messages.");
}
+ return CompletableFuture.completedFuture(null);
},
processingPool)
.thenRun(() -> {
// SUCCESS: Reset retry counter and schedule next poll
pollBatch(
- client, processingPool, totalReceived, emptyPolls, consumedBatches, offset, 0, loopFuture);
+ client,
+ processingPool,
+ totalReceived,
+ lastProgressNanos,
+ consumedBatches,
+ offset,
+ 0,
+ loopFuture);
})
.exceptionally(e -> {
// ERROR RECOVERY WITH EXPONENTIAL BACKOFF:
@@ -294,7 +300,7 @@ public final class AsyncConsumer {
client,
processingPool,
totalReceived,
- emptyPolls,
+ lastProgressNanos,
consumedBatches,
offset,
retryAttempt + 1,
diff --git a/examples/java/src/test/java/org/apache/iggy/examples/ConsumerOffsetTest.java b/examples/java/src/test/java/org/apache/iggy/examples/ConsumerOffsetTest.java
index 2472c1340..e0e795461 100644
--- a/examples/java/src/test/java/org/apache/iggy/examples/ConsumerOffsetTest.java
+++ b/examples/java/src/test/java/org/apache/iggy/examples/ConsumerOffsetTest.java
@@ -32,6 +32,7 @@ import org.apache.iggy.examples.streambuilder.StreamBasic;
import org.apache.iggy.examples.tcptls.consumer.TcpTlsConsumer;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
import org.apache.iggy.message.HeaderKey;
import org.apache.iggy.message.HeaderValue;
import org.apache.iggy.message.Message;
@@ -40,6 +41,7 @@ import org.apache.iggy.message.Partitioning;
import org.apache.iggy.message.PolledMessages;
import org.apache.iggy.message.PollingStrategy;
import org.apache.iggy.message.SendMessagesResponse;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
@@ -57,15 +59,18 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
-@Timeout(15)
+@Timeout(30)
class ConsumerOffsetTest {
private static final int MAX_BATCH_SIZE = 2;
private static final int MESSAGES_COUNT = 10;
+ private static final int CONSUMER_TIMEOUT_SECONDS = 20;
+ private static final long IDLE_POLL_MILLIS = 100;
@ParameterizedTest(name = "{0}, first retained offset {1}")
@MethodSource("blockingConsumers")
@@ -90,36 +95,70 @@ class ConsumerOffsetTest {
void shouldAdvanceAsynchronouslyOnlyAfterProcessing(int firstOffset, boolean failFirstProcessing) throws Exception {
var retained = new RetainedMessages(firstOffset, MESSAGES_COUNT, false);
List<BigInteger> requestedOffsets = new ArrayList<>();
+ List<DeferredPollOptions> requestedOptions = new ArrayList<>();
+ var messages = new DeferredMessages((strategy, count, options) -> {
+ requestedOffsets.add(strategy.value());
+ requestedOptions.add(options);
+ if (failFirstProcessing && requestedOffsets.size() == 1) {
+ var message = retained.retained.get(0);
+ var invalid = new Message(message.header(), null, message.userHeaders());
+ return CompletableFuture.completedFuture(
+ new PolledMessages(0L, message.header().offset(), 1L, List.of(invalid)));
+ }
+ return CompletableFuture.completedFuture(retained.poll(strategy, count));
+ });
+ runConsumer(messages);
+ retained.assertConsumedOnce();
+ assertThat(requestedOptions)
+ .as("the example asks the server to hold the poll instead of sleeping")
+ .isNotEmpty()
+ .allMatch(DeferredPollOptions.defaults()::equals);
+ if (failFirstProcessing) {
+ assertThat(requestedOffsets)
+ .as("processing failure must retry the same polling offset")
+ .startsWith(BigInteger.ZERO, BigInteger.ZERO);
+ }
+ }
+
+ @Test
+ void shouldNotAdvanceOnEmptyDeferredPolls() throws Exception {
+ var retained = new RetainedMessages(0, MESSAGES_COUNT, false);
+ List<BigInteger> requestedOffsets = new ArrayList<>();
+ var emptyPolls = new AtomicInteger();
+ var messages = new DeferredMessages((strategy, count, options) -> {
+ requestedOffsets.add(strategy.value());
+ // Two quiet replies before every batch: an empty deferred reply
+ // advances nothing, so the next poll repeats the same offset.
+ if (emptyPolls.incrementAndGet() % 3 != 0) {
+ return CompletableFuture.completedFuture(new PolledMessages(0L, BigInteger.ZERO, 0L, List.of()));
+ }
+ return CompletableFuture.completedFuture(retained.poll(strategy, count));
+ });
+ runConsumer(messages);
+ retained.assertConsumedOnce();
+ assertThat(requestedOffsets).startsWith(BigInteger.ZERO, BigInteger.ZERO, BigInteger.ZERO);
+ }
+
+ @Test
+ void shouldStopAfterTheIdleLimitWithoutMessages() throws Exception {
+ var polls = new AtomicInteger();
+ // A held poll costs real time on a server, so the double spends some
+ // too; otherwise the idle limit would be measured against a hot loop.
+ var messages = new DeferredMessages((strategy, count, options) -> {
+ polls.incrementAndGet();
+ return CompletableFuture.supplyAsync(
+ () -> new PolledMessages(0L, BigInteger.ZERO, 0L, List.of()),
+ CompletableFuture.delayedExecutor(IDLE_POLL_MILLIS, TimeUnit.MILLISECONDS));
+ });
+ runConsumer(messages);
+ assertThat(polls.get()).isGreaterThan(1);
+ }
+
+ private static void runConsumer(DeferredMessages messages) throws Exception {
var client = new AsyncIggyTcpClient("localhost", 8090) {
@Override
public org.apache.iggy.client.async.MessagesClient messages() {
- return new org.apache.iggy.client.async.MessagesClient() {
- @Override
- public CompletableFuture<PolledMessages> pollMessages(
- StreamId streamId,
- TopicId topicId,
- Optional<Long> partitionId,
- Consumer consumer,
- PollingStrategy strategy,
- Long count,
- boolean autoCommit) {
- requestedOffsets.add(strategy.value());
- if (failFirstProcessing && requestedOffsets.size() == 1) {
- var message = retained.retained.get(0);
- var invalid = new Message(message.header(), null, message.userHeaders());
- return CompletableFuture.completedFuture(
- new PolledMessages(0L, message.header().offset(), 1L, List.of(invalid)));
- }
- return CompletableFuture.completedFuture(retained.pollMessages(
- streamId, topicId, partitionId, consumer, strategy, count, autoCommit));
- }
-
- @Override
- public CompletableFuture<SendMessagesResponse> sendMessages(
- StreamId streamId, TopicId topicId, Partitioning partitioning, List<Message> messages) {
- throw new AssertionError("Consumer must not send messages");
- }
- };
+ return messages;
}
};
ExecutorService processingPool = Executors.newSingleThreadExecutor();
@@ -128,13 +167,7 @@ class ConsumerOffsetTest {
"pollMessagesAsync", AsyncIggyTcpClient.class, ExecutorService.class);
consume.setAccessible(true);
var completed = (CompletableFuture<?>) consume.invoke(null, client, processingPool);
- completed.get(10, TimeUnit.SECONDS);
- retained.assertConsumedOnce();
- if (failFirstProcessing) {
- assertThat(requestedOffsets)
- .as("processing failure must retry the same polling offset")
- .startsWith(BigInteger.ZERO, BigInteger.ZERO);
- }
+ completed.get(CONSUMER_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} finally {
processingPool.shutdownNow();
assertThat(processingPool.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
@@ -152,6 +185,54 @@ class ConsumerOffsetTest {
.flatMap(consumer -> Stream.of(0, 25).map(firstOffset -> Arguments.of(consumer, firstOffset)));
}
+ /**
+ * An async client that only answers deferred polls. An immediate poll fails
+ * the test, so a consumer that stops asking the server to wait is caught.
+ */
+ private static final class DeferredMessages implements org.apache.iggy.client.async.MessagesClient {
+ private final Reply reply;
+
+ private DeferredMessages(Reply reply) {
+ this.reply = reply;
+ }
+
+ @Override
+ public CompletableFuture<PolledMessages> pollMessagesDeferred(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit,
+ DeferredPollOptions options) {
+ return reply.poll(strategy, count, options);
+ }
+
+ @Override
+ public CompletableFuture<PolledMessages> pollMessages(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit) {
+ throw new AssertionError("Consumer must poll with an explicit readiness wait");
+ }
+
+ @Override
+ public CompletableFuture<SendMessagesResponse> sendMessages(
+ StreamId streamId, TopicId topicId, Partitioning partitioning, List<Message> messages) {
+ throw new AssertionError("Consumer must not send messages");
+ }
+
+ @FunctionalInterface
+ private interface Reply {
+ CompletableFuture<PolledMessages> poll(PollingStrategy strategy, Long count, DeferredPollOptions options);
+ }
+ }
+
private static final class RetainedMessages implements MessagesClient {
private final List<Message> retained;
private final List<BigInteger> delivered = new ArrayList<>();
@@ -186,6 +267,10 @@ class ConsumerOffsetTest {
PollingStrategy strategy,
Long count,
boolean autoCommit) {
+ return poll(strategy, count);
+ }
+
+ private PolledMessages poll(PollingStrategy strategy, Long count) {
var batch = retained.stream()
.filter(message -> message.header().offset().compareTo(strategy.value()) >= 0)
.limit(Math.min(count, MAX_BATCH_SIZE))
diff --git a/foreign/java/README.md b/foreign/java/README.md
index f64f75615..1ec87af88 100644
--- a/foreign/java/README.md
+++ b/foreign/java/README.md
@@ -241,6 +241,36 @@ info.getGitCommit(); // Git commit hash
info.getUserAgent(); // User-Agent string for HTTP
```
+## Polling
+
+`pollMessages` returns whatever the server already holds. An empty topic answers
+at once, so the caller has to poll again.
+
+`pollMessagesDeferred` lets the server hold the request instead. It waits up to
+`maxWait` for `minCount` messages, caps the encoded response at `maxBytes`, and
+bounds the whole call by `requestTimeout`.
+
+```java
+import org.apache.iggy.message.DeferredPollOptions;
+
+// Defaults: 1 s wait, 1 message, 1 MiB, 11 s budget.
+var options = DeferredPollOptions.defaults()
+ .withMaxWait(Duration.ofSeconds(5))
+ .withMinCount(10);
+
+var polled = client.messages().pollMessagesDeferred(
+ streamId, topicId, Optional.of(0L), Consumer.of(1L),
+ PollingStrategy.next(), 100L, false, options);
+```
+
+If the readiness wait expires, the server still answers, with a partial batch or
+an empty one. If the request timeout expires, the call fails.
+
+A deferred poll runs on a connection of its own, so a held request never delays
+other traffic. The TCP client must come from `Iggy.tcpClientBuilder()`, because a
+client built directly on a single connection rejects the call. The HTTP client
+accepts a plain consumer only, because its poll query carries no consumer kind.
+
## Exception Handling
The SDK's custom exception types inherit from `IggyException`. Joining a failed future can wrap the cause in `CompletionException`; the HTTP client's `close()` method declares `IOException`. Handle those boundaries as well as specific SDK errors.
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java
index b9e0440c0..55355c6c5 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java
@@ -20,8 +20,10 @@
package org.apache.iggy.client.async;
import org.apache.iggy.consumergroup.Consumer;
+import org.apache.iggy.exception.IggyOperationNotSupportedException;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
import org.apache.iggy.message.Message;
import org.apache.iggy.message.Partitioning;
import org.apache.iggy.message.PolledMessages;
@@ -139,6 +141,83 @@ public interface MessagesClient {
autoCommit);
}
+ /**
+ * Polls messages asynchronously, letting the server hold the request until data is ready.
+ *
+ * <p>Unlike {@link #pollMessages}, which reads whatever is resident and returns, this
+ * waits up to {@link DeferredPollOptions#maxWait()} for {@link DeferredPollOptions#minCount()}
+ * messages, caps the encoded response at {@link DeferredPollOptions#maxBytes()}, and bounds
+ * the whole exchange by {@link DeferredPollOptions#requestTimeout()}. An expired readiness
+ * wait can still return partial or empty data; an expired request timeout is an error.
+ *
+ * <p>The server must support command 105. An older one answers
+ * {@link org.apache.iggy.exception.IggyErrorCode#INVALID_COMMAND}, which also covers an
+ * operator wait limit below the requested wait and a request this client encoded wrongly.
+ * Never retry it and never infer the cause from the code alone.
+ *
+ * @param streamId the stream identifier (numeric or string-based)
+ * @param topicId the topic identifier (numeric or string-based)
+ * @param partitionId optional partition ID to poll from; when empty and polling with a
+ * group consumer, the client selects the partition round-robin from
+ * the member's synced group assignment
+ * @param consumer the consumer identity, either individual or group
+ * @param strategy the polling strategy controlling where to start reading
+ * @param count the maximum number of messages to return
+ * @param autoCommit whether the server should automatically commit the consumer offset
+ * @param options the readiness, byte and request-time limits
+ * @return a {@link CompletableFuture} that completes with the {@link PolledMessages}
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ default CompletableFuture<PolledMessages> pollMessagesDeferred(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit,
+ DeferredPollOptions options) {
+ return CompletableFuture.failedFuture(new IggyOperationNotSupportedException(
+ "pollMessagesDeferred", getClass().getSimpleName()));
+ }
+
+ /**
+ * Polls messages asynchronously with a deferred wait using numeric identifiers.
+ *
+ * <p>See {@link #pollMessagesDeferred(StreamId, TopicId, Optional, Consumer, PollingStrategy,
+ * Long, boolean, DeferredPollOptions)} for full documentation.
+ *
+ * @param streamId the numeric stream ID
+ * @param topicId the numeric topic ID
+ * @param partitionId optional partition ID
+ * @param consumerId the numeric consumer ID
+ * @param strategy the polling strategy
+ * @param count the maximum number of messages to return
+ * @param autoCommit whether to auto-commit offsets
+ * @param options the readiness, byte and request-time limits
+ * @return a {@link CompletableFuture} that completes with the {@link PolledMessages}
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ default CompletableFuture<PolledMessages> pollMessagesDeferred(
+ Long streamId,
+ Long topicId,
+ Optional<Long> partitionId,
+ Long consumerId,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit,
+ DeferredPollOptions options) {
+ return pollMessagesDeferred(
+ StreamId.of(streamId),
+ TopicId.of(topicId),
+ partitionId,
+ Consumer.of(consumerId),
+ strategy,
+ count,
+ autoCommit,
+ options);
+ }
+
/**
* Sends messages to a topic asynchronously.
*
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java
index bc981f502..164106514 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java
@@ -49,6 +49,7 @@ import io.netty.util.concurrent.ScheduledFuture;
import org.apache.iggy.client.ConnectionInfo;
import org.apache.iggy.client.async.tcp.vsr.ConsensusSession;
import org.apache.iggy.client.async.tcp.vsr.VsrFrameDecoder;
+import org.apache.iggy.client.async.tcp.vsr.VsrHeaders;
import org.apache.iggy.client.async.tcp.vsr.VsrRequestEncoder;
import org.apache.iggy.client.async.tcp.vsr.VsrResponseHandler;
import org.apache.iggy.exception.IggyClientException;
@@ -82,6 +83,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.IntConsumer;
+import java.util.function.IntSupplier;
/**
* Async TCP connection using Netty for non-blocking I/O.
@@ -124,6 +126,8 @@ public class AsyncTcpConnection {
private final TransientFailoverHandler transientFailoverHandler;
private final IntConsumer sessionResetListener;
private final Consumer<Throwable> connectionFailureListener;
+ private final ConnectionInfo endpoint;
+ private final int maxVsrFrameSize;
private final long requestTimeoutNanos;
private final long heartbeatIntervalNanos;
private final Object heartbeatLock = new Object();
@@ -131,6 +135,13 @@ public class AsyncTcpConnection {
private ScheduledFuture<?> heartbeatTask;
private boolean heartbeatRunning;
+ /**
+ * Frames a deferred exchange may accept on this connection, header
+ * included. A held poll declares its own response limit, so the channel
+ * must refuse a larger body before accumulating it.
+ */
+ private volatile int deferredFrameLimit = Integer.MAX_VALUE;
+
private volatile int loginCommandCode;
private volatile boolean authenticated = false;
private volatile long authenticatedUserId;
@@ -175,6 +186,8 @@ public class AsyncTcpConnection {
TransientFailoverHandler transientFailoverHandler,
IntConsumer sessionResetListener,
Consumer<Throwable> connectionFailureListener) {
+ this.endpoint = new ConnectionInfo(host, port);
+ this.maxVsrFrameSize = maxVsrFrameSize;
this.transientFailoverHandler = transientFailoverHandler;
this.sessionResetListener = sessionResetListener;
this.connectionFailureListener = connectionFailureListener;
@@ -222,7 +235,7 @@ public class AsyncTcpConnection {
sslContext,
dialTimeoutMillis,
consensusSession,
- maxVsrFrameSize,
+ this::currentMaxFrameSize,
this::onSessionEvicted,
channels::add),
ChannelHealthChecker.ACTIVE,
@@ -238,12 +251,18 @@ public class AsyncTcpConnection {
* Validates server reachability by eagerly acquiring and releasing one connection.
*/
public CompletableFuture<Void> connect() {
+ return connect(true);
+ }
+
+ private CompletableFuture<Void> connect(boolean withHeartbeat) {
CompletableFuture<Void> future = new CompletableFuture<>();
channelPool.acquire().addListener((FutureListener<Channel>) f -> {
if (f.isSuccess()) {
channelPool.release(f.getNow()).addListener(release -> {
if (release.isSuccess()) {
- startHeartbeat();
+ if (withHeartbeat) {
+ startHeartbeat();
+ }
future.complete(null);
} else {
future.completeExceptionally(release.cause());
@@ -261,6 +280,19 @@ public class AsyncTcpConnection {
return future;
}
+ /**
+ * Connects a deferred data connection. A held poll occupies the server's
+ * per-connection drain, so an automatic ping would queue behind it and
+ * time out against its own shorter budget, closing a valid exchange. The
+ * parent keeps its heartbeat and the server's active-request protection
+ * covers this one.
+ *
+ * @return a future completing once the connection is usable
+ */
+ CompletableFuture<Void> connectWithoutHeartbeat() {
+ return connect(false);
+ }
+
private void startHeartbeat() {
synchronized (heartbeatLock) {
if (heartbeatRunning || isClosed.get()) {
@@ -340,6 +372,18 @@ public class AsyncTcpConnection {
return consensusSession.metadataWatermark();
}
+ ConnectionInfo endpoint() {
+ return endpoint;
+ }
+
+ Optional<ConsensusSession.Snapshot> sessionSnapshot() {
+ return consensusSession.snapshot();
+ }
+
+ private int currentMaxFrameSize() {
+ return Math.min(maxVsrFrameSize, deferredFrameLimit);
+ }
+
long sessionGeneration() {
return consensusSession.generation();
}
@@ -465,6 +509,24 @@ public class AsyncTcpConnection {
return callerFuture;
}
+ /**
+ * Sends one deferred poll under the caller's absolute deadline, refusing a
+ * reply larger than the response limit the request declared.
+ *
+ * @param commandCode 105 or 106
+ * @param payload the encoded deferred poll body
+ * @param deadlineNanos the caller's absolute {@link System#nanoTime()} deadline
+ * @param maxResponseBytes the response body limit the request declared
+ * @return a future completing with the reply body
+ */
+ CompletableFuture<ByteBuf> sendDeferredPoll(
+ int commandCode, ByteBuf payload, long deadlineNanos, long maxResponseBytes) {
+ deferredFrameLimit = (int) Math.min(VsrHeaders.HEADER_SIZE + maxResponseBytes, maxVsrFrameSize);
+ CompletableFuture<ByteBuf> sent = send(commandCode, payload, deadlineNanos);
+ sent.whenComplete((response, error) -> deferredFrameLimit = Integer.MAX_VALUE);
+ return sent;
+ }
+
CompletableFuture<ByteBuf> sendPrimaryPoll(ByteBuf payload, long sessionGeneration) {
return send(
CommandCode.Messages.POLL_ON_PRIMARY.getValue(),
@@ -764,7 +826,9 @@ public class AsyncTcpConnection {
return commandCode == CommandCode.System.GET_CLUSTER_METADATA.getValue()
|| commandCode == CommandCode.System.ATTACH_CONSUMER_SESSION.getValue()
|| commandCode == CommandCode.Messages.GET_POLL_ROUTING.getValue()
- || commandCode == CommandCode.Messages.POLL_ON_PRIMARY.getValue();
+ || commandCode == CommandCode.Messages.POLL_ON_PRIMARY.getValue()
+ || commandCode == CommandCode.Messages.POLL_DEFERRED.getValue()
+ || commandCode == CommandCode.Messages.POLL_DEFERRED_ON_PRIMARY.getValue();
}
private static boolean mutatesSessionState(int commandCode) {
@@ -1096,7 +1160,7 @@ public class AsyncTcpConnection {
private final SslContext sslContext;
private final long dialTimeoutMillis;
private final ConsensusSession consensusSession;
- private final int maxVsrFrameSize;
+ private final IntSupplier maxVsrFrameSize;
private final IntConsumer onEviction;
private final Consumer<Channel> onChannelCreated;
@@ -1108,7 +1172,7 @@ public class AsyncTcpConnection {
SslContext sslContext,
long dialTimeoutMillis,
ConsensusSession consensusSession,
- int maxVsrFrameSize,
+ IntSupplier maxVsrFrameSize,
IntConsumer onEviction,
Consumer<Channel> onChannelCreated) {
this.host = host;
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java
index 6c87f0a8d..a86c745b1 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java
@@ -26,11 +26,14 @@ import org.apache.iggy.client.async.MessagesClient;
import org.apache.iggy.client.async.TopicsClient;
import org.apache.iggy.consumergroup.Consumer;
import org.apache.iggy.exception.IggyErrorCode;
+import org.apache.iggy.exception.IggyMalformedResponseException;
+import org.apache.iggy.exception.IggyOperationNotSupportedException;
import org.apache.iggy.exception.IggyResourceNotFoundException;
import org.apache.iggy.exception.IggyServerException;
import org.apache.iggy.hash.XxHash32;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
import org.apache.iggy.message.Message;
import org.apache.iggy.message.Partitioning;
import org.apache.iggy.message.PartitioningKind;
@@ -50,6 +53,7 @@ import java.util.Optional;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
+import java.util.function.LongFunction;
import java.util.function.Supplier;
import static org.apache.iggy.serde.BytesSerializer.encodeMessagesBatchInto;
@@ -73,6 +77,9 @@ public class MessagesTcpClient implements MessagesClient {
private static final int GROUP_POLL_MAX_ATTEMPTS = 2;
private static final int PARTITION_NOT_OWNED_ERROR_CODE = 5009;
+ /** {@code partition_id:u32}, {@code current_offset:u64}, {@code count:u32}. */
+ private static final int POLL_RESPONSE_PREFIX_BYTES = 16;
+
/**
* Staleness budget for the client-side routing caches: group assignments
* and topic partition counts. Both re-fetch lazily on the next use once
@@ -131,50 +138,183 @@ public class MessagesTcpClient implements MessagesClient {
// The VSR broker fences group polls against unowned partitions
// instead of picking one, so the partition is selected here from
// the member's synced assignment, matching the Rust SDK.
- PollCancellation cancellation = new PollCancellation();
- CompletableFuture<PolledMessages> result = pollGroupMessages(
- streamId, topicId, consumer, strategy, count, autoCommit, GROUP_POLL_MAX_ATTEMPTS, cancellation);
- result.whenComplete((response, error) -> {
- if (result.isCancelled()) {
- cancellation.cancel();
- }
- });
- return result;
+ return pollAssignedPartition(
+ streamId,
+ topicId,
+ consumer,
+ partition -> pollPartition(
+ streamId, topicId, Optional.of(partition), consumer, strategy, count, autoCommit));
}
return pollPartition(streamId, topicId, partitionId, consumer, strategy, count, autoCommit);
}
- private CompletableFuture<PolledMessages> pollPartition(
+ @Override
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public CompletableFuture<PolledMessages> pollMessagesDeferred(
StreamId streamId,
TopicId topicId,
Optional<Long> partitionId,
Consumer consumer,
PollingStrategy strategy,
Long count,
- boolean autoCommit) {
-
- var payload = Unpooled.buffer();
+ boolean autoCommit,
+ DeferredPollOptions options) {
+ try {
+ options.validate(count);
+ } catch (RuntimeException invalid) {
+ return CompletableFuture.failedFuture(invalid);
+ }
+ if (pollRouter == null) {
+ return CompletableFuture.failedFuture(
+ new IggyOperationNotSupportedException(
+ "A deferred poll runs on a dedicated data connection; build the client with Iggy.tcpClientBuilder()"));
+ }
+ // One monotonic budget for the whole call, taken before group sync,
+ // route lookup, queuing, connection setup, login and attachment.
+ long startNanos = System.nanoTime();
+ long waitDeadlineNanos = startNanos + options.maxWait().toNanos();
+ long requestDeadlineNanos = startNanos + options.requestTimeout().toNanos();
+ if (consumer.kind() == Consumer.Kind.ConsumerGroup && partitionId.isEmpty()) {
+ return pollAssignedPartition(
+ streamId,
+ topicId,
+ consumer,
+ partition -> pollDeferredPartition(
+ streamId,
+ topicId,
+ Optional.of(partition),
+ consumer,
+ strategy,
+ count,
+ autoCommit,
+ options,
+ waitDeadlineNanos,
+ requestDeadlineNanos));
+ }
+ return pollDeferredPartition(
+ streamId,
+ topicId,
+ partitionId,
+ consumer,
+ strategy,
+ count,
+ autoCommit,
+ options,
+ waitDeadlineNanos,
+ requestDeadlineNanos);
+ }
- var consumerBytes = toBytes(consumer);
- payload.writeBytes(consumerBytes);
+ private CompletableFuture<PolledMessages> pollAssignedPartition(
+ StreamId streamId,
+ TopicId topicId,
+ Consumer consumer,
+ LongFunction<CompletableFuture<PolledMessages>> pollPartition) {
+ PollCancellation cancellation = new PollCancellation();
+ CompletableFuture<PolledMessages> result =
+ pollGroupMessages(streamId, topicId, consumer, pollPartition, GROUP_POLL_MAX_ATTEMPTS, cancellation);
+ result.whenComplete((response, error) -> {
+ if (result.isCancelled()) {
+ cancellation.cancel();
+ }
+ });
+ return result;
+ }
- var streamBytes = toBytes(streamId);
- payload.writeBytes(streamBytes);
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ private CompletableFuture<PolledMessages> pollDeferredPartition(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit,
+ DeferredPollOptions options,
+ long waitDeadlineNanos,
+ long requestDeadlineNanos) {
+ ByteBuf payload = encodePoll(streamId, topicId, partitionId, consumer, strategy, count, autoCommit);
+ PollCancellation cancellation = new PollCancellation();
+ // Only an auto-commit poll needs the primary route; a manual one attaches
+ // the parent session on the coordinator's own endpoint.
+ CompletableFuture<Boolean> clusteredRoute =
+ autoCommit ? clustered.get() : CompletableFuture.completedFuture(false);
+ CompletableFuture<ByteBuf> sent = clusteredRoute
+ .handle((isClustered, error) -> {
+ if (error != null || cancellation.isCancelled()) {
+ payload.release();
+ return CompletableFuture.<ByteBuf>failedFuture(
+ error != null ? error : new CancellationException());
+ }
+ CompletableFuture<ByteBuf> deferred = pollRouter.pollDeferred(
+ payload, options, isClustered, waitDeadlineNanos, requestDeadlineNanos);
+ cancellation.track(deferred);
+ return deferred;
+ })
+ .thenCompose(deferred -> deferred);
+ CompletableFuture<PolledMessages> result = sent.thenApply(response -> {
+ try {
+ return readDeferredResponse(response, count);
+ } finally {
+ response.release();
+ }
+ });
+ result.whenComplete((response, error) -> {
+ if (result.isCancelled()) {
+ cancellation.cancel();
+ }
+ });
+ return result;
+ }
- var topicBytes = toBytes(topicId);
- payload.writeBytes(topicBytes);
+ /**
+ * A deferred reply is bounded before it is accumulated, so what remains to
+ * check is that the body describes itself consistently: a reply claiming
+ * more messages than it carries, or more than the caller asked for, must not
+ * escape as a partially decoded batch.
+ */
+ private static PolledMessages readDeferredResponse(ByteBuf response, Long count) {
+ if (response.readableBytes() < POLL_RESPONSE_PREFIX_BYTES) {
+ throw new IggyMalformedResponseException(
+ "Truncated deferred poll response: " + response.readableBytes() + " bytes");
+ }
+ PolledMessages polled = BytesDeserializer.readPolledMessages(response);
+ if (polled.messages().size() != polled.count() || polled.count() > count || response.isReadable()) {
+ throw new IggyMalformedResponseException(
+ "Deferred poll response advertised " + polled.count() + " messages for a request of " + count
+ + " but carried " + polled.messages().size());
+ }
+ return polled;
+ }
+ private static ByteBuf encodePoll(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit) {
+ var payload = Unpooled.buffer();
+ payload.writeBytes(toBytes(consumer));
+ payload.writeBytes(toBytes(streamId));
+ payload.writeBytes(toBytes(topicId));
payload.writeBytes(toBytes(partitionId));
-
- var strategyBytes = toBytes(strategy);
- payload.writeBytes(strategyBytes);
-
+ payload.writeBytes(toBytes(strategy));
payload.writeIntLE(count.intValue());
-
payload.writeByte(autoCommit ? 1 : 0);
+ return payload;
+ }
- // Send async request and transform response
- CompletableFuture<ByteBuf> sent = sendPoll(payload, autoCommit);
+ private CompletableFuture<PolledMessages> pollPartition(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit) {
+ CompletableFuture<ByteBuf> sent =
+ sendPoll(encodePoll(streamId, topicId, partitionId, consumer, strategy, count, autoCommit), autoCommit);
CompletableFuture<PolledMessages> result = sent.thenApply(response -> {
try {
return BytesDeserializer.readPolledMessages(response);
@@ -293,14 +433,11 @@ public class MessagesTcpClient implements MessagesClient {
* re-sync after the coordinator rejects a stale assignment, then one
* retry; an exhausted budget is an empty poll, not an error.
*/
- @SuppressWarnings("checkstyle:ParameterNumber")
private CompletableFuture<PolledMessages> pollGroupMessages(
StreamId streamId,
TopicId topicId,
Consumer consumer,
- PollingStrategy strategy,
- Long count,
- boolean autoCommit,
+ LongFunction<CompletableFuture<PolledMessages>> pollPartition,
int attemptsLeft,
PollCancellation cancellation) {
if (cancellation.isCancelled()) {
@@ -328,22 +465,14 @@ public class MessagesTcpClient implements MessagesClient {
Optional.empty(),
Optional.empty()));
}
- CompletableFuture<PolledMessages> polledPartition = pollPartition(
- streamId, topicId, Optional.of(partitionId.getAsLong()), consumer, strategy, count, autoCommit);
+ CompletableFuture<PolledMessages> polledPartition = pollPartition.apply(partitionId.getAsLong());
cancellation.track(polledPartition);
return polledPartition
.thenCompose(polled -> {
if (polled.messages().isEmpty() && polled.partitionId() == RESYNC_REQUIRED_PARTITION_SENTINEL) {
routingState.invalidateAssignment(groupKey);
return pollGroupMessages(
- streamId,
- topicId,
- consumer,
- strategy,
- count,
- autoCommit,
- attemptsLeft - 1,
- cancellation);
+ streamId, topicId, consumer, pollPartition, attemptsLeft - 1, cancellation);
}
return CompletableFuture.completedFuture(polled);
})
@@ -353,14 +482,7 @@ public class MessagesTcpClient implements MessagesClient {
}
routingState.invalidateAssignment(groupKey);
return pollGroupMessages(
- streamId,
- topicId,
- consumer,
- strategy,
- count,
- autoCommit,
- attemptsLeft - 1,
- cancellation);
+ streamId, topicId, consumer, pollPartition, attemptsLeft - 1, cancellation);
});
});
}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PollRouter.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PollRouter.java
index 6664dfefe..e90964257 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PollRouter.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PollRouter.java
@@ -30,15 +30,22 @@ import org.apache.iggy.exception.IggyMalformedResponseException;
import org.apache.iggy.exception.IggyNotConnectedException;
import org.apache.iggy.exception.IggyServerException;
import org.apache.iggy.exception.IggyTimeoutException;
+import org.apache.iggy.message.DeferredPollOptions;
import org.apache.iggy.serde.BytesDeserializer;
import org.apache.iggy.serde.CommandCode;
import java.io.IOException;
import java.time.Duration;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.TimeUnit;
@@ -58,11 +65,25 @@ final class PollRouter {
private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30);
private static final long RETRY_INTERVAL_MILLIS = 50;
+ /**
+ * Deferred exchanges hold their connection for the whole readiness wait, so
+ * each is an exclusive permit rather than a Netty pool lease. The bound
+ * matches the Rust SDK's {@code MAX_DEFERRED_CONNECTIONS}.
+ */
+ private static final int MAX_DEFERRED_LEASES = 16;
+
+ private static final int DEFERRED_TRAILER_BYTES = 24;
+ private static final long NANOS_PER_MICRO = 1000;
+ private static final long DEFERRED_RETRY_MAX_INTERVAL_MILLIS = 1000;
+
private final Supplier<AsyncTcpConnection> coordinator;
private final Function<ConnectionInfo, AsyncTcpConnection> connectData;
private final Map<String, Route> routes = new HashMap<>();
private final Map<ConnectionInfo, Slot> connections = new HashMap<>();
private final Set<Poll> pending = new HashSet<>();
+ private final Set<DeferredPoll> pendingDeferred = new HashSet<>();
+ private final List<DeferredLease> deferredLeases = new ArrayList<>();
+ private final Deque<LeaseWaiter> leaseWaiters = new ArrayDeque<>();
private long metadataWatermark;
PollRouter(Supplier<AsyncTcpConnection> coordinator, Function<ConnectionInfo, AsyncTcpConnection> connectData) {
@@ -87,12 +108,12 @@ final class PollRouter {
}
var timeout =
coordinator.get().eventLoop().schedule(poll::expire, POLL_TIMEOUT.toNanos(), TimeUnit.NANOSECONDS);
- poll.result.whenComplete((response, error) -> {
+ poll.result().whenComplete((response, error) -> {
timeout.cancel(false);
synchronized (this) {
pending.remove(poll);
if (error != null) {
- routes.remove(poll.key);
+ routes.remove(poll.key());
}
}
if (error != null) {
@@ -100,7 +121,64 @@ final class PollRouter {
}
});
attempt(poll);
- return poll.result;
+ return poll.result();
+ }
+
+ /**
+ * Runs one deferred poll on a dedicated data connection, so a readiness wait
+ * never occupies the coordinator's request drain.
+ *
+ * <p>A clustered auto-commit poll resolves its primary through command 103 and
+ * sends 106. Every other deferred poll reaches the coordinator's own endpoint
+ * on an auxiliary channel, attaches the parent's logical session, and sends
+ * 105. Both deadlines are the caller's: they are never restarted by a retry,
+ * and only an explicit non-admission permits one.
+ *
+ * @param payload the ordinary poll body, without the deferred trailer
+ * @param options the readiness, byte and request-time limits
+ * @param primary whether to route to the partition primary
+ * @param waitDeadlineNanos absolute {@link System#nanoTime()} readiness deadline
+ * @param requestDeadlineNanos absolute {@link System#nanoTime()} request deadline
+ * @return a future completing with the reply body
+ */
+ CompletableFuture<ByteBuf> pollDeferred(
+ ByteBuf payload,
+ DeferredPollOptions options,
+ boolean primary,
+ long waitDeadlineNanos,
+ long requestDeadlineNanos) {
+ DeferredPoll poll;
+ synchronized (this) {
+ if (pending.size() + pendingDeferred.size() >= MAX_PENDING_POLLS) {
+ payload.release();
+ return CompletableFuture.failedFuture(new IggyClientException("Too many pending deferred polls"));
+ }
+ try {
+ poll = new DeferredPoll(
+ deferredRouteKey(payload, primary),
+ ByteBufUtil.getBytes(payload),
+ options,
+ primary,
+ waitDeadlineNanos,
+ requestDeadlineNanos);
+ } finally {
+ payload.release();
+ }
+ pendingDeferred.add(poll);
+ }
+ poll.result().whenComplete((response, error) -> {
+ synchronized (this) {
+ pendingDeferred.remove(poll);
+ if (error != null && poll.key() != null) {
+ routes.remove(poll.key());
+ }
+ }
+ if (error != null) {
+ abandonLeaseWork(poll);
+ }
+ });
+ attemptDeferred(poll);
+ return poll.result();
}
synchronized CompletableFuture<Void> clearSession(AsyncTcpConnection previous) {
@@ -108,34 +186,59 @@ final class PollRouter {
observeMetadata(previous.metadataWatermark());
}
routes.clear();
+ failPending();
+ return CompletableFuture.allOf(closeSlots(), closeDeferredLeases());
+ }
+
+ private void failPending() {
for (Poll poll : Set.copyOf(pending)) {
- poll.result.completeExceptionally(uncommitted());
+ poll.result().completeExceptionally(uncommitted());
+ }
+ for (DeferredPoll poll : Set.copyOf(pendingDeferred)) {
+ poll.result().completeExceptionally(uncommitted());
}
+ for (LeaseWaiter waiter : List.copyOf(leaseWaiters)) {
+ waiter.acquired().completeExceptionally(uncommitted());
+ }
+ leaseWaiters.clear();
+ }
+
+ private CompletableFuture<Void> closeSlots() {
CompletableFuture<?>[] closing =
connections.values().stream().map(Slot::close).toArray(CompletableFuture[]::new);
connections.clear();
return CompletableFuture.allOf(closing);
}
+ private CompletableFuture<Void> closeDeferredLeases() {
+ CompletableFuture<?>[] closing = deferredLeases.stream()
+ .map(DeferredLease::take)
+ .filter(Objects::nonNull)
+ .map(AsyncTcpConnection::close)
+ .toArray(CompletableFuture[]::new);
+ deferredLeases.clear();
+ return CompletableFuture.allOf(closing);
+ }
+
private void attempt(Poll poll) {
if (!poll.beginAttempt()) {
return;
}
route(poll).thenCompose(route -> enqueue(route, poll)).whenComplete((response, error) -> {
- if (poll.result.isDone()) {
+ if (poll.result().isDone()) {
if (response != null) {
response.release();
}
return;
}
if (error == null) {
- if (!poll.result.complete(response)) {
+ if (!poll.result().complete(response)) {
response.release();
}
return;
}
synchronized (this) {
- routes.remove(poll.key);
+ routes.remove(poll.key());
}
if (isNotAccepted(error)) {
if (!poll.retryRefusal()) {
@@ -146,27 +249,365 @@ final class PollRouter {
.eventLoop()
.schedule(() -> attempt(poll), RETRY_INTERVAL_MILLIS, TimeUnit.MILLISECONDS);
} else {
- poll.result.completeExceptionally(unwrap(error));
+ poll.result().completeExceptionally(unwrap(error));
+ }
+ });
+ }
+
+ private void attemptDeferred(DeferredPoll poll) {
+ if (poll.result().isDone()) {
+ return;
+ }
+ if (poll.requestDeadlineNanos - System.nanoTime() <= 0) {
+ poll.result().completeExceptionally(deferredTimeout());
+ return;
+ }
+ deferredRoute(poll)
+ .thenCompose(route -> runDeferred(route, poll))
+ .whenComplete((response, error) -> completeDeferredAttempt(poll, response, error));
+ }
+
+ private void completeDeferredAttempt(DeferredPoll poll, ByteBuf response, Throwable error) {
+ if (poll.result().isDone()) {
+ if (response != null) {
+ response.release();
+ }
+ return;
+ }
+ if (error == null) {
+ if (!poll.result().complete(response)) {
+ response.release();
+ }
+ return;
+ }
+ synchronized (this) {
+ if (poll.key() != null) {
+ routes.remove(poll.key());
+ }
+ }
+ // Only an explicit refusal proves the poll was never admitted. Anything
+ // else - a lost reply, a disconnect, a malformed body - may have been
+ // served and auto-committed, so replaying it would skip messages.
+ if (!isNotAccepted(error) || !retryAfterRefusal(poll)) {
+ poll.result().completeExceptionally(unwrap(error));
+ }
+ }
+
+ /** Schedules the next attempt inside the original deadline, never past it. */
+ private boolean retryAfterRefusal(DeferredPoll poll) {
+ long backoffMillis = poll.nextBackoffMillis();
+ AsyncTcpConnection parent = coordinator.get();
+ long backoffNanos = TimeUnit.MILLISECONDS.toNanos(backoffMillis);
+ if (parent == null || poll.requestDeadlineNanos - System.nanoTime() <= backoffNanos) {
+ return false;
+ }
+ try {
+ parent.eventLoop().schedule(() -> attemptDeferred(poll), backoffMillis, TimeUnit.MILLISECONDS);
+ return true;
+ } catch (RuntimeException loopGone) {
+ return false;
+ }
+ }
+
+ /**
+ * A clustered auto-commit poll asks the coordinator which node owns the
+ * partition. Every other deferred poll stays on the coordinator's endpoint
+ * and builds its attachment locally: command 103 refuses a manual poll, so
+ * it cannot supply one.
+ */
+ private CompletableFuture<Route> deferredRoute(DeferredPoll poll) {
+ if (poll.primary) {
+ return route(poll);
+ }
+ AsyncTcpConnection parent = coordinator.get();
+ if (parent == null || !parent.isAuthenticated()) {
+ return CompletableFuture.failedFuture(new IggyNotConnectedException("Not authenticated, call login first"));
+ }
+ var snapshot = parent.sessionSnapshot();
+ if (snapshot.isEmpty()) {
+ return CompletableFuture.failedFuture(new IggyNotConnectedException("Not authenticated, call login first"));
+ }
+ var session = snapshot.get();
+ synchronized (this) {
+ observeMetadata(session.metadataWatermark());
+ Attachment attachment = new Attachment(
+ session.clientIdLow(),
+ session.clientIdHigh(),
+ session.session(),
+ session.metadataWatermark())
+ .withWatermark(metadataWatermark);
+ return CompletableFuture.completedFuture(
+ new Route(parent.endpoint(), attachment, parent, session.generation()));
+ }
+ }
+
+ private CompletableFuture<ByteBuf> runDeferred(Route route, DeferredPoll poll) {
+ return acquireLease(route, poll).thenCompose(lease -> prepareLease(lease, route, poll)
+ .thenCompose(ignored -> sendDeferred(lease, route, poll))
+ .whenComplete((response, error) -> releaseLease(lease, poll, error)));
+ }
+
+ /**
+ * Takes one of the 16 exclusive deferred permits. The Netty pool lease is
+ * released once a frame is dispatched, so it cannot express ownership that
+ * spans a readiness wait and its reply body.
+ */
+ private CompletableFuture<DeferredLease> acquireLease(Route route, DeferredPoll poll) {
+ CompletableFuture<DeferredLease> acquired = new CompletableFuture<>();
+ AsyncTcpConnection stale = null;
+ DeferredLease ready;
+ LeaseWaiter waiter = null;
+ synchronized (this) {
+ ready = idleLease(route.endpoint());
+ if (ready == null && deferredLeases.size() < MAX_DEFERRED_LEASES) {
+ ready = new DeferredLease(route.endpoint());
+ deferredLeases.add(ready);
+ }
+ if (ready == null) {
+ ready = idleLease(null);
+ if (ready != null) {
+ stale = ready.repoint(route.endpoint());
+ }
+ }
+ if (ready == null) {
+ waiter = new LeaseWaiter(route.endpoint(), poll, acquired);
+ leaseWaiters.add(waiter);
+ poll.waiter = waiter;
+ } else {
+ ready.owner = poll;
+ poll.lease = ready;
}
+ }
+ if (stale != null) {
+ stale.close();
+ }
+ if (ready != null) {
+ acquired.complete(ready);
+ return acquired;
+ }
+ return awaitLease(poll, waiter, acquired);
+ }
+
+ /** Requires the monitor. A null endpoint matches any idle lease. */
+ private DeferredLease idleLease(ConnectionInfo endpoint) {
+ for (DeferredLease lease : deferredLeases) {
+ if (lease.owner == null && (endpoint == null || endpoint.equals(lease.endpoint))) {
+ return lease;
+ }
+ }
+ return null;
+ }
+
+ private CompletableFuture<DeferredLease> awaitLease(
+ DeferredPoll poll, LeaseWaiter waiter, CompletableFuture<DeferredLease> acquired) {
+ AsyncTcpConnection parent = coordinator.get();
+ if (parent == null) {
+ failWaiter(waiter, notAccepted());
+ return acquired;
+ }
+ long remaining = Math.max(0, poll.requestDeadlineNanos - System.nanoTime());
+ try {
+ var expiry = parent.eventLoop()
+ .schedule(() -> failWaiter(waiter, deferredTimeout()), remaining, TimeUnit.NANOSECONDS);
+ acquired.whenComplete((lease, error) -> expiry.cancel(false));
+ } catch (RuntimeException loopGone) {
+ failWaiter(waiter, notAccepted());
+ }
+ return acquired;
+ }
+
+ private void failWaiter(LeaseWaiter waiter, Throwable error) {
+ boolean queued;
+ synchronized (this) {
+ queued = leaseWaiters.remove(waiter);
+ }
+ if (queued) {
+ waiter.acquired().completeExceptionally(error);
+ }
+ }
+
+ private CompletableFuture<Void> prepareLease(DeferredLease lease, Route route, DeferredPoll poll) {
+ if (poll.result().isDone() || !routeIsCurrent(route)) {
+ return CompletableFuture.failedFuture(notAccepted());
+ }
+ CompletableFuture<Void> ready;
+ if (lease.connection == null) {
+ AsyncTcpConnection data = connectData.apply(route.endpoint());
+ lease.connection = data;
+ ready = data.connectWithoutHeartbeat().thenCompose(ignored -> {
+ if (poll.result().isDone() || lease.connection != data) {
+ // A connection that finished dialling after the caller gave
+ // up is closed here rather than installed in the lease.
+ data.close();
+ return CompletableFuture.failedFuture(notAccepted());
+ }
+ var authentication = route.parent().authenticationSnapshot();
+ if (authentication.isEmpty()) {
+ return CompletableFuture.failedFuture(new IggyNotConnectedException("Not authenticated"));
+ }
+ var login = authentication.get();
+ return data.send(login.commandCode(), login.payload()).thenAccept(ByteBuf::release);
+ });
+ } else {
+ ready = CompletableFuture.completedFuture(null);
+ }
+ return ready.thenCompose(ignored -> attachLease(lease, route, poll)).exceptionallyCompose(error -> {
+ // Setup failed before the poll was written, so nothing was
+ // admitted and the caller may route again.
+ lease.discard(poll);
+ return CompletableFuture.failedFuture(connectionFailed(error) ? notAccepted() : unwrap(error));
});
}
- private CompletableFuture<Route> route(Poll poll) {
+ private CompletableFuture<Void> attachLease(DeferredLease lease, Route route, DeferredPoll poll) {
+ if (poll.result().isDone() || !routeIsCurrent(route)) {
+ return CompletableFuture.failedFuture(notAccepted());
+ }
+ AsyncTcpConnection data = lease.connection;
+ if (data == null) {
+ return CompletableFuture.failedFuture(notAccepted());
+ }
+ AttachedSession current = lease.attachment;
+ if (current != null && current.covers(data, route.attachment())) {
+ return CompletableFuture.completedFuture(null);
+ }
+ return data.send(
+ CommandCode.System.ATTACH_CONSUMER_SESSION,
+ route.attachment().encode())
+ .thenAccept(response -> {
+ response.release();
+ if (lease.connection == data
+ && lease.owner == poll
+ && !poll.result().isDone()) {
+ lease.attachment = new AttachedSession(data, data.sessionGeneration(), route.attachment());
+ }
+ });
+ }
+
+ private CompletableFuture<ByteBuf> sendDeferred(DeferredLease lease, Route route, DeferredPoll poll) {
+ if (poll.result().isDone() || !routeIsCurrent(route)) {
+ return CompletableFuture.failedFuture(notAccepted());
+ }
+ AsyncTcpConnection data = lease.connection;
+ if (data == null) {
+ return CompletableFuture.failedFuture(notAccepted());
+ }
+ // Both budgets are reduced by the same elapsed time before every send,
+ // so a retry inherits what is left instead of restarting the wait.
+ long now = System.nanoTime();
+ long requestTimeoutMicros = Math.max(0, poll.requestDeadlineNanos - now) / NANOS_PER_MICRO;
+ if (requestTimeoutMicros == 0) {
+ return CompletableFuture.failedFuture(deferredTimeout());
+ }
+ long waitMicros = Math.max(0, poll.waitDeadlineNanos - now) / NANOS_PER_MICRO;
+ int code = poll.primary
+ ? CommandCode.Messages.POLL_DEFERRED_ON_PRIMARY.getValue()
+ : CommandCode.Messages.POLL_DEFERRED.getValue();
+ ByteBuf wire = encodeDeferred(poll, waitMicros, requestTimeoutMicros);
+ // Once the poll is on the wire its outcome is reported as it happened: a
+ // locally expired budget stays a timeout and a server code stays that
+ // code. Nothing here is replayed, so no failure needs relabelling.
+ return data.sendDeferredPoll(code, wire, poll.requestDeadlineNanos, poll.options.maxBytes());
+ }
+
+ private static ByteBuf encodeDeferred(DeferredPoll poll, long waitMicros, long requestTimeoutMicros) {
+ return Unpooled.buffer(poll.payload().length + DEFERRED_TRAILER_BYTES)
+ .writeBytes(poll.payload())
+ .writeLongLE(waitMicros)
+ .writeIntLE((int) poll.options.minCount())
+ .writeIntLE((int) poll.options.maxBytes())
+ .writeLongLE(requestTimeoutMicros);
+ }
+
+ private void releaseLease(DeferredLease lease, DeferredPoll poll, Throwable error) {
+ AsyncTcpConnection discarded;
+ LeaseWaiter next;
+ synchronized (this) {
+ if (lease.owner != poll) {
+ return;
+ }
+ discarded = settleLease(lease, poll, error);
+ lease.owner = null;
+ next = leaseWaiters.poll();
+ if (next != null) {
+ discarded = handOverLease(lease, next, discarded);
+ }
+ }
+ if (discarded != null) {
+ discarded.close();
+ }
+ if (next != null) {
+ next.acquired().complete(lease);
+ }
+ }
+
+ /** Requires the monitor. Returns a connection the caller must close. */
+ private static AsyncTcpConnection settleLease(DeferredLease lease, DeferredPoll poll, Throwable error) {
+ if (error == null) {
+ return null;
+ }
+ if (connectionFailed(error) || poll.result().isCancelled()) {
+ return lease.take();
+ }
+ if (isNotAccepted(error)) {
+ // The server refused this exchange, so the attachment it was made
+ // under is no longer known to be installed.
+ lease.attachment = null;
+ }
+ return null;
+ }
+
+ /**
+ * Requires the monitor. A repointed lease has no connection left to take, so
+ * at most one connection needs closing.
+ */
+ private static AsyncTcpConnection handOverLease(
+ DeferredLease lease, LeaseWaiter next, AsyncTcpConnection discarded) {
+ AsyncTcpConnection repointed = next.endpoint().equals(lease.endpoint) ? null : lease.repoint(next.endpoint());
+ lease.owner = next.poll();
+ next.poll().lease = lease;
+ return discarded != null ? discarded : repointed;
+ }
+
+ private void abandonLeaseWork(DeferredPoll poll) {
+ LeaseWaiter waiter = poll.waiter;
+ if (waiter != null) {
+ failWaiter(waiter, new CancellationException());
+ }
+ DeferredLease lease = poll.lease;
+ if (lease != null) {
+ // A reply that arrives for a poll nobody waits for would desynchronize
+ // the next exchange on this channel, so the channel goes with it.
+ lease.discard(poll);
+ }
+ }
+
+ private static String deferredRouteKey(ByteBuf payload, boolean primary) {
+ return primary
+ ? ByteBufUtil.hexDump(payload, payload.readerIndex(), payload.readableBytes() - POLL_PARAMETERS_BYTES)
+ : null;
+ }
+
+ private static IggyTimeoutException deferredTimeout() {
+ return new IggyTimeoutException("Deferred poll exceeded its request timeout");
+ }
+
+ private CompletableFuture<Route> route(RoutedPoll poll) {
AsyncTcpConnection parent = coordinator.get();
if (!parent.isAuthenticated()) {
return CompletableFuture.failedFuture(new IggyNotConnectedException("Not authenticated, call login first"));
}
synchronized (this) {
observeMetadata(parent.metadataWatermark());
- Route cached = routes.get(poll.key);
+ Route cached = routes.get(poll.key());
if (cached != null && routeIsCurrent(cached)) {
return CompletableFuture.completedFuture(cached);
}
}
- return parent.send(CommandCode.Messages.GET_POLL_ROUTING, Unpooled.wrappedBuffer(poll.payload))
+ return parent.send(CommandCode.Messages.GET_POLL_ROUTING, Unpooled.wrappedBuffer(poll.payload()))
.thenApply(response -> decodeRoute(parent, poll, response))
.exceptionallyCompose(error -> {
- if (!connectionFailed(error) || poll.result.isDone()) {
+ if (!connectionFailed(error) || poll.result().isDone()) {
return CompletableFuture.failedFuture(unwrap(error));
}
return coordinator
@@ -179,7 +620,7 @@ final class PollRouter {
});
}
- private Route decodeRoute(AsyncTcpConnection parent, Poll poll, ByteBuf response) {
+ private Route decodeRoute(AsyncTcpConnection parent, RoutedPoll poll, ByteBuf response) {
try {
if (response.readableBytes() < ATTACHMENT_BYTES) {
throw new IggyClientException("Truncated primary poll session attachment");
@@ -200,8 +641,8 @@ final class PollRouter {
if (routes.size() >= MAX_ROUTES) {
routes.clear();
}
- if (!poll.result.isDone() && coordinator.get() == parent) {
- routes.put(poll.key, route);
+ if (!poll.result().isDone() && coordinator.get() == parent) {
+ routes.put(poll.key(), route);
}
return route;
}
@@ -305,7 +746,7 @@ final class PollRouter {
}
private CompletableFuture<ByteBuf> pollOnConnection(Route route, Poll poll) {
- if (poll.result.isDone() || !routeIsCurrent(route)) {
+ if (poll.result().isDone() || !routeIsCurrent(route)) {
return CompletableFuture.failedFuture(notAccepted());
}
synchronized (this) {
@@ -327,7 +768,7 @@ final class PollRouter {
}
private CompletableFuture<ByteBuf> sendPoll(Route route, Poll poll) {
- if (poll.result.isDone() || !routeIsCurrent(route)) {
+ if (poll.result().isDone() || !routeIsCurrent(route)) {
return CompletableFuture.failedFuture(notAccepted());
}
AsyncTcpConnection data = connection;
@@ -335,7 +776,7 @@ final class PollRouter {
if (data == null || attached == null) {
return CompletableFuture.failedFuture(notAccepted());
}
- return data.sendPrimaryPoll(Unpooled.wrappedBuffer(poll.payload), attached.generation)
+ return data.sendPrimaryPoll(Unpooled.wrappedBuffer(poll.payload()), attached.generation)
.whenComplete((response, error) -> {
if (error != null) {
if (isNotAccepted(error)) {
@@ -354,12 +795,12 @@ final class PollRouter {
if (connection == null) {
AsyncTcpConnection data = connectData.apply(route.endpoint);
connection = data;
- if (poll.result.isDone()) {
+ if (poll.result().isDone()) {
close();
return CompletableFuture.failedFuture(uncommitted());
}
ready = data.connect().thenCompose(ignored -> {
- if (poll.result.isDone()) {
+ if (poll.result().isDone()) {
return CompletableFuture.failedFuture(uncommitted());
}
var authentication = route.parent.authenticationSnapshot();
@@ -376,7 +817,7 @@ final class PollRouter {
}
private CompletableFuture<Void> attach(Route route, Poll poll) {
- if (poll.result.isDone() || !routeIsCurrent(route)) {
+ if (poll.result().isDone() || !routeIsCurrent(route)) {
return CompletableFuture.failedFuture(notAccepted());
}
AsyncTcpConnection data = connection;
@@ -391,7 +832,9 @@ final class PollRouter {
.thenAccept(response -> {
response.release();
synchronized (this) {
- if (connection == data && activePoll == poll && !poll.result.isDone()) {
+ if (connection == data
+ && activePoll == poll
+ && !poll.result().isDone()) {
attachment = new AttachedSession(data, data.sessionGeneration(), route.attachment);
}
}
@@ -424,17 +867,40 @@ final class PollRouter {
}
}
- private static final class Poll {
+ /**
+ * A poll that resolves a route through command 103 and replays nothing on
+ * its own: the key identifies the route cache entry, never a dedup slot.
+ */
+ private abstract static class RoutedPoll {
private final String key;
private final byte[] payload;
private final CompletableFuture<ByteBuf> result = new CompletableFuture<>();
+
+ RoutedPoll(String key, byte[] payload) {
+ this.key = key;
+ this.payload = payload;
+ }
+
+ String key() {
+ return key;
+ }
+
+ byte[] payload() {
+ return payload;
+ }
+
+ CompletableFuture<ByteBuf> result() {
+ return result;
+ }
+ }
+
+ private static final class Poll extends RoutedPoll {
private final long deadline = System.nanoTime() + POLL_TIMEOUT.toNanos();
private volatile Slot activeSlot;
private boolean retryingRefusal;
private Poll(String key, byte[] payload) {
- this.key = key;
- this.payload = payload;
+ super(key, payload);
}
private void discardConnection() {
@@ -445,7 +911,7 @@ final class PollRouter {
}
private synchronized boolean beginAttempt() {
- if (result.isDone()) {
+ if (result().isDone()) {
return false;
}
retryingRefusal = false;
@@ -455,17 +921,90 @@ final class PollRouter {
private synchronized boolean retryRefusal() {
retryingRefusal = true;
if (deadline - System.nanoTime() <= TimeUnit.MILLISECONDS.toNanos(RETRY_INTERVAL_MILLIS)) {
- result.completeExceptionally(notAccepted());
+ result().completeExceptionally(notAccepted());
return false;
}
- return !result.isDone();
+ return !result().isDone();
}
private synchronized void expire() {
- result.completeExceptionally(retryingRefusal ? notAccepted() : uncommitted());
+ result().completeExceptionally(retryingRefusal ? notAccepted() : uncommitted());
+ }
+ }
+
+ private static final class DeferredPoll extends RoutedPoll {
+ private final DeferredPollOptions options;
+ private final boolean primary;
+ private final long waitDeadlineNanos;
+ private final long requestDeadlineNanos;
+ private long retryIntervalMillis = RETRY_INTERVAL_MILLIS;
+ private volatile LeaseWaiter waiter;
+ private volatile DeferredLease lease;
+
+ private DeferredPoll(
+ String key,
+ byte[] payload,
+ DeferredPollOptions options,
+ boolean primary,
+ long waitDeadlineNanos,
+ long requestDeadlineNanos) {
+ super(key, payload);
+ this.options = options;
+ this.primary = primary;
+ this.waitDeadlineNanos = waitDeadlineNanos;
+ this.requestDeadlineNanos = requestDeadlineNanos;
+ }
+
+ private synchronized long nextBackoffMillis() {
+ long current = retryIntervalMillis;
+ retryIntervalMillis = Math.min(retryIntervalMillis * 2, DEFERRED_RETRY_MAX_INTERVAL_MILLIS);
+ return current;
+ }
+ }
+
+ /**
+ * One exclusive deferred permit and the connection it currently owns. The
+ * owner and endpoint are guarded by the router's monitor; the connection and
+ * attachment are read by the owning exchange and cleared by a cancellation.
+ */
+ private static final class DeferredLease {
+ private volatile ConnectionInfo endpoint;
+ private volatile AsyncTcpConnection connection;
+ private volatile AttachedSession attachment;
+ private volatile DeferredPoll owner;
+
+ private DeferredLease(ConnectionInfo endpoint) {
+ this.endpoint = endpoint;
+ }
+
+ private AsyncTcpConnection take() {
+ AsyncTcpConnection previous = connection;
+ connection = null;
+ attachment = null;
+ return previous;
+ }
+
+ private AsyncTcpConnection repoint(ConnectionInfo next) {
+ endpoint = next;
+ return take();
+ }
+
+ private void discard(DeferredPoll poll) {
+ AsyncTcpConnection previous;
+ synchronized (this) {
+ if (owner != poll) {
+ return;
+ }
+ previous = take();
+ }
+ if (previous != null) {
+ previous.close();
+ }
}
}
+ private record LeaseWaiter(ConnectionInfo endpoint, DeferredPoll poll, CompletableFuture<DeferredLease> acquired) {}
+
private record AttachedSession(AsyncTcpConnection connection, long generation, Attachment attachment) {
boolean covers(AsyncTcpConnection data, Attachment required) {
return connection == data && generation == data.sessionGeneration() && attachment.covers(required);
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java
index 21586f33a..668f8da43 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java
@@ -22,6 +22,7 @@ package org.apache.iggy.client.async.tcp.vsr;
import org.apache.iggy.exception.IggyNotConnectedException;
import java.security.SecureRandom;
+import java.util.Optional;
/**
* VSR client identity and dedup state, mirroring
@@ -151,6 +152,21 @@ public final class ConsensusSession {
}
}
+ /**
+ * The identity a deferred data connection attaches to its parent, read in
+ * one critical section. Taken field by field, a re-login in between would
+ * pair a fresh client id with the previous epoch and the server would
+ * refuse the attachment, or worse accept a mismatched one.
+ *
+ * @return the snapshot, or empty while no session is bound
+ */
+ public synchronized Optional<Snapshot> snapshot() {
+ if (session == null) {
+ return Optional.empty();
+ }
+ return Optional.of(new Snapshot(clientIdLow, clientIdHigh, session, metadataWatermark, generation));
+ }
+
synchronized long clientIdLow() {
return clientIdLow;
}
@@ -165,4 +181,17 @@ public final class ConsensusSession {
clientIdHigh = RANDOM.nextLong();
} while (clientIdLow == 0 && clientIdHigh == 0);
}
+
+ /**
+ * A bound session, its client id halves and the metadata watermark the
+ * client has seen. The generation is local fencing state, not a wire field.
+ *
+ * @param clientIdLow low half of the {@code u128} client id
+ * @param clientIdHigh high half of the {@code u128} client id
+ * @param session the bound fence epoch
+ * @param metadataWatermark the highest observed metadata commit
+ * @param generation the local session generation
+ */
+ public record Snapshot(
+ long clientIdLow, long clientIdHigh, long session, long metadataWatermark, long generation) {}
}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java
index 7398f527f..d939e7dfb 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java
@@ -25,6 +25,7 @@ import io.netty.handler.codec.ByteToMessageDecoder;
import io.netty.handler.codec.DecoderException;
import java.util.List;
+import java.util.function.IntSupplier;
/**
* Decoder for VSR response frames: a 256-byte consensus header whose total
@@ -36,18 +37,34 @@ public class VsrFrameDecoder extends ByteToMessageDecoder {
/** Matches the server's default {@code max_message_size} (64 MB). */
public static final int DEFAULT_MAX_FRAME_SIZE = 64 * 1024 * 1024;
- private final int maxFrameSize;
+ private final IntSupplier maxFrameSize;
public VsrFrameDecoder() {
this(DEFAULT_MAX_FRAME_SIZE);
}
public VsrFrameDecoder(int maxFrameSize) {
+ this(fixed(maxFrameSize));
+ }
+
+ /**
+ * A deferred poll declares how many response bytes it will accept, and that
+ * bound has to reach the decoder before it accumulates a body, not after.
+ * The supplier is read per frame so one exchange can tighten the limit and
+ * the connection's ordinary replies keep the configured ceiling.
+ *
+ * @param maxFrameSize the current frame ceiling, never below the header size
+ */
+ public VsrFrameDecoder(IntSupplier maxFrameSize) {
+ this.maxFrameSize = maxFrameSize;
+ }
+
+ private static IntSupplier fixed(int maxFrameSize) {
if (maxFrameSize < VsrHeaders.HEADER_SIZE) {
throw new IllegalArgumentException(
"Maximum VSR frame size must be at least " + VsrHeaders.HEADER_SIZE + " bytes");
}
- this.maxFrameSize = maxFrameSize;
+ return () -> maxFrameSize;
}
@Override
@@ -56,7 +73,7 @@ public class VsrFrameDecoder extends ByteToMessageDecoder {
return;
}
long totalSize = VsrHeaders.readSize(in);
- if (totalSize < VsrHeaders.HEADER_SIZE || totalSize > maxFrameSize) {
+ if (totalSize < VsrHeaders.HEADER_SIZE || totalSize > maxFrameSize.getAsInt()) {
throw new DecoderException("Invalid VSR frame size " + totalSize + ", connection is desynchronized");
}
if (in.readableBytes() < totalSize) {
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java
index 809912676..c8f8a580b 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java
@@ -20,8 +20,10 @@
package org.apache.iggy.client.blocking;
import org.apache.iggy.consumergroup.Consumer;
+import org.apache.iggy.exception.IggyOperationNotSupportedException;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
import org.apache.iggy.message.Message;
import org.apache.iggy.message.Partitioning;
import org.apache.iggy.message.PolledMessages;
@@ -60,6 +62,79 @@ public interface MessagesClient {
Long count,
boolean autoCommit);
+ /**
+ * Polls messages with a deferred wait using numeric identifiers.
+ *
+ * <p>See {@link #pollMessagesDeferred(StreamId, TopicId, Optional, Consumer, PollingStrategy,
+ * Long, boolean, DeferredPollOptions)} for full documentation.
+ *
+ * @param streamId the numeric stream ID
+ * @param topicId the numeric topic ID
+ * @param partitionId optional partition ID
+ * @param consumerId the numeric consumer ID
+ * @param strategy the polling strategy
+ * @param count the maximum number of messages to return
+ * @param autoCommit whether to auto-commit offsets
+ * @param options the readiness, byte and request-time limits
+ * @return the polled messages
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ default PolledMessages pollMessagesDeferred(
+ Long streamId,
+ Long topicId,
+ Optional<Long> partitionId,
+ Long consumerId,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit,
+ DeferredPollOptions options) {
+ return pollMessagesDeferred(
+ StreamId.of(streamId),
+ TopicId.of(topicId),
+ partitionId,
+ Consumer.of(consumerId),
+ strategy,
+ count,
+ autoCommit,
+ options);
+ }
+
+ /**
+ * Polls messages, letting the server hold the request until data is ready.
+ *
+ * <p>Unlike {@link #pollMessages}, which reads whatever is resident and returns, this
+ * waits up to {@link DeferredPollOptions#maxWait()} for {@link DeferredPollOptions#minCount()}
+ * messages, caps the encoded response at {@link DeferredPollOptions#maxBytes()}, and bounds
+ * the whole exchange by {@link DeferredPollOptions#requestTimeout()}. An expired readiness
+ * wait can still return partial or empty data; an expired request timeout is an error.
+ *
+ * <p>The HTTP client accepts only a plain consumer, because its JSON poll query cannot
+ * carry a consumer kind. A group consumer is rejected before any request is sent.
+ *
+ * @param streamId the stream identifier (numeric or string-based)
+ * @param topicId the topic identifier (numeric or string-based)
+ * @param partitionId optional partition ID to poll from
+ * @param consumer the consumer identity
+ * @param strategy the polling strategy controlling where to start reading
+ * @param count the maximum number of messages to return
+ * @param autoCommit whether the server should automatically commit the consumer offset
+ * @param options the readiness, byte and request-time limits
+ * @return the polled messages
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ default PolledMessages pollMessagesDeferred(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit,
+ DeferredPollOptions options) {
+ throw new IggyOperationNotSupportedException(
+ "pollMessagesDeferred", getClass().getSimpleName());
+ }
+
default SendMessagesResponse sendMessages(
Long streamId, Long topicId, Partitioning partitioning, List<Message> messages) {
return sendMessages(StreamId.of(streamId), TopicId.of(topicId), partitioning, messages);
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/InternalHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/InternalHttpClient.java
index 77e2a64ad..1184eb080 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/InternalHttpClient.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/InternalHttpClient.java
@@ -24,16 +24,22 @@ import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.ssl.DefaultClientTlsStrategy;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
+import org.apache.iggy.exception.IggyClientException;
import org.apache.iggy.exception.IggyConnectionException;
+import org.apache.iggy.exception.IggyMalformedResponseException;
import org.apache.iggy.exception.IggyServerException;
+import org.apache.iggy.exception.IggyTimeoutException;
import org.apache.iggy.exception.IggyTlsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,21 +48,39 @@ import tools.jackson.databind.JavaType;
import tools.jackson.databind.ObjectMapper;
import tools.jackson.databind.node.ObjectNode;
+import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.security.GeneralSecurityException;
import java.time.Duration;
import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
final class InternalHttpClient implements Closeable {
private static final Logger log = LoggerFactory.getLogger(InternalHttpClient.class);
private static final String AUTHORIZATION = "Authorization";
+
+ /**
+ * Deferred requests occupy their connection for the whole readiness wait, so
+ * they get their own pool and can never starve control traffic. The bound
+ * matches the binary client's deferred lease count.
+ */
+ private static final int MAX_DEFERRED_CONNECTIONS = 16;
+
+ /** Mirrors the binary client's pending-poll admission cap. */
+ private static final int MAX_PENDING_DEFERRED_REQUESTS = 4096;
+
+ private static final int BODY_CHUNK_BYTES = 8 * 1024;
+
private final String url;
private final ObjectMapper objectMapper = ObjectMapperFactory.getInstance();
private final CloseableHttpClient httpClient;
+ private final CloseableHttpClient deferredHttpClient;
+ private final AtomicInteger pendingDeferred = new AtomicInteger();
private Optional<String> token = Optional.empty();
InternalHttpClient(
@@ -67,6 +91,7 @@ final class InternalHttpClient implements Closeable {
UrlValidator.validateHttpUrl(url);
this.url = url;
this.httpClient = createHttpClient(connectionTimeout, requestTimeout, tlsCertificate);
+ this.deferredHttpClient = createDeferredHttpClient(connectionTimeout, tlsCertificate);
}
private static CloseableHttpClient createHttpClient(
@@ -77,16 +102,7 @@ final class InternalHttpClient implements Closeable {
var connectionManagerBuilder = PoolingHttpClientConnectionManagerBuilder.create()
.setDefaultConnectionConfig(connectionConfigBuilder.build());
- tlsCertificate.ifPresent(cert -> {
- try {
- var sslContext =
- SSLContextBuilder.create().loadTrustMaterial(cert, null).build();
- var tlsStrategy = new DefaultClientTlsStrategy(sslContext);
- connectionManagerBuilder.setTlsSocketStrategy(tlsStrategy);
- } catch (GeneralSecurityException | IOException e) {
- throw new IggyTlsException("Failed to configure TLS certificate", e);
- }
- });
+ applyTlsCertificate(connectionManagerBuilder, tlsCertificate);
var requestConfigBuilder = RequestConfig.custom();
requestTimeout.ifPresent(timeout -> requestConfigBuilder.setResponseTimeout(Timeout.of(timeout)));
@@ -97,9 +113,119 @@ final class InternalHttpClient implements Closeable {
.build();
}
+ /**
+ * A deferred poll declares its own total budget, so the library's response
+ * timeout is set from the caller's remaining time rather than a shared
+ * setting, and automatic retries are off: the server answers both
+ * {@code TRANSIENT_NOT_ACCEPTED} and the ambiguous
+ * {@code TRANSIENT_NOT_COMMITTED} with HTTP 503, and a proxy can answer it
+ * too, so only a parsed server code may authorize a replay.
+ *
+ * @param request the prepared deferred request
+ * @param clazz the success body type
+ * @param deadlineNanos absolute {@link System#nanoTime()} deadline for the whole exchange
+ * @param maxBodyBytes the largest encoded body this client will accumulate
+ * @param <T> the success body type
+ * @return the decoded body
+ */
+ <T> T executeDeferred(ClassicHttpRequest request, Class<T> clazz, long deadlineNanos, long maxBodyBytes) {
+ if (pendingDeferred.incrementAndGet() > MAX_PENDING_DEFERRED_REQUESTS) {
+ pendingDeferred.decrementAndGet();
+ throw new IggyClientException("Too many pending deferred HTTP polls");
+ }
+ try {
+ long remainingNanos = deadlineNanos - System.nanoTime();
+ if (remainingNanos <= 0) {
+ throw new IggyTimeoutException("Deferred poll exceeded its request timeout");
+ }
+ var context = HttpClientContext.create();
+ context.setRequestConfig(RequestConfig.custom()
+ .setConnectionRequestTimeout(Timeout.ofNanoseconds(remainingNanos))
+ .setResponseTimeout(Timeout.ofNanoseconds(remainingNanos))
+ .build());
+ return deferredHttpClient.execute(request, context, response -> {
+ byte[] body = readBounded(response.getEntity(), deadlineNanos, maxBodyBytes);
+ if (!isSuccessful(response.getCode())) {
+ throw toServerException(body);
+ }
+ return objectMapper.readValue(body, clazz);
+ });
+ } catch (IOException e) {
+ throw new IggyConnectionException("Deferred HTTP request failed", e);
+ } finally {
+ pendingDeferred.decrementAndGet();
+ }
+ }
+
+ /**
+ * Reads the whole body under the caller's deadline. A response timeout only
+ * bounds inactivity, so a peer that keeps trickling bytes would otherwise
+ * hold the caller past its budget. Throwing here aborts the exchange, which
+ * discards the connection and frees its pool slot.
+ */
+ private static byte[] readBounded(HttpEntity entity, long deadlineNanos, long maxBodyBytes) throws IOException {
+ if (entity == null) {
+ return new byte[0];
+ }
+ if (entity.getContentLength() > maxBodyBytes) {
+ throw oversized(entity.getContentLength(), maxBodyBytes);
+ }
+ var body = new ByteArrayOutputStream();
+ byte[] chunk = new byte[BODY_CHUNK_BYTES];
+ try (InputStream content = entity.getContent()) {
+ int read;
+ while ((read = content.read(chunk)) >= 0) {
+ if (Thread.interrupted()) {
+ Thread.currentThread().interrupt();
+ throw new IggyClientException("Interrupted while reading a deferred poll response");
+ }
+ if (deadlineNanos - System.nanoTime() <= 0) {
+ throw new IggyTimeoutException("Deferred poll exceeded its request timeout while reading");
+ }
+ if (body.size() + read > maxBodyBytes) {
+ throw oversized((long) body.size() + read, maxBodyBytes);
+ }
+ body.write(chunk, 0, read);
+ }
+ }
+ return body.toByteArray();
+ }
+
+ private static IggyMalformedResponseException oversized(long size, long limit) {
+ return new IggyMalformedResponseException(
+ "Deferred poll response body of " + size + " bytes exceeds the limit of " + limit);
+ }
+
+ private IggyServerException toServerException(byte[] body) {
+ ObjectNode errorNode = objectMapper.readValue(body, ObjectNode.class);
+ return IggyServerException.fromHttpResponse(
+ textOrNull(errorNode, "id"),
+ textOrNull(errorNode, "code"),
+ textOrNull(errorNode, "reason"),
+ textOrNull(errorNode, "field"));
+ }
+
+ private static String textOrNull(ObjectNode node, String field) {
+ return node.has(field) ? node.get(field).asString() : null;
+ }
+
@Override
public void close() throws IOException {
httpClient.close();
+ deferredHttpClient.close();
+ }
+
+ private static void applyTlsCertificate(
+ PoolingHttpClientConnectionManagerBuilder connectionManagerBuilder, Optional<File> tlsCertificate) {
+ tlsCertificate.ifPresent(cert -> {
+ try {
+ var sslContext =
+ SSLContextBuilder.create().loadTrustMaterial(cert, null).build();
+ connectionManagerBuilder.setTlsSocketStrategy(new DefaultClientTlsStrategy(sslContext));
+ } catch (GeneralSecurityException | IOException e) {
+ throw new IggyTlsException("Failed to configure TLS certificate", e);
+ }
+ });
}
void setToken(Optional<String> token) {
@@ -145,6 +271,22 @@ final class InternalHttpClient implements Closeable {
});
}
+ private static CloseableHttpClient createDeferredHttpClient(
+ Optional<Duration> connectionTimeout, Optional<File> tlsCertificate) {
+ var connectionConfigBuilder = ConnectionConfig.custom();
+ connectionTimeout.ifPresent(timeout -> connectionConfigBuilder.setConnectTimeout(Timeout.of(timeout)));
+ var connectionManagerBuilder = PoolingHttpClientConnectionManagerBuilder.create()
+ .setDefaultConnectionConfig(connectionConfigBuilder.build())
+ .setMaxConnTotal(MAX_DEFERRED_CONNECTIONS)
+ .setMaxConnPerRoute(MAX_DEFERRED_CONNECTIONS);
+ applyTlsCertificate(connectionManagerBuilder, tlsCertificate);
+ return HttpClients.custom()
+ .setConnectionManager(connectionManagerBuilder.build())
+ .disableAutomaticRetries()
+ .evictIdleConnections(TimeValue.ofSeconds(30))
+ .build();
+ }
+
private <T> T executeRequest(ClassicHttpRequest request, HttpClientResponseHandler<T> responseHandler) {
try {
return httpClient.execute(request, responseHandler);
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java
index 998637f4c..db08dcae2 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java
@@ -19,11 +19,18 @@
package org.apache.iggy.client.blocking.http;
+import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.iggy.client.blocking.MessagesClient;
import org.apache.iggy.consumergroup.Consumer;
+import org.apache.iggy.exception.IggyClientException;
+import org.apache.iggy.exception.IggyErrorCode;
+import org.apache.iggy.exception.IggyOperationNotSupportedException;
+import org.apache.iggy.exception.IggyServerException;
+import org.apache.iggy.exception.IggyTimeoutException;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
import org.apache.iggy.message.Message;
import org.apache.iggy.message.Partitioning;
import org.apache.iggy.message.PolledMessages;
@@ -32,9 +39,21 @@ import org.apache.iggy.message.SendMessagesResponse;
import java.util.List;
import java.util.Optional;
+import java.util.concurrent.TimeUnit;
class MessagesHttpClient implements MessagesClient {
+ /**
+ * The server applies the byte limit to the equivalent binary selection and
+ * then encodes it as JSON, which is larger. Rust bounds the JSON body at the
+ * same multiple.
+ */
+ private static final long JSON_BODY_EXPANSION = 16;
+
+ private static final long INITIAL_RETRY_INTERVAL_MILLIS = 50;
+ private static final long MAX_RETRY_INTERVAL_MILLIS = 1000;
+ private static final long NANOS_PER_MICRO = 1000;
+
private final InternalHttpClient httpClient;
public MessagesHttpClient(InternalHttpClient httpClient) {
@@ -63,6 +82,102 @@ class MessagesHttpClient implements MessagesClient {
return httpClient.execute(request, PolledMessages.class);
}
+ @Override
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public PolledMessages pollMessagesDeferred(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit,
+ DeferredPollOptions options) {
+ options.validate(count);
+ if (consumer.kind() == Consumer.Kind.ConsumerGroup) {
+ // The poll query has no consumer kind field, so the server would read
+ // a group id as a plain consumer and poll the wrong thing.
+ throw new IggyOperationNotSupportedException(
+ "A consumer group cannot poll over HTTP; the poll query carries no consumer kind");
+ }
+ long startNanos = System.nanoTime();
+ long waitDeadlineNanos = startNanos + options.maxWait().toNanos();
+ long requestDeadlineNanos = startNanos + options.requestTimeout().toNanos();
+ long maxBodyBytes = Math.multiplyExact(options.maxBytes(), JSON_BODY_EXPANSION);
+ long retryIntervalMillis = INITIAL_RETRY_INTERVAL_MILLIS;
+ while (true) {
+ ClassicHttpRequest request = deferredRequest(
+ streamId,
+ topicId,
+ partitionId,
+ consumer,
+ strategy,
+ count,
+ autoCommit,
+ options,
+ waitDeadlineNanos,
+ requestDeadlineNanos);
+ try {
+ return httpClient.executeDeferred(request, PolledMessages.class, requestDeadlineNanos, maxBodyBytes);
+ } catch (IggyServerException refused) {
+ // Only an explicit non-admission proves nothing was served. An
+ // opaque 503 can be an ambiguous not-committed or a proxy fault.
+ if (refused.getRawErrorCode() != IggyErrorCode.TRANSIENT_NOT_ACCEPTED.getCode()) {
+ throw refused;
+ }
+ sleepBeforeRetry(retryIntervalMillis, requestDeadlineNanos, refused);
+ retryIntervalMillis = Math.min(retryIntervalMillis * 2, MAX_RETRY_INTERVAL_MILLIS);
+ }
+ }
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ private ClassicHttpRequest deferredRequest(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit,
+ DeferredPollOptions options,
+ long waitDeadlineNanos,
+ long requestDeadlineNanos) {
+ // Both budgets carry across retries rather than restarting.
+ long now = System.nanoTime();
+ long requestTimeoutMicros = Math.max(0, requestDeadlineNanos - now) / NANOS_PER_MICRO;
+ if (requestTimeoutMicros == 0) {
+ throw new IggyTimeoutException("Deferred poll exceeded its request timeout");
+ }
+ long waitMicros = Math.max(0, waitDeadlineNanos - now) / NANOS_PER_MICRO;
+ return httpClient.prepareGetRequest(
+ path(streamId, topicId) + "/deferred",
+ new BasicNameValuePair("consumer_id", consumer.id().toString()),
+ partitionId
+ .map(id -> new BasicNameValuePair("partition_id", id.toString()))
+ .orElse(null),
+ new BasicNameValuePair("kind", strategy.kind().name().toLowerCase()),
+ new BasicNameValuePair("value", strategy.value().toString()),
+ new BasicNameValuePair("count", count.toString()),
+ new BasicNameValuePair("auto_commit", Boolean.toString(autoCommit)),
+ new BasicNameValuePair("wait_us", Long.toString(waitMicros)),
+ new BasicNameValuePair("min_count", Long.toString(options.minCount())),
+ new BasicNameValuePair("max_bytes", Long.toString(options.maxBytes())),
+ new BasicNameValuePair("request_timeout_us", Long.toString(requestTimeoutMicros)));
+ }
+
+ private static void sleepBeforeRetry(long intervalMillis, long requestDeadlineNanos, IggyServerException refused) {
+ if (requestDeadlineNanos - System.nanoTime() <= TimeUnit.MILLISECONDS.toNanos(intervalMillis)) {
+ throw refused;
+ }
+ try {
+ Thread.sleep(intervalMillis);
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IggyClientException("Interrupted while retrying a deferred poll", interrupted);
+ }
+ }
+
@Override
public SendMessagesResponse sendMessages(
StreamId streamId, TopicId topicId, Partitioning partitioning, List<Message> messages) {
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/FutureUtil.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/FutureUtil.java
index 3dfa7e276..b7969f652 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/FutureUtil.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/FutureUtil.java
@@ -24,6 +24,7 @@ import org.apache.iggy.exception.IggyClientException;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
final class FutureUtil {
@@ -39,6 +40,30 @@ final class FutureUtil {
}
}
+ /**
+ * Waits for a call that can be held by the server for a long time. Unlike
+ * {@link #resolve}, an interrupt reaches the caller: it cancels the
+ * exchange, restores the interrupt flag and reports it, instead of parking
+ * until the server answers.
+ *
+ * @param future the pending call
+ * @param <T> the result type
+ * @return the resolved value
+ */
+ static <T> T resolveInterruptibly(CompletableFuture<T> future) {
+ try {
+ return future.get();
+ } catch (InterruptedException interrupted) {
+ future.cancel(false);
+ Thread.currentThread().interrupt();
+ throw new IggyClientException("Interrupted while waiting for a response", interrupted);
+ } catch (ExecutionException e) {
+ throw unwrap(e.getCause());
+ } catch (CancellationException e) {
+ throw new IggyClientException("Operation was cancelled", e);
+ }
+ }
+
private static RuntimeException unwrap(Throwable cause) {
if (cause instanceof RuntimeException re) {
return re;
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java
index 47e561b0d..f8baa26a4 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java
@@ -23,6 +23,7 @@ import org.apache.iggy.client.blocking.MessagesClient;
import org.apache.iggy.consumergroup.Consumer;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
import org.apache.iggy.message.Message;
import org.apache.iggy.message.Partitioning;
import org.apache.iggy.message.PolledMessages;
@@ -53,6 +54,21 @@ final class MessagesTcpClient implements MessagesClient {
delegate.pollMessages(streamId, topicId, partitionId, consumer, strategy, count, autoCommit));
}
+ @Override
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public PolledMessages pollMessagesDeferred(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit,
+ DeferredPollOptions options) {
+ return FutureUtil.resolveInterruptibly(delegate.pollMessagesDeferred(
+ streamId, topicId, partitionId, consumer, strategy, count, autoCommit, options));
+ }
+
@Override
public SendMessagesResponse sendMessages(
StreamId streamId, TopicId topicId, Partitioning partitioning, List<Message> messages) {
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/DeferredPollOptions.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/DeferredPollOptions.java
new file mode 100644
index 000000000..518834e28
--- /dev/null
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/DeferredPollOptions.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iggy.message;
+
+import org.apache.iggy.exception.IggyInvalidArgumentException;
+
+import java.time.Duration;
+
+/**
+ * Readiness and response limits for a deferred (long) poll, independent of the
+ * maximum message count. Mirrors {@code DeferredPollOptions} in
+ * {@code core/common/src/types/message/deferred_poll_options.rs}.
+ *
+ * <p>A deferred poll returns as soon as {@code minCount} messages are readable,
+ * when {@code maxBytes} prevents selecting more, or when {@code maxWait} expires.
+ * Readiness expiry permits a final bounded read, so it can return partial or
+ * empty data. {@code requestTimeout} is the total budget for routing, waiting,
+ * reading and the reply; exhausting it is an error, never an empty success.
+ *
+ * <p>{@code maxBytes} bounds the encoded response body, including the 16-byte
+ * poll prefix and all batch framing, but excluding the 256-byte transport
+ * header. It is not a payload-only limit, and not a bound on server disk work.
+ *
+ * @param maxWait readiness wait, zero through 600 s; the server's
+ * configured ceiling can be lower, 30 s by default
+ * @param minCount messages that make the poll ready, at least one and no
+ * greater than the poll's {@code count}
+ * @param maxBytes maximum encoded response body, at least 16
+ * @param requestTimeout total request budget, at least {@code maxWait} and no
+ * more than 630 s
+ */
+public record DeferredPollOptions(Duration maxWait, long minCount, long maxBytes, Duration requestTimeout) {
+
+ /** Finite protocol ceiling; a server may admit less. */
+ public static final Duration MAX_WAIT = Duration.ofSeconds(600);
+
+ public static final Duration MAX_REQUEST_TIMEOUT = MAX_WAIT.plusSeconds(30);
+
+ /** The poll response prefix ({@code partition:u32}, {@code offset:u64}, {@code count:u32}). */
+ public static final long MIN_MAX_BYTES = 16;
+
+ public static final Duration DEFAULT_MAX_WAIT = Duration.ofSeconds(1);
+ public static final long DEFAULT_MIN_COUNT = 1;
+ public static final long DEFAULT_MAX_BYTES = 1024L * 1024L;
+ public static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(11);
+
+ private static final DeferredPollOptions DEFAULTS =
+ new DeferredPollOptions(DEFAULT_MAX_WAIT, DEFAULT_MIN_COUNT, DEFAULT_MAX_BYTES, DEFAULT_REQUEST_TIMEOUT);
+
+ private static final long MAX_UNSIGNED_INT = 0xFFFF_FFFFL;
+ private static final long NANOS_PER_MICRO = 1000;
+
+ public DeferredPollOptions {
+ requireMicrosecondDuration(maxWait, "maxWait");
+ requireMicrosecondDuration(requestTimeout, "requestTimeout");
+ if (maxWait.compareTo(MAX_WAIT) > 0) {
+ throw new IggyInvalidArgumentException("Deferred poll maxWait " + maxWait + " exceeds the protocol maximum "
+ + MAX_WAIT + "; a server can admit less");
+ }
+ if (requestTimeout.isZero() || requestTimeout.compareTo(MAX_REQUEST_TIMEOUT) > 0) {
+ throw new IggyInvalidArgumentException("Deferred poll requestTimeout " + requestTimeout
+ + " must be positive and no more than " + MAX_REQUEST_TIMEOUT);
+ }
+ if (requestTimeout.compareTo(maxWait) < 0) {
+ throw new IggyInvalidArgumentException(
+ "Deferred poll requestTimeout " + requestTimeout + " is shorter than maxWait " + maxWait);
+ }
+ if (minCount < 1 || minCount > MAX_UNSIGNED_INT) {
+ throw new IggyInvalidArgumentException(
+ "Deferred poll minCount " + minCount + " must be between 1 and " + MAX_UNSIGNED_INT);
+ }
+ if (maxBytes < MIN_MAX_BYTES || maxBytes > MAX_UNSIGNED_INT) {
+ throw new IggyInvalidArgumentException("Deferred poll maxBytes " + maxBytes + " must be between "
+ + MIN_MAX_BYTES + " and " + MAX_UNSIGNED_INT);
+ }
+ }
+
+ /** Wake on the first readable message within one second, up to 1 MiB. */
+ public static DeferredPollOptions defaults() {
+ return DEFAULTS;
+ }
+
+ public DeferredPollOptions withMaxWait(Duration maxWait) {
+ return new DeferredPollOptions(maxWait, minCount, maxBytes, requestTimeout);
+ }
+
+ public DeferredPollOptions withMinCount(long minCount) {
+ return new DeferredPollOptions(maxWait, minCount, maxBytes, requestTimeout);
+ }
+
+ public DeferredPollOptions withMaxBytes(long maxBytes) {
+ return new DeferredPollOptions(maxWait, minCount, maxBytes, requestTimeout);
+ }
+
+ public DeferredPollOptions withRequestTimeout(Duration requestTimeout) {
+ return new DeferredPollOptions(maxWait, minCount, maxBytes, requestTimeout);
+ }
+
+ /**
+ * Checks the one rule that depends on the poll itself. Call it before
+ * allocating buffers or acquiring a connection.
+ *
+ * @param count the maximum message count the poll requests
+ * @throws IggyInvalidArgumentException if the readiness target cannot be met
+ */
+ public void validate(long count) {
+ if (count < 1 || count > MAX_UNSIGNED_INT) {
+ throw new IggyInvalidArgumentException(
+ "Poll count " + count + " must be between 1 and " + MAX_UNSIGNED_INT);
+ }
+ if (minCount > count) {
+ throw new IggyInvalidArgumentException(
+ "Deferred poll minCount " + minCount + " exceeds the requested count " + count);
+ }
+ }
+
+ public long maxWaitMicros() {
+ return maxWait.toNanos() / NANOS_PER_MICRO;
+ }
+
+ public long requestTimeoutMicros() {
+ return requestTimeout.toNanos() / NANOS_PER_MICRO;
+ }
+
+ /**
+ * Rejects what the wire cannot carry exactly. Silently truncating a
+ * sub-microsecond wait would change a caller's readiness budget, and a
+ * duration too large for nanosecond arithmetic cannot be compared either.
+ */
+ private static void requireMicrosecondDuration(Duration duration, String name) {
+ if (duration.isNegative()) {
+ throw new IggyInvalidArgumentException("Deferred poll " + name + " " + duration + " must not be negative");
+ }
+ long nanos;
+ try {
+ nanos = duration.toNanos();
+ } catch (ArithmeticException overflow) {
+ throw new IggyInvalidArgumentException(
+ "Deferred poll " + name + " " + duration + " is too large to express", overflow);
+ }
+ if (nanos % NANOS_PER_MICRO != 0) {
+ throw new IggyInvalidArgumentException(
+ "Deferred poll " + name + " " + duration + " must be a whole number of microseconds");
+ }
+ }
+}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java
index 8e7fb2968..08605414c 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java
@@ -96,7 +96,9 @@ public interface CommandCode {
POLL(100),
SEND(101),
GET_POLL_ROUTING(103),
- POLL_ON_PRIMARY(104);
+ POLL_ON_PRIMARY(104),
+ POLL_DEFERRED(105),
+ POLL_DEFERRED_ON_PRIMARY(106);
private final int value;
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/DeferredPollCompatibilityTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/DeferredPollCompatibilityTest.java
new file mode 100644
index 000000000..4466f5dcb
--- /dev/null
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/DeferredPollCompatibilityTest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iggy.client;
+
+import org.apache.iggy.consumergroup.Consumer;
+import org.apache.iggy.exception.IggyOperationNotSupportedException;
+import org.apache.iggy.identifier.StreamId;
+import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
+import org.apache.iggy.message.Message;
+import org.apache.iggy.message.Partitioning;
+import org.apache.iggy.message.PolledMessages;
+import org.apache.iggy.message.PollingStrategy;
+import org.apache.iggy.message.SendMessagesResponse;
+import org.junit.jupiter.api.Test;
+
+import java.math.BigInteger;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * An implementation written before deferred polling existed still compiles and
+ * runs. Its inherited default reports the missing support instead of silently
+ * degrading to an immediate poll, which would drop the readiness, byte and
+ * request-time guarantees the caller asked for.
+ */
+class DeferredPollCompatibilityTest {
+
+ private static final PolledMessages EMPTY = new PolledMessages(0L, BigInteger.ZERO, 0L, List.of());
+
+ @Test
+ void shouldReportMissingSupportFromABlockingImplementation() {
+ var client = new LegacyBlockingClient();
+
+ assertThatThrownBy(() -> client.pollMessagesDeferred(
+ StreamId.of(1L),
+ TopicId.of(1L),
+ Optional.of(0L),
+ Consumer.of(0L),
+ PollingStrategy.next(),
+ 10L,
+ false,
+ DeferredPollOptions.defaults()))
+ .isInstanceOf(IggyOperationNotSupportedException.class);
+ assertThat(client.immediatePolls).isZero();
+ }
+
+ @Test
+ void shouldReportMissingSupportFromAnAsyncImplementation() {
+ var client = new LegacyAsyncClient();
+
+ assertThatThrownBy(() -> client.pollMessagesDeferred(
+ 1L,
+ 1L,
+ Optional.of(0L),
+ 0L,
+ PollingStrategy.next(),
+ 10L,
+ false,
+ DeferredPollOptions.defaults())
+ .get())
+ .isInstanceOf(ExecutionException.class)
+ .hasCauseInstanceOf(IggyOperationNotSupportedException.class);
+ assertThat(client.immediatePolls).isZero();
+ }
+
+ @Test
+ void shouldDelegateNumericOverloadsToTheTypedMethod() {
+ var client = new RecordingBlockingClient();
+
+ client.pollMessagesDeferred(
+ 7L, 9L, Optional.of(3L), 5L, PollingStrategy.next(), 10L, true, DeferredPollOptions.defaults());
+
+ assertThat(client.streamId).hasToString("7");
+ assertThat(client.topicId).hasToString("9");
+ assertThat(client.consumer.id()).hasToString("5");
+ assertThat(client.consumer.kind()).isEqualTo(Consumer.Kind.Consumer);
+ }
+
+ private static class LegacyBlockingClient implements org.apache.iggy.client.blocking.MessagesClient {
+ private int immediatePolls;
+
+ @Override
+ public PolledMessages pollMessages(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit) {
+ immediatePolls++;
+ return EMPTY;
+ }
+
+ @Override
+ public SendMessagesResponse sendMessages(
+ StreamId streamId, TopicId topicId, Partitioning partitioning, List<Message> messages) {
+ return SendMessagesResponse.empty();
+ }
+ }
+
+ private static final class LegacyAsyncClient implements org.apache.iggy.client.async.MessagesClient {
+ private int immediatePolls;
+
+ @Override
+ public CompletableFuture<PolledMessages> pollMessages(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit) {
+ immediatePolls++;
+ return CompletableFuture.completedFuture(EMPTY);
+ }
+
+ @Override
+ public CompletableFuture<SendMessagesResponse> sendMessages(
+ StreamId streamId, TopicId topicId, Partitioning partitioning, List<Message> messages) {
+ return CompletableFuture.completedFuture(SendMessagesResponse.empty());
+ }
+ }
+
+ private static final class RecordingBlockingClient extends LegacyBlockingClient {
+ private StreamId streamId;
+ private TopicId topicId;
+ private Consumer consumer;
+
+ @Override
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public PolledMessages pollMessagesDeferred(
+ StreamId streamId,
+ TopicId topicId,
+ Optional<Long> partitionId,
+ Consumer consumer,
+ PollingStrategy strategy,
+ Long count,
+ boolean autoCommit,
+ DeferredPollOptions options) {
+ this.streamId = streamId;
+ this.topicId = topicId;
+ this.consumer = consumer;
+ return EMPTY;
+ }
+ }
+}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java
index dbc62bf70..2ee170d36 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java
@@ -21,6 +21,9 @@ package org.apache.iggy.client.async.tcp;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
+import org.apache.iggy.client.async.tcp.VsrLoopbackPeer.Request;
+import org.apache.iggy.client.async.tcp.VsrLoopbackPeer.RequestHandler;
+import org.apache.iggy.client.async.tcp.VsrLoopbackPeer.Response;
import org.apache.iggy.consumergroup.Consumer;
import org.apache.iggy.exception.IggyClientException;
import org.apache.iggy.exception.IggyErrorCode;
@@ -34,16 +37,10 @@ import org.apache.iggy.message.PollingStrategy;
import org.apache.iggy.serde.CommandCode;
import org.junit.jupiter.api.Test;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.net.InetAddress;
import java.net.ServerSocket;
-import java.net.Socket;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
@@ -56,25 +53,17 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.clusterMetadata;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.registerBody;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.serve;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.singleNodeMetadata;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.threeNodeMetadata;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.transientResult;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.writeNode;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
class AsyncIggyTcpClientTransientFailoverTest {
- private static final int HEADER_SIZE = 256;
- private static final int SIZE_OFFSET = 48;
- private static final int COMMAND_OFFSET = 60;
- private static final int REQUEST_ID_OFFSET = 168;
- private static final int REQUEST_OPERATION_OFFSET = 176;
- private static final int REQUEST_CODE_OFFSET = 196;
- private static final int REPLY_REQUEST_ID_OFFSET = 200;
- private static final int REPLY_OPERATION_OFFSET = 208;
- private static final int REPLY_STATUS_OFFSET = 216;
- private static final int REPLY_COMMIT_OFFSET = 184;
- private static final int REQUEST_CLIENT_OFFSET = 128;
- private static final int EVICTION_REASON_OFFSET = 255;
-
- private static final int COMMAND_REPLY = 8;
- private static final int COMMAND_EVICTION = 13;
private static final int OPERATION_REGISTER = 1;
private static final int OPERATION_NON_REPLICATED = 2;
private static final int OPERATION_LOGOUT = 3;
@@ -1429,212 +1418,4 @@ class AsyncIggyTcpClientTransientFailoverTest {
}
throw new IllegalStateException("Unexpected request to new leader: " + request);
}
-
- private static CompletableFuture<Void> serve(ServerSocket server, RequestHandler handler) {
- return serve(server, 1, handler);
- }
-
- /**
- * Runs blocking socket I/O on one dedicated daemon thread per mock node.
- * The common fork-join pool has only cores minus one workers on small CI
- * runners, so three blocking nodes can starve the client continuations the
- * test is waiting for when the full suite runs concurrently.
- */
- private static CompletableFuture<Void> serve(ServerSocket server, int connectionCount, RequestHandler handler) {
- CompletableFuture<Void> serving = new CompletableFuture<>();
- Thread serverThread = new Thread(
- () -> {
- try {
- for (int connection = 0; connection < connectionCount; connection++) {
- try (Socket socket = server.accept()) {
- InputStream input = socket.getInputStream();
- OutputStream output = socket.getOutputStream();
- Request request;
- while ((request = readRequest(input)) != null) {
- Response response = handler.handle(request);
- if (response.command() == -1) {
- break;
- }
- writeResponse(output, request, response);
- if (response.closeAfterReply()) {
- break;
- }
- }
- }
- }
- serving.complete(null);
- } catch (IOException error) {
- serving.completeExceptionally(new IllegalStateException("Mock VSR server failed", error));
- } catch (RuntimeException error) {
- serving.completeExceptionally(error);
- }
- },
- "transient-failover-server-" + server.getLocalPort());
- serverThread.setDaemon(true);
- serverThread.start();
- return serving;
- }
-
- private static Request readRequest(InputStream input) throws IOException {
- byte[] header = input.readNBytes(HEADER_SIZE);
- if (header.length == 0) {
- return null;
- }
- if (header.length != HEADER_SIZE) {
- throw new EOFException("Truncated VSR request header");
- }
- ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN);
- int size = fields.getInt(SIZE_OFFSET);
- byte[] body = input.readNBytes(size - HEADER_SIZE);
- if (body.length != size - HEADER_SIZE) {
- throw new EOFException("Truncated VSR request body");
- }
- return new Request(
- Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]),
- fields.getInt(REQUEST_CODE_OFFSET),
- fields.getLong(REQUEST_ID_OFFSET),
- fields.getLong(REQUEST_CLIENT_OFFSET),
- fields.getLong(REQUEST_CLIENT_OFFSET + Long.BYTES),
- body);
- }
-
- private static void writeResponse(OutputStream output, Request request, Response response) throws IOException {
- if (response.command() == -2) {
- return;
- }
- byte[] body = new byte[response.body().readableBytes()];
- response.body().readBytes(body);
- response.body().release();
- byte[] header = new byte[HEADER_SIZE];
- ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN);
- fields.putInt(SIZE_OFFSET, HEADER_SIZE + body.length);
- header[COMMAND_OFFSET] = (byte) response.command();
- if (response.command() == COMMAND_EVICTION) {
- header[EVICTION_REASON_OFFSET] = (byte) response.evictionReason();
- } else {
- fields.putLong(REPLY_REQUEST_ID_OFFSET, request.requestId());
- header[REPLY_OPERATION_OFFSET] = (byte) response.operation();
- fields.putInt(REPLY_STATUS_OFFSET, response.status());
- fields.putLong(REPLY_COMMIT_OFFSET, response.commit());
- }
- output.write(header);
- output.write(body);
- output.flush();
- }
-
- private static ByteBuf registerBody(long session) {
- ByteBuf body = Unpooled.buffer();
- body.writeIntLE(0);
- body.writeIntLE(1);
- body.writeLongLE(session);
- body.writeIntLE(11 << 10);
- body.writeByte(0);
- return body;
- }
-
- private static ByteBuf transientResult(int errorCode) {
- ByteBuf body = Unpooled.buffer(3 * Integer.BYTES);
- body.writeIntLE(1);
- body.writeIntLE(0);
- body.writeIntLE(errorCode);
- return body;
- }
-
- private static ByteBuf singleNodeMetadata(int port) {
- ByteBuf body = Unpooled.buffer();
- writeString(body, "test-cluster");
- body.writeIntLE(1);
- writeNode(body, "node", port, true);
- return body;
- }
-
- private static ByteBuf clusterMetadata(int oldLeaderPort, int newLeaderPort, int leaderPort) {
- ByteBuf body = Unpooled.buffer();
- writeString(body, "test-cluster");
- body.writeIntLE(2);
- writeNode(body, "old-node", oldLeaderPort, oldLeaderPort == leaderPort);
- writeNode(body, "new-node", newLeaderPort, newLeaderPort == leaderPort);
- return body;
- }
-
- private static ByteBuf threeNodeMetadata(int firstPort, int secondPort, int thirdPort) {
- ByteBuf body = Unpooled.buffer();
- writeString(body, "test-cluster");
- body.writeIntLE(3);
- writeNode(body, "metadata-leader", firstPort, true);
- writeNode(body, "follower", secondPort, false);
- writeNode(body, "partition-primary", thirdPort, false);
- return body;
- }
-
- private static void writeNode(ByteBuf body, String name, int port, boolean leader) {
- writeString(body, name);
- writeString(body, InetAddress.getLoopbackAddress().getHostAddress());
- body.writeShortLE(port);
- body.writeShortLE(0);
- body.writeShortLE(0);
- body.writeShortLE(0);
- body.writeByte(leader ? 0 : 1);
- body.writeByte(0);
- }
-
- private static void writeString(ByteBuf body, String value) {
- byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
- body.writeIntLE(bytes.length);
- body.writeBytes(bytes);
- }
-
- private record Request(
- int operation, int commandCode, long requestId, long clientLow, long clientHigh, byte[] body) {
- boolean is(int expectedCode, int expectedOperation) {
- return commandCode == expectedCode && operation == expectedOperation;
- }
-
- /** The request body as text, for asserting which user a login names. */
- String bodyAsText() {
- return new String(body, StandardCharsets.UTF_8);
- }
- }
-
- private record Response(
- int command,
- int operation,
- int status,
- int evictionReason,
- long commit,
- ByteBuf body,
- boolean closeAfterReply) {
- static Response success(int operation, ByteBuf body) {
- return new Response(COMMAND_REPLY, operation, 0, 0, 0, body, false);
- }
-
- static Response error(int operation, int status) {
- return new Response(COMMAND_REPLY, operation, status, 0, 0, Unpooled.EMPTY_BUFFER, false);
- }
-
- static Response eviction(int reason) {
- return new Response(COMMAND_EVICTION, 0, 0, reason, 0, Unpooled.EMPTY_BUFFER, false);
- }
-
- static Response committed(int operation, long commit, ByteBuf body) {
- return new Response(COMMAND_REPLY, operation, 0, 0, commit, body, false);
- }
-
- static Response disconnect() {
- return new Response(-1, 0, 0, 0, 0, Unpooled.EMPTY_BUFFER, false);
- }
-
- static Response noReply() {
- return new Response(-2, 0, 0, 0, 0, Unpooled.EMPTY_BUFFER, false);
- }
-
- static Response successAndDisconnect(int operation, ByteBuf body) {
- return new Response(COMMAND_REPLY, operation, 0, 0, 0, body, true);
- }
- }
-
- @FunctionalInterface
- private interface RequestHandler {
- Response handle(Request request);
- }
}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/PollRouterTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/PollRouterTest.java
new file mode 100644
index 000000000..1fa146a4f
--- /dev/null
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/PollRouterTest.java
@@ -0,0 +1,560 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iggy.client.async.tcp;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.apache.iggy.client.async.tcp.VsrLoopbackPeer.Request;
+import org.apache.iggy.client.async.tcp.VsrLoopbackPeer.Response;
+import org.apache.iggy.client.async.tcp.vsr.VsrHeaders;
+import org.apache.iggy.client.async.tcp.vsr.VsrOperation;
+import org.apache.iggy.consumergroup.Consumer;
+import org.apache.iggy.exception.IggyServerException;
+import org.apache.iggy.exception.IggyTimeoutException;
+import org.apache.iggy.identifier.StreamId;
+import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
+import org.apache.iggy.message.PolledMessages;
+import org.apache.iggy.message.PollingStrategy;
+import org.apache.iggy.serde.CommandCode;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.clusterMetadata;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.registerBody;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.serveConcurrently;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.singleNodeMetadata;
+import static org.apache.iggy.client.async.tcp.VsrLoopbackPeer.writeNode;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * Deferred polling through {@link PollRouter}: the wire trailer, the dedicated
+ * data connection and its exclusive lease, one monotonic budget, and the single
+ * retry rule.
+ */
+class PollRouterTest {
+
+ private static final int TRAILER_BYTES = 24;
+ private static final int ATTACHMENT_BYTES = 32;
+ private static final int PING_CODE = CommandCode.System.PING.getValue();
+ private static final int POLL_CODE = CommandCode.Messages.POLL.getValue();
+ private static final int POLL_DEFERRED_CODE = CommandCode.Messages.POLL_DEFERRED.getValue();
+ private static final int POLL_DEFERRED_ON_PRIMARY_CODE = CommandCode.Messages.POLL_DEFERRED_ON_PRIMARY.getValue();
+ private static final int GET_POLL_ROUTING_CODE = CommandCode.Messages.GET_POLL_ROUTING.getValue();
+ private static final int GET_CLUSTER_METADATA_CODE = CommandCode.System.GET_CLUSTER_METADATA.getValue();
+ private static final int ATTACH_CONSUMER_SESSION_CODE = CommandCode.System.ATTACH_CONSUMER_SESSION.getValue();
+ private static final int TRANSIENT_NOT_COMMITTED = 57;
+ private static final int TRANSIENT_NOT_ACCEPTED = 58;
+ private static final long SESSION_EPOCH = 7;
+ private static final int TEST_TIMEOUT_SECONDS = 15;
+
+ @Test
+ void shouldAppendTheTrailerToAnUnchangedPollBodyOnADedicatedConnection() throws Exception {
+ var peer = new DeferredPeer();
+ try (ServerSocket socket = peer.listen()) {
+ serveConcurrently(socket, peer::handle);
+ AsyncIggyTcpClient client = client(socket, Duration.ofMinutes(1));
+ try {
+ connect(client);
+ poll(client);
+ Request immediate = peer.immediatePolls.get(0);
+
+ var options = DeferredPollOptions.defaults();
+ PolledMessages polled =
+ deferredPoll(client, false, options).get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ assertThat(polled.messages()).isEmpty();
+ Request deferred = peer.deferredPolls.get(0);
+ assertThat(deferred.operation()).isEqualTo(VsrOperation.NON_REPLICATED);
+ assertThat(deferred.body()).hasSize(immediate.body().length + TRAILER_BYTES);
+ assertThat(Arrays.copyOf(deferred.body(), immediate.body().length))
+ .as("the immediate poll body travels unchanged")
+ .isEqualTo(immediate.body());
+
+ var trailer = trailer(deferred.body());
+ assertThat(trailer.waitMicros()).isPositive().isLessThanOrEqualTo(options.maxWaitMicros());
+ assertThat(trailer.minCount()).isEqualTo(options.minCount());
+ assertThat(trailer.maxBytes()).isEqualTo(options.maxBytes());
+ assertThat(trailer.requestTimeoutMicros())
+ .isPositive()
+ .isLessThanOrEqualTo(options.requestTimeoutMicros());
+ } finally {
+ close(client);
+ }
+ }
+ }
+
+ @Test
+ void shouldAttachTheParentSessionToTheDataConnection() throws Exception {
+ var peer = new DeferredPeer();
+ try (ServerSocket socket = peer.listen()) {
+ serveConcurrently(socket, peer::handle);
+ AsyncIggyTcpClient client = client(socket, Duration.ofMinutes(1));
+ try {
+ connect(client);
+ poll(client);
+ Request parentRequest = peer.immediatePolls.get(0);
+
+ deferredPoll(client, false, DeferredPollOptions.defaults()).get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ assertThat(peer.attachments).hasSize(1);
+ byte[] body = peer.attachments.get(0).body();
+ assertThat(body).hasSize(ATTACHMENT_BYTES);
+ var attachment = ByteBuffer.wrap(body).order(ByteOrder.LITTLE_ENDIAN);
+ assertThat(attachment.getLong(0)).isEqualTo(parentRequest.clientLow());
+ assertThat(attachment.getLong(Long.BYTES)).isEqualTo(parentRequest.clientHigh());
+ assertThat(attachment.getLong(2 * Long.BYTES)).isEqualTo(SESSION_EPOCH);
+
+ assertThat(peer.deferredPolls.get(0).clientLow())
+ .as("the data connection has its own client id, so the session travels in the attachment")
+ .isNotEqualTo(parentRequest.clientLow());
+
+ assertThat(peer.connections())
+ .as("a held poll never occupies the coordinator's own drain")
+ .anyMatch(connection ->
+ connection.contains(POLL_DEFERRED_CODE) && !connection.contains(POLL_CODE));
+ } finally {
+ close(client);
+ }
+ }
+ }
+
+ @Test
+ void shouldRouteAClusteredAutoCommitPollToThePrimaryWithCommand106() throws Exception {
+ var coordinator = new DeferredPeer();
+ var primary = new DeferredPeer();
+ try (ServerSocket coordinatorSocket = coordinator.listen();
+ ServerSocket primarySocket = primary.listen()) {
+ coordinator.clusterPeerPort = primary.port;
+ serveConcurrently(coordinatorSocket, coordinator::handle);
+ serveConcurrently(primarySocket, primary::handle);
+ AsyncIggyTcpClient client = client(coordinatorSocket, Duration.ofMinutes(1));
+ try {
+ connect(client);
+
+ PolledMessages polled = deferredPoll(client, true, DeferredPollOptions.defaults())
+ .get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ assertThat(polled.messages()).isEmpty();
+ assertThat(coordinator.routingRequests).hasValue(1);
+ assertThat(coordinator.deferredPolls).isEmpty();
+ assertThat(primary.deferredPolls).hasSize(1);
+ assertThat(primary.deferredPolls.get(0).commandCode()).isEqualTo(POLL_DEFERRED_ON_PRIMARY_CODE);
+ assertThat(primary.attachments).hasSize(1);
+ } finally {
+ close(client);
+ }
+ }
+ }
+
+ @Test
+ void shouldNotPingTheConnectionHoldingAPoll() throws Exception {
+ var peer = new DeferredPeer();
+ peer.holdMillis = 900;
+ try (ServerSocket socket = peer.listen()) {
+ serveConcurrently(socket, peer::handle);
+ AsyncIggyTcpClient client = client(socket, Duration.ofMillis(100));
+ try {
+ connect(client);
+
+ deferredPoll(client, false, DeferredPollOptions.defaults()).get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ List<List<Integer>> dataConnections = peer.connections().stream()
+ .filter(connection -> connection.contains(POLL_DEFERRED_CODE))
+ .toList();
+ assertThat(dataConnections).hasSize(1);
+ assertThat(dataConnections.get(0))
+ .as("a ping queued behind a held reply would expire and close the channel")
+ .doesNotContain(PING_CODE);
+ assertThat(peer.pings)
+ .as("the coordinator keeps its own heartbeat")
+ .hasValueGreaterThan(0);
+ } finally {
+ close(client);
+ }
+ }
+ }
+
+ @Test
+ void shouldGiveEachConcurrentPollItsOwnConnection() throws Exception {
+ var peer = new DeferredPeer();
+ peer.holdMillis = 400;
+ try (ServerSocket socket = peer.listen()) {
+ serveConcurrently(socket, peer::handle);
+ AsyncIggyTcpClient client = client(socket, Duration.ofMinutes(1));
+ try {
+ connect(client);
+
+ var first = deferredPoll(client, false, DeferredPollOptions.defaults());
+ var second = deferredPoll(client, false, DeferredPollOptions.defaults());
+ first.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ second.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ assertThat(peer.deferredPolls).hasSize(2);
+ assertThat(peer.maxConcurrentPolls)
+ .as("a lease is exclusive, so two waits can only overlap on two connections")
+ .hasValue(2);
+ assertThat(peer.connections().stream()
+ .filter(connection -> connection.contains(POLL_DEFERRED_CODE))
+ .count())
+ .isEqualTo(2);
+ } finally {
+ close(client);
+ }
+ }
+ }
+
+ @Test
+ void shouldRetryAnExplicitRefusalInsideTheOriginalBudget() throws Exception {
+ var peer = new DeferredPeer();
+ peer.refusals = 2;
+ try (ServerSocket socket = peer.listen()) {
+ serveConcurrently(socket, peer::handle);
+ AsyncIggyTcpClient client = client(socket, Duration.ofMinutes(1));
+ try {
+ connect(client);
+
+ deferredPoll(client, false, DeferredPollOptions.defaults()).get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ assertThat(peer.deferredPolls).hasSize(3);
+ var first = trailer(peer.deferredPolls.get(0).body());
+ var last = trailer(peer.deferredPolls.get(2).body());
+ assertThat(last.waitMicros())
+ .as("a retry inherits what is left of the readiness wait")
+ .isLessThan(first.waitMicros());
+ assertThat(last.requestTimeoutMicros())
+ .as("and what is left of the total budget")
+ .isLessThan(first.requestTimeoutMicros());
+ } finally {
+ close(client);
+ }
+ }
+ }
+
+ @Test
+ void shouldNotReplayAnAmbiguousRefusal() throws Exception {
+ var peer = new DeferredPeer();
+ peer.uncommitted = true;
+ try (ServerSocket socket = peer.listen()) {
+ serveConcurrently(socket, peer::handle);
+ AsyncIggyTcpClient client = client(socket, Duration.ofMinutes(1));
+ try {
+ connect(client);
+
+ assertThatThrownBy(() -> deferredPoll(client, false, DeferredPollOptions.defaults())
+ .get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class)
+ .cause()
+ .isInstanceOf(IggyServerException.class)
+ .extracting(error -> ((IggyServerException) error).getRawErrorCode())
+ .isEqualTo(TRANSIENT_NOT_COMMITTED);
+ assertThat(peer.deferredPolls)
+ .as("a poll that may have been served and auto-committed is never replayed")
+ .hasSize(1);
+ } finally {
+ close(client);
+ }
+ }
+ }
+
+ @Test
+ void shouldExpireOnTheCallersBudgetAsALocalTimeout() throws Exception {
+ var peer = new DeferredPeer();
+ peer.silentPolls = true;
+ try (ServerSocket socket = peer.listen()) {
+ serveConcurrently(socket, peer::handle);
+ AsyncIggyTcpClient client = client(socket, Duration.ofMinutes(1));
+ try {
+ connect(client);
+ var options = DeferredPollOptions.defaults()
+ .withMaxWait(Duration.ofMillis(200))
+ .withRequestTimeout(Duration.ofMillis(400));
+
+ long startedNanos = System.nanoTime();
+ assertThatThrownBy(
+ () -> deferredPoll(client, false, options).get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class)
+ .hasCauseInstanceOf(IggyTimeoutException.class);
+
+ assertThat(Duration.ofNanos(System.nanoTime() - startedNanos))
+ .as("the caller's budget bounds the call, not the transport's own timeout")
+ .isLessThan(Duration.ofSeconds(4));
+ } finally {
+ close(client);
+ }
+ }
+ }
+
+ @Test
+ void shouldDiscardTheDataConnectionWhenACallerCancels() throws Exception {
+ var peer = new DeferredPeer();
+ peer.holdMillis = 2000;
+ try (ServerSocket socket = peer.listen()) {
+ serveConcurrently(socket, peer::handle);
+ AsyncIggyTcpClient client = client(socket, Duration.ofMinutes(1));
+ try {
+ connect(client);
+ var cancelled = deferredPoll(client, false, DeferredPollOptions.defaults());
+ peer.awaitFirstPoll();
+
+ assertThat(cancelled.cancel(false)).isTrue();
+
+ peer.holdMillis = 0;
+ deferredPoll(client, false, DeferredPollOptions.defaults()).get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ assertThat(peer.connections().stream()
+ .filter(connection -> connection.contains(POLL_DEFERRED_CODE))
+ .count())
+ .as("a late reply on the abandoned channel would desynchronize the next exchange")
+ .isEqualTo(2);
+ assertThat(client.system().ping().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+ .isNotNull();
+ } finally {
+ close(client);
+ }
+ }
+ }
+
+ @Test
+ void shouldRefuseAReplyLargerThanTheDeclaredByteLimit() throws Exception {
+ var peer = new DeferredPeer();
+ peer.replyPaddingBytes = 4096;
+ try (ServerSocket socket = peer.listen()) {
+ serveConcurrently(socket, peer::handle);
+ AsyncIggyTcpClient client = client(socket, Duration.ofMinutes(1));
+ try {
+ connect(client);
+ var options = DeferredPollOptions.defaults().withMaxBytes(1024);
+
+ assertThatThrownBy(
+ () -> deferredPoll(client, false, options).get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class)
+ .rootCause()
+ .as("the cap is the header plus maxBytes, checked before the body is accumulated")
+ .hasMessageContaining("Invalid VSR frame size " + (VsrHeaders.HEADER_SIZE + 16 + 4096));
+ } finally {
+ close(client);
+ }
+ }
+ }
+
+ private static AsyncIggyTcpClient client(ServerSocket coordinator, Duration heartbeatInterval) {
+ return AsyncIggyTcpClient.builder()
+ .host(coordinator.getInetAddress().getHostAddress())
+ .port(coordinator.getLocalPort())
+ .credentials("iggy", "iggy")
+ .requestTimeout(Duration.ofSeconds(5))
+ .heartbeatInterval(heartbeatInterval)
+ .build();
+ }
+
+ private static void connect(AsyncIggyTcpClient client) throws Exception {
+ client.connect().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ client.login().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+
+ private static void close(AsyncIggyTcpClient client) throws Exception {
+ client.close().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+
+ private static void poll(AsyncIggyTcpClient client) throws Exception {
+ client.messages()
+ .pollMessages(
+ StreamId.of(1L),
+ TopicId.of(1L),
+ Optional.of(0L),
+ Consumer.of(3L),
+ PollingStrategy.next(),
+ 10L,
+ false)
+ .get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+
+ private static CompletableFuture<PolledMessages> deferredPoll(
+ AsyncIggyTcpClient client, boolean autoCommit, DeferredPollOptions options) {
+ return client.messages()
+ .pollMessagesDeferred(
+ StreamId.of(1L),
+ TopicId.of(1L),
+ Optional.of(0L),
+ Consumer.of(3L),
+ PollingStrategy.next(),
+ 10L,
+ autoCommit,
+ options);
+ }
+
+ private static Trailer trailer(byte[] body) {
+ var fields = ByteBuffer.wrap(body, body.length - TRAILER_BYTES, TRAILER_BYTES)
+ .order(ByteOrder.LITTLE_ENDIAN);
+ return new Trailer(
+ fields.getLong(),
+ Integer.toUnsignedLong(fields.getInt()),
+ Integer.toUnsignedLong(fields.getInt()),
+ fields.getLong());
+ }
+
+ private record Trailer(long waitMicros, long minCount, long maxBytes, long requestTimeoutMicros) {}
+
+ /**
+ * A node that records what each connection was asked for, so a test can tell
+ * the coordinator's own drain from a deferred data connection.
+ */
+ private static final class DeferredPeer {
+ private final ThreadLocal<List<Integer>> connection = ThreadLocal.withInitial(CopyOnWriteArrayList::new);
+ private final List<List<Integer>> connections = new CopyOnWriteArrayList<>();
+ private final List<Request> deferredPolls = new CopyOnWriteArrayList<>();
+ private final List<Request> immediatePolls = new CopyOnWriteArrayList<>();
+ private final List<Request> attachments = new CopyOnWriteArrayList<>();
+ private final AtomicInteger routingRequests = new AtomicInteger();
+ private final AtomicInteger pings = new AtomicInteger();
+ private final AtomicInteger activePolls = new AtomicInteger();
+ private final AtomicInteger maxConcurrentPolls = new AtomicInteger();
+ private final CompletableFuture<Void> firstPoll = new CompletableFuture<>();
+ private int port;
+ private int clusterPeerPort;
+ private volatile long holdMillis;
+ private int refusals;
+ private boolean uncommitted;
+ private boolean silentPolls;
+ private int replyPaddingBytes;
+
+ private ServerSocket listen() throws IOException {
+ ServerSocket socket = new ServerSocket(0, 16, InetAddress.getLoopbackAddress());
+ port = socket.getLocalPort();
+ return socket;
+ }
+
+ private void awaitFirstPoll() throws Exception {
+ firstPoll.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+
+ private List<List<Integer>> connections() {
+ return List.copyOf(connections);
+ }
+
+ private Response handle(Request request) {
+ List<Integer> seen = connection.get();
+ if (seen.isEmpty()) {
+ connections.add(seen);
+ }
+ seen.add(request.commandCode());
+ if (request.is(POLL_DEFERRED_CODE, VsrOperation.NON_REPLICATED)
+ || request.is(POLL_DEFERRED_ON_PRIMARY_CODE, VsrOperation.NON_REPLICATED)) {
+ return handleDeferred(request);
+ }
+ if (request.is(POLL_CODE, VsrOperation.NON_REPLICATED)) {
+ immediatePolls.add(request);
+ return Response.success(VsrOperation.NON_REPLICATED, emptyPoll());
+ }
+ return handleSession(request);
+ }
+
+ private Response handleSession(Request request) {
+ if (request.operation() == VsrOperation.REGISTER) {
+ return Response.success(VsrOperation.REGISTER, registerBody(SESSION_EPOCH));
+ }
+ if (request.is(PING_CODE, VsrOperation.NON_REPLICATED)) {
+ pings.incrementAndGet();
+ return Response.success(VsrOperation.NON_REPLICATED, Unpooled.EMPTY_BUFFER);
+ }
+ if (request.is(GET_CLUSTER_METADATA_CODE, VsrOperation.NON_REPLICATED)) {
+ return Response.success(
+ VsrOperation.NON_REPLICATED,
+ clusterPeerPort == 0 ? singleNodeMetadata(port) : clusterMetadata(port, clusterPeerPort, port));
+ }
+ if (request.is(GET_POLL_ROUTING_CODE, VsrOperation.NON_REPLICATED)) {
+ routingRequests.incrementAndGet();
+ return Response.success(VsrOperation.NON_REPLICATED, route(request));
+ }
+ if (request.is(ATTACH_CONSUMER_SESSION_CODE, VsrOperation.NON_REPLICATED)) {
+ attachments.add(request);
+ return Response.success(VsrOperation.NON_REPLICATED, Unpooled.EMPTY_BUFFER);
+ }
+ throw new IllegalStateException("Unexpected request: " + request.commandCode());
+ }
+
+ private Response handleDeferred(Request request) {
+ deferredPolls.add(request);
+ firstPoll.complete(null);
+ if (silentPolls) {
+ return Response.noReply();
+ }
+ if (uncommitted) {
+ return Response.error(VsrOperation.NON_REPLICATED, TRANSIENT_NOT_COMMITTED);
+ }
+ if (refusals > 0) {
+ refusals--;
+ return Response.error(VsrOperation.NON_REPLICATED, TRANSIENT_NOT_ACCEPTED);
+ }
+ hold();
+ ByteBuf body = emptyPoll();
+ if (replyPaddingBytes > 0) {
+ body.writeZero(replyPaddingBytes);
+ }
+ return Response.success(VsrOperation.NON_REPLICATED, body);
+ }
+
+ private void hold() {
+ long millis = holdMillis;
+ if (millis == 0) {
+ return;
+ }
+ int active = activePolls.incrementAndGet();
+ maxConcurrentPolls.accumulateAndGet(active, Math::max);
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ } finally {
+ activePolls.decrementAndGet();
+ }
+ }
+
+ private ByteBuf route(Request request) {
+ ByteBuf body = Unpooled.buffer()
+ .writeLongLE(request.clientLow())
+ .writeLongLE(request.clientHigh())
+ .writeLongLE(SESSION_EPOCH)
+ .writeLongLE(0);
+ writeNode(body, "primary", clusterPeerPort, false);
+ return body;
+ }
+
+ private static ByteBuf emptyPoll() {
+ return Unpooled.buffer().writeIntLE(0).writeLongLE(0).writeIntLE(0);
+ }
+ }
+}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/VsrLoopbackPeer.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/VsrLoopbackPeer.java
new file mode 100644
index 000000000..51fef4d06
--- /dev/null
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/VsrLoopbackPeer.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iggy.client.async.tcp;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * A loopback server speaking the VSR frame format, for tests that need to steer
+ * a real client through routing, attachment and poll exchanges.
+ *
+ * <p>A handler answers each decoded {@link Request} with a {@link Response}. It
+ * may block to hold a reply, which is how a test keeps an exchange open.
+ */
+final class VsrLoopbackPeer {
+
+ static final int HEADER_SIZE = 256;
+ static final int COMMAND_REPLY = 8;
+ static final int COMMAND_EVICTION = 13;
+
+ /** Tells {@link #serve} to drop the connection instead of replying. */
+ static final int COMMAND_DISCONNECT = -1;
+
+ /** Tells {@link #serve} to keep the request open with no reply at all. */
+ static final int COMMAND_SILENCE = -2;
+
+ private static final int SIZE_OFFSET = 48;
+ private static final int COMMAND_OFFSET = 60;
+ private static final int REQUEST_CLIENT_OFFSET = 128;
+ private static final int REQUEST_ID_OFFSET = 168;
+ private static final int REQUEST_OPERATION_OFFSET = 176;
+ private static final int REPLY_COMMIT_OFFSET = 184;
+ private static final int REQUEST_CODE_OFFSET = 196;
+ private static final int REPLY_REQUEST_ID_OFFSET = 200;
+ private static final int REPLY_OPERATION_OFFSET = 208;
+ private static final int REPLY_STATUS_OFFSET = 216;
+ private static final int EVICTION_REASON_OFFSET = 255;
+
+ private VsrLoopbackPeer() {}
+
+ static CompletableFuture<Void> serve(ServerSocket server, RequestHandler handler) {
+ return serve(server, 1, handler);
+ }
+
+ /**
+ * Runs blocking socket I/O on one dedicated daemon thread per mock node.
+ * The common fork-join pool has only cores minus one workers on small CI
+ * runners, so three blocking nodes can starve the client continuations the
+ * test is waiting for when the full suite runs concurrently.
+ *
+ * @param server the listening socket
+ * @param connectionCount how many connections to accept in sequence
+ * @param handler answers each decoded request
+ * @return a future completing once every accepted connection has ended
+ */
+ static CompletableFuture<Void> serve(ServerSocket server, int connectionCount, RequestHandler handler) {
+ CompletableFuture<Void> serving = new CompletableFuture<>();
+ Thread serverThread = new Thread(
+ () -> {
+ try {
+ for (int connection = 0; connection < connectionCount; connection++) {
+ serveConnection(server, handler);
+ }
+ serving.complete(null);
+ } catch (IOException error) {
+ serving.completeExceptionally(new IllegalStateException("Mock VSR server failed", error));
+ } catch (RuntimeException error) {
+ serving.completeExceptionally(error);
+ }
+ },
+ "vsr-loopback-peer-" + server.getLocalPort());
+ serverThread.setDaemon(true);
+ serverThread.start();
+ return serving;
+ }
+
+ /**
+ * Accepts connections until the socket closes, each on its own thread, for
+ * tests that keep several client connections open at the same time.
+ *
+ * @param server the listening socket
+ * @param handler answers each decoded request
+ */
+ static void serveConcurrently(ServerSocket server, RequestHandler handler) {
+ Thread acceptor = new Thread(
+ () -> {
+ while (!server.isClosed()) {
+ Socket socket;
+ try {
+ socket = server.accept();
+ } catch (IOException closed) {
+ return;
+ }
+ Thread worker = new Thread(() -> serveSocket(socket, handler), "vsr-loopback-connection");
+ worker.setDaemon(true);
+ worker.start();
+ }
+ },
+ "vsr-loopback-acceptor-" + server.getLocalPort());
+ acceptor.setDaemon(true);
+ acceptor.start();
+ }
+
+ static ByteBuf registerBody(long session) {
+ ByteBuf body = Unpooled.buffer();
+ body.writeIntLE(0);
+ body.writeIntLE(1);
+ body.writeLongLE(session);
+ body.writeIntLE(11 << 10);
+ body.writeByte(0);
+ return body;
+ }
+
+ static ByteBuf transientResult(int errorCode) {
+ ByteBuf body = Unpooled.buffer(3 * Integer.BYTES);
+ body.writeIntLE(1);
+ body.writeIntLE(0);
+ body.writeIntLE(errorCode);
+ return body;
+ }
+
+ static ByteBuf singleNodeMetadata(int port) {
+ ByteBuf body = Unpooled.buffer();
+ writeString(body, "test-cluster");
+ body.writeIntLE(1);
+ writeNode(body, "node", port, true);
+ return body;
+ }
+
+ static ByteBuf clusterMetadata(int oldLeaderPort, int newLeaderPort, int leaderPort) {
+ ByteBuf body = Unpooled.buffer();
+ writeString(body, "test-cluster");
+ body.writeIntLE(2);
+ writeNode(body, "old-node", oldLeaderPort, oldLeaderPort == leaderPort);
+ writeNode(body, "new-node", newLeaderPort, newLeaderPort == leaderPort);
+ return body;
+ }
+
+ static ByteBuf threeNodeMetadata(int firstPort, int secondPort, int thirdPort) {
+ ByteBuf body = Unpooled.buffer();
+ writeString(body, "test-cluster");
+ body.writeIntLE(3);
+ writeNode(body, "metadata-leader", firstPort, true);
+ writeNode(body, "follower", secondPort, false);
+ writeNode(body, "partition-primary", thirdPort, false);
+ return body;
+ }
+
+ static void writeNode(ByteBuf body, String name, int port, boolean leader) {
+ writeString(body, name);
+ writeString(body, InetAddress.getLoopbackAddress().getHostAddress());
+ body.writeShortLE(port);
+ body.writeShortLE(0);
+ body.writeShortLE(0);
+ body.writeShortLE(0);
+ body.writeByte(leader ? 0 : 1);
+ body.writeByte(0);
+ }
+
+ private static void serveConnection(ServerSocket server, RequestHandler handler) throws IOException {
+ try (Socket socket = server.accept()) {
+ pump(socket, handler);
+ }
+ }
+
+ private static void serveSocket(Socket socket, RequestHandler handler) {
+ try (socket) {
+ pump(socket, handler);
+ } catch (IOException | RuntimeException ignored) {
+ // A client that closes mid-exchange is the behaviour under test.
+ }
+ }
+
+ private static void pump(Socket socket, RequestHandler handler) throws IOException {
+ InputStream input = socket.getInputStream();
+ OutputStream output = socket.getOutputStream();
+ Request request;
+ while ((request = readRequest(input)) != null) {
+ Response response = handler.handle(request);
+ if (response.command() == COMMAND_DISCONNECT) {
+ return;
+ }
+ writeResponse(output, request, response);
+ if (response.closeAfterReply()) {
+ return;
+ }
+ }
+ }
+
+ private static Request readRequest(InputStream input) throws IOException {
+ byte[] header = input.readNBytes(HEADER_SIZE);
+ if (header.length == 0) {
+ return null;
+ }
+ if (header.length != HEADER_SIZE) {
+ throw new EOFException("Truncated VSR request header");
+ }
+ ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN);
+ int size = fields.getInt(SIZE_OFFSET);
+ byte[] body = input.readNBytes(size - HEADER_SIZE);
+ if (body.length != size - HEADER_SIZE) {
+ throw new EOFException("Truncated VSR request body");
+ }
+ return new Request(
+ Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]),
+ fields.getInt(REQUEST_CODE_OFFSET),
+ fields.getLong(REQUEST_ID_OFFSET),
+ fields.getLong(REQUEST_CLIENT_OFFSET),
+ fields.getLong(REQUEST_CLIENT_OFFSET + Long.BYTES),
+ body);
+ }
+
+ private static void writeResponse(OutputStream output, Request request, Response response) throws IOException {
+ if (response.command() == COMMAND_SILENCE) {
+ return;
+ }
+ byte[] body = new byte[response.body().readableBytes()];
+ response.body().readBytes(body);
+ response.body().release();
+ byte[] header = new byte[HEADER_SIZE];
+ ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN);
+ fields.putInt(SIZE_OFFSET, HEADER_SIZE + body.length);
+ header[COMMAND_OFFSET] = (byte) response.command();
+ if (response.command() == COMMAND_EVICTION) {
+ header[EVICTION_REASON_OFFSET] = (byte) response.evictionReason();
+ } else {
+ fields.putLong(REPLY_REQUEST_ID_OFFSET, request.requestId());
+ header[REPLY_OPERATION_OFFSET] = (byte) response.operation();
+ fields.putInt(REPLY_STATUS_OFFSET, response.status());
+ fields.putLong(REPLY_COMMIT_OFFSET, response.commit());
+ }
+ output.write(header);
+ output.write(body);
+ output.flush();
+ }
+
+ private static void writeString(ByteBuf body, String value) {
+ byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+ body.writeIntLE(bytes.length);
+ body.writeBytes(bytes);
+ }
+
+ @FunctionalInterface
+ interface RequestHandler {
+ Response handle(Request request);
+ }
+
+ record Request(int operation, int commandCode, long requestId, long clientLow, long clientHigh, byte[] body) {
+ boolean is(int expectedCode, int expectedOperation) {
+ return commandCode == expectedCode && operation == expectedOperation;
+ }
+
+ /** The request body as text, for asserting which user a login names. */
+ String bodyAsText() {
+ return new String(body, StandardCharsets.UTF_8);
+ }
+ }
+
+ record Response(
+ int command,
+ int operation,
+ int status,
+ int evictionReason,
+ long commit,
+ ByteBuf body,
+ boolean closeAfterReply) {
+
+ static Response success(int operation, ByteBuf body) {
+ return new Response(COMMAND_REPLY, operation, 0, 0, 0, body, false);
+ }
+
+ static Response error(int operation, int status) {
+ return new Response(COMMAND_REPLY, operation, status, 0, 0, Unpooled.EMPTY_BUFFER, false);
+ }
+
+ static Response eviction(int reason) {
+ return new Response(COMMAND_EVICTION, 0, 0, reason, 0, Unpooled.EMPTY_BUFFER, false);
+ }
+
+ static Response committed(int operation, long commit, ByteBuf body) {
+ return new Response(COMMAND_REPLY, operation, 0, 0, commit, body, false);
+ }
+
+ static Response successAndDisconnect(int operation, ByteBuf body) {
+ return new Response(COMMAND_REPLY, operation, 0, 0, 0, body, true);
+ }
+
+ static Response disconnect() {
+ return new Response(COMMAND_DISCONNECT, 0, 0, 0, 0, Unpooled.EMPTY_BUFFER, false);
+ }
+
+ static Response noReply() {
+ return new Response(COMMAND_SILENCE, 0, 0, 0, 0, Unpooled.EMPTY_BUFFER, false);
+ }
+ }
+}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java
index fe916cd9f..5f5c50d45 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java
@@ -26,6 +26,8 @@ import io.netty.handler.codec.DecoderException;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
+import java.util.concurrent.atomic.AtomicInteger;
+
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -115,4 +117,44 @@ class VsrFrameDecoderTest {
}
}
}
+
+ @Test
+ void shouldFollowALimitTightenedForOneExchange() {
+ int tightened = VsrHeaders.HEADER_SIZE + 16;
+ AtomicInteger limit = new AtomicInteger(VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE);
+ EmbeddedChannel tightenedChannel = new EmbeddedChannel(new VsrFrameDecoder(limit::get));
+ try {
+ limit.set(tightened);
+ ByteBuf oversized = Unpooled.buffer(VsrHeaders.HEADER_SIZE);
+ oversized.writeZero(VsrHeaders.HEADER_SIZE);
+ oversized.setIntLE(VsrHeaders.SIZE_OFFSET, tightened + 1);
+
+ assertThatThrownBy(() -> tightenedChannel.writeInbound(oversized)).isInstanceOf(DecoderException.class);
+ } finally {
+ try {
+ tightenedChannel.finishAndReleaseAll();
+ } catch (DecoderException ignored) {
+ // Closing a failed decoder can replay its exception.
+ }
+ }
+ }
+
+ @Test
+ void shouldAcceptAFrameInsideTheTightenedLimitBeforeAccumulatingIt() {
+ int tightened = VsrHeaders.HEADER_SIZE + 16;
+ AtomicInteger limit = new AtomicInteger(tightened);
+ EmbeddedChannel tightenedChannel = new EmbeddedChannel(new VsrFrameDecoder(limit::get));
+ try {
+ ByteBuf frame = Unpooled.buffer(tightened);
+ frame.writeZero(tightened);
+ frame.setIntLE(VsrHeaders.SIZE_OFFSET, tightened);
+
+ assertThat(tightenedChannel.writeInbound(frame)).isTrue();
+ ByteBuf decoded = tightenedChannel.readInbound();
+ assertThat(decoded.readableBytes()).isEqualTo(tightened);
+ decoded.release();
+ } finally {
+ tightenedChannel.finishAndReleaseAll();
+ }
+ }
}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java
index fd4034f2b..d5bf0176a 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java
@@ -20,16 +20,20 @@
package org.apache.iggy.client.blocking;
import org.apache.iggy.consumergroup.Consumer;
+import org.apache.iggy.message.DeferredPollOptions;
import org.apache.iggy.message.Message;
import org.apache.iggy.message.Partitioning;
+import org.apache.iggy.message.PolledMessages;
import org.apache.iggy.message.PollingKind;
import org.apache.iggy.message.PollingStrategy;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.math.BigInteger;
+import java.time.Duration;
import java.util.List;
import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
import static java.util.Optional.empty;
import static org.apache.iggy.TestConstants.STREAM_NAME;
@@ -38,6 +42,11 @@ import static org.assertj.core.api.Assertions.assertThat;
public abstract class MessagesClientBaseTest extends IntegrationTest {
+ private static final Duration READINESS_WAIT = Duration.ofSeconds(4);
+ private static final Duration SHORT_READINESS_WAIT = Duration.ofMillis(400);
+ private static final Duration SEND_DELAY = Duration.ofMillis(200);
+ private static final int BYTE_BOUND_PAYLOAD_SIZE = 400;
+
protected MessagesClient messagesClient;
@BeforeEach
@@ -213,4 +222,125 @@ public abstract class MessagesClientBaseTest extends IntegrationTest {
assertThat(polledMessages.messages()).hasSize(1);
assertThat(new String(polledMessages.messages().get(0).payload())).isEqualTo(content);
}
+
+ @Test
+ void shouldWakeADeferredPollWhenAMessageArrives() {
+ // given
+ setUpStreamAndTopic();
+ var options = DeferredPollOptions.defaults().withMaxWait(READINESS_WAIT);
+ CompletableFuture<Void> sender = CompletableFuture.runAsync(() -> {
+ sleep(SEND_DELAY);
+ messagesClient.sendMessages(
+ STREAM_NAME, TOPIC_NAME, Partitioning.partitionId(0L), List.of(Message.of("late arrival")));
+ });
+
+ // when
+ long startedNanos = System.nanoTime();
+ PolledMessages polled = messagesClient.pollMessagesDeferred(
+ STREAM_NAME,
+ TOPIC_NAME,
+ Optional.of(0L),
+ Consumer.of(0L),
+ PollingStrategy.first(),
+ 10L,
+ false,
+ options);
+ Duration elapsed = Duration.ofNanos(System.nanoTime() - startedNanos);
+ sender.join();
+
+ // then
+ assertThat(polled.messages()).hasSize(1);
+ assertThat(new String(polled.messages().get(0).payload())).isEqualTo("late arrival");
+ assertThat(elapsed)
+ .as("the poll woke on the message, not on the readiness deadline")
+ .isLessThan(READINESS_WAIT);
+ }
+
+ @Test
+ void shouldReturnEmptyWhenTheReadinessWaitExpires() {
+ // given
+ setUpStreamAndTopic();
+ var options = DeferredPollOptions.defaults().withMaxWait(SHORT_READINESS_WAIT);
+
+ // when
+ long startedNanos = System.nanoTime();
+ PolledMessages polled = messagesClient.pollMessagesDeferred(
+ STREAM_NAME,
+ TOPIC_NAME,
+ Optional.of(0L),
+ Consumer.of(0L),
+ PollingStrategy.first(),
+ 10L,
+ false,
+ options);
+ Duration elapsed = Duration.ofNanos(System.nanoTime() - startedNanos);
+
+ // then
+ assertThat(polled.messages()).isEmpty();
+ assertThat(elapsed).isGreaterThanOrEqualTo(SHORT_READINESS_WAIT.dividedBy(2));
+ }
+
+ @Test
+ void shouldReturnAPartialBatchWhenTheReadinessTargetIsNotMet() {
+ // given
+ setUpStreamAndTopic();
+ messagesClient.sendMessages(
+ STREAM_NAME, TOPIC_NAME, Partitioning.partitionId(0L), List.of(Message.of("only one")));
+ var options =
+ DeferredPollOptions.defaults().withMaxWait(SHORT_READINESS_WAIT).withMinCount(3);
+
+ // when
+ PolledMessages polled = messagesClient.pollMessagesDeferred(
+ STREAM_NAME,
+ TOPIC_NAME,
+ Optional.of(0L),
+ Consumer.of(0L),
+ PollingStrategy.first(),
+ 10L,
+ false,
+ options);
+
+ // then
+ assertThat(polled.messages()).hasSize(1);
+ }
+
+ @Test
+ void shouldReturnADeferredBatchBoundedByBytes() {
+ // given
+ setUpStreamAndTopic();
+ String payload = "x".repeat(BYTE_BOUND_PAYLOAD_SIZE);
+ messagesClient.sendMessages(
+ STREAM_NAME,
+ TOPIC_NAME,
+ Partitioning.partitionId(0L),
+ List.of(Message.of(payload), Message.of(payload), Message.of(payload)));
+ // Room for a message and its framing, far short of all three.
+ var options = DeferredPollOptions.defaults()
+ .withMaxWait(SHORT_READINESS_WAIT)
+ .withMaxBytes(2L * BYTE_BOUND_PAYLOAD_SIZE);
+
+ // when
+ PolledMessages polled = messagesClient.pollMessagesDeferred(
+ STREAM_NAME,
+ TOPIC_NAME,
+ Optional.of(0L),
+ Consumer.of(0L),
+ PollingStrategy.first(),
+ 10L,
+ false,
+ options);
+
+ // then
+ assertThat(polled.messages()).hasSizeBetween(1, 2);
+ assertThat(new String(polled.messages().get(0).payload())).isEqualTo(payload);
+ }
+
+ private static void sleep(Duration duration) {
+ try {
+ Thread.sleep(duration.toMillis());
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(interrupted);
+ }
+ }
}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/MessagesHttpDeferredTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/MessagesHttpDeferredTest.java
new file mode 100644
index 000000000..1cf1c9e0b
--- /dev/null
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/MessagesHttpDeferredTest.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iggy.client.blocking.http;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.iggy.consumergroup.Consumer;
+import org.apache.iggy.exception.IggyException;
+import org.apache.iggy.exception.IggyOperationNotSupportedException;
+import org.apache.iggy.exception.IggyServerException;
+import org.apache.iggy.identifier.StreamId;
+import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
+import org.apache.iggy.message.PolledMessages;
+import org.apache.iggy.message.PollingStrategy;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * The HTTP deferred poll against a peer that can misbehave: the route and its
+ * query, the local refusal a stateless transport forces, the retry rule, and
+ * the body bound a well-behaved server never exercises.
+ */
+class MessagesHttpDeferredTest {
+
+ private static final String EMPTY_POLL = "{\"partition_id\":0,\"current_offset\":0,\"count\":0,\"messages\":[]}";
+ private static final String NOT_ACCEPTED =
+ "{\"id\":58,\"code\":\"transient_not_accepted\",\"reason\":\"no admission\"}";
+ private static final String NOT_COMMITTED =
+ "{\"id\":57,\"code\":\"transient_not_committed\",\"reason\":\"unknown outcome\"}";
+ private static final int SERVICE_UNAVAILABLE = 503;
+ private static final int TRANSIENT_NOT_COMMITTED = 57;
+
+ @Test
+ void shouldSendTheDeclaredLimitsToTheDeferredRoute() throws Exception {
+ var peer = new FakePeer(exchange -> respond(exchange, 200, EMPTY_POLL));
+ try (var server = peer.start();
+ var client = client(peer)) {
+ var options = DeferredPollOptions.defaults()
+ .withMaxWait(Duration.ofSeconds(2))
+ .withMinCount(4)
+ .withMaxBytes(2048)
+ .withRequestTimeout(Duration.ofSeconds(9));
+
+ PolledMessages polled = poll(client, Consumer.of(3L), options);
+
+ assertThat(polled.messages()).isEmpty();
+ assertThat(peer.paths).containsExactly("/streams/1/topics/1/messages/deferred");
+ Map<String, String> query = peer.queries.get(0);
+ assertThat(Long.parseLong(query.get("wait_us")))
+ .as("both budgets are reduced by the time already spent")
+ .isPositive()
+ .isLessThanOrEqualTo(options.maxWaitMicros());
+ assertThat(query.get("min_count")).isEqualTo("4");
+ assertThat(query.get("max_bytes")).isEqualTo("2048");
+ assertThat(Long.parseLong(query.get("request_timeout_us")))
+ .isPositive()
+ .isLessThanOrEqualTo(options.requestTimeoutMicros());
+ assertThat(query.get("consumer_id")).isEqualTo("3");
+ assertThat(query.get("partition_id")).isEqualTo("0");
+ assertThat(query.get("count")).isEqualTo("10");
+ assertThat(query.get("auto_commit")).isEqualTo("false");
+ }
+ }
+
+ @Test
+ void shouldRefuseAGroupConsumerWithoutReachingTheServer() throws Exception {
+ var peer = new FakePeer(exchange -> respond(exchange, 200, EMPTY_POLL));
+ try (var server = peer.start();
+ var client = client(peer)) {
+
+ assertThatThrownBy(() -> poll(client, Consumer.group(3L), DeferredPollOptions.defaults()))
+ .isInstanceOf(IggyOperationNotSupportedException.class);
+
+ assertThat(peer.paths)
+ .as("the poll query has no consumer kind, so a group id would poll the wrong thing")
+ .isEmpty();
+ }
+ }
+
+ @Test
+ void shouldRetryAnExplicitRefusalWithWhatIsLeftOfTheBudget() throws Exception {
+ var refusals = new AtomicInteger(2);
+ var peer = new FakePeer(exchange -> {
+ if (refusals.getAndDecrement() > 0) {
+ respond(exchange, SERVICE_UNAVAILABLE, NOT_ACCEPTED);
+ } else {
+ respond(exchange, 200, EMPTY_POLL);
+ }
+ });
+ try (var server = peer.start();
+ var client = client(peer)) {
+
+ poll(client, Consumer.of(3L), DeferredPollOptions.defaults());
+
+ assertThat(peer.queries).hasSize(3);
+ long firstWait = Long.parseLong(peer.queries.get(0).get("wait_us"));
+ long lastWait = Long.parseLong(peer.queries.get(2).get("wait_us"));
+ assertThat(lastWait).isLessThan(firstWait);
+ }
+ }
+
+ @Test
+ void shouldNotReplayAnAmbiguousRefusal() throws Exception {
+ var peer = new FakePeer(exchange -> respond(exchange, SERVICE_UNAVAILABLE, NOT_COMMITTED));
+ try (var server = peer.start();
+ var client = client(peer)) {
+
+ assertThatThrownBy(() -> poll(client, Consumer.of(3L), DeferredPollOptions.defaults()))
+ .isInstanceOf(IggyServerException.class)
+ .extracting(error -> ((IggyServerException) error).getRawErrorCode())
+ .isEqualTo(TRANSIENT_NOT_COMMITTED);
+ assertThat(peer.queries)
+ .as("an auto-commit poll that may have been served is never replayed")
+ .hasSize(1);
+ }
+ }
+
+ @Test
+ void shouldRefuseABodyLargerThanTheDeclaredBound() throws Exception {
+ var peer = new FakePeer(exchange -> {
+ // Chunked, so the bound has to hold while the body is read.
+ exchange.sendResponseHeaders(200, 0);
+ try (OutputStream body = exchange.getResponseBody()) {
+ byte[] chunk = new byte[8 * 1024];
+ for (int written = 0; written < 1024 * 1024; written += chunk.length) {
+ body.write(chunk);
+ }
+ }
+ });
+ try (var server = peer.start();
+ var client = client(peer)) {
+ var options = DeferredPollOptions.defaults().withMaxBytes(1024);
+
+ assertThatThrownBy(() -> poll(client, Consumer.of(3L), options))
+ .isInstanceOf(IggyException.class)
+ .hasMessageContaining("bytes");
+ }
+ }
+
+ private static IggyHttpClient client(FakePeer peer) {
+ return new IggyHttpClient("http://" + InetAddress.getLoopbackAddress().getHostAddress() + ":" + peer.port);
+ }
+
+ private static PolledMessages poll(IggyHttpClient client, Consumer consumer, DeferredPollOptions options) {
+ return client.messages()
+ .pollMessagesDeferred(
+ StreamId.of(1L),
+ TopicId.of(1L),
+ Optional.of(0L),
+ consumer,
+ PollingStrategy.first(),
+ 10L,
+ false,
+ options);
+ }
+
+ private static void respond(HttpExchange exchange, int status, String body) throws IOException {
+ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
+ exchange.getResponseHeaders().add("Content-Type", "application/json");
+ exchange.sendResponseHeaders(status, bytes.length);
+ try (OutputStream out = exchange.getResponseBody()) {
+ out.write(bytes);
+ }
+ }
+
+ private static final class FakePeer implements HttpHandler {
+ private final List<String> paths = new CopyOnWriteArrayList<>();
+ private final List<Map<String, String>> queries = new CopyOnWriteArrayList<>();
+ private final HttpHandler handler;
+ private int port;
+
+ private FakePeer(HttpHandler handler) {
+ this.handler = handler;
+ }
+
+ private ServerHandle start() throws IOException {
+ HttpServer server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+ server.createContext("/", this);
+ server.start();
+ port = server.getAddress().getPort();
+ return new ServerHandle(server);
+ }
+
+ @Override
+ public void handle(HttpExchange exchange) throws IOException {
+ paths.add(exchange.getRequestURI().getPath());
+ queries.add(parseQuery(exchange.getRequestURI().getRawQuery()));
+ try {
+ handler.handle(exchange);
+ } finally {
+ exchange.close();
+ }
+ }
+
+ private static Map<String, String> parseQuery(String rawQuery) {
+ Map<String, String> parsed = new HashMap<>();
+ if (rawQuery == null) {
+ return parsed;
+ }
+ for (String pair : rawQuery.split("&")) {
+ int separator = pair.indexOf('=');
+ if (separator > 0) {
+ parsed.put(pair.substring(0, separator), pair.substring(separator + 1));
+ }
+ }
+ return parsed;
+ }
+ }
+
+ private record ServerHandle(HttpServer server) implements AutoCloseable {
+ @Override
+ public void close() {
+ server.stop(0);
+ }
+ }
+}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java
index f284ad5c8..c76f8b005 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java
@@ -22,6 +22,7 @@ package org.apache.iggy.client.blocking.tcp;
import org.apache.iggy.client.BaseIntegrationTest;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.DeferredPollOptions;
import org.apache.iggy.message.Message;
import org.apache.iggy.message.Partitioning;
import org.apache.iggy.message.PolledMessages;
@@ -157,6 +158,58 @@ class TlsConnectionTest extends BaseIntegrationTest {
}
}
+ /**
+ * A deferred poll runs on its own data connection, so it repeats the TLS
+ * handshake with the configured certificate rather than reusing the
+ * coordinator's channel.
+ */
+ @Test
+ void deferredPollOverTlsShouldWork() {
+ var client = IggyTcpClient.builder()
+ .host(TLS_HOST)
+ .port(serverTcpPort())
+ .enableTls()
+ .tlsCertificate(CERTS_DIR.resolve("iggy_ca_cert.pem").toFile())
+ .credentials("iggy", "iggy")
+ .buildAndLogin();
+
+ String streamName = "tls-deferred-stream";
+ StreamId streamId = StreamId.of(streamName);
+ String topicName = "tls-deferred-topic";
+ TopicId topicId = TopicId.of(topicName);
+
+ try {
+ client.streams().createStream(streamName);
+ client.topics()
+ .createTopic(streamId, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, topicName);
+ client.messages()
+ .sendMessages(
+ streamId,
+ topicId,
+ Partitioning.partitionId(0L),
+ List.of(Message.of("tls-deferred-1"), Message.of("tls-deferred-2")));
+
+ PolledMessages polled = client.messages()
+ .pollMessagesDeferred(
+ streamId,
+ topicId,
+ Optional.of(0L),
+ org.apache.iggy.consumergroup.Consumer.of(0L),
+ PollingStrategy.first(),
+ 10L,
+ false,
+ DeferredPollOptions.defaults().withMinCount(2));
+
+ assertThat(polled.messages()).hasSize(2);
+ } finally {
+ try {
+ client.streams().deleteStream(streamId);
+ } catch (RuntimeException e) {
+ log.debug("Cleanup failed: {}", e.getMessage());
+ }
+ }
+ }
+
private static Path findCertsDir() {
File dir = new File(System.getProperty("user.dir"));
while (dir != null) {
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/message/DeferredPollOptionsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/message/DeferredPollOptionsTest.java
new file mode 100644
index 000000000..21e52ad35
--- /dev/null
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/message/DeferredPollOptionsTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iggy.message;
+
+import org.apache.iggy.exception.IggyInvalidArgumentException;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class DeferredPollOptionsTest {
+
+ @Test
+ void shouldMatchTheServerDefaults() {
+ var defaults = DeferredPollOptions.defaults();
+
+ assertThat(defaults.maxWait()).isEqualTo(Duration.ofSeconds(1));
+ assertThat(defaults.minCount()).isEqualTo(1);
+ assertThat(defaults.maxBytes()).isEqualTo(1024 * 1024);
+ assertThat(defaults.requestTimeout()).isEqualTo(Duration.ofSeconds(11));
+ assertThat(defaults.maxWaitMicros()).isEqualTo(1_000_000);
+ assertThat(defaults.requestTimeoutMicros()).isEqualTo(11_000_000);
+ }
+
+ @Test
+ void shouldAcceptTheProtocolBoundaries() {
+ var atLimit = new DeferredPollOptions(
+ DeferredPollOptions.MAX_WAIT,
+ 0xFFFF_FFFFL,
+ DeferredPollOptions.MIN_MAX_BYTES,
+ DeferredPollOptions.MAX_REQUEST_TIMEOUT);
+
+ assertThat(atLimit.maxWaitMicros()).isEqualTo(600_000_000L);
+ assertThat(atLimit.requestTimeoutMicros()).isEqualTo(630_000_000L);
+ }
+
+ @Test
+ void shouldAcceptAZeroWaitAsAnImmediateBoundedRead() {
+ var immediate = DeferredPollOptions.defaults().withMaxWait(Duration.ZERO);
+
+ assertThat(immediate.maxWaitMicros()).isZero();
+ }
+
+ @Test
+ void shouldRejectLimitsTheServerWouldRefuse() {
+ var defaults = DeferredPollOptions.defaults();
+
+ assertThatThrownBy(() -> defaults.withMaxWait(DeferredPollOptions.MAX_WAIT.plusMillis(1)))
+ .isInstanceOf(IggyInvalidArgumentException.class);
+ assertThatThrownBy(() -> defaults.withMaxWait(Duration.ofSeconds(-1)))
+ .isInstanceOf(IggyInvalidArgumentException.class);
+ assertThatThrownBy(() -> defaults.withRequestTimeout(Duration.ZERO))
+ .isInstanceOf(IggyInvalidArgumentException.class);
+ assertThatThrownBy(() -> defaults.withRequestTimeout(DeferredPollOptions.MAX_REQUEST_TIMEOUT.plusMillis(1)))
+ .isInstanceOf(IggyInvalidArgumentException.class);
+ assertThatThrownBy(() -> defaults.withMinCount(0)).isInstanceOf(IggyInvalidArgumentException.class);
+ assertThatThrownBy(() -> defaults.withMinCount(0x1_0000_0000L))
+ .isInstanceOf(IggyInvalidArgumentException.class);
+ assertThatThrownBy(() -> defaults.withMaxBytes(DeferredPollOptions.MIN_MAX_BYTES - 1))
+ .isInstanceOf(IggyInvalidArgumentException.class);
+ assertThatThrownBy(() -> defaults.withMaxBytes(0x1_0000_0000L))
+ .isInstanceOf(IggyInvalidArgumentException.class);
+ }
+
+ @Test
+ void shouldRejectAWaitLongerThanTheTotalBudget() {
+ assertThatThrownBy(() -> new DeferredPollOptions(Duration.ofSeconds(12), 1, 1024, Duration.ofSeconds(11)))
+ .isInstanceOf(IggyInvalidArgumentException.class)
+ .hasMessageContaining("shorter than maxWait");
+ }
+
+ @Test
+ void shouldAcceptAWaitEqualToTheTotalBudget() {
+ var equal = new DeferredPollOptions(Duration.ofSeconds(11), 1, 1024, Duration.ofSeconds(11));
+
+ assertThat(equal.maxWaitMicros()).isEqualTo(equal.requestTimeoutMicros());
+ }
+
+ @Test
+ void shouldRejectFractionalMicrosecondsInsteadOfTruncatingThem() {
+ assertThatThrownBy(() -> DeferredPollOptions.defaults().withMaxWait(Duration.ofNanos(1500)))
+ .isInstanceOf(IggyInvalidArgumentException.class)
+ .hasMessageContaining("whole number of microseconds");
+ assertThatThrownBy(() -> DeferredPollOptions.defaults().withRequestTimeout(Duration.ofNanos(11_000_000_001L)))
+ .isInstanceOf(IggyInvalidArgumentException.class)
+ .hasMessageContaining("whole number of microseconds");
+ }
+
+ @Test
+ void shouldRejectAReadinessTargetTheCountCannotMeet() {
+ var options = DeferredPollOptions.defaults().withMinCount(10);
+
+ options.validate(10);
+
+ assertThatThrownBy(() -> options.validate(9))
+ .isInstanceOf(IggyInvalidArgumentException.class)
+ .hasMessageContaining("exceeds the requested count");
+ assertThatThrownBy(() -> options.validate(0)).isInstanceOf(IggyInvalidArgumentException.class);
+ assertThatThrownBy(() -> options.validate(0x1_0000_0000L)).isInstanceOf(IggyInvalidArgumentException.class);
+ }
+}