You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/09/01 21:16:16 UTC
(iggy) branch master updated: chore(sdk): IggyConsumer docs (#3913)
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new a7d5a2894 chore(sdk): IggyConsumer docs (#3913)
a7d5a2894 is described below
commit a7d5a2894953cfac073a45741bc6cd5851d9a2ff
Author: haubur <ha...@mailbox.org>
AuthorDate: Tue Sep 1 23:16:11 2026 +0200
chore(sdk): IggyConsumer docs (#3913)
Co-authored-by: Hubert Gruszecki <h....@gmail.com>
---
core/sdk/src/clients/consumer.rs | 563 ++++++++++++++++++++++++++++++-
core/sdk/src/clients/consumer_builder.rs | 7 +-
2 files changed, 551 insertions(+), 19 deletions(-)
diff --git a/core/sdk/src/clients/consumer.rs b/core/sdk/src/clients/consumer.rs
index a4477e6c8..a7bf54983 100644
--- a/core/sdk/src/clients/consumer.rs
+++ b/core/sdk/src/clients/consumer.rs
@@ -270,6 +270,355 @@ impl IggyConsumerState {
// 4. All `&self` methods only access Sync-safe fields
unsafe impl Sync for IggyConsumer {}
+/// Reads messages from the partitions of one topic and yields them one at a time.
+///
+/// A topic is split into partitions, and a partition is an ordered log that producers append to.
+/// Every message sits at an *offset*, its position in that log. Reading is therefore always the
+/// same three decisions: which partition to read, where in it to start, and how to keep track of
+/// how far you got so the next run can continue there.
+///
+/// `IggyConsumer` handles all three. It fetches batches of messages from the server, keeps them in
+/// an in-memory buffer, decrypts them when it has an encryptor, and records how far it has read.
+/// **It implements [`Stream`], so consuming is a loop over [`StreamExt::next`].**
+///
+/// You can use a consumer as a worker draining a topic, a reader that replays a
+/// partition from a chosen point, and a pool of consumers sharing a workload through a consumer
+/// group.
+///
+/// # Creating a consumer
+///
+/// Easiest way is to use the [`IggyClient`] with a configured connection. Then:
+/// - [`IggyClient::consumer()`] builds a standalone consumer, bound to the one partition passed
+/// in.
+/// - [`IggyClient::consumer_group()`] builds a member of a consumer group. The server gives every
+/// partition to exactly one member, so several consumers using the same group name split the
+/// topic between them and share one set of offsets.
+///
+/// Note, building never talks to the server. [`init()`](Self::init) must be awaited once before the
+/// first message is read.
+///
+/// # Examples
+///
+/// A standalone consumer reading partition 1 with the defaults:
+///
+/// ```rust,no_run
+/// use futures_util::StreamExt;
+/// use iggy::prelude::*;
+///
+/// # async fn example() -> Result<(), IggyError> {
+/// let client = IggyClient::from_connection_string("iggy://iggy:iggy@localhost:8090")?;
+/// client.connect().await?;
+///
+/// let mut consumer = client
+/// .consumer("my-consumer", "my-stream", "my-topic", 1)?
+/// .batch_length(100)
+/// .poll_interval(IggyDuration::new_from_secs(1))
+/// .build();
+/// consumer.init().await?;
+///
+/// while let Some(received) = consumer.next().await {
+/// match received {
+/// Ok(received) => println!("Offset: {}", received.message.header.offset),
+/// Err(error) => eprintln!("Failed to read a message: {error}"),
+/// }
+/// }
+/// # Ok(())
+/// # }
+/// ```
+///
+/// A group member that queues a commit for every message just before handing it over, and shuts
+/// down cleanly:
+///
+/// ```rust,no_run
+/// use futures_util::StreamExt;
+/// use iggy::prelude::*;
+///
+/// # async fn handle(message: &IggyMessage) {}
+/// # async fn example() -> Result<(), IggyError> {
+/// let client = IggyClient::from_connection_string("iggy://iggy:iggy@localhost:8090")?;
+/// client.connect().await?;
+///
+/// let mut consumer = client
+/// .consumer_group("order-workers", "my-stream", "my-topic")?
+/// .auto_commit(AutoCommit::When(AutoCommitWhen::ConsumingEachMessage))
+/// .polling_strategy(PollingStrategy::next())
+/// .build();
+/// consumer.init().await?;
+///
+/// let mut consumed = 0;
+/// while let Some(received) = consumer.next().await {
+/// match received {
+/// Ok(received) => {
+/// handle(&received.message).await;
+/// consumed += 1;
+/// }
+/// Err(error) => eprintln!("Failed to read a message: {error}"),
+/// }
+/// if consumed == 100 {
+/// break;
+/// }
+/// }
+///
+/// consumer.shutdown().await?;
+/// # Ok(())
+/// # }
+/// ```
+///
+/// Committing by hand, so that a message the handler could not process comes back on the next
+/// run. Every commit is one round trip, and no auto-commit setting substitutes, since each of them
+/// also commits a message whose handler failed:
+///
+/// ```rust,no_run
+/// use futures_util::StreamExt;
+/// use iggy::prelude::*;
+///
+/// # async fn handle(message: &IggyMessage) -> Result<(), IggyError> { Ok(()) }
+/// # async fn example() -> Result<(), IggyError> {
+/// let client = IggyClient::from_connection_string("iggy://iggy:iggy@localhost:8090")?;
+/// client.connect().await?;
+///
+/// let mut consumer = client
+/// .consumer("my-consumer", "my-stream", "my-topic", 1)?
+/// .auto_commit(AutoCommit::Disabled)
+/// .polling_strategy(PollingStrategy::next())
+/// .build();
+/// consumer.init().await?;
+///
+/// while let Some(received) = consumer.next().await {
+/// let received = match received {
+/// Ok(received) => received,
+/// Err(error) => {
+/// eprintln!("Failed to read a message: {error}");
+/// continue;
+/// }
+/// };
+/// // Leaving a failed message uncommitted is what brings it back on the next run.
+/// if handle(&received.message).await.is_err() {
+/// break;
+/// }
+/// consumer
+/// .store_offset(received.message.header.offset, Some(received.partition_id))
+/// .await?;
+/// }
+/// // No shutdown() here: it would commit the reading position, failed message included.
+/// # Ok(())
+/// # }
+/// ```
+///
+/// # Which partitions are read
+///
+/// A **standalone consumer** reads exactly one partition, the one passed to
+/// [`IggyClient::consumer()`]. Covering a whole topic with several partitions
+/// means running one consumer per partition and dividing the work yourself.
+///
+/// A **consumer group member** does not choose. The server hands every partition of the topic to
+/// exactly one member, so consumers sharing a group name split the topic between them without
+/// coordinating. [`ReceivedMessage::partition_id`] tells where a message came from.
+///
+/// What to know when working with consumer groups:
+/// - With [`auto_join_consumer_group()`] (the default) a member joins during [`init()`](Self::init),
+/// creating the group first if [`create_consumer_group_if_not_exists()`] is set (the default).
+/// It rejoins on its own after a reconnect and whenever the server reports that its membership
+/// is gone.
+/// - Until the join has succeeded the consumer does not poll. It re-checks every
+/// [`polling_retry_interval()`] and polls once joined.
+/// - Partitions are redistributed whenever members join or leave, so a member reads different
+/// partitions over time and messages from several partitions interleave in its stream.
+/// - More members than partitions leaves the surplus members without partitions. Such a member
+/// still polls, one group sync round trip per attempt, so give it a [`poll_interval()`]. The
+/// partition count of the topic is the ceiling on how far one group can be scaled out.
+/// - The group shares one set of stored offsets, kept under the group name. Thus,
+/// a partition taken over by another member continues where the previous one
+/// committed.
+///
+/// # How messages are read
+///
+/// Reading is done by polling. One request fetches up to [`batch_length()`] messages. The consumer
+/// passes the first one to the caller and buffers the rest. The next request is sent once that buffer
+/// is empty.
+///
+/// [`poll_interval()`] sets the smallest gap between two requests, measured from one send to the
+/// next. Without it the next request goes out as soon as the previous one is answered, which is
+/// the fastest option but keeps a busy loop running against an idle topic.
+///
+/// [`polling_strategy()`] decides **where** in the partition reading begins:
+///
+/// | Strategy | Starts at |
+/// | --- | --- |
+/// | [`PollingStrategy::next()`] (default) | the message after the offset stored on the server, or the first message when nothing is stored yet |
+/// | [`PollingStrategy::first()`] | the oldest message in the partition |
+/// | [`PollingStrategy::last()`] | the end of the partition (returns up to [`batch_length()`] of the most recent messages) |
+/// | [`PollingStrategy::offset()`] | a custom offset |
+/// | [`PollingStrategy::timestamp()`] | the first message at or after a given point in time |
+///
+/// Only [`PollingStrategy::next()`] consults the offset stored on the server.
+/// Use this if you want to resume where a previous run stopped. The other four are starting points
+/// for the first request only. From the second request onwards, the consumer asks for whatever
+/// follows the last message it handed over, and it keeps one such continuation point for all
+/// partitions. That makes them fit for standalone consumers only: a group member polls a different
+/// partition on every request, so a continuation point taken from one partition is applied to the
+/// next, where it skips or repeats messages, and under the default [`auto_commit()`] the skipped
+/// range is committed as read.
+///
+/// [`StreamExt::next`] yields `None` once [`shutdown()`](Self::shutdown) has been called, and never
+/// otherwise: not when the topic is empty and not while the client is disconnected. A request that
+/// comes back empty is not an error and not the end of the stream, it just means nothing new has
+/// arrived yet.
+///
+/// A failed request is yielded as `Some(Err(..))` and leaves the consumer usable, while the next call
+/// retries. Connection and authentication failures pause polling until the client has reconnected
+/// and signed in again, which the consumer handles automatically. Hence, deciding when to give up on
+/// repeated errors is up to you.
+///
+/// For a boilerplate implementation of such a loop Iggy provides [`IggyConsumerMessageExt::consume_messages`].
+///
+/// # Tracking what has been read
+///
+/// An offset is the index tracking what has been already read from a partition by the consumer.
+/// Managing the offset has implications on where consumers resume reading messages.
+///
+/// There are two positions (offsets) tracked in two different places:
+/// - The **reading position** is held by the consumer, one per partition, and is the offset of the
+/// last message handed over
+/// ([`get_last_consumed_offset()`](Self::get_last_consumed_offset)). It dies with the process.
+/// - The **stored offset** lives on the server under the consumer name, or the group name for a
+/// group. This offset survives restarts. Writing it is called *storing* or *committing* an offset.
+///
+/// Committing matters because [`PollingStrategy::next()`] resumes from the stored offset. A
+/// consumer that never commits starts over from the same place on every run. Within a run it
+/// stalls instead: the server serves the same messages again on every request, messages at or
+/// below the reading position are dropped (see [Guarantees](#guarantees)), so the stream goes
+/// quiet once the reading position is [`batch_length()`] or more ahead of the stored offset. Under
+/// [`PollingStrategy::next()`], keep commits within [`batch_length()`] of the reading position.
+///
+/// [`auto_commit()`] decides when the consumer commits by itself:
+///
+/// | Setting | Commits |
+/// | --- | --- |
+/// | [`AutoCommit::Disabled`] | never on its own, decide manually with [`store_offset()`](Self::store_offset). [`shutdown()`](Self::shutdown) still commits the reading position |
+/// | [`AutoCommit::Interval`] | on every tick, the reading position of every partition read so far |
+/// | [`AutoCommitWhen::PollingMessages`] | sends the commit with the poll request itself, before your code sees the batch |
+/// | [`AutoCommitWhen::ConsumingEachMessage`] | queued just before every message is handed over to the calling code, at one round trip per message and without backpressure |
+/// | [`AutoCommitWhen::ConsumingEveryNthMessage`] | queued just before a message whose offset divides by `n` is handed over |
+/// | [`AutoCommitWhen::ConsumingAllMessages`] | queued when the buffer of the current batch runs empty |
+/// | [`AutoCommitAfter`] variants | once the handler returned, `Ok` or `Err`, and only under [`IggyConsumerMessageExt::consume_messages`], see below |
+///
+/// [`AutoCommit::IntervalOrWhen`] and [`AutoCommit::IntervalOrAfter`] combine an interval with a
+/// message trigger. The default is [`AutoCommit::IntervalOrWhen`] with one second and
+/// [`AutoCommitWhen::PollingMessages`].
+/// Important implications of these settings:
+/// - [`AutoCommitWhen::PollingMessages`] marks a batch as consumed while it is being delivered,
+/// before your code has seen any of it. For a crash-safe option configure with [`AutoCommit::Disabled`]
+/// and manually store the offset with [`Self::store_offset()`].
+/// - [`AutoCommitWhen::ConsumingEveryNthMessage`] tests the offset of a message, not a counter of
+/// messages this process handled, so it commits at every `n`-th offset of the partition. With
+/// `n = 0` the trigger never fires, so without an interval only [`shutdown()`](Self::shutdown)
+/// commits.
+/// - [`AutoCommitAfter::ConsumingAllMessages`] fires for the message whose offset equals the
+/// partition head seen by the poll ([`ReceivedMessage::current_offset`]), not when the buffer
+/// runs empty, so a consumer that lags behind commits nothing until it has caught up. Every
+/// [`AutoCommitAfter`] variant commits after a handler that returned `Err` as well.
+///
+/// ## Guarantees
+///
+/// - **Each message is handed over once per consumer.** Messages whose offset is not greater than
+/// the reading position of their partition are dropped before they reach the stream. Re-reading
+/// a partition, or seeing a failed message again within the same consumer, needs
+/// [`allow_replay()`], which turns that filter off. A new consumer starts with an empty filter.
+/// - **Delivering at-least-once.** If you cannot tolerate missing any messages, use [`AutoCommit::Disabled`]
+/// and store the offset using [`Self::store_offset()`] after handling a message. Every other
+/// setting except the plain [`AutoCommit::After`] variants can commit a message before your
+/// handler is done with it, so a crash in the handler loses it. [`AutoCommit::IntervalOrAfter`]
+/// still commits on its interval tick. [`shutdown()`](Self::shutdown) commits the reading
+/// position under every setting, [`AutoCommit::Disabled`] included, so it also commits a message
+/// whose handler failed.
+///
+/// # Options and defaults
+///
+/// Everything is configured on the [`IggyConsumerBuilder`] before [`build()`] and is fixed
+/// afterwards.
+///
+/// | Option | Default | Controls |
+/// | --- | --- | --- |
+/// | [`stream()`], [`topic()`], [`partition()`] | the values passed to the entry point | what is read. [`partition()`] is for standalone consumers, on a group member it pins every poll to that partition instead of the server's assignment |
+/// | [`batch_length()`] | 1000 | messages fetched per request |
+/// | [`poll_interval()`] | none | smallest gap between two requests |
+/// | [`polling_strategy()`] | [`PollingStrategy::next()`] | where reading starts. Anything but [`PollingStrategy::next()`] is for standalone consumers only |
+/// | [`auto_commit()`] | [`AutoCommit::IntervalOrWhen`], one second, [`AutoCommitWhen::PollingMessages`] | when offsets are committed. [`commit_failed_messages()`] is a synonym for [`AutoCommit::Disabled`] |
+/// | [`allow_replay()`] | off | whether a message can be handed over again |
+/// | [`auto_join_consumer_group()`] | on | joining the group during [`init()`](Self::init) and after a reconnect. A group member built with [`do_not_auto_join_consumer_group()`] never polls, since polling waits for the join |
+/// | [`create_consumer_group_if_not_exists()`] | on | creating the group when it is missing |
+/// | [`polling_retry_interval()`] | one second | wait between attempts while polling is blocked |
+/// | [`init_retries()`] | none, one second apart | retries when the stream or topic is missing at [`init()`](Self::init) |
+/// | [`offset_drain_timeout()`] | five seconds | how long [`shutdown()`](Self::shutdown) waits for pending commits |
+/// | [`encryptor()`] | inherited from the client | decrypting payloads and user headers |
+///
+/// The switches have inverse setters as well, such as [`without_poll_interval()`],
+/// [`without_encryptor()`], [`do_not_auto_join_consumer_group()`] and
+/// [`do_not_create_consumer_group_if_not_exists()`].
+///
+/// # Encryption
+///
+/// A consumer with an encryptor, inherited from the [`IggyClient`] or set with [`encryptor()`],
+/// decrypts payloads and user headers before a message is yielded. That only works if the producer
+/// encrypted them with the same key, which an [`IggyProducer`] and an `IggyConsumer` from the same
+/// client share unless one of them overrides it on its builder. Without an encryptor the consumer
+/// yields payloads as stored, encrypted or not.
+///
+/// A message that cannot be decrypted is yielded as an `Err` and the whole batch is dropped. What
+/// happens next depends on [`auto_commit()`]. Under [`AutoCommitWhen::PollingMessages`] (the
+/// default) the server committed the batch with the poll, so it is skipped for good. Under every
+/// other setting the next request fetches the same batch and fails the same way until
+/// [`store_offset()`](Self::store_offset) moves the offset past it. Pick a setting other than
+/// [`AutoCommitWhen::PollingMessages`] if a batch that fails to decrypt must not be lost silently.
+///
+/// # Concurrency
+///
+/// `IggyConsumer` is `Send` and `Sync` but not `Clone`. Driving the stream
+/// ([`StreamExt::next`]) and [`shutdown()`](Self::shutdown) take `&mut self`, so one task owns and
+/// drives a consumer end to end. To read offsets or commit from another task, take an
+/// [`IggyConsumerState`] via [`state()`](Self::state): it is a cheap clone of the shared
+/// bookkeeping and needs no lock.
+///
+/// Besides the stream, a consumer runs background tasks: one watching the connection lifecycle,
+/// one sending queued commits, and an interval commit task for the [`AutoCommit`] variants that
+/// carry an interval. See [`init()`](Self::init).
+///
+/// # Shutting down
+///
+/// Call [`shutdown()`](Self::shutdown) once done consuming. It drains the commit tasks, commits
+/// the reading position of every partition, [`AutoCommit::Disabled`] included, and leaves the
+/// consumer group. Dropping an `IggyConsumer` instead skips that final commit and the group
+/// leave, so the server reassigns the member's partitions only once the connection is gone.
+/// Commits already queued are still sent. Neither stops the lifecycle task, which runs until the
+/// client shuts down.
+///
+/// [`IggyClient`]: crate::prelude::IggyClient
+/// [`IggyClient::consumer()`]: crate::prelude::IggyClient::consumer
+/// [`IggyClient::consumer_group()`]: crate::prelude::IggyClient::consumer_group
+/// [`IggyProducer`]: crate::prelude::IggyProducer
+/// [`IggyConsumerBuilder`]: crate::prelude::IggyConsumerBuilder
+/// [`IggyConsumerMessageExt::consume_messages`]: crate::prelude::IggyConsumerMessageExt::consume_messages
+/// [`allow_replay()`]: crate::prelude::IggyConsumerBuilder::allow_replay
+/// [`auto_commit()`]: crate::prelude::IggyConsumerBuilder::auto_commit
+/// [`auto_join_consumer_group()`]: crate::prelude::IggyConsumerBuilder::auto_join_consumer_group
+/// [`batch_length()`]: crate::prelude::IggyConsumerBuilder::batch_length
+/// [`build()`]: crate::prelude::IggyConsumerBuilder::build
+/// [`commit_failed_messages()`]: crate::prelude::IggyConsumerBuilder::commit_failed_messages
+/// [`create_consumer_group_if_not_exists()`]: crate::prelude::IggyConsumerBuilder::create_consumer_group_if_not_exists
+/// [`do_not_auto_join_consumer_group()`]: crate::prelude::IggyConsumerBuilder::do_not_auto_join_consumer_group
+/// [`do_not_create_consumer_group_if_not_exists()`]: crate::prelude::IggyConsumerBuilder::do_not_create_consumer_group_if_not_exists
+/// [`encryptor()`]: crate::prelude::IggyConsumerBuilder::encryptor
+/// [`init_retries()`]: crate::prelude::IggyConsumerBuilder::init_retries
+/// [`offset_drain_timeout()`]: crate::prelude::IggyConsumerBuilder::offset_drain_timeout
+/// [`partition()`]: crate::prelude::IggyConsumerBuilder::partition
+/// [`poll_interval()`]: crate::prelude::IggyConsumerBuilder::poll_interval
+/// [`polling_retry_interval()`]: crate::prelude::IggyConsumerBuilder::polling_retry_interval
+/// [`polling_strategy()`]: crate::prelude::IggyConsumerBuilder::polling_strategy
+/// [`stream()`]: crate::prelude::IggyConsumerBuilder::stream
+/// [`topic()`]: crate::prelude::IggyConsumerBuilder::topic
+/// [`without_encryptor()`]: crate::prelude::IggyConsumerBuilder::without_encryptor
+/// [`without_poll_interval()`]: crate::prelude::IggyConsumerBuilder::without_poll_interval
pub struct IggyConsumer {
initialized: bool,
shutdown: Arc<AtomicBool>,
@@ -407,21 +756,28 @@ impl IggyConsumer {
}
/// Returns the name of the consumer.
+ ///
+ /// For a consumer group this is also the name of the group.
pub fn name(&self) -> &str {
&self.consumer_name
}
- /// Returns the topic ID of the consumer.
+ /// Returns the identifier of the topic this consumer reads from.
pub fn topic(&self) -> &Identifier {
&self.topic_id
}
- /// Returns the stream ID of the consumer.
+ /// Returns the identifier of the stream this consumer reads from.
pub fn stream(&self) -> &Identifier {
&self.stream_id
}
- /// Returns the current partition ID of the consumer.
+ /// Returns the partition the most recent poll response came from.
+ ///
+ /// This is `0` before the first response, and an empty response can report `0` as well. For a
+ /// consumer group the value changes over time, as the server hands different partitions to
+ /// this member. To commit for the partition a message came from, pass
+ /// [`ReceivedMessage::partition_id`] to [`store_offset()`](Self::store_offset) instead.
pub fn partition_id(&self) -> u32 {
self.state.partition_id()
}
@@ -431,7 +787,28 @@ impl IggyConsumer {
self.state.clone()
}
- /// Stores the consumer offset on the server either for the current partition or the provided partition ID.
+ /// Stores an offset on the server, marking every message up to and including it as consumed.
+ ///
+ /// This is the manual counterpart to [`AutoCommit`] and is meant for
+ /// [`AutoCommit::Disabled`].
+ ///
+ /// Pass `None` as `partition_id` to use [`partition_id()`](Self::partition_id), which for a
+ /// consumer group can already point at another partition. Prefer passing
+ /// [`ReceivedMessage::partition_id`].
+ ///
+ /// An offset that is not ahead of the last one this consumer stored for that partition is
+ /// skipped and `Ok(())` is returned without a request, unless the consumer was built with
+ /// [`allow_replay`](crate::prelude::IggyConsumerBuilder::allow_replay). Offset `0` is always
+ /// sent, yet [`PollingStrategy::next()`] then resumes at offset `1`, so it does not rewind to
+ /// the start. To start over from the first message, delete the stored offset with
+ /// [`delete_offset()`](Self::delete_offset) after [`shutdown()`](Self::shutdown) and build a new
+ /// consumer. A new consumer built with [`PollingStrategy::offset()`] re-reads from any point.
+ ///
+ /// # Errors
+ ///
+ /// Returns any error the server raised while storing the offset, for example
+ /// [`IggyError::Disconnected`] or a permission error. The offset is then not stored and the
+ /// call can be retried.
pub async fn store_offset(
&self,
offset: u64,
@@ -440,26 +817,102 @@ impl IggyConsumer {
self.state.store_offset(offset, partition_id).await
}
- /// Retrieves the last consumed offset for the specified partition ID.
- /// To get the current partition ID use `partition_id()`
+ /// Returns the offset of the last message this consumer handed over for the given partition,
+ /// or `None` while it has not polled that partition yet. The first poll of a partition seeds
+ /// the entry with `0`, so `Some(0)` also covers "polled, nothing handed over yet".
+ ///
+ /// This is the local reading position, which can be ahead of what has been stored on the
+ /// server.
pub fn get_last_consumed_offset(&self, partition_id: u32) -> Option<u64> {
self.state.get_last_consumed_offset(partition_id)
}
- /// Deletes the consumer offset on the server either for the current partition or the provided partition ID.
+ /// Deletes the offset stored on the server, so that the next consumer polling with
+ /// [`PollingStrategy::next()`] starts at the first message.
+ ///
+ /// `None` as `partition_id` means [`partition_id()`](Self::partition_id) for a standalone
+ /// consumer. A consumer group passes `None` through to the server. This consumer's own records
+ /// are untouched, so its auto-commit or [`shutdown()`](Self::shutdown) can store the offset
+ /// again. When starting over, call it after [`shutdown()`](Self::shutdown).
+ ///
+ /// # Errors
+ ///
+ /// Returns [`IggyError::ConsumerOffsetNotFound`] when nothing is stored for that partition, or
+ /// any other error the server raised, for example [`IggyError::Disconnected`].
pub async fn delete_offset(&self, partition_id: Option<u32>) -> Result<(), IggyError> {
self.state.delete_offset(partition_id).await
}
- /// Retrieves the last stored offset (on the server) for the specified partition ID.
- /// To get the current partition ID use `partition_id()`
+ /// Returns the offset this consumer last stored on the server for the given partition, or
+ /// `None` while it has neither polled nor stored for that partition. The first poll or store
+ /// seeds the entry, so `Some(0)` also covers "seen, nothing stored yet".
+ ///
+ /// The value is this consumer's own record of what it committed, kept in memory rather than
+ /// read back from the server.
+ /// Under auto-commit-on-poll (the default) this can trail the server by up to one batch.
pub fn get_last_stored_offset(&self, partition_id: u32) -> Option<u64> {
self.state.get_last_stored_offset(partition_id)
}
- /// Initializes the consumer by subscribing to diagnostic events, initializing the consumer group if needed, storing the offsets in the background etc.
+ /// Initializes the consumer and makes it ready to poll messages.
+ ///
+ /// This must be called before the consumer can start polling messages. Calling it again on an
+ /// initialized consumer does nothing and returns immediately.
+ ///
+ /// Initialization ensures that:
+ /// - the consumer's `stream_id` and `topic_id` exist on the server.
+ /// It retries for a number of `init_retries` (defaults to `None`, which is treated as no
+ /// retry) with `init_retry_interval` (defaults to one
+ /// second) time in between retries. Both can be set together through
+ /// [`IggyConsumerBuilder::init_retries`](crate::prelude::IggyConsumerBuilder::init_retries).
+ /// - the consumer subscribes to connection lifecycle events ([`DiagnosticEvent`]) in order to
+ /// update its state, should it receive a shutdown, connected, disconnected, log in or log out event.
+ /// - if the consumer belongs to a group and `auto_join_consumer_group` is enabled, the group is
+ /// initialized if it does not exist yet, and the consumer joins that group.
+ /// - the tasks that store the offset on the server are spawned.
+ ///
+ /// # Lifecycle events
///
- /// Note: This method must be called before polling messages.
+ /// Calling init spawns a background task that listens for lifecycle changes ([`DiagnosticEvent`]s) of the
+ /// client connection. It runs until the client shuts down and is not stopped by
+ /// [`shutdown()`](Self::shutdown).
+ /// - [`DiagnosticEvent::Connected`]: a fresh connection has not joined anything yet.
+ /// Polling resumes immediately only for a consumer that is not a group member.
+ /// - [`DiagnosticEvent::SignedIn`]: re-enables polling. A group member signing in after a
+ /// reconnect rejoins its group first and only polls once that succeeded. A failed rejoin is
+ /// logged and leaves polling disabled until the next reconnect or an explicit login.
+ /// - [`DiagnosticEvent::Disconnected`] and [`DiagnosticEvent::SignedOut`] disable polling.
+ /// - [`DiagnosticEvent::Shutdown`] disables polling and terminates the background task listening
+ /// for lifecycle changes. It does not flush in-flight commits; that only happens when
+ /// [`shutdown()`](Self::shutdown) itself is called.
+ ///
+ /// # Storing offsets
+ ///
+ /// When the consumer commits is decided by
+ /// [`auto_commit()`](crate::prelude::IggyConsumerBuilder::auto_commit), see
+ /// [Tracking what has been read](IggyConsumer#tracking-what-has-been-read). `init()` spawns the
+ /// tasks behind it:
+ /// - An interval task, only for the variants that carry an interval ([`AutoCommit::Interval`],
+ /// [`AutoCommit::IntervalOrWhen`], [`AutoCommit::IntervalOrAfter`]). Every tick it stores the
+ /// reading position of every partition read so far.
+ /// - An offset store task, always. It sends the commits queued by the [`AutoCommitWhen`] and
+ /// [`AutoCommitAfter`] triggers one at a time and stays idle under [`AutoCommit::Disabled`].
+ ///
+ /// Both skip an offset that is not ahead of this consumer's own record of what it stored
+ /// ([`get_last_stored_offset()`](Self::get_last_stored_offset)). Only offset `0` is always sent.
+ /// Under auto-commit-on-poll (the default) that record trails the server by one batch, so every
+ /// tick re-sends the reading position and the server, which takes an explicit store as is,
+ /// moves its offset back to it until the next poll.
+ ///
+ /// # Errors
+ ///
+ /// - [`IggyError::StreamNameNotFound`] or [`IggyError::TopicNameNotFound`] when the
+ /// stream or the topic still does not exist once the retries are exhausted.
+ /// - [`IggyError::ConsumerGroupNameNotFound`] when the consumer group does not exist
+ /// and its auto creation is disabled.
+ /// - Any error returned by the server while looking up the stream or the topic, or
+ /// while creating or joining the consumer group. Such an error ends initialization
+ /// immediately instead of consuming a retry.
pub async fn init(&mut self) -> Result<(), IggyError> {
if self.initialized {
return Ok(());
@@ -485,6 +938,9 @@ impl IggyConsumer {
let mut stream_exists = client.get_stream(&stream_id).await?.is_some();
let mut topic_exists = client.get_topic(&stream_id, &topic_id).await?.is_some();
+ // Absent streams or topics are not necessarily permanent failures.
+ // It may happen that get_stream/ get_topic races the initial setup of the stream/ topic.
+ // Retry for init_retries times, while waiting interval between retries.
loop {
if stream_exists && topic_exists {
info!(
@@ -538,6 +994,7 @@ impl IggyConsumer {
}
self.subscribe_events().await;
+ // No-op if either is_consumer_group or auto_join_consumer_group is false
self.init_consumer_group().await?;
match self.auto_commit {
@@ -553,6 +1010,9 @@ impl IggyConsumer {
let (store_offset_sender, store_offset_receiver) = flume::unbounded();
self.store_offset_sender = store_offset_sender;
+ // Message-triggered commits from `poll_next` and `consume_messages` queue here and go out
+ // one at a time. The interval task above and the poll request's own `auto_commit` flag
+ // are the other commit paths.
self.store_offset_task = Some(tokio::spawn(async move {
while let Ok((partition_id, offset)) = store_offset_receiver.recv_async().await {
trace!(
@@ -579,13 +1039,16 @@ impl IggyConsumer {
let notify = self.background_commit_notify.clone();
tokio::spawn(async move {
loop {
+ // Wait for the task until either the interval has passed or
+ // the task is explicitly notified, which happens when shutdown() is called.
tokio::select! {
_ = sleep(interval.get_duration()) => {}
_ = notify.notified() => {}
}
- // Checked before storing: `shutdown` already ran its own final
- // flush as a group member, so a store past that point would
- // hit a group we've since left.
+
+ // Checked before storing: `shutdown()` runs its own final flush as a
+ // group member and then leaves, so a store past this point would hit
+ // a group we've since left. After a bare `Drop` nothing flushes.
if shutdown.load(ORDERING) {
trace!("Shutdown signal received, stopping background offset storage");
break;
@@ -1035,13 +1498,29 @@ impl IggyConsumer {
}
}
+/// A single message handed over by an [`IggyConsumer`].
pub struct ReceivedMessage {
+ /// The message itself, with its payload already decrypted when the client uses an encryptor.
+ ///
+ /// Its own offset is `message.header.offset`, which is the value to pass to
+ /// [`IggyConsumer::store_offset`] when committing by hand.
pub message: IggyMessage,
+ /// The offset of the newest message in the partition at the time it was polled.
+ ///
+ /// Comparing it with `message.header.offset` shows how far this consumer lags behind the end
+ /// of the partition. It is a snapshot taken per request, so it does not change while the
+ /// buffered messages of that request are handed over.
pub current_offset: u64,
+ /// The partition this message was read from.
+ ///
+ /// For a consumer group this varies between messages, since the server hands different
+ /// partitions to the same member.
pub partition_id: u32,
}
impl ReceivedMessage {
+ /// Creates a received message from a message, the partition head at poll time and the
+ /// partition it was read from.
pub fn new(message: IggyMessage, current_offset: u64, partition_id: u32) -> Self {
Self {
message,
@@ -1051,6 +1530,9 @@ impl ReceivedMessage {
}
}
+/// Yields messages one at a time, from the buffer first and from a fresh poll once it is empty.
+///
+/// See [How messages are read](IggyConsumer#how-messages-are-read) for errors, `None` and polling.
impl Stream for IggyConsumer {
type Item = Result<ReceivedMessage, IggyError>;
@@ -1062,6 +1544,9 @@ impl Stream for IggyConsumer {
let partition_id = self.state.partition_id();
if let Some(message) = self.buffered_messages.pop_front() {
{
+ // Since a consumer can be standalone or a member of a consumer group, in which case
+ // it can be reassigned to another partition, either update the offset of a partition
+ // the consumer already worked with or add a new record, if it got reassigned.
if let Some(last_consumed_offset_entry) =
self.state.last_consumed_offsets.get(&partition_id)
{
@@ -1080,6 +1565,11 @@ impl Stream for IggyConsumer {
}
}
+ // Popping above may have left the buffer empty.
+ // The next turn will therefore poll messages from the server.
+ // With `PollingStrategy` the user defines the starting point where to poll from.
+ // After that, each poll must read the next sequential offset. Hence, strategy is
+ // set to `PollingKind::Offset` and the next offset to read from is the last consumed message + 1.
if self.buffered_messages.is_empty() {
if self.polling_strategy.kind != PollingKind::Next {
self.polling_strategy = PollingStrategy::offset(message.header.offset + 1);
@@ -1090,6 +1580,8 @@ impl Stream for IggyConsumer {
}
}
+ // Not the position of this message but the newest offset the partition had when the
+ // batch was polled. So every message of a batch reports the same value.
let current_offset;
if let Some(current_offset_entry) = self.current_offsets.get(&partition_id) {
current_offset = current_offset_entry.load(ORDERING);
@@ -1104,6 +1596,7 @@ impl Stream for IggyConsumer {
))));
}
+ // A used (and therefore invalid) future was dropped, thus create a fresh one.
if self.poll_future.is_none() {
let future = self.create_poll_messages_future();
self.poll_future = Some(Box::pin(future));
@@ -1192,6 +1685,7 @@ impl Stream for IggyConsumer {
);
}
+ // Drop future since it is [invalid after being ready](https://doc.rust-lang.org/std/future/trait.Future.html#panics)
self.poll_future = None;
return Poll::Ready(Some(Ok(ReceivedMessage::new(
message,
@@ -1213,17 +1707,42 @@ impl Stream for IggyConsumer {
}
impl IggyConsumer {
+ /// Shuts the consumer down.
+ ///
+ /// Specifically, run shutdown and await before dropping the consumer to
+ /// - finish storing the offsets that are currently in-flight.
+ /// The interval task and the offset store task (see [`init()`](Self::init)) can both have
+ /// commits in flight. The consumer waits for `offset_drain_timeout` on each in turn before
+ /// forcing it to abort.
+ /// - commit the reading position of every partition where it is ahead of this consumer's own
+ /// record of what it stored, under every [`AutoCommit`] setting, [`AutoCommit::Disabled`]
+ /// included. Under auto-commit-on-poll (the default) the poll already committed the whole
+ /// batch, so this store moves the server offset back to the last message handed over, and
+ /// the next run resumes right after it instead of after the last batch fetched.
+ /// - leave the consumer group, if this consumer is a group member. This lets the server give its partitions to
+ /// the remaining members immediately instead of waiting for the connection to time out.
+ ///
+ /// The lifecycle event task is not stopped. It runs until the client shuts down.
+ ///
+ /// # Errors
+ ///
+ /// Returns `Ok(())` even when the final commits or the group leave failed, since those
+ /// failures are logged and do not leave anything for the caller to undo. The
+ /// [`Result`] is part of the signature for forward compatibility.
pub async fn shutdown(&mut self) -> Result<(), IggyError> {
+ // Swap so background tasks see that the consumer got shut down.
if self.shutdown.swap(true, ORDERING) {
return Ok(());
}
info!("Shutting down consumer: {}...", self.consumer_name);
- // Drain the background commit tasks while still a group member,
- // before leaving below — otherwise a store they send afterward hits
- // a group we've already left.
+ // Drain the background commit tasks while still a group member, before
+ // leaving below. Otherwise a store they send afterward hits a group
+ // we've already left.
self.background_commit_notify.notify_one();
+
+ // A background_commit_task exists, if auto_commit is configured with an interval option.
if let Some(mut task) = self.background_commit_task.take()
&& time::timeout(self.offset_drain_timeout.get_duration(), &mut task)
.await
@@ -1238,11 +1757,17 @@ impl IggyConsumer {
);
}
+ // Drop the sending end of the store offset task to end the `recv_async()` loop in `init()`.
+ // Offsets in queue will still be committed. This prevents loading additional offsets into a channel
+ // that is not read anymore.
+ // Replace with a new (hanging) channel, since `store_offset_sender` is not optional.
let (closed_sender, _) = flume::bounded(0);
drop(std::mem::replace(
&mut self.store_offset_sender,
closed_sender,
));
+
+ // This task never sleeps, so no need to notify.
if let Some(mut task) = self.store_offset_task.take()
&& time::timeout(self.offset_drain_timeout.get_duration(), &mut task)
.await
@@ -1300,6 +1825,10 @@ impl IggyConsumer {
}
}
+/// Wakes the interval commit task so it exits. Commits already queued still go out.
+///
+/// Nothing is flushed and the consumer group is not left. Await [`IggyConsumer::shutdown`] first,
+/// see [Shutting down](IggyConsumer#shutting-down).
impl Drop for IggyConsumer {
fn drop(&mut self) {
self.shutdown.store(true, ORDERING);
diff --git a/core/sdk/src/clients/consumer_builder.rs b/core/sdk/src/clients/consumer_builder.rs
index 5f31cdd7a..87b427d83 100644
--- a/core/sdk/src/clients/consumer_builder.rs
+++ b/core/sdk/src/clients/consumer_builder.rs
@@ -92,7 +92,9 @@ impl IggyConsumerBuilder {
Self { topic, ..self }
}
- /// Sets the partition identifier.
+ /// Sets the partition to read. `None` lets a consumer group read its assigned partitions and
+ /// makes the server read partition `0` for a standalone consumer. `Some(n)` on a group member
+ /// pins every poll to that partition instead of the assignment.
pub fn partition(self, partition: Option<u32>) -> Self {
Self { partition, ..self }
}
@@ -105,7 +107,7 @@ impl IggyConsumerBuilder {
}
}
- /// Sets the batch size for polling messages.
+ /// Sets how many messages one poll request fetches at most. Defaults to 1000.
pub fn batch_length(self, batch_length: u32) -> Self {
Self {
batch_length,
@@ -121,6 +123,7 @@ impl IggyConsumerBuilder {
}
}
+ /// Same as [`auto_commit`](Self::auto_commit) with [`AutoCommit::Disabled`].
pub fn commit_failed_messages(self) -> Self {
Self {
auto_commit: AutoCommit::Disabled,