You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2025/04/03 10:51:36 UTC
(iggy) branch zero-copy-no-batching-rebase updated (bfc5b3ba -> f32bac2a)
This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a change to branch zero-copy-no-batching-rebase
in repository https://gitbox.apache.org/repos/asf/iggy.git
discard bfc5b3ba zero copy
new f32bac2a zero copy
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (bfc5b3ba)
\
N -- N -- N refs/heads/zero-copy-no-batching-rebase (f32bac2a)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
configs/server.toml | 8 ++-
integration/tests/streaming/get_by_offset.rs | 25 ++++----
integration/tests/streaming/get_by_timestamp.rs | 25 ++++----
server/src/configs/cache_indexes.rs | 72 +++++++++++++++++++++++
server/src/configs/config_provider.rs | 5 +-
server/src/configs/defaults.rs | 2 +-
server/src/configs/mod.rs | 1 +
server/src/configs/system.rs | 3 +-
server/src/streaming/partitions/storage.rs | 24 +++++++-
server/src/streaming/segments/reading_messages.rs | 4 +-
server/src/streaming/segments/segment.rs | 11 ++--
server/src/streaming/segments/writing_messages.rs | 3 +-
12 files changed, 147 insertions(+), 36 deletions(-)
create mode 100644 server/src/configs/cache_indexes.rs
(iggy) 01/01: zero copy
Posted by hg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a commit to branch zero-copy-no-batching-rebase
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit f32bac2a9c60b05c2355cd16af9b294c582a9564
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Wed Apr 2 14:27:39 2025 +0200
zero copy
---
.config/nextest.toml | 5 +-
Cargo.lock | 191 ++++-
bench/Cargo.toml | 5 +-
bench/report/Cargo.toml | 2 +-
bench/src/actors/consumer.rs | 13 +-
bench/src/actors/producer.rs | 30 +-
bench/src/actors/producing_consumer.rs | 14 +-
bench/src/actors/utils.rs | 16 +-
bench/src/benchmarks/benchmark.rs | 15 +-
bench/src/benchmarks/consumer_benchmark.rs | 20 +-
bench/src/benchmarks/consumer_group_benchmark.rs | 21 +-
.../benchmarks/producer_and_consumer_benchmark.rs | 23 +-
.../producer_and_consumer_group_benchmark.rs | 27 +-
bench/src/benchmarks/producer_benchmark.rs | 17 +-
.../src/benchmarks/producing_consumer_benchmark.rs | 20 +-
.../producing_consumer_group_benchmark.rs | 18 +-
bench/src/runner.rs | 12 +-
cli/src/args/message.rs | 9 +-
configs/server.toml | 22 +-
examples/src/basic/consumer/main.rs | 10 +-
examples/src/basic/producer/main.rs | 8 +-
examples/src/getting-started/consumer/main.rs | 12 +-
examples/src/getting-started/producer/main.rs | 12 +-
examples/src/message-envelope/consumer/main.rs | 10 +-
examples/src/message-envelope/producer/main.rs | 8 +-
examples/src/message-headers/consumer/main.rs | 20 +-
examples/src/message-headers/producer/main.rs | 13 +-
examples/src/multi-tenant/consumer/main.rs | 15 +-
examples/src/multi-tenant/producer/main.rs | 18 +-
examples/src/new-sdk/consumer/main.rs | 14 +-
examples/src/new-sdk/producer/main.rs | 12 +-
examples/src/shared/system.rs | 12 +-
examples/src/stream-builder/stream-basic/main.rs | 17 +-
.../stream-builder/stream-consumer-config/main.rs | 10 +-
.../src/stream-builder/stream-consumer/main.rs | 5 +-
.../stream-builder/stream-producer-config/main.rs | 11 +-
.../src/stream-builder/stream-producer/main.rs | 17 +-
integration/Cargo.toml | 4 +-
.../test_consumer_offset_get_command.rs | 9 +-
.../test_consumer_offset_set_command.rs | 9 +-
.../tests/cli/message/test_message_poll_command.rs | 9 +-
.../message/test_message_poll_to_file_command.rs | 10 +-
.../cli/message/test_message_reply_via_file.rs | 3 +-
.../tests/cli/message/test_message_send_command.rs | 13 +-
.../message/test_message_send_from_file_command.rs | 25 +-
integration/tests/cli/personal_access_token/mod.rs | 5 +-
.../tests/cli/stream/test_stream_purge_command.rs | 7 +-
integration/tests/cli/system/mod.rs | 11 +-
.../tests/cli/topic/test_topic_purge_command.rs | 7 +-
integration/tests/examples/mod.rs | 17 +-
integration/tests/mod.rs | 12 +-
...h_multiple_clients_polling_messages_scenario.rs | 18 +-
...with_single_client_polling_messages_scenario.rs | 25 +-
.../server/scenarios/create_message_payload.rs | 21 +-
.../server/scenarios/message_headers_scenario.rs | 22 +-
.../server/scenarios/message_size_scenario.rs | 58 +-
integration/tests/server/scenarios/mod.rs | 2 +-
.../scenarios/stream_size_validation_scenario.rs | 19 +-
.../tests/server/scenarios/system_scenario.rs | 43 +-
integration/tests/state/file.rs | 8 +-
integration/tests/state/mod.rs | 9 +-
integration/tests/state/system.rs | 16 +-
integration/tests/streaming/get_by_offset.rs | 358 ++++++----
integration/tests/streaming/get_by_timestamp.rs | 336 +++++----
integration/tests/streaming/messages.rs | 203 +++---
integration/tests/streaming/mod.rs | 17 +-
integration/tests/streaming/partition.rs | 32 +-
integration/tests/streaming/segment.rs | 244 +++----
integration/tests/streaming/snapshot.rs | 2 +-
integration/tests/streaming/stream.rs | 33 +-
integration/tests/streaming/topic.rs | 29 +-
integration/tests/streaming/topic_messages.rs | 125 +---
sdk/Cargo.toml | 6 +-
sdk/src/binary/mapper.rs | 102 ---
sdk/src/binary/messages.rs | 18 +-
sdk/src/bytes_serializable.rs | 13 +-
sdk/src/cli/message/poll_messages.rs | 71 +-
sdk/src/cli/message/send_messages.rs | 29 +-
sdk/src/client.rs | 8 +-
sdk/src/clients/client.rs | 13 +-
sdk/src/clients/consumer.rs | 54 +-
sdk/src/clients/producer.rs | 25 +-
sdk/src/consumer_ext/consumer_message_ext.rs | 2 +-
sdk/src/consumer_ext/mod.rs | 3 +-
sdk/src/error.rs | 14 +-
sdk/src/http/messages.rs | 12 +-
sdk/src/identifier.rs | 28 +
sdk/src/lib.rs | 1 +
sdk/src/messages/flush_unsaved_buffer.rs | 8 +-
sdk/src/messages/mod.rs | 22 +-
sdk/src/messages/partitioning.rs | 202 ++++++
sdk/src/messages/partitioning_kind.rs | 56 ++
sdk/src/messages/poll_messages.rs | 301 ++------
sdk/src/messages/polled_messages.rs | 111 +++
sdk/src/messages/polling_kind.rs | 72 ++
sdk/src/messages/polling_strategy.rs | 129 ++++
sdk/src/messages/send_messages.rs | 790 ++++++---------------
sdk/src/models/messages.rs | 182 -----
sdk/src/models/messaging/index.rs | 6 +
sdk/src/models/messaging/index_view.rs | 62 ++
sdk/src/models/messaging/indexes.rs | 185 +++++
sdk/src/models/messaging/message.rs | 574 +++++++++++++++
sdk/src/models/messaging/message_header.rs | 154 ++++
sdk/src/models/messaging/message_header_view.rs | 99 +++
sdk/src/models/messaging/message_view.rs | 157 ++++
sdk/src/models/messaging/messages_batch.rs | 586 +++++++++++++++
sdk/src/models/messaging/mod.rs | 27 +
.../{header.rs => messaging/user_headers.rs} | 7 +-
sdk/src/models/mod.rs | 4 +-
sdk/src/partitioner.rs | 4 +-
sdk/src/prelude.rs | 66 ++
.../stream_builder/config/config_iggy_consumer.rs | 5 +-
.../stream_builder/config/config_iggy_producer.rs | 5 +-
sdk/src/tcp/client.rs | 3 +-
sdk/src/utils/byte_size.rs | 5 +
sdk/src/utils/duration.rs | 11 +-
sdk/src/utils/timestamp.rs | 30 +-
server/Cargo.toml | 13 +-
server/src/binary/command.rs | 585 +++++++++------
.../create_consumer_group_handler.rs | 103 +--
.../delete_consumer_group_handler.rs | 97 ++-
.../consumer_groups/get_consumer_group_handler.rs | 69 +-
.../consumer_groups/get_consumer_groups_handler.rs | 59 +-
.../consumer_groups/join_consumer_group_handler.rs | 69 +-
.../leave_consumer_group_handler.rs | 70 +-
.../delete_consumer_offset_handler.rs | 67 +-
.../get_consumer_offset_handler.rs | 77 +-
.../store_consumer_offset_handler.rs | 67 +-
.../messages/flush_unsaved_buffer_handler.rs | 74 +-
.../handlers/messages/poll_messages_handler.rs | 126 +++-
.../handlers/messages/send_messages_handler.rs | 117 ++-
server/src/binary/handlers/mod.rs | 1 +
.../partitions/create_partitions_handler.rs | 63 +-
.../partitions/delete_partitions_handler.rs | 61 +-
.../create_personal_access_token_handler.rs | 95 ++-
.../delete_personal_access_token_handler.rs | 77 +-
.../get_personal_access_tokens_handler.rs | 57 +-
.../login_with_personal_access_token_handler.rs | 65 +-
.../handlers/streams/create_stream_handler.rs | 76 +-
.../handlers/streams/delete_stream_handler.rs | 68 +-
.../binary/handlers/streams/get_stream_handler.rs | 60 +-
.../binary/handlers/streams/get_streams_handler.rs | 50 +-
.../handlers/streams/purge_stream_handler.rs | 72 +-
.../handlers/streams/update_stream_handler.rs | 68 +-
.../binary/handlers/system/get_client_handler.rs | 48 +-
.../binary/handlers/system/get_clients_handler.rs | 57 +-
.../src/binary/handlers/system/get_me_handler.rs | 45 +-
server/src/binary/handlers/system/get_snapshot.rs | 50 +-
.../binary/handlers/system/get_stats_handler.rs | 51 +-
server/src/binary/handlers/system/ping_handler.rs | 53 +-
.../binary/handlers/topics/create_topic_handler.rs | 103 +--
.../binary/handlers/topics/delete_topic_handler.rs | 69 +-
.../binary/handlers/topics/get_topic_handler.rs | 59 +-
.../binary/handlers/topics/get_topics_handler.rs | 59 +-
.../binary/handlers/topics/purge_topic_handler.rs | 81 ++-
.../binary/handlers/topics/update_topic_handler.rs | 97 ++-
.../handlers/users/change_password_handler.rs | 97 ++-
.../binary/handlers/users/create_user_handler.rs | 103 +--
.../binary/handlers/users/delete_user_handler.rs | 73 +-
.../src/binary/handlers/users/get_user_handler.rs | 59 +-
.../src/binary/handlers/users/get_users_handler.rs | 55 +-
.../binary/handlers/users/login_user_handler.rs | 63 +-
.../binary/handlers/users/logout_user_handler.rs | 57 +-
.../handlers/users/update_permissions_handler.rs | 71 +-
.../binary/handlers/users/update_user_handler.rs | 87 ++-
server/src/binary/handlers/utils.rs | 20 +
server/src/binary/macros.rs | 96 +++
server/src/binary/mapper.rs | 21 -
server/src/binary/mod.rs | 7 +-
server/src/binary/sender.rs | 7 +
server/src/channels/commands/archive_state.rs | 8 +-
.../commands/clean_personal_access_tokens.rs | 6 +-
server/src/channels/commands/maintain_messages.rs | 22 +-
server/src/channels/commands/print_sysinfo.rs | 4 +-
server/src/channels/commands/save_messages.rs | 4 +-
server/src/channels/commands/verify_heartbeats.rs | 4 +-
server/src/channels/handler.rs | 8 +-
server/src/channels/server_command.rs | 5 +-
server/src/command.rs | 683 ------------------
.../src/compat/index_rebuilding/index_rebuilder.rs | 64 +-
server/src/configs/cache_indexes.rs | 72 ++
server/src/configs/config_provider.rs | 5 +-
server/src/configs/defaults.rs | 8 +-
server/src/configs/mod.rs | 1 +
server/src/configs/system.rs | 6 +-
server/src/http/consumer_groups.rs | 7 +-
server/src/http/messages.rs | 36 +-
server/src/http/partitions.rs | 4 +-
server/src/http/personal_access_tokens.rs | 4 +-
server/src/http/streams.rs | 8 +-
server/src/http/system.rs | 2 +-
server/src/http/topics.rs | 8 +-
server/src/http/users.rs | 10 +-
server/src/lib.rs | 1 -
server/src/log/logger.rs | 65 +-
server/src/main.rs | 4 +-
server/src/quic/listener.rs | 91 ++-
server/src/quic/quic_sender.rs | 62 +-
server/src/state/file.rs | 2 +-
server/src/state/mod.rs | 4 +-
.../streaming/batching/appendable_batch_info.rs | 16 -
server/src/streaming/batching/batch_accumulator.rs | 120 ----
server/src/streaming/batching/batch_filter.rs | 8 -
server/src/streaming/batching/iterator.rs | 64 --
server/src/streaming/batching/message_batch.rs | 86 ---
server/src/streaming/batching/mod.rs | 5 -
server/src/streaming/cache/buffer.rs | 177 -----
server/src/streaming/cache/memory_tracker.rs | 101 ---
server/src/streaming/cache/mod.rs | 3 -
server/src/streaming/mod.rs | 4 +-
server/src/streaming/models/messages.rs | 205 ------
server/src/streaming/models/mod.rs | 3 -
server/src/streaming/partitions/messages.rs | 474 +++++--------
server/src/streaming/partitions/mod.rs | 20 -
server/src/streaming/partitions/partition.rs | 73 +-
server/src/streaming/partitions/persistence.rs | 4 +-
server/src/streaming/partitions/segments.rs | 18 +-
server/src/streaming/partitions/storage.rs | 84 +--
server/src/streaming/segments/indexes/index.rs | 199 ------
.../src/streaming/segments/indexes/index_reader.rs | 374 ++++++----
.../src/streaming/segments/indexes/index_writer.rs | 45 +-
.../src/streaming/segments/indexes/indexes_mut.rs | 333 +++++++++
server/src/streaming/segments/indexes/mod.rs | 12 +-
server/src/streaming/segments/logs/log_reader.rs | 340 ---------
server/src/streaming/segments/logs/mod.rs | 7 -
.../streaming/segments/messages/messages_reader.rs | 164 +++++
.../log_writer.rs => messages/messages_writer.rs} | 115 ++-
server/src/streaming/segments/messages/mod.rs | 41 ++
.../segments/{logs => messages}/persister_task.rs | 119 +---
.../src/streaming/segments/messages_accumulator.rs | 185 +++++
server/src/streaming/segments/mod.rs | 11 +-
server/src/streaming/segments/reading_messages.rs | 527 ++++++--------
server/src/streaming/segments/segment.rs | 253 ++++---
.../segments/types/message_header_view_mut.rs | 85 +++
.../streaming/segments/types/message_view_mut.rs | 101 +++
.../streaming/segments/types/messages_batch_mut.rs | 405 +++++++++++
.../streaming/segments/types/messages_batch_set.rs | 350 +++++++++
server/src/streaming/segments/types/mod.rs | 9 +
server/src/streaming/segments/writing_messages.rs | 179 +++--
server/src/streaming/streams/topics.rs | 2 +-
server/src/streaming/systems/messages.rs | 203 +++---
server/src/streaming/systems/snapshot/mod.rs | 6 +-
server/src/streaming/systems/snapshot/procdump.rs | 50 +-
server/src/streaming/systems/stats.rs | 19 +-
server/src/streaming/systems/system.rs | 27 +-
server/src/streaming/systems/users.rs | 2 +-
server/src/streaming/topics/messages.rs | 211 ++----
server/src/streaming/topics/storage.rs | 6 -
server/src/streaming/utils/file.rs | 30 +-
server/src/tcp/connection_handler.rs | 57 +-
server/src/tcp/sender.rs | 46 +-
server/src/tcp/tcp_sender.rs | 8 +
server/src/tcp/tcp_tls_sender.rs | 8 +
tools/src/data-seeder/seeder.rs | 85 ++-
254 files changed, 10325 insertions(+), 7683 deletions(-)
diff --git a/.config/nextest.toml b/.config/nextest.toml
index 6a53fe0b..a99f1b10 100644
--- a/.config/nextest.toml
+++ b/.config/nextest.toml
@@ -1,5 +1,8 @@
[[profile.default.overrides]]
# This is a solution (or actually a workaround) for the problem that nextest does not support
-# #[serial] macro which shall enfroce sequential execution of the test case.
+# #[serial] macro which shall enforce sequential execution of the test case.
filter = 'package(integration) and test(cli::system::test_cli_session_scenario::should_be_successful)'
threads-required = "num-cpus"
+
+[profile.default]
+slow-timeout = { period = "30s", terminate-after = 4 }
diff --git a/Cargo.lock b/Cargo.lock
index 08eb5a77..acef200a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -408,9 +408,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.12.3"
+version = "1.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c6a895b664295a4ba0c2c0203c7075ea585dd75cd5c37a8efac829e13e460ef"
+checksum = "4cd755adf9707cf671e31d944a189be3deaaeee11c8bc1d669bb8022ac90fbd0"
dependencies = [
"aws-lc-sys",
"paste",
@@ -607,7 +607,7 @@ dependencies = [
[[package]]
name = "bench"
-version = "0.2.3"
+version = "0.2.4"
dependencies = [
"async-trait",
"atomic-time",
@@ -616,7 +616,6 @@ dependencies = [
"chrono",
"clap",
"figlet-rs",
- "futures",
"hostname",
"human-repr",
"iggy",
@@ -834,9 +833,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
-version = "1.10.0"
+version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9"
+checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
+dependencies = [
+ "serde",
+]
[[package]]
name = "bzip2"
@@ -893,9 +895,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.14"
+version = "1.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9"
+checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af"
dependencies = [
"jobserver",
"libc",
@@ -1576,6 +1578,18 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "enum_dispatch"
+version = "0.3.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
+dependencies = [
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.98",
+]
+
[[package]]
name = "env_filter"
version = "0.1.3"
@@ -1669,6 +1683,35 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "ext-trait"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d772df1c1a777963712fb68e014235e80863d6a91a85c4e06ba2d16243a310e5"
+dependencies = [
+ "ext-trait-proc_macros",
+]
+
+[[package]]
+name = "ext-trait-proc_macros"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ab7934152eaf26aa5aa9f7371408ad5af4c31357073c9e84c3b9d7f11ad639a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "extension-traits"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a296e5a895621edf9fa8329c83aa1cb69a964643e36cf54d8d7a69b789089537"
+dependencies = [
+ "ext-trait",
+]
+
[[package]]
name = "fast-async-mutex"
version = "0.6.7"
@@ -1966,6 +2009,15 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
+[[package]]
+name = "gxhash"
+version = "3.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a197c9b654827513cf53842c5c6d3da2b4b35a785f8e0eff78bdf8e445aba1bb"
+dependencies = [
+ "rustversion",
+]
+
[[package]]
name = "h2"
version = "0.4.8"
@@ -2471,6 +2523,7 @@ dependencies = [
"flume",
"futures",
"futures-util",
+ "gxhash",
"humantime",
"keyring",
"passterm",
@@ -2584,9 +2637,9 @@ checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb"
[[package]]
name = "inout"
-version = "0.1.3"
+version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
+checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
"generic-array",
]
@@ -2748,6 +2801,31 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+[[package]]
+name = "lending-iterator"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc07588c853b50689205fb5c00498aa681d89828e0ce8cbd965ebc7a5d8ae260"
+dependencies = [
+ "extension-traits",
+ "lending-iterator-proc_macros",
+ "macro_rules_attribute",
+ "never-say-never",
+ "nougat",
+ "polonius-the-crab",
+]
+
+[[package]]
+name = "lending-iterator-proc_macros"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5445dd1c0deb1e97b8a16561d17fc686ca83e8411128fb036e9668a72d51b1d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
[[package]]
name = "libc"
version = "0.2.169"
@@ -2848,9 +2926,9 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
[[package]]
name = "log"
-version = "0.4.25"
+version = "0.4.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"
+checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
[[package]]
name = "loom"
@@ -2886,6 +2964,22 @@ dependencies = [
"pkg-config",
]
+[[package]]
+name = "macro_rules_attribute"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862"
+dependencies = [
+ "macro_rules_attribute-proc_macro",
+ "paste",
+]
+
+[[package]]
+name = "macro_rules_attribute-proc_macro"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
+
[[package]]
name = "matchers"
version = "0.1.0"
@@ -2971,9 +3065,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
-version = "0.8.4"
+version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3b1c9bd4fe1f0f8b387f6eb9eb3b4a1aa26185e5750efb9140301703f62cd1b"
+checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
dependencies = [
"adler2",
]
@@ -3063,6 +3157,12 @@ dependencies = [
"tempfile",
]
+[[package]]
+name = "never-say-never"
+version = "6.6.666"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf5a574dadd7941adeaa71823ecba5e28331b8313fb2e1c6a5c7e5981ea53ad6"
+
[[package]]
name = "nix"
version = "0.29.0"
@@ -3097,6 +3197,27 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
+[[package]]
+name = "nougat"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97b57b9ced431322f054fc673f1d3c7fa52d80efd9df74ad2fc759f044742510"
+dependencies = [
+ "macro_rules_attribute",
+ "nougat-proc_macros",
+]
+
+[[package]]
+name = "nougat-proc_macros"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c84f77a45e99a2f9b492695d99e1c23844619caa5f3e57647cffacad773ca257"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
[[package]]
name = "ntapi"
version = "0.4.1"
@@ -3451,7 +3572,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
dependencies = [
"cfg-if",
"libc",
- "redox_syscall 0.5.8",
+ "redox_syscall 0.5.9",
"smallvec",
"windows-targets 0.52.6",
]
@@ -3507,9 +3628,9 @@ dependencies = [
[[package]]
name = "pem"
-version = "3.0.4"
+version = "3.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae"
+checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3"
dependencies = [
"base64 0.22.1",
"serde",
@@ -3604,6 +3725,12 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
+[[package]]
+name = "polonius-the-crab"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a69ee997a6282f8462abf1e0d8c38c965e968799e912b3bed8c9e8a28c2f9f"
+
[[package]]
name = "polyval"
version = "0.6.2"
@@ -3990,9 +4117,9 @@ dependencies = [
[[package]]
name = "redox_syscall"
-version = "0.5.8"
+version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"
+checksum = "82b568323e98e49e2a0899dcee453dd679fae22d69adf9b11dd508d1549b7e2f"
dependencies = [
"bitflags 2.8.0",
]
@@ -4159,9 +4286,9 @@ dependencies = [
[[package]]
name = "ring"
-version = "0.17.9"
+version = "0.17.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e75ec5e92c4d8aede845126adc388046234541629e76029599ed35a003c7ed24"
+checksum = "d34b5020fcdea098ef7d95e9f89ec15952123a4a039badd09fabebe9e963e839"
dependencies = [
"cc",
"cfg-if",
@@ -4623,7 +4750,7 @@ dependencies = [
[[package]]
name = "server"
-version = "0.4.213"
+version = "0.4.214"
dependencies = [
"ahash 0.8.11",
"anyhow",
@@ -4641,14 +4768,18 @@ dependencies = [
"dashmap",
"derive_more",
"dotenvy",
+ "enum_dispatch",
"error_set",
"figlet-rs",
"figment",
"flume",
"futures",
+ "gxhash",
"human-repr",
"iggy",
"jsonwebtoken",
+ "lending-iterator",
+ "memmap2",
"mimalloc",
"mockall",
"moka",
@@ -5572,9 +5703,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
-version = "1.13.2"
+version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c1f41ffb7cf259f1ecc2876861a17e7142e63ead296f671f81f6ae85903e0d6"
+checksum = "93d59ca99a559661b96bf898d8fce28ed87935fd2bea9f05983c1464dd6c71b1"
dependencies = [
"getrandom 0.3.1",
"rand 0.9.0",
@@ -6384,27 +6515,27 @@ dependencies = [
[[package]]
name = "zstd"
-version = "0.13.2"
+version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
-version = "7.2.1"
+version = "7.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
+checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
-version = "2.0.13+zstd.1.5.6"
+version = "2.0.14+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
+checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5"
dependencies = [
"cc",
"pkg-config",
diff --git a/bench/Cargo.toml b/bench/Cargo.toml
index e31482b3..ea88d38e 100644
--- a/bench/Cargo.toml
+++ b/bench/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "bench"
-version = "0.2.3"
+version = "0.2.4"
edition = "2021"
license = "Apache-2.0"
# Due to dependency to integration, which has a dependency to server, setting
@@ -14,7 +14,6 @@ charming = "0.4.0"
chrono = "0.4.39"
clap = { version = "4.5.30", features = ["derive"] }
figlet-rs = "0.1.5"
-futures = "0.3.31"
hostname = "0.4.0"
human-repr = "1.1.0"
iggy = { path = "../sdk" }
@@ -28,7 +27,7 @@ toml = "0.8.20"
tracing = { version = "0.1.41" }
tracing-appender = "0.2.3"
tracing-subscriber = { version = "0.3.19", features = ["fmt", "env-filter"] }
-uuid = { version = "1.13.2", features = ["serde"] }
+uuid = { version = "1.14.0", features = ["serde"] }
[[bin]]
name = "iggy-bench"
diff --git a/bench/report/Cargo.toml b/bench/report/Cargo.toml
index 9f8a13ca..d95197f9 100644
--- a/bench/report/Cargo.toml
+++ b/bench/report/Cargo.toml
@@ -16,4 +16,4 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
sysinfo = "0.33.1"
tracing = "0.1"
-uuid = { version = "1.13.2", features = ["serde"] }
+uuid = { version = "1.14.0", features = ["serde"] }
diff --git a/bench/src/actors/consumer.rs b/bench/src/actors/consumer.rs
index ce2b3e83..887715c5 100644
--- a/bench/src/actors/consumer.rs
+++ b/bench/src/actors/consumer.rs
@@ -7,7 +7,8 @@ use iggy::client::{ConsumerGroupClient, MessageClient};
use iggy::clients::client::IggyClient;
use iggy::consumer::Consumer as IggyConsumer;
use iggy::error::IggyError;
-use iggy::messages::poll_messages::{PollingKind, PollingStrategy};
+use iggy::messages::PollingKind;
+use iggy::prelude::PollingStrategy;
use iggy::utils::byte_size::IggyByteSize;
use iggy::utils::duration::IggyDuration;
use iggy::utils::sizeable::Sizeable;
@@ -74,7 +75,7 @@ impl Consumer {
}
}
- pub async fn run(&self) -> Result<BenchmarkIndividualMetrics, IggyError> {
+ pub async fn run(self) -> Result<BenchmarkIndividualMetrics, IggyError> {
let topic_id: u32 = 1;
let default_partition_id: u32 = 1;
let message_batches = self.message_batches as u64;
@@ -129,6 +130,7 @@ impl Consumer {
self.consumer_id, self.warmup_time
);
}
+
let warmup_end = Instant::now() + self.warmup_time.get_duration();
while Instant::now() < warmup_end {
let offset = current_iteration * messages_per_batch as u64;
@@ -228,7 +230,6 @@ impl Consumer {
let polled_messages = polled_messages.unwrap();
if polled_messages.messages.is_empty() {
- // Store initial poll timestamp if this is the first attempt
if initial_poll_timestamp.is_none() {
initial_poll_timestamp = Some(before_poll);
}
@@ -287,11 +288,11 @@ impl Consumer {
// We don't need to calculate the size whole batch every time by iterating over it - just always use the size of the first message
if batch_user_size_bytes == 0 || batch_size_total_bytes == 0 {
- batch_user_size_bytes =
- polled_messages.messages[0].payload.len() as u64 * messages_per_batch as u64;
+ batch_user_size_bytes = polled_messages.messages[0].payload.len() as u64
+ * polled_messages.messages.len() as u64;
batch_size_total_bytes =
polled_messages.messages[0].get_size_bytes().as_bytes_u64()
- * messages_per_batch as u64;
+ * polled_messages.messages.len() as u64;
}
total_user_data_bytes += IggyByteSize::from(batch_user_size_bytes);
diff --git a/bench/src/actors/producer.rs b/bench/src/actors/producer.rs
index 04d867c3..affbd457 100644
--- a/bench/src/actors/producer.rs
+++ b/bench/src/actors/producer.rs
@@ -6,7 +6,8 @@ use human_repr::HumanCount;
use iggy::client::MessageClient;
use iggy::clients::client::IggyClient;
use iggy::error::IggyError;
-use iggy::messages::send_messages::{Message, Partitioning};
+use iggy::prelude::IggyMessage;
+use iggy::prelude::Partitioning;
use iggy::utils::byte_size::IggyByteSize;
use iggy::utils::duration::IggyDuration;
use iggy::utils::sizeable::Sizeable;
@@ -70,7 +71,7 @@ impl Producer {
}
}
- pub async fn run(&self) -> Result<BenchmarkIndividualMetrics, IggyError> {
+ pub async fn run(self) -> Result<BenchmarkIndividualMetrics, IggyError> {
let topic_id: u32 = 1;
let default_partition_id: u32 = 1;
let partitions_count = self.partitions_count;
@@ -91,8 +92,8 @@ impl Producer {
let mut batch_total_bytes = 0;
let mut messages = Vec::with_capacity(messages_per_batch as usize);
for _ in 0..messages_per_batch {
- let message = Message::from_str(&payload).unwrap();
- batch_user_data_bytes += message.length as u64;
+ let message = IggyMessage::from_str(&payload).unwrap();
+ batch_user_data_bytes += message.payload.len() as u64;
batch_total_bytes += message.get_size_bytes().as_bytes_u64();
messages.push(message);
}
@@ -136,7 +137,10 @@ impl Producer {
let start_timestamp = Instant::now();
let mut latencies: Vec<Duration> = Vec::with_capacity(message_batches as usize);
let mut records = Vec::with_capacity(message_batches as usize);
+ let mut total_overhead_time = Duration::from_secs(0);
for i in 1..=message_batches {
+ let iteration_start = Instant::now();
+
// Apply rate limiting if configured
if let Some(limiter) = &self.rate_limiter {
limiter.throttle(batch_total_bytes).await;
@@ -150,6 +154,10 @@ impl Producer {
.await?;
let latency = before_send.elapsed();
+ let iteration_time = iteration_start.elapsed();
+ let overhead_time = iteration_time - latency;
+ total_overhead_time += overhead_time;
+
let messages_processed = (i * messages_per_batch) as u64;
let batches_processed = i as u64;
let user_data_bytes = batches_processed * batch_user_data_bytes;
@@ -182,6 +190,20 @@ impl Producer {
&metrics,
);
+ let avg_overhead_per_batch =
+ total_overhead_time.as_micros() as f64 / message_batches as f64;
+ let total_elapsed = start_timestamp.elapsed();
+ let send_time = total_elapsed - total_overhead_time;
+ info!(
+ "Producer #{} → Overhead stats: Total time: {} ms, Send time: {} ms, Overhead time: {} ms ({:.2}%), Avg overhead per batch: {:.2} μs",
+ self.producer_id,
+ total_elapsed.as_millis(),
+ send_time.as_millis(),
+ total_overhead_time.as_millis(),
+ (total_overhead_time.as_micros() as f64 / total_elapsed.as_micros() as f64) * 100.0,
+ avg_overhead_per_batch
+ );
+
Ok(metrics)
}
diff --git a/bench/src/actors/producing_consumer.rs b/bench/src/actors/producing_consumer.rs
index 7865e6d8..07ae2978 100644
--- a/bench/src/actors/producing_consumer.rs
+++ b/bench/src/actors/producing_consumer.rs
@@ -1,4 +1,5 @@
-use crate::actors::utils::{calculate_latency_from_first_message, put_timestamp_in_first_message};
+use crate::actors::utils::calculate_latency_from_first_message;
+use crate::actors::utils::put_timestamp_in_first_message;
use crate::analytics::metrics::individual::from_records;
use crate::analytics::record::BenchmarkRecord;
use crate::rate_limiter::RateLimiter;
@@ -8,8 +9,7 @@ use iggy::client::MessageClient;
use iggy::clients::client::IggyClient;
use iggy::consumer::Consumer as IggyConsumer;
use iggy::error::IggyError;
-use iggy::messages::poll_messages::{PollingKind, PollingStrategy};
-use iggy::messages::send_messages::{Message, Partitioning};
+use iggy::prelude::*;
use iggy::utils::byte_size::IggyByteSize;
use iggy::utils::duration::IggyDuration;
use iggy::utils::sizeable::Sizeable;
@@ -83,7 +83,7 @@ impl ProducingConsumer {
}
}
- pub async fn run(&self) -> Result<BenchmarkIndividualMetrics, IggyError> {
+ pub async fn run(self) -> Result<BenchmarkIndividualMetrics, IggyError> {
let topic_id: u32 = 1;
let default_partition_id: u32 = 1;
let message_batches = self.message_batches;
@@ -104,8 +104,8 @@ impl ProducingConsumer {
let mut batch_total_bytes = 0;
let mut messages = Vec::with_capacity(messages_per_batch as usize);
for _ in 0..messages_per_batch {
- let message = Message::from_str(&payload).unwrap();
- batch_user_data_bytes += message.length as u64;
+ let message = IggyMessage::from_str(&payload).unwrap();
+ batch_user_data_bytes += message.payload.len() as u64;
batch_total_bytes += message.get_size_bytes().as_bytes_u64();
messages.push(message);
}
@@ -172,6 +172,7 @@ impl ProducingConsumer {
self.polling_kind
),
};
+
let polled_messages = client
.poll_messages(
&stream_id,
@@ -205,6 +206,7 @@ impl ProducingConsumer {
continue;
}
+
current_offset += messages_per_batch as u64;
}
}
diff --git a/bench/src/actors/utils.rs b/bench/src/actors/utils.rs
index 90edf19b..3ec1e8d9 100644
--- a/bench/src/actors/utils.rs
+++ b/bench/src/actors/utils.rs
@@ -1,33 +1,25 @@
-use std::time::Duration;
-
use bytes::Bytes;
-use iggy::{messages::send_messages::Message, models::messages::PolledMessage};
+use iggy::prelude::IggyMessage;
+use std::time::Duration;
-pub fn put_timestamp_in_first_message(message: &mut Message) {
+pub fn put_timestamp_in_first_message(message: &mut IggyMessage) {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_micros() as u64;
- // Convert current payload to Vec<u8>
let mut payload_vec = message.payload.to_vec();
-
- // Ensure payload is at least 8 bytes
if payload_vec.len() < 8 {
let mut new_payload = vec![0u8; 8];
new_payload.extend_from_slice(&payload_vec);
payload_vec = new_payload;
}
-
- // Put timestamp in first 8 bytes
let timestamp_bytes = now.to_le_bytes();
payload_vec[0..8].copy_from_slice(×tamp_bytes);
-
- // Convert back to Bytes
message.payload = Bytes::from(payload_vec);
}
-pub fn calculate_latency_from_first_message(message: &PolledMessage) -> Duration {
+pub fn calculate_latency_from_first_message(message: &IggyMessage) -> Duration {
let send_timestamp = u64::from_le_bytes(message.payload[0..8].try_into().unwrap());
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
diff --git a/bench/src/benchmarks/benchmark.rs b/bench/src/benchmarks/benchmark.rs
index be87d3c8..acfb441b 100644
--- a/bench/src/benchmarks/benchmark.rs
+++ b/bench/src/benchmarks/benchmark.rs
@@ -1,7 +1,6 @@
use crate::args::kind::BenchmarkKindCommand;
use crate::{args::common::IggyBenchArgs, utils::client_factory::create_client_factory};
use async_trait::async_trait;
-use futures::Future;
use iggy::client::{StreamClient, TopicClient};
use iggy::clients::client::IggyClient;
use iggy::compression::compression_algorithm::CompressionAlgorithm;
@@ -11,7 +10,8 @@ use iggy::utils::topic_size::MaxTopicSize;
use iggy_bench_report::benchmark_kind::BenchmarkKind;
use iggy_bench_report::individual_metrics::BenchmarkIndividualMetrics;
use integration::test_server::{login_root, ClientFactory};
-use std::{pin::Pin, sync::Arc};
+use std::sync::Arc;
+use tokio::task::JoinSet;
use tracing::info;
use super::consumer_benchmark::ConsumerBenchmark;
@@ -22,11 +22,6 @@ use super::producer_benchmark::ProducerBenchmark;
use super::producing_consumer_benchmark::EndToEndProducingConsumerBenchmark;
use super::producing_consumer_group_benchmark::EndToEndProducingConsumerGroupBenchmark;
-pub type BenchmarkFutures = Result<
- Vec<Pin<Box<dyn Future<Output = Result<BenchmarkIndividualMetrics, IggyError>> + Send>>>,
- IggyError,
->;
-
impl From<IggyBenchArgs> for Box<dyn Benchmarkable> {
fn from(args: IggyBenchArgs) -> Self {
let client_factory = create_client_factory(&args);
@@ -56,14 +51,16 @@ impl From<IggyBenchArgs> for Box<dyn Benchmarkable> {
BenchmarkKindCommand::EndToEndProducingConsumerGroup(_) => Box::new(
EndToEndProducingConsumerGroupBenchmark::new(Arc::new(args), client_factory),
),
- _ => todo!(),
+ _ => unreachable!(),
}
}
}
#[async_trait]
pub trait Benchmarkable {
- async fn run(&mut self) -> BenchmarkFutures;
+ async fn run(
+ &mut self,
+ ) -> Result<JoinSet<Result<BenchmarkIndividualMetrics, IggyError>>, IggyError>;
fn kind(&self) -> BenchmarkKind;
fn args(&self) -> &IggyBenchArgs;
fn client_factory(&self) -> &Arc<dyn ClientFactory>;
diff --git a/bench/src/benchmarks/consumer_benchmark.rs b/bench/src/benchmarks/consumer_benchmark.rs
index 59574cf9..c5fffdd6 100644
--- a/bench/src/benchmarks/consumer_benchmark.rs
+++ b/bench/src/benchmarks/consumer_benchmark.rs
@@ -1,13 +1,16 @@
use crate::actors::consumer::Consumer;
use crate::args::common::IggyBenchArgs;
-use crate::benchmarks::benchmark::{BenchmarkFutures, Benchmarkable};
+use crate::benchmarks::benchmark::Benchmarkable;
use crate::rate_limiter::RateLimiter;
use async_trait::async_trait;
-use iggy::messages::poll_messages::PollingKind;
+use iggy::error::IggyError;
+use iggy::messages::PollingKind;
use iggy_bench_report::benchmark_kind::BenchmarkKind;
+use iggy_bench_report::individual_metrics::BenchmarkIndividualMetrics;
use integration::test_server::ClientFactory;
use std::sync::atomic::AtomicI64;
use std::sync::Arc;
+use tokio::task::JoinSet;
use tracing::info;
pub struct ConsumerBenchmark {
@@ -26,14 +29,16 @@ impl ConsumerBenchmark {
#[async_trait]
impl Benchmarkable for ConsumerBenchmark {
- async fn run(&mut self) -> BenchmarkFutures {
+ async fn run(
+ &mut self,
+ ) -> Result<JoinSet<Result<BenchmarkIndividualMetrics, IggyError>>, IggyError> {
self.check_streams().await?;
let consumers_count = self.args.consumers();
info!("Creating {} consumer(s)...", consumers_count);
let messages_per_batch = self.args.messages_per_batch();
let message_batches = self.args.message_batches();
- let mut futures: BenchmarkFutures = Ok(Vec::with_capacity(consumers_count as usize));
+ let mut set = JoinSet::new();
for consumer_id in 1..=consumers_count {
let args = self.args.clone();
let client_factory = self.client_factory.clone();
@@ -76,12 +81,11 @@ impl Benchmarkable for ConsumerBenchmark {
args.rate_limit()
.map(|rl| RateLimiter::new(rl.as_bytes_u64())),
);
-
- let future = Box::pin(async move { consumer.run().await });
- futures.as_mut().unwrap().push(future);
+ set.spawn(consumer.run());
}
+
info!("Created {} consumer(s).", consumers_count);
- futures
+ Ok(set)
}
fn kind(&self) -> BenchmarkKind {
diff --git a/bench/src/benchmarks/consumer_group_benchmark.rs b/bench/src/benchmarks/consumer_group_benchmark.rs
index daa67f33..42195f56 100644
--- a/bench/src/benchmarks/consumer_group_benchmark.rs
+++ b/bench/src/benchmarks/consumer_group_benchmark.rs
@@ -1,4 +1,4 @@
-use super::benchmark::{BenchmarkFutures, Benchmarkable};
+use super::benchmark::Benchmarkable;
use crate::{
actors::consumer::Consumer,
args::common::IggyBenchArgs,
@@ -8,11 +8,14 @@ use crate::{
use async_trait::async_trait;
use iggy::{
client::ConsumerGroupClient, clients::client::IggyClient, error::IggyError,
- messages::poll_messages::PollingKind,
+ messages::PollingKind,
+};
+use iggy_bench_report::{
+ benchmark_kind::BenchmarkKind, individual_metrics::BenchmarkIndividualMetrics,
};
-use iggy_bench_report::benchmark_kind::BenchmarkKind;
use integration::test_server::{login_root, ClientFactory};
use std::sync::{atomic::AtomicI64, Arc};
+use tokio::task::JoinSet;
use tracing::{error, info};
pub struct ConsumerGroupBenchmark {
@@ -69,7 +72,9 @@ impl ConsumerGroupBenchmark {
#[async_trait]
impl Benchmarkable for ConsumerGroupBenchmark {
- async fn run(&mut self) -> BenchmarkFutures {
+ async fn run(
+ &mut self,
+ ) -> Result<JoinSet<Result<BenchmarkIndividualMetrics, IggyError>>, IggyError> {
self.check_streams().await?;
let consumer_groups_count = self.args.number_of_consumer_groups();
self.init_consumer_groups(consumer_groups_count)
@@ -81,11 +86,11 @@ impl Benchmarkable for ConsumerGroupBenchmark {
let consumers = self.args.consumers();
let messages_per_batch = self.args.messages_per_batch();
let warmup_time = self.args.warmup_time();
- let mut futures: BenchmarkFutures = Ok(Vec::with_capacity((consumers) as usize));
let polling_kind = PollingKind::Next;
let message_batches = self.args.message_batches();
let total_message_batches = Arc::new(AtomicI64::new((message_batches * consumers) as i64));
+ let mut set = JoinSet::new();
for consumer_id in 1..=consumers {
let consumer_group_id =
start_consumer_group_id + 1 + (consumer_id % consumer_groups_count);
@@ -110,14 +115,14 @@ impl Benchmarkable for ConsumerGroupBenchmark {
.rate_limit()
.map(|rl| RateLimiter::new(rl.as_bytes_u64())),
);
- let future = Box::pin(async move { consumer.run().await });
- futures.as_mut().unwrap().push(future);
+ set.spawn(consumer.run());
}
+
info!(
"Starting consumer group benchmark with {} messages",
self.total_messages()
);
- futures
+ Ok(set)
}
fn kind(&self) -> BenchmarkKind {
diff --git a/bench/src/benchmarks/producer_and_consumer_benchmark.rs b/bench/src/benchmarks/producer_and_consumer_benchmark.rs
index b26f5a7b..6bfe6bee 100644
--- a/bench/src/benchmarks/producer_and_consumer_benchmark.rs
+++ b/bench/src/benchmarks/producer_and_consumer_benchmark.rs
@@ -1,14 +1,17 @@
use crate::actors::consumer::Consumer;
use crate::actors::producer::Producer;
use crate::args::common::IggyBenchArgs;
-use crate::benchmarks::benchmark::{BenchmarkFutures, Benchmarkable};
+use crate::benchmarks::benchmark::Benchmarkable;
use crate::rate_limiter::RateLimiter;
use async_trait::async_trait;
-use iggy::messages::poll_messages::PollingKind;
+use iggy::error::IggyError;
+use iggy::messages::PollingKind;
use iggy_bench_report::benchmark_kind::BenchmarkKind;
+use iggy_bench_report::individual_metrics::BenchmarkIndividualMetrics;
use integration::test_server::ClientFactory;
use std::sync::atomic::AtomicI64;
use std::sync::Arc;
+use tokio::task::JoinSet;
use tracing::info;
pub struct ProducerAndConsumerBenchmark {
@@ -27,7 +30,9 @@ impl ProducerAndConsumerBenchmark {
#[async_trait]
impl Benchmarkable for ProducerAndConsumerBenchmark {
- async fn run(&mut self) -> BenchmarkFutures {
+ async fn run(
+ &mut self,
+ ) -> Result<JoinSet<Result<BenchmarkIndividualMetrics, IggyError>>, IggyError> {
self.init_streams().await.expect("Failed to init streams!");
let start_stream_id = self.args.start_stream_id();
let producers = self.args.producers();
@@ -38,8 +43,7 @@ impl Benchmarkable for ProducerAndConsumerBenchmark {
let partitions_count = self.args.number_of_partitions();
let warmup_time = self.args.warmup_time();
- let mut futures: BenchmarkFutures =
- Ok(Vec::with_capacity((producers + consumers) as usize));
+ let mut set = JoinSet::new();
for producer_id in 1..=producers {
let stream_id = start_stream_id + producer_id;
let producer = Producer::new(
@@ -59,8 +63,7 @@ impl Benchmarkable for ProducerAndConsumerBenchmark {
.map(|rl| RateLimiter::new(rl.as_bytes_u64())),
false, // TODO: put timestamp into first message, it should be an argument to iggy-bench
);
- let future = Box::pin(async move { producer.run().await });
- futures.as_mut().unwrap().push(future);
+ set.spawn(producer.run());
}
let polling_kind = PollingKind::Offset;
@@ -85,14 +88,14 @@ impl Benchmarkable for ProducerAndConsumerBenchmark {
.rate_limit()
.map(|rl| RateLimiter::new(rl.as_bytes_u64())),
);
- let future = Box::pin(async move { consumer.run().await });
- futures.as_mut().unwrap().push(future);
+ set.spawn(consumer.run());
}
+
info!(
"Starting to send and poll {} messages",
self.total_messages()
);
- futures
+ Ok(set)
}
fn kind(&self) -> BenchmarkKind {
diff --git a/bench/src/benchmarks/producer_and_consumer_group_benchmark.rs b/bench/src/benchmarks/producer_and_consumer_group_benchmark.rs
index 9117cb1a..7fee789b 100644
--- a/bench/src/benchmarks/producer_and_consumer_group_benchmark.rs
+++ b/bench/src/benchmarks/producer_and_consumer_group_benchmark.rs
@@ -1,4 +1,4 @@
-use super::benchmark::{BenchmarkFutures, Benchmarkable};
+use super::benchmark::Benchmarkable;
use crate::{
actors::{consumer::Consumer, producer::Producer},
args::common::IggyBenchArgs,
@@ -6,13 +6,14 @@ use crate::{
rate_limiter::RateLimiter,
};
use async_trait::async_trait;
-use iggy::{
- client::ConsumerGroupClient, clients::client::IggyClient, error::IggyError,
- messages::poll_messages::PollingKind,
+use iggy::messages::PollingKind;
+use iggy::{client::ConsumerGroupClient, clients::client::IggyClient, error::IggyError};
+use iggy_bench_report::{
+ benchmark_kind::BenchmarkKind, individual_metrics::BenchmarkIndividualMetrics,
};
-use iggy_bench_report::benchmark_kind::BenchmarkKind;
use integration::test_server::{login_root, ClientFactory};
use std::sync::{atomic::AtomicI64, Arc};
+use tokio::task::JoinSet;
use tracing::{error, info};
pub struct ProducerAndConsumerGroupBenchmark {
@@ -69,7 +70,9 @@ impl ProducerAndConsumerGroupBenchmark {
#[async_trait]
impl Benchmarkable for ProducerAndConsumerGroupBenchmark {
- async fn run(&mut self) -> BenchmarkFutures {
+ async fn run(
+ &mut self,
+ ) -> Result<JoinSet<Result<BenchmarkIndividualMetrics, IggyError>>, IggyError> {
self.init_streams().await.expect("Failed to init streams!");
let consumer_groups_count = self.args.number_of_consumer_groups();
self.init_consumer_groups(consumer_groups_count)
@@ -98,9 +101,7 @@ impl Benchmarkable for ProducerAndConsumerGroupBenchmark {
message_batches * producers,
);
- let mut futures: BenchmarkFutures =
- Ok(Vec::with_capacity((producers + consumers) as usize));
-
+ let mut set = JoinSet::new();
for producer_id in 1..=producers {
info!("Executing the benchmark on producer #{}...", producer_id);
let stream_id = self.args.start_stream_id() + 1 + (producer_id % streams_number);
@@ -122,8 +123,7 @@ impl Benchmarkable for ProducerAndConsumerGroupBenchmark {
.map(|rl| RateLimiter::new(rl.as_bytes_u64())),
false, // TODO: Put latency into payload of first message, it should be an argument to iggy-bench
);
- let future = Box::pin(async move { producer.run().await });
- futures.as_mut().unwrap().push(future);
+ set.spawn(producer.run());
}
info!("Created {} producer(s).", producers);
@@ -150,15 +150,14 @@ impl Benchmarkable for ProducerAndConsumerGroupBenchmark {
.rate_limit()
.map(|rl| RateLimiter::new(rl.as_bytes_u64())),
);
- let future = Box::pin(async move { consumer.run().await });
- futures.as_mut().unwrap().push(future);
+ set.spawn(consumer.run());
}
info!(
"Starting to send and poll {} messages",
self.total_messages()
);
- futures
+ Ok(set)
}
fn kind(&self) -> BenchmarkKind {
diff --git a/bench/src/benchmarks/producer_benchmark.rs b/bench/src/benchmarks/producer_benchmark.rs
index 0e9379e0..7453508b 100644
--- a/bench/src/benchmarks/producer_benchmark.rs
+++ b/bench/src/benchmarks/producer_benchmark.rs
@@ -1,11 +1,14 @@
use crate::actors::producer::Producer;
use crate::args::common::IggyBenchArgs;
-use crate::benchmarks::benchmark::{BenchmarkFutures, Benchmarkable};
+use crate::benchmarks::benchmark::Benchmarkable;
use crate::rate_limiter::RateLimiter;
use async_trait::async_trait;
+use iggy::error::IggyError;
use iggy_bench_report::benchmark_kind::BenchmarkKind;
+use iggy_bench_report::individual_metrics::BenchmarkIndividualMetrics;
use integration::test_server::ClientFactory;
use std::sync::Arc;
+use tokio::task::JoinSet;
use tracing::info;
pub struct ProducerBenchmark {
@@ -24,7 +27,9 @@ impl ProducerBenchmark {
#[async_trait]
impl Benchmarkable for ProducerBenchmark {
- async fn run(&mut self) -> BenchmarkFutures {
+ async fn run(
+ &mut self,
+ ) -> Result<JoinSet<Result<BenchmarkIndividualMetrics, IggyError>>, IggyError> {
self.init_streams().await.expect("Failed to init streams!");
let producers_count = self.args.producers();
info!("Creating {} producer(s)...", producers_count);
@@ -35,7 +40,7 @@ impl Benchmarkable for ProducerBenchmark {
let partitions_count = self.args.number_of_partitions();
let warmup_time = self.args.warmup_time();
- let mut futures: BenchmarkFutures = Ok(Vec::with_capacity(producers_count as usize));
+ let mut set = JoinSet::new();
for producer_id in 1..=producers_count {
let args = self.args.clone();
let client_factory = self.client_factory.clone();
@@ -58,11 +63,11 @@ impl Benchmarkable for ProducerBenchmark {
.map(|rl| RateLimiter::new(rl.as_bytes_u64())),
false, // TODO: Put timestamp in payload of first message, it should be an argument to iggy-bench
);
- let future = Box::pin(async move { producer.run().await });
- futures.as_mut().unwrap().push(future);
+ set.spawn(producer.run());
}
+
info!("Created {} producer(s).", producers_count);
- futures
+ Ok(set)
}
fn kind(&self) -> BenchmarkKind {
diff --git a/bench/src/benchmarks/producing_consumer_benchmark.rs b/bench/src/benchmarks/producing_consumer_benchmark.rs
index 1f4b9d67..4692280b 100644
--- a/bench/src/benchmarks/producing_consumer_benchmark.rs
+++ b/bench/src/benchmarks/producing_consumer_benchmark.rs
@@ -1,13 +1,16 @@
use crate::actors::producing_consumer::ProducingConsumer;
use crate::args::common::IggyBenchArgs;
-use crate::benchmarks::benchmark::{BenchmarkFutures, Benchmarkable};
+use crate::benchmarks::benchmark::Benchmarkable;
use crate::rate_limiter::RateLimiter;
use async_trait::async_trait;
-use iggy::messages::poll_messages::PollingKind;
+use iggy::error::IggyError;
+use iggy::messages::PollingKind;
use iggy_bench_report::benchmark_kind::BenchmarkKind;
+use iggy_bench_report::individual_metrics::BenchmarkIndividualMetrics;
use integration::test_server::ClientFactory;
use std::sync::atomic::AtomicI64;
use std::sync::Arc;
+use tokio::task::JoinSet;
use tracing::info;
pub struct EndToEndProducingConsumerBenchmark {
@@ -26,7 +29,9 @@ impl EndToEndProducingConsumerBenchmark {
#[async_trait]
impl Benchmarkable for EndToEndProducingConsumerBenchmark {
- async fn run(&mut self) -> BenchmarkFutures {
+ async fn run(
+ &mut self,
+ ) -> Result<JoinSet<Result<BenchmarkIndividualMetrics, IggyError>>, IggyError> {
self.init_streams().await.expect("Failed to init streams!");
let actors_count = self.args.producers();
info!("Creating {} producing consumer(s)...", actors_count);
@@ -36,7 +41,8 @@ impl Benchmarkable for EndToEndProducingConsumerBenchmark {
let message_size = self.args.message_size();
let warmup_time = self.args.warmup_time();
let polling_kind = PollingKind::Offset;
- let mut futures: BenchmarkFutures = Ok(Vec::with_capacity(actors_count as usize));
+
+ let mut set = JoinSet::new();
for actor_id in 1..=actors_count {
let args = self.args.clone();
let client_factory = self.client_factory.clone();
@@ -65,11 +71,11 @@ impl Benchmarkable for EndToEndProducingConsumerBenchmark {
polling_kind,
false,
);
- let future = Box::pin(async move { actor.run().await });
- futures.as_mut().unwrap().push(future);
+ set.spawn(actor.run());
}
+
info!("Created {} producing consumer(s).", actors_count);
- futures
+ Ok(set)
}
fn kind(&self) -> BenchmarkKind {
diff --git a/bench/src/benchmarks/producing_consumer_group_benchmark.rs b/bench/src/benchmarks/producing_consumer_group_benchmark.rs
index 6553ce7f..b5d57c7e 100644
--- a/bench/src/benchmarks/producing_consumer_group_benchmark.rs
+++ b/bench/src/benchmarks/producing_consumer_group_benchmark.rs
@@ -1,17 +1,19 @@
use crate::actors::producing_consumer::ProducingConsumer;
use crate::args::common::IggyBenchArgs;
-use crate::benchmarks::benchmark::{BenchmarkFutures, Benchmarkable};
+use crate::benchmarks::benchmark::Benchmarkable;
use crate::benchmarks::{CONSUMER_GROUP_BASE_ID, CONSUMER_GROUP_NAME_PREFIX};
use crate::rate_limiter::RateLimiter;
use async_trait::async_trait;
use iggy::client::ConsumerGroupClient;
use iggy::clients::client::IggyClient;
use iggy::error::IggyError;
-use iggy::messages::poll_messages::PollingKind;
+use iggy::messages::PollingKind;
use iggy_bench_report::benchmark_kind::BenchmarkKind;
+use iggy_bench_report::individual_metrics::BenchmarkIndividualMetrics;
use integration::test_server::{login_root, ClientFactory};
use std::sync::atomic::AtomicI64;
use std::sync::Arc;
+use tokio::task::JoinSet;
use tracing::{error, info};
pub struct EndToEndProducingConsumerGroupBenchmark {
@@ -68,7 +70,9 @@ impl EndToEndProducingConsumerGroupBenchmark {
#[async_trait]
impl Benchmarkable for EndToEndProducingConsumerGroupBenchmark {
- async fn run(&mut self) -> BenchmarkFutures {
+ async fn run(
+ &mut self,
+ ) -> Result<JoinSet<Result<BenchmarkIndividualMetrics, IggyError>>, IggyError> {
self.init_streams().await.expect("Failed to init streams!");
let consumer_groups_count = self.args.number_of_consumer_groups();
self.init_consumer_groups(consumer_groups_count)
@@ -87,11 +91,11 @@ impl Benchmarkable for EndToEndProducingConsumerGroupBenchmark {
let partitions_count = self.args.number_of_partitions();
let warmup_time = self.args.warmup_time();
let polling_kind = PollingKind::Next;
- let mut futures: BenchmarkFutures = Ok(Vec::with_capacity(actors_count as usize));
let consumer_groups_count = self.args.number_of_consumer_groups();
let total_message_batches =
Arc::new(AtomicI64::new((message_batches * actors_count) as i64));
+ let mut set = JoinSet::new();
for actor_id in 1..=actors_count {
let args = self.args.clone();
let client_factory = self.client_factory.clone();
@@ -122,11 +126,11 @@ impl Benchmarkable for EndToEndProducingConsumerGroupBenchmark {
polling_kind,
false,
);
- let future = Box::pin(async move { actor.run().await });
- futures.as_mut().unwrap().push(future);
+ set.spawn(actor.run());
}
+
info!("Created {actors_count} producing consumer(s) which would be part of {consumer_groups_count} consumer groups");
- futures
+ Ok(set)
}
fn kind(&self) -> BenchmarkKind {
diff --git a/bench/src/runner.rs b/bench/src/runner.rs
index 71cbfb68..60732720 100644
--- a/bench/src/runner.rs
+++ b/bench/src/runner.rs
@@ -5,7 +5,6 @@ use crate::plot::{plot_chart, ChartType};
use crate::utils::collect_server_logs_and_save_to_file;
use crate::utils::cpu_name::append_cpu_name_lowercase;
use crate::utils::server_starter::start_server_if_needed;
-use futures::future::select_all;
use iggy::error::IggyError;
use iggy_bench_report::hardware::BenchmarkHardware;
use iggy_bench_report::params::BenchmarkParams;
@@ -42,18 +41,15 @@ impl BenchmarkRunner {
let mut individual_metrics = Vec::new();
- while !join_handles.is_empty() {
- let (result, _index, remaining) = select_all(join_handles).await;
- join_handles = remaining;
-
- match result {
- Ok(r) => individual_metrics.push(r),
+ while let Some(individual_metric) = join_handles.join_next().await {
+ let individual_metric = individual_metric.expect("Failed to join actor!");
+ match individual_metric {
+ Ok(individual_metric) => individual_metrics.push(individual_metric),
Err(e) => return Err(e),
}
}
info!("All actors joined!");
-
let hardware =
BenchmarkHardware::get_system_info_with_identifier(benchmark.args().identifier());
let params = BenchmarkParams::from(benchmark.args());
diff --git a/cli/src/args/message.rs b/cli/src/args/message.rs
index d334de05..e7cea251 100644
--- a/cli/src/args/message.rs
+++ b/cli/src/args/message.rs
@@ -1,9 +1,6 @@
use clap::builder::NonEmptyStringValueParser;
use clap::{ArgGroup, Args, Subcommand};
-use iggy::error::IggyError;
-use iggy::error::IggyError::InvalidFormat;
-use iggy::identifier::Identifier;
-use iggy::models::header::{HeaderKey, HeaderValue};
+use iggy::prelude::*;
use std::str::FromStr;
#[derive(Debug, Clone, Subcommand)]
@@ -84,7 +81,7 @@ pub(crate) struct SendMessagesArgs {
pub(crate) messages: Option<Vec<String>>,
/// Comma separated list of key:kind:value, sent as header with the message
///
- /// Headers are comma seperated key-value pairs that can be sent with the message.
+ /// Headers are comma separated key-value pairs that can be sent with the message.
/// Kind can be one of the following: raw, string, bool, int8, int16, int32, int64,
/// int128, uint8, uint16, uint32, uint64, uint128, float32, float64
#[clap(verbatim_doc_comment)]
@@ -109,7 +106,7 @@ fn parse_key_val(s: &str) -> Result<(HeaderKey, HeaderValue), IggyError> {
let parts = lower.split(':').collect::<Vec<_>>();
if parts.len() != 3 {
- Err(InvalidFormat)?;
+ return Err(IggyError::InvalidFormat);
}
let key = HeaderKey::from_str(parts[0])?;
diff --git a/configs/server.toml b/configs/server.toml
index 0ea8afe9..527fc06c 100644
--- a/configs/server.toml
+++ b/configs/server.toml
@@ -374,7 +374,7 @@ sysinfo_print_interval = "10 s"
# Enables or disables the system cache.
# `true` activates caching for frequently accessed data.
# `false` disables caching, data is always read from the source.
-enabled = true
+enabled = false
# Maximum size of the cache, e.g. "4GB".
size = "4 GB"
@@ -444,17 +444,27 @@ enforce_fsync = false
# `false` skips these checks for faster loading at the risk of undetected corruption.
validate_checksum = false
-# The threshold of buffered messages before triggering a save to disk (integer).
+# The count threshold of buffered messages before triggering a save to disk (integer).
# Specifies how many messages accumulate before persisting to storage.
# Adjusting this can balance between write performance and data durability.
+# This is soft limit, actual number of messages may be higher, depending on last batch size.
+# Together with `size_of_messages_required_to_save` it defines the threshold of buffered messages.
messages_required_to_save = 1000
+# The size threshold of buffered messages before triggering a save to disk (string).
+# Specifies how much size of messages accumulate before persisting to storage.
+# Adjusting this can balance between write performance and data durability.
+# This is soft limit, actual number of messages may be higher, depending on last batch size.
+# Together with `messages_required_to_save` it defines the threshold of buffered messages.
+size_of_messages_required_to_save = "1 MB"
+
# Segment configuration
[system.segment]
# Defines the soft limit for the size of a storage segment.
# When a segment reaches this size, a new segment is created for subsequent data.
# Example: if `size` is set "1GB", the actual segment size may be 1GB + the size of remaining messages in received batch.
size = "1 GB"
+
# Configures the message time-based expiry setting.
# "none" means messages are kept indefinitely.
# A time value in human-readable format determines the lifespan of messages.
@@ -472,9 +482,11 @@ server_confirmation = "wait"
archive_expired = false
# Controls whether to cache indexes (time and positional) for segment access (boolean).
-# `true` keeps indexes in memory, speeding up data retrieval.
-# `false` reads indexes from disk, which can conserve memory at the cost of access speed.
-cache_indexes = true
+# Possible values:
+# - "true" or "all": keeps indexes in memory, speeding up data retrieval at the cost of memory
+# - "open_segment": keeps indexes in memory only for the currently open segment
+# - "false" or "none": reads indexes from disk, which can conserve memory at the cost of access speed
+cache_indexes = "all"
# Message deduplication configuration
[system.message_deduplication]
diff --git a/examples/src/basic/consumer/main.rs b/examples/src/basic/consumer/main.rs
index 627b6ccb..4e3cb594 100644
--- a/examples/src/basic/consumer/main.rs
+++ b/examples/src/basic/consumer/main.rs
@@ -1,9 +1,5 @@
use clap::Parser;
-use iggy::client::Client;
-use iggy::client_provider;
-use iggy::client_provider::ClientProviderConfig;
-use iggy::clients::client::IggyClient;
-use iggy::models::messages::PolledMessage;
+use iggy::prelude::*;
use iggy_examples::shared::args::Args;
use iggy_examples::shared::system;
use std::error::Error;
@@ -32,12 +28,12 @@ async fn main() -> Result<(), Box<dyn Error>> {
system::consume_messages(&args, &client, &handle_message).await
}
-fn handle_message(message: &PolledMessage) -> Result<(), Box<dyn Error>> {
+fn handle_message(message: &IggyMessage) -> Result<(), Box<dyn Error>> {
// The payload can be of any type as it is a raw byte array. In this case it's a simple string.
let payload = std::str::from_utf8(&message.payload)?;
info!(
"Handling message at offset: {}, payload: {}...",
- message.offset, payload
+ message.header.offset, payload
);
Ok(())
}
diff --git a/examples/src/basic/producer/main.rs b/examples/src/basic/producer/main.rs
index 25c32119..3cf307bd 100644
--- a/examples/src/basic/producer/main.rs
+++ b/examples/src/basic/producer/main.rs
@@ -1,9 +1,5 @@
use clap::Parser;
-use iggy::client::Client;
-use iggy::client_provider;
-use iggy::client_provider::ClientProviderConfig;
-use iggy::clients::client::IggyClient;
-use iggy::messages::send_messages::{Message, Partitioning};
+use iggy::prelude::*;
use iggy_examples::shared::args::Args;
use iggy_examples::shared::system;
use std::error::Error;
@@ -64,7 +60,7 @@ async fn produce_messages(args: &Args, client: &dyn Client) -> Result<(), Box<dy
for _ in 0..args.messages_per_batch {
current_id += 1;
let payload = format!("message-{current_id}");
- let message = Message::from_str(&payload)?;
+ let message = IggyMessage::from_str(&payload)?;
messages.push(message);
sent_messages.push(payload);
}
diff --git a/examples/src/getting-started/consumer/main.rs b/examples/src/getting-started/consumer/main.rs
index 3b94f988..64979c6f 100644
--- a/examples/src/getting-started/consumer/main.rs
+++ b/examples/src/getting-started/consumer/main.rs
@@ -1,10 +1,4 @@
-use iggy::client::{Client, UserClient};
-use iggy::clients::builder::IggyClientBuilder;
-use iggy::consumer::Consumer;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::models::messages::PolledMessage;
-use iggy::users::defaults::*;
-use iggy::utils::duration::IggyDuration;
+use iggy::prelude::*;
use std::env;
use std::error::Error;
use std::str::FromStr;
@@ -89,12 +83,12 @@ async fn consume_messages(client: &dyn Client) -> Result<(), Box<dyn Error>> {
}
}
-fn handle_message(message: &PolledMessage) -> Result<(), Box<dyn Error>> {
+fn handle_message(message: &IggyMessage) -> Result<(), Box<dyn Error>> {
// The payload can be of any type as it is a raw byte array. In this case it's a simple string.
let payload = std::str::from_utf8(&message.payload)?;
info!(
"Handling message at offset: {}, payload: {}...",
- message.offset, payload
+ message.header.offset, payload
);
Ok(())
}
diff --git a/examples/src/getting-started/producer/main.rs b/examples/src/getting-started/producer/main.rs
index 601ead7a..16831c96 100644
--- a/examples/src/getting-started/producer/main.rs
+++ b/examples/src/getting-started/producer/main.rs
@@ -1,12 +1,4 @@
-use iggy::client::{Client, StreamClient, TopicClient, UserClient};
-use iggy::clients::builder::IggyClientBuilder;
-use iggy::clients::client::IggyClient;
-use iggy::compression::compression_algorithm::CompressionAlgorithm;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::users::defaults::*;
-use iggy::utils::duration::IggyDuration;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use std::env;
use std::error::Error;
use std::str::FromStr;
@@ -94,7 +86,7 @@ async fn produce_messages(client: &dyn Client) -> Result<(), Box<dyn Error>> {
for _ in 0..messages_per_batch {
current_id += 1;
let payload = format!("message-{current_id}");
- let message = Message::from_str(&payload)?;
+ let message = IggyMessage::from_str(&payload)?;
messages.push(message);
}
client
diff --git a/examples/src/message-envelope/consumer/main.rs b/examples/src/message-envelope/consumer/main.rs
index 4b23fb69..a0974696 100644
--- a/examples/src/message-envelope/consumer/main.rs
+++ b/examples/src/message-envelope/consumer/main.rs
@@ -1,10 +1,6 @@
use anyhow::Result;
use clap::Parser;
-use iggy::client::Client;
-use iggy::client_provider;
-use iggy::client_provider::ClientProviderConfig;
-use iggy::clients::client::IggyClient;
-use iggy::models::messages::PolledMessage;
+use iggy::prelude::*;
use iggy_examples::shared::args::Args;
use iggy_examples::shared::messages::*;
use iggy_examples::shared::system;
@@ -34,14 +30,14 @@ async fn main() -> Result<(), Box<dyn Error>> {
system::consume_messages(&args, &client, &handle_message).await
}
-fn handle_message(message: &PolledMessage) -> Result<(), Box<dyn Error>> {
+fn handle_message(message: &IggyMessage) -> Result<(), Box<dyn Error>> {
// The payload can be of any type as it is a raw byte array. In this case it's a JSON string.
let json = std::str::from_utf8(&message.payload)?;
// The message envelope can be used to send the different types of messages to the same topic.
let envelope = serde_json::from_str::<Envelope>(json)?;
info!(
"Handling message type: {} at offset: {}...",
- envelope.message_type, message.offset
+ envelope.message_type, message.header.offset
);
match envelope.message_type.as_str() {
ORDER_CREATED_TYPE => {
diff --git a/examples/src/message-envelope/producer/main.rs b/examples/src/message-envelope/producer/main.rs
index bafb1a49..55245ad6 100644
--- a/examples/src/message-envelope/producer/main.rs
+++ b/examples/src/message-envelope/producer/main.rs
@@ -1,10 +1,6 @@
use anyhow::Result;
use clap::Parser;
-use iggy::client::{Client, MessageClient};
-use iggy::client_provider;
-use iggy::client_provider::ClientProviderConfig;
-use iggy::clients::client::IggyClient;
-use iggy::messages::send_messages::{Message, Partitioning};
+use iggy::prelude::*;
use iggy_examples::shared::args::Args;
use iggy_examples::shared::messages_generator::MessagesGenerator;
use iggy_examples::shared::system;
@@ -66,7 +62,7 @@ async fn produce_messages(args: &Args, client: &IggyClient) -> Result<(), Box<dy
let serializable_message = message_generator.generate();
// You can send the different message types to the same partition, or stick to the single type.
let json_envelope = serializable_message.to_json_envelope();
- let message = Message::from_str(&json_envelope)?;
+ let message = IggyMessage::from_str(&json_envelope)?;
messages.push(message);
// This is used for the logging purposes only.
serializable_messages.push(serializable_message);
diff --git a/examples/src/message-headers/consumer/main.rs b/examples/src/message-headers/consumer/main.rs
index 0f09b829..d7e05335 100644
--- a/examples/src/message-headers/consumer/main.rs
+++ b/examples/src/message-headers/consumer/main.rs
@@ -1,11 +1,6 @@
use anyhow::Result;
use clap::Parser;
-use iggy::client::Client;
-use iggy::client_provider;
-use iggy::client_provider::ClientProviderConfig;
-use iggy::clients::client::IggyClient;
-use iggy::models::header::HeaderKey;
-use iggy::models::messages::PolledMessage;
+use iggy::prelude::*;
use iggy_examples::shared::args::Args;
use iggy_examples::shared::messages::*;
use iggy_examples::shared::system;
@@ -35,21 +30,16 @@ async fn main() -> Result<(), Box<dyn Error>> {
system::consume_messages(&args, &client, &handle_message).await
}
-fn handle_message(message: &PolledMessage) -> Result<(), Box<dyn Error>> {
+fn handle_message(message: &IggyMessage) -> Result<(), Box<dyn Error>> {
// The payload can be of any type as it is a raw byte array. In this case it's a JSON string.
let payload = std::str::from_utf8(&message.payload)?;
// The message type is stored in the custom message header.
let header_key = HeaderKey::new("message_type").unwrap();
- let message_type = message
- .headers
- .as_ref()
- .unwrap()
- .get(&header_key)
- .unwrap()
- .as_str()?;
+ let headers_map = message.user_headers_map()?.unwrap();
+ let message_type = headers_map.get(&header_key).unwrap().as_str()?;
info!(
"Handling message type: {} at offset: {}...",
- message_type, message.offset
+ message_type, message.header.offset
);
match message_type {
ORDER_CREATED_TYPE => {
diff --git a/examples/src/message-headers/producer/main.rs b/examples/src/message-headers/producer/main.rs
index 5d38381c..9a3e649e 100644
--- a/examples/src/message-headers/producer/main.rs
+++ b/examples/src/message-headers/producer/main.rs
@@ -1,12 +1,7 @@
use anyhow::Result;
use bytes::Bytes;
use clap::Parser;
-use iggy::client::{Client, MessageClient};
-use iggy::client_provider;
-use iggy::client_provider::ClientProviderConfig;
-use iggy::clients::client::IggyClient;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::models::header::{HeaderKey, HeaderValue};
+use iggy::prelude::*;
use iggy_examples::shared::args::Args;
use iggy_examples::shared::messages_generator::MessagesGenerator;
use iggy_examples::shared::system;
@@ -78,7 +73,11 @@ async fn produce_messages(args: &Args, client: &IggyClient) -> Result<(), Box<dy
HeaderValue::from_str(message_type).unwrap(),
);
- let message = Message::new(None, Bytes::from(json), Some(headers));
+ let message = IggyMessage::builder()
+ .payload(Bytes::from(json))
+ .headers(headers)
+ .build()
+ .unwrap();
messages.push(message);
// This is used for the logging purposes only.
serializable_messages.push(serializable_message);
diff --git a/examples/src/multi-tenant/consumer/main.rs b/examples/src/multi-tenant/consumer/main.rs
index d70a3c5a..3231a8c1 100644
--- a/examples/src/multi-tenant/consumer/main.rs
+++ b/examples/src/multi-tenant/consumer/main.rs
@@ -2,16 +2,7 @@ use ahash::AHashMap;
use clap::Parser;
use futures_util::future::join_all;
use futures_util::StreamExt;
-use iggy::client::{Client, StreamClient, TopicClient, UserClient};
-use iggy::clients::client::IggyClient;
-use iggy::clients::consumer::{AutoCommit, AutoCommitWhen, IggyConsumer};
-use iggy::error::IggyError;
-use iggy::identifier::Identifier;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::models::permissions::{Permissions, StreamPermissions, TopicPermissions};
-use iggy::models::user_status::UserStatus;
-use iggy::users::defaults::{DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME};
-use iggy::utils::duration::IggyDuration;
+use iggy::prelude::*;
use iggy_examples::shared::args::Args;
use std::collections::HashMap;
use std::env;
@@ -108,7 +99,7 @@ async fn main() -> anyhow::Result<(), Box<dyn Error>> {
}
print_info("Disconnecting root client");
- root_client.disconnect().await?;
+ root_client.shutdown().await?;
print_info("Creating clients for each tenant");
let mut tenants = Vec::new();
@@ -233,7 +224,7 @@ fn start_consumers(tenant_id: u32, consumers: Vec<TenantConsumer>) -> Vec<JoinHa
if let Ok(message) = message {
let current_offset = message.current_offset;
let partition_id = message.partition_id;
- let offset = message.message.offset;
+ let offset = message.message.header.offset;
let payload = std::str::from_utf8(&message.message.payload);
if payload.is_err() {
let error = payload.unwrap_err();
diff --git a/examples/src/multi-tenant/producer/main.rs b/examples/src/multi-tenant/producer/main.rs
index 35a34d5c..33d2ed7c 100644
--- a/examples/src/multi-tenant/producer/main.rs
+++ b/examples/src/multi-tenant/producer/main.rs
@@ -1,19 +1,7 @@
use ahash::AHashMap;
use clap::Parser;
use futures_util::future::join_all;
-use iggy::client::{Client, StreamClient, UserClient};
-use iggy::clients::builder::IggyClientBuilder;
-use iggy::clients::client::IggyClient;
-use iggy::clients::producer::IggyProducer;
-use iggy::error::IggyError;
-use iggy::identifier::Identifier;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::models::permissions::{Permissions, StreamPermissions};
-use iggy::models::user_status::UserStatus;
-use iggy::users::defaults::{DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME};
-use iggy::utils::duration::IggyDuration;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use iggy_examples::shared::args::Args;
use std::collections::HashMap;
use std::env;
@@ -114,7 +102,7 @@ async fn main() -> anyhow::Result<(), Box<dyn Error>> {
}
print_info("Disconnecting root client");
- root_client.disconnect().await?;
+ root_client.shutdown().await?;
print_info("Creating clients for each tenant");
let mut tenants = Vec::new();
@@ -216,7 +204,7 @@ fn start_producers(
let mut messages = Vec::with_capacity(batch_size as usize);
for _ in 1..=batch_size {
let payload = format!("{message}-{producer_id}-{message_id}");
- let message = Message::from_str(&payload).expect("Invalid message");
+ let message = IggyMessage::from_str(&payload).expect("Invalid message");
messages.push(message);
}
diff --git a/examples/src/new-sdk/consumer/main.rs b/examples/src/new-sdk/consumer/main.rs
index 22ff0e7e..91e5f601 100644
--- a/examples/src/new-sdk/consumer/main.rs
+++ b/examples/src/new-sdk/consumer/main.rs
@@ -1,13 +1,6 @@
use clap::Parser;
use futures_util::StreamExt;
-use iggy::client::Client;
-use iggy::client_provider;
-use iggy::client_provider::ClientProviderConfig;
-use iggy::clients::client::IggyClient;
-use iggy::clients::consumer::{AutoCommit, AutoCommitWhen, IggyConsumer, ReceivedMessage};
-use iggy::consumer::ConsumerKind;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::utils::duration::IggyDuration;
+use iggy::prelude::*;
use iggy_examples::shared::args::Args;
use iggy_examples::shared::messages::{
Envelope, OrderConfirmed, OrderCreated, OrderRejected, ORDER_CONFIRMED_TYPE,
@@ -94,7 +87,10 @@ fn handle_message(message: &ReceivedMessage) -> anyhow::Result<(), Box<dyn Error
let envelope = serde_json::from_str::<Envelope>(json)?;
info!(
"Handling message type: {} at offset: {} in partition ID: {} with current offset: {}",
- envelope.message_type, message.message.offset, message.partition_id, message.current_offset,
+ envelope.message_type,
+ message.message.header.offset,
+ message.partition_id,
+ message.current_offset,
);
match envelope.message_type.as_str() {
ORDER_CREATED_TYPE => {
diff --git a/examples/src/new-sdk/producer/main.rs b/examples/src/new-sdk/producer/main.rs
index bee171f4..7a40802e 100644
--- a/examples/src/new-sdk/producer/main.rs
+++ b/examples/src/new-sdk/producer/main.rs
@@ -1,13 +1,5 @@
use clap::Parser;
-use iggy::client::Client;
-use iggy::client_provider;
-use iggy::client_provider::ClientProviderConfig;
-use iggy::clients::client::IggyClient;
-use iggy::clients::producer::IggyProducer;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::utils::duration::IggyDuration;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use iggy_examples::shared::args::Args;
use iggy_examples::shared::messages_generator::MessagesGenerator;
use std::error::Error;
@@ -75,7 +67,7 @@ async fn produce_messages(
for _ in 0..args.messages_per_batch {
let serializable_message = message_generator.generate();
let json_envelope = serializable_message.to_json_envelope();
- let message = Message::from_str(&json_envelope)?;
+ let message = IggyMessage::from_str(&json_envelope)?;
messages.push(message);
}
producer.send(messages).await?;
diff --git a/examples/src/shared/system.rs b/examples/src/shared/system.rs
index 26660c96..c6586c03 100644
--- a/examples/src/shared/system.rs
+++ b/examples/src/shared/system.rs
@@ -1,16 +1,8 @@
use crate::shared::args::Args;
-use iggy::client::Client;
-use iggy::compression::compression_algorithm::CompressionAlgorithm;
-use iggy::consumer::{Consumer, ConsumerKind};
-use iggy::error::IggyError;
-use iggy::identifier::Identifier;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::models::messages::PolledMessage;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use tracing::info;
-type MessageHandler = dyn Fn(&PolledMessage) -> Result<(), Box<dyn std::error::Error>>;
+type MessageHandler = dyn Fn(&IggyMessage) -> Result<(), Box<dyn std::error::Error>>;
pub async fn init_by_consumer(args: &Args, client: &dyn Client) {
let (stream_id, topic_id, partition_id) = (
diff --git a/examples/src/stream-builder/stream-basic/main.rs b/examples/src/stream-builder/stream-basic/main.rs
index 18117572..08f75f74 100644
--- a/examples/src/stream-builder/stream-basic/main.rs
+++ b/examples/src/stream-builder/stream-basic/main.rs
@@ -1,9 +1,6 @@
use crate::shared::stream::PrintEventConsumer;
-use iggy::client::{Client, StreamClient};
use iggy::consumer_ext::IggyConsumerMessageExt;
-use iggy::error::IggyError;
-use iggy::messages::send_messages::Message;
-use iggy::stream_builder::{IggyStream, IggyStreamConfig};
+use iggy::prelude::*;
use iggy_examples::shared;
use std::str::FromStr;
use tokio::sync::oneshot;
@@ -32,9 +29,15 @@ async fn main() -> Result<(), IggyError> {
});
println!("Send 3 test messages...");
- producer.send_one(Message::from_str("Hello World")?).await?;
- producer.send_one(Message::from_str("Hola Iggy")?).await?;
- producer.send_one(Message::from_str("Hi Apache")?).await?;
+ producer
+ .send_one(IggyMessage::from_str("Hello World")?)
+ .await?;
+ producer
+ .send_one(IggyMessage::from_str("Hola Iggy")?)
+ .await?;
+ producer
+ .send_one(IggyMessage::from_str("Hi Apache")?)
+ .await?;
// Wait a bit for all messages to arrive.
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
diff --git a/examples/src/stream-builder/stream-consumer-config/main.rs b/examples/src/stream-builder/stream-consumer-config/main.rs
index c0fb5828..f2a1be32 100644
--- a/examples/src/stream-builder/stream-consumer-config/main.rs
+++ b/examples/src/stream-builder/stream-consumer-config/main.rs
@@ -1,12 +1,4 @@
-use iggy::client::Client;
-use iggy::clients::consumer::{AutoCommit, AutoCommitWhen};
-use iggy::consumer::ConsumerKind;
-use iggy::consumer_ext::IggyConsumerMessageExt;
-use iggy::error::IggyError;
-use iggy::identifier::Identifier;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::stream_builder::{IggyConsumerConfig, IggyStreamConsumer};
-use iggy::utils::duration::IggyDuration;
+use iggy::prelude::*;
use iggy_examples::shared::stream::PrintEventConsumer;
use std::str::FromStr;
use tokio::sync::oneshot;
diff --git a/examples/src/stream-builder/stream-consumer/main.rs b/examples/src/stream-builder/stream-consumer/main.rs
index 0ba6b154..74dbd4e5 100644
--- a/examples/src/stream-builder/stream-consumer/main.rs
+++ b/examples/src/stream-builder/stream-consumer/main.rs
@@ -1,7 +1,4 @@
-use iggy::client::Client;
-use iggy::consumer_ext::IggyConsumerMessageExt;
-use iggy::error::IggyError;
-use iggy::stream_builder::{IggyConsumerConfig, IggyStreamConsumer};
+use iggy::prelude::*;
use iggy_examples::shared::stream::PrintEventConsumer;
use tokio::sync::oneshot;
diff --git a/examples/src/stream-builder/stream-producer-config/main.rs b/examples/src/stream-builder/stream-producer-config/main.rs
index 8db647d0..521615ab 100644
--- a/examples/src/stream-builder/stream-producer-config/main.rs
+++ b/examples/src/stream-builder/stream-producer-config/main.rs
@@ -1,9 +1,4 @@
-use iggy::client::{Client, StreamClient};
-use iggy::error::IggyError;
-use iggy::identifier::Identifier;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::stream_builder::{IggyProducerConfig, IggyStreamProducer};
-use iggy::utils::duration::IggyDuration;
+use iggy::prelude::*;
use std::str::FromStr;
const IGGY_URL: &str = "iggy://iggy:iggy@localhost:8090";
@@ -53,7 +48,9 @@ async fn main() -> Result<(), IggyError> {
let (client, producer) = IggyStreamProducer::with_client_from_url(IGGY_URL, &config).await?;
println!("Send 3 test messages...");
- producer.send_one(Message::from_str("Hello World")?).await?;
+ producer
+ .send_one(IggyMessage::from_str("Hello World")?)
+ .await?;
// Wait a bit for all messages to arrive.
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
diff --git a/examples/src/stream-builder/stream-producer/main.rs b/examples/src/stream-builder/stream-producer/main.rs
index 16e66d1f..7779f0d8 100644
--- a/examples/src/stream-builder/stream-producer/main.rs
+++ b/examples/src/stream-builder/stream-producer/main.rs
@@ -1,7 +1,4 @@
-use iggy::client::{Client, StreamClient};
-use iggy::error::IggyError;
-use iggy::messages::send_messages::Message;
-use iggy::stream_builder::{IggyProducerConfig, IggyStreamProducer};
+use iggy::prelude::*;
use std::str::FromStr;
const IGGY_URL: &str = "iggy://iggy:iggy@localhost:8090";
@@ -14,9 +11,15 @@ async fn main() -> Result<(), IggyError> {
let (client, producer) = IggyStreamProducer::with_client_from_url(IGGY_URL, &config).await?;
println!("Send 3 test messages...");
- producer.send_one(Message::from_str("Hello World")?).await?;
- producer.send_one(Message::from_str("Hola Iggy")?).await?;
- producer.send_one(Message::from_str("Hi Apache")?).await?;
+ producer
+ .send_one(IggyMessage::from_str("Hello World")?)
+ .await?;
+ producer
+ .send_one(IggyMessage::from_str("Hola Iggy")?)
+ .await?;
+ producer
+ .send_one(IggyMessage::from_str("Hi Apache")?)
+ .await?;
// Wait a bit for all messages to arrive.
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
diff --git a/integration/Cargo.toml b/integration/Cargo.toml
index 547fd79a..47e647c6 100644
--- a/integration/Cargo.toml
+++ b/integration/Cargo.toml
@@ -19,7 +19,7 @@ iggy = { path = "../sdk", features = ["iggy-cli"] }
keyring = "3.6.1"
lazy_static = "1.5.0"
libc = "0.2.169"
-log = "0.4.25"
+log = "0.4.26"
predicates = "3.1.3"
regex = "1.11.1"
serial_test = "3.2.0"
@@ -29,7 +29,7 @@ test-case = "3.3.1"
tokio = { version = "1.43.0", features = ["full"] }
tracing-subscriber = { version = "0.3.19", features = ["fmt", "env-filter"] }
twox-hash = { version = "2.1.0", features = ["xxhash32"] }
-uuid = { version = "1.13.2", features = ["v7", "fast-rng", "zerocopy"] }
+uuid = { version = "1.14.0", features = ["v7", "fast-rng", "zerocopy"] }
zip = "2.2.2"
# Some tests are failing in CI due to lack of IPv6 interfaces
diff --git a/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs b/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs
index bec85ff1..d515dd8d 100644
--- a/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs
+++ b/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs
@@ -4,12 +4,7 @@ use crate::cli::common::{
};
use assert_cmd::assert::Assert;
use async_trait::async_trait;
-use iggy::client::Client;
-use iggy::consumer::{Consumer, ConsumerKind};
-use iggy::identifier::Identifier;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use predicates::str::{contains, starts_with};
use serial_test::parallel;
use std::str::FromStr;
@@ -104,7 +99,7 @@ impl IggyCmdTestCase for TestConsumerOffsetGetCmd {
assert!(topic.is_ok());
let mut messages = (1..=self.messages_count)
- .filter_map(|id| Message::from_str(format!("Test message {id}").as_str()).ok())
+ .filter_map(|id| IggyMessage::from_str(format!("Test message {id}").as_str()).ok())
.collect::<Vec<_>>();
let send_status = client
diff --git a/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs b/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs
index 8db7e333..85aca91f 100644
--- a/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs
+++ b/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs
@@ -4,12 +4,7 @@ use crate::cli::common::{
};
use assert_cmd::assert::Assert;
use async_trait::async_trait;
-use iggy::client::Client;
-use iggy::consumer::{Consumer, ConsumerKind};
-use iggy::identifier::Identifier;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use predicates::str::diff;
use serial_test::parallel;
use std::str::FromStr;
@@ -104,7 +99,7 @@ impl IggyCmdTestCase for TestConsumerOffsetSetCmd {
assert!(topic.is_ok());
let mut messages = (1..=self.stored_offset + 1)
- .filter_map(|id| Message::from_str(format!("Test message {id}").as_str()).ok())
+ .filter_map(|id| IggyMessage::from_str(format!("Test message {id}").as_str()).ok())
.collect::<Vec<_>>();
let send_status = client
diff --git a/integration/tests/cli/message/test_message_poll_command.rs b/integration/tests/cli/message/test_message_poll_command.rs
index 80ac5206..36beef76 100644
--- a/integration/tests/cli/message/test_message_poll_command.rs
+++ b/integration/tests/cli/message/test_message_poll_command.rs
@@ -5,12 +5,7 @@ use crate::cli::common::{
use assert_cmd::assert::Assert;
use async_trait::async_trait;
use bytes::Bytes;
-use iggy::client::Client;
-use iggy::messages::poll_messages::{PollingKind, PollingStrategy};
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::models::header::{HeaderKey, HeaderValue};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use predicates::str::{contains, starts_with};
use serial_test::parallel;
use std::collections::HashMap;
@@ -132,7 +127,7 @@ impl IggyCmdTestCase for TestMessagePollCmd {
.iter()
.map(|s| {
let payload = Bytes::from(s.as_bytes().to_vec());
- Message::new(None, payload, Some(HashMap::from([self.headers.clone()])))
+ IggyMessage::with_headers(payload, HashMap::from([self.headers.clone()]))
})
.collect::<Vec<_>>();
diff --git a/integration/tests/cli/message/test_message_poll_to_file_command.rs b/integration/tests/cli/message/test_message_poll_to_file_command.rs
index 88635fb0..984db924 100644
--- a/integration/tests/cli/message/test_message_poll_to_file_command.rs
+++ b/integration/tests/cli/message/test_message_poll_to_file_command.rs
@@ -2,13 +2,7 @@ use crate::cli::common::{IggyCmdCommand, IggyCmdTest, IggyCmdTestCase};
use assert_cmd::assert::Assert;
use async_trait::async_trait;
use bytes::Bytes;
-use iggy::client::Client;
-use iggy::identifier::Identifier;
-use iggy::messages::poll_messages::{PollingKind, PollingStrategy};
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::models::header::{HeaderKey, HeaderValue};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use predicates::str::{contains, is_match, starts_with};
use serial_test::parallel;
use std::collections::HashMap;
@@ -112,7 +106,7 @@ impl IggyCmdTestCase for TestMessagePollToFileCmd<'_> {
.iter()
.map(|s| {
let payload = Bytes::from(s.as_bytes().to_vec());
- Message::new(None, payload, Some(self.headers.clone()))
+ IggyMessage::with_headers(payload, self.headers.clone())
})
.collect::<Vec<_>>();
diff --git a/integration/tests/cli/message/test_message_reply_via_file.rs b/integration/tests/cli/message/test_message_reply_via_file.rs
index 5c27d4d0..62e78fba 100644
--- a/integration/tests/cli/message/test_message_reply_via_file.rs
+++ b/integration/tests/cli/message/test_message_reply_via_file.rs
@@ -1,8 +1,7 @@
use crate::cli::common::IggyCmdTest;
use crate::cli::message::test_message_poll_to_file_command::TestMessagePollToFileCmd;
use crate::cli::message::test_message_send_from_file_command::TestMessageSendFromFileCmd;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::models::header::{HeaderKey, HeaderValue};
+use iggy::prelude::*;
use serial_test::parallel;
use std::collections::HashMap;
use std::str::FromStr;
diff --git a/integration/tests/cli/message/test_message_send_command.rs b/integration/tests/cli/message/test_message_send_command.rs
index 69694225..b686d449 100644
--- a/integration/tests/cli/message/test_message_send_command.rs
+++ b/integration/tests/cli/message/test_message_send_command.rs
@@ -4,12 +4,7 @@ use crate::cli::common::{
};
use assert_cmd::assert::Assert;
use async_trait::async_trait;
-use iggy::client::Client;
-use iggy::consumer::Consumer;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::models::header::{HeaderKey, HeaderValue};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use predicates::str::diff;
use serial_test::parallel;
use std::collections::HashMap;
@@ -230,8 +225,8 @@ impl IggyCmdTestCase for TestMessageSendCmd {
if let Some(expected_header) = &self.header {
polled_messages.messages.iter().for_each(|m| {
- assert!(m.headers.is_some());
- assert_eq!(expected_header, m.headers.as_ref().unwrap());
+ assert!(m.user_headers.is_some());
+ assert_eq!(expected_header, &m.user_headers_map().unwrap().unwrap());
})
};
@@ -417,7 +412,7 @@ Options:
-H, --headers <HEADERS>
Comma separated list of key:kind:value, sent as header with the message
{CLAP_INDENT}
- Headers are comma seperated key-value pairs that can be sent with the message.
+ Headers are comma separated key-value pairs that can be sent with the message.
Kind can be one of the following: raw, string, bool, int8, int16, int32, int64,
int128, uint8, uint16, uint32, uint64, uint128, float32, float64
diff --git a/integration/tests/cli/message/test_message_send_from_file_command.rs b/integration/tests/cli/message/test_message_send_from_file_command.rs
index 739b94b4..e58b8067 100644
--- a/integration/tests/cli/message/test_message_send_from_file_command.rs
+++ b/integration/tests/cli/message/test_message_send_from_file_command.rs
@@ -2,15 +2,7 @@ use crate::cli::common::{IggyCmdCommand, IggyCmdTest, IggyCmdTestCase};
use assert_cmd::assert::Assert;
use async_trait::async_trait;
use bytes::Bytes;
-use iggy::bytes_serializable::BytesSerializable;
-use iggy::client::Client;
-use iggy::consumer::Consumer;
-use iggy::identifier::Identifier;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::messages::send_messages::Message;
-use iggy::models::header::{HeaderKey, HeaderValue};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use predicates::str::{ends_with, is_match, starts_with};
use serial_test::parallel;
use std::collections::HashMap;
@@ -109,15 +101,15 @@ impl IggyCmdTestCase for TestMessageSendFromFileCmd<'_> {
.iter()
.map(|s| {
let payload = Bytes::from(s.as_bytes().to_vec());
- Message::new(None, payload, Some(self.headers.clone()))
+ IggyMessage::with_headers(payload, self.headers.clone())
})
.collect::<Vec<_>>();
for message in messages.iter() {
- let message = Message::new(
- Some(message.id),
+ let message = IggyMessage::with_id_and_headers(
+ message.header.id,
message.payload.clone(),
- message.headers.clone(),
+ message.user_headers_map().unwrap().unwrap(),
);
let write_result = file.write_all(&message.to_bytes()).await;
@@ -196,8 +188,11 @@ impl IggyCmdTestCase for TestMessageSendFromFileCmd<'_> {
message.payload,
Bytes::from(self.messages[i].as_bytes().to_vec())
);
- assert_eq!(message.headers.is_some(), !self.headers.is_empty());
- assert_eq!(message.headers.as_ref().unwrap(), &self.headers);
+ assert_eq!(
+ message.user_headers_map().unwrap().is_some(),
+ !self.headers.is_empty()
+ );
+ assert_eq!(&message.user_headers_map().unwrap().unwrap(), &self.headers);
}
let topic_delete = client.delete_topic(&stream_id, &topic_id).await;
diff --git a/integration/tests/cli/personal_access_token/mod.rs b/integration/tests/cli/personal_access_token/mod.rs
index 2ab7449c..22074ea0 100644
--- a/integration/tests/cli/personal_access_token/mod.rs
+++ b/integration/tests/cli/personal_access_token/mod.rs
@@ -2,6 +2,7 @@ mod test_pat_create_command;
mod test_pat_delete_command;
mod test_pat_help_command;
mod test_pat_list_command;
-// Disable tests due to missing keyring on macOS until #794 is implemented
-#[cfg(not(target_os = "macos"))]
+// Disable tests due to missing keyring on macOS until #794 is implemented and skip for musl targets
+// due to missing keyring support while running tests under cross
+#[cfg(not(any(target_os = "macos", target_env = "musl")))]
mod test_pat_login_options;
diff --git a/integration/tests/cli/stream/test_stream_purge_command.rs b/integration/tests/cli/stream/test_stream_purge_command.rs
index bceaf7b2..f8f3cbaf 100644
--- a/integration/tests/cli/stream/test_stream_purge_command.rs
+++ b/integration/tests/cli/stream/test_stream_purge_command.rs
@@ -4,10 +4,7 @@ use crate::cli::common::{
};
use assert_cmd::assert::Assert;
use async_trait::async_trait;
-use iggy::client::Client;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use predicates::str::diff;
use serial_test::parallel;
use std::str::FromStr;
@@ -63,7 +60,7 @@ impl IggyCmdTestCase for TestStreamPurgeCmd {
let mut messages = (1..100)
.map(|n| format!("message {}", n))
- .filter_map(|s| Message::from_str(s.as_str()).ok())
+ .filter_map(|s| IggyMessage::from_str(s.as_str()).ok())
.collect::<Vec<_>>();
let send_status = client
diff --git a/integration/tests/cli/system/mod.rs b/integration/tests/cli/system/mod.rs
index 51969c73..d3b0e96e 100644
--- a/integration/tests/cli/system/mod.rs
+++ b/integration/tests/cli/system/mod.rs
@@ -1,13 +1,14 @@
-// Disable tests due to missing keyring on macOS until #794 is implemented
-#[cfg(not(target_os = "macos"))]
+// Disable tests due to missing keyring on macOS until #794 is implemented and skip for musl targets
+// due to missing keyring support while running tests under cross
+#[cfg(not(any(target_os = "macos", target_env = "musl")))]
mod test_cli_session_scenario;
-#[cfg(not(target_os = "macos"))]
+#[cfg(not(any(target_os = "macos", target_env = "musl")))]
mod test_login_cmd;
mod test_login_command;
-#[cfg(not(target_os = "macos"))]
+#[cfg(not(any(target_os = "macos", target_env = "musl")))]
mod test_logout_cmd;
mod test_logout_command;
-#[cfg(not(target_os = "macos"))]
+#[cfg(not(any(target_os = "macos", target_env = "musl")))]
mod test_me_command;
mod test_ping_command;
mod test_snapshot_cmd;
diff --git a/integration/tests/cli/topic/test_topic_purge_command.rs b/integration/tests/cli/topic/test_topic_purge_command.rs
index 66eb5f80..f518ede4 100644
--- a/integration/tests/cli/topic/test_topic_purge_command.rs
+++ b/integration/tests/cli/topic/test_topic_purge_command.rs
@@ -4,10 +4,7 @@ use crate::cli::common::{
};
use assert_cmd::assert::Assert;
use async_trait::async_trait;
-use iggy::client::Client;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use predicates::str::diff;
use serial_test::parallel;
use std::str::FromStr;
@@ -79,7 +76,7 @@ impl IggyCmdTestCase for TestTopicPurgeCmd {
let mut messages = (1..100)
.map(|n| format!("message {}", n))
- .filter_map(|s| Message::from_str(s.as_str()).ok())
+ .filter_map(|s| IggyMessage::from_str(s.as_str()).ok())
.collect::<Vec<_>>();
let send_status = client
diff --git a/integration/tests/examples/mod.rs b/integration/tests/examples/mod.rs
index 2ac6d49d..7d2aa010 100644
--- a/integration/tests/examples/mod.rs
+++ b/integration/tests/examples/mod.rs
@@ -100,7 +100,7 @@ impl<'a> IggyExampleTest<'a> {
pub(crate) async fn setup(&mut self, existing_stream_and_topic: bool) {
self.client.connect().await.unwrap();
let ping_result = self.client.ping().await;
- assert!(ping_result.is_ok());
+ assert!(ping_result.is_ok(), "Failed to ping server");
self.client
.login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
.await
@@ -167,6 +167,11 @@ impl IggyExampleTest<'_> {
.as_ref()
.to_string()
});
+
+ let producer_stdout = producer_handle
+ .await
+ .expect("Failed to join producer, check producer logs");
+
let consumer_handle = tokio::spawn(async move {
let consumer_assert = consumer_cmd
.timeout(Duration::from_secs(10))
@@ -177,9 +182,11 @@ impl IggyExampleTest<'_> {
.as_ref()
.to_string()
});
- (
- producer_handle.await.unwrap(),
- consumer_handle.await.unwrap(),
- )
+
+ let consumer_stdout = consumer_handle
+ .await
+ .expect("Failed to join consumer, check consumer logs");
+
+ (producer_stdout, consumer_stdout)
}
}
diff --git a/integration/tests/mod.rs b/integration/tests/mod.rs
index dd153591..46821879 100644
--- a/integration/tests/mod.rs
+++ b/integration/tests/mod.rs
@@ -1,6 +1,6 @@
use ctor::{ctor, dtor};
use lazy_static::lazy_static;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
use std::io::Write;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::RwLock;
@@ -21,7 +21,8 @@ lazy_static! {
static ref TESTS_FAILED: AtomicBool = AtomicBool::new(false);
static ref LOGS_BUFFER: Arc<RwLock<HashMap<String, Vec<u8>>>> =
Arc::new(RwLock::new(HashMap::new()));
- static ref FAILED_TEST_CASES: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
+ static ref FAILED_TEST_CASES: Arc<RwLock<HashSet<String>>> =
+ Arc::new(RwLock::new(HashSet::new()));
}
static INIT: Once = Once::new();
@@ -32,7 +33,7 @@ fn setup() {
let mut logger = env_logger::builder();
logger.is_test(true);
- logger.filter(None, log::LevelFilter::Debug);
+ logger.filter(None, log::LevelFilter::Trace); // todo
logger.target(env_logger::Target::Pipe(Box::new(LogWriter(log_buffer))));
logger.format(move |buf, record| {
let timestamp = chrono::Local::now().format("%Y-%m-%d %H:%M:%S%.6f");
@@ -49,7 +50,10 @@ fn setup() {
let thread = thread::current();
let thread_name = thread.name().unwrap_or(UNKNOWN_TEST_NAME);
let failed_tests = FAILED_TEST_CASES.clone();
- failed_tests.write().unwrap().push(thread_name.to_string());
+ failed_tests
+ .write()
+ .unwrap()
+ .insert(thread_name.to_string());
// If a test panics, set the failure flag to true
TESTS_FAILED.store(true, Ordering::SeqCst);
diff --git a/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs b/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs
index e0bc1580..4081216f 100644
--- a/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs
+++ b/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs
@@ -3,16 +3,7 @@ use crate::server::scenarios::{
CONSUMER_GROUP_NAME, MESSAGES_COUNT, PARTITIONS_COUNT, STREAM_ID, STREAM_NAME, TOPIC_ID,
TOPIC_NAME,
};
-use iggy::client::{ConsumerGroupClient, MessageClient, StreamClient, SystemClient, TopicClient};
-use iggy::clients::client::IggyClient;
-use iggy::compression::compression_algorithm::CompressionAlgorithm;
-use iggy::consumer::Consumer;
-use iggy::identifier::Identifier;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::models::consumer_group::ConsumerGroupDetails;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use integration::test_server::{
assert_clean_system, create_user, login_root, login_user, ClientFactory,
};
@@ -104,7 +95,7 @@ async fn execute_using_messages_key_key(
) {
// 1. Send messages to the calculated partition ID on the server side by using entity ID as a key
for entity_id in 1..=MESSAGES_COUNT {
- let message = Message::from_str(&create_message_payload(entity_id)).unwrap();
+ let message = IggyMessage::from_str(&create_message_payload(entity_id)).unwrap();
let mut messages = vec![message];
system_client
.send_messages(
@@ -167,7 +158,8 @@ async fn execute_using_none_key(
}
let message =
- Message::from_str(&create_extended_message_payload(partition_id, entity_id)).unwrap();
+ IggyMessage::from_str(&create_extended_message_payload(partition_id, entity_id))
+ .unwrap();
let mut messages = vec![message];
system_client
.send_messages(
@@ -221,7 +213,7 @@ async fn validate_message_polling(client: &IggyClient, consumer_group: &Consumer
assert_eq!(polled_messages.messages.len(), 1);
let message = &polled_messages.messages[0];
let offset = (i - 1) as u64;
- assert_eq!(message.offset, offset);
+ assert_eq!(message.header.offset, offset);
let entity_id = start_entity_id + ((i - 1) * PARTITIONS_COUNT);
let payload = from_utf8(&message.payload).unwrap();
assert_eq!(
diff --git a/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs b/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs
index e6f08758..19878f4b 100644
--- a/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs
+++ b/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs
@@ -3,15 +3,7 @@ use crate::server::scenarios::{
CONSUMER_GROUP_NAME, MESSAGES_COUNT, PARTITIONS_COUNT, STREAM_ID, STREAM_NAME, TOPIC_ID,
TOPIC_NAME,
};
-use iggy::client::{ConsumerGroupClient, MessageClient, StreamClient, SystemClient, TopicClient};
-use iggy::clients::client::IggyClient;
-use iggy::compression::compression_algorithm::CompressionAlgorithm;
-use iggy::consumer::Consumer;
-use iggy::identifier::Identifier;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use integration::test_server::{assert_clean_system, login_root, ClientFactory};
use std::str::{from_utf8, FromStr};
@@ -83,7 +75,7 @@ async fn init_system(client: &IggyClient) {
async fn execute_using_messages_key_key(client: &IggyClient) {
// 1. Send messages to the calculated partition ID on the server side by using entity ID as a key
for entity_id in 1..=MESSAGES_COUNT {
- let message = Message::from_str(&create_message_payload(entity_id)).unwrap();
+ let message = IggyMessage::from_str(&create_message_payload(entity_id)).unwrap();
let mut messages = vec![message];
client
.send_messages(
@@ -132,7 +124,8 @@ async fn execute_using_none_key(client: &IggyClient) {
}
let message =
- Message::from_str(&create_extended_message_payload(partition_id, entity_id)).unwrap();
+ IggyMessage::from_str(&create_extended_message_payload(partition_id, entity_id))
+ .unwrap();
let mut messages = vec![message];
client
.send_messages(
@@ -150,7 +143,7 @@ async fn execute_using_none_key(client: &IggyClient) {
let mut partition_id = 1;
let mut offset = 0;
let mut entity_id = 1;
- for _ in 1..=PARTITIONS_COUNT * MESSAGES_COUNT {
+ for i in 1..=PARTITIONS_COUNT * MESSAGES_COUNT {
let polled_messages = client
.poll_messages(
&Identifier::numeric(STREAM_ID).unwrap(),
@@ -164,9 +157,13 @@ async fn execute_using_none_key(client: &IggyClient) {
.await
.unwrap();
- assert_eq!(polled_messages.messages.len(), 1);
+ assert_eq!(
+ polled_messages.messages.len(),
+ 1,
+ "polled messages count is not 1 at iteration {i}"
+ );
let message = &polled_messages.messages[0];
- assert_eq!(message.offset, offset);
+ assert_eq!(message.header.offset, offset);
let payload = from_utf8(&message.payload).unwrap();
assert_eq!(
payload,
diff --git a/integration/tests/server/scenarios/create_message_payload.rs b/integration/tests/server/scenarios/create_message_payload.rs
index bce6c92c..652fdd0d 100644
--- a/integration/tests/server/scenarios/create_message_payload.rs
+++ b/integration/tests/server/scenarios/create_message_payload.rs
@@ -1,12 +1,5 @@
use bytes::Bytes;
-use iggy::client::{MessageClient, StreamClient, TopicClient};
-use iggy::clients::client::IggyClient;
-use iggy::consumer::Consumer;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::models::header::{HeaderKey, HeaderValue};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use integration::test_server::{assert_clean_system, login_root, ClientFactory};
use std::collections::HashMap;
use std::str::FromStr;
@@ -32,12 +25,8 @@ pub async fn run(client_factory: &dyn ClientFactory) {
let id = (offset + 1) as u128;
let payload = create_message_payload(offset as u64);
let headers = create_message_headers();
- messages.push(Message {
- id,
- length: payload.len() as u32,
- payload,
- headers: Some(headers),
- });
+ let message = IggyMessage::with_id_and_headers(id, payload, headers);
+ messages.push(message);
}
client
@@ -67,8 +56,8 @@ pub async fn run(client_factory: &dyn ClientFactory) {
assert_eq!(polled_messages.messages.len() as u32, MESSAGES_COUNT);
for i in 0..MESSAGES_COUNT {
let message = polled_messages.messages.get(i as usize).unwrap();
- assert!(message.headers.is_some());
- let headers = message.headers.as_ref().unwrap();
+ assert!(message.user_headers.is_some());
+ let headers = message.user_headers_map().unwrap().unwrap();
assert_eq!(headers.len(), 3);
assert_eq!(
headers
diff --git a/integration/tests/server/scenarios/message_headers_scenario.rs b/integration/tests/server/scenarios/message_headers_scenario.rs
index 3718394c..12c72379 100644
--- a/integration/tests/server/scenarios/message_headers_scenario.rs
+++ b/integration/tests/server/scenarios/message_headers_scenario.rs
@@ -3,16 +3,7 @@ use crate::server::scenarios::{
TOPIC_ID, TOPIC_NAME,
};
use bytes::Bytes;
-use iggy::client::{MessageClient, StreamClient, TopicClient};
-use iggy::clients::client::IggyClient;
-use iggy::compression::compression_algorithm::CompressionAlgorithm;
-use iggy::consumer::Consumer;
-use iggy::identifier::Identifier;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::models::header::{HeaderKey, HeaderValue};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use integration::test_server::{assert_clean_system, login_root, ClientFactory};
use std::collections::HashMap;
use std::str::FromStr;
@@ -28,12 +19,7 @@ pub async fn run(client_factory: &dyn ClientFactory) {
let id = (offset + 1) as u128;
let payload = create_message_payload(offset as u64);
let headers = create_message_headers();
- messages.push(Message {
- id,
- length: payload.len() as u32,
- payload,
- headers: Some(headers),
- });
+ messages.push(IggyMessage::with_id_and_headers(id, payload, headers));
}
client
@@ -64,8 +50,8 @@ pub async fn run(client_factory: &dyn ClientFactory) {
assert_eq!(polled_messages.messages.len() as u32, MESSAGES_COUNT);
for i in 0..MESSAGES_COUNT {
let message = polled_messages.messages.get(i as usize).unwrap();
- assert!(message.headers.is_some());
- let headers = message.headers.as_ref().unwrap();
+ assert!(message.user_headers.is_some());
+ let headers = message.user_headers_map().unwrap().unwrap();
assert_eq!(headers.len(), 3);
assert_eq!(
headers
diff --git a/integration/tests/server/scenarios/message_size_scenario.rs b/integration/tests/server/scenarios/message_size_scenario.rs
index 4612f44d..c0d9f909 100644
--- a/integration/tests/server/scenarios/message_size_scenario.rs
+++ b/integration/tests/server/scenarios/message_size_scenario.rs
@@ -1,13 +1,5 @@
use bytes::Bytes;
-use iggy::client::{MessageClient, StreamClient, TopicClient};
-use iggy::clients::client::IggyClient;
-use iggy::consumer::Consumer;
-use iggy::error::IggyError;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::models::header::{HeaderKey, HeaderValue};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use integration::test_server::{assert_clean_system, login_root, ClientFactory};
use std::collections::HashMap;
use std::str::FromStr;
@@ -18,7 +10,7 @@ const STREAM_NAME: &str = "test-stream";
const TOPIC_NAME: &str = "test-topic";
const PARTITIONS_COUNT: u32 = 3;
const PARTITION_ID: u32 = 1;
-const MAX_HEADER_SIZE: usize = 255;
+const MAX_SINGLE_HEADER_SIZE: usize = 200;
enum MessageToSend {
NoMessage,
@@ -172,29 +164,41 @@ fn create_string_of_size(size: usize) -> String {
"x".repeat(size)
}
-fn create_message_header_of_size(size: usize) -> HashMap<HeaderKey, HeaderValue> {
+fn create_message_header_of_size(target_size: usize) -> HashMap<HeaderKey, HeaderValue> {
let mut headers = HashMap::new();
let mut header_id = 1;
- let mut size = size;
+ let mut current_size = 0;
- while size > 0 {
- let header_size = if size > MAX_HEADER_SIZE {
- MAX_HEADER_SIZE
+ while current_size < target_size {
+ let remaining_size = target_size - current_size;
+
+ let key_str = format!("header-{header_id}");
+ let key_overhead = 4; // 4 bytes for key length
+ let value_overhead = 5; // 1 byte for type + 4 bytes for value length
+ let total_overhead = key_overhead + key_str.len() + value_overhead;
+
+ let value_size = if remaining_size <= total_overhead {
+ break;
+ } else if remaining_size - total_overhead > MAX_SINGLE_HEADER_SIZE {
+ MAX_SINGLE_HEADER_SIZE
} else {
- size
+ remaining_size - total_overhead
};
- headers.insert(
- HeaderKey::new(format!("header-{header_id}").as_str()).unwrap(),
- HeaderValue::from_str(create_string_of_size(header_size).as_str()).unwrap(),
- );
+
+ let key = HeaderKey::new(key_str.as_str()).unwrap();
+ let value = HeaderValue::from_str(create_string_of_size(value_size).as_str()).unwrap();
+
+ let actual_header_size = 4 + key_str.len() + 1 + 4 + value_size;
+ current_size += actual_header_size;
+
+ headers.insert(key, value);
header_id += 1;
- size -= header_size;
}
headers
}
-fn create_message(header_size: Option<usize>, payload_size: usize) -> Message {
+fn create_message(header_size: Option<usize>, payload_size: usize) -> IggyMessage {
let headers = match header_size {
Some(header_size) => {
if header_size > 0 {
@@ -207,10 +211,10 @@ fn create_message(header_size: Option<usize>, payload_size: usize) -> Message {
};
let payload = create_string_of_size(payload_size);
- Message {
- id: 1u128,
- length: payload.len() as u32,
- payload: Bytes::from(payload),
- headers,
+
+ if let Some(headers) = headers {
+ IggyMessage::with_id_and_headers(1u128, Bytes::from(payload), headers)
+ } else {
+ IggyMessage::with_id(1u128, Bytes::from(payload))
}
}
diff --git a/integration/tests/server/scenarios/mod.rs b/integration/tests/server/scenarios/mod.rs
index d37a81d7..eafe2c46 100644
--- a/integration/tests/server/scenarios/mod.rs
+++ b/integration/tests/server/scenarios/mod.rs
@@ -28,7 +28,7 @@ const USERNAME_2: &str = "user2";
const USERNAME_3: &str = "user3";
const CONSUMER_ID: u32 = 1;
const CONSUMER_KIND: ConsumerKind = ConsumerKind::Consumer;
-const MESSAGES_COUNT: u32 = 1000;
+const MESSAGES_COUNT: u32 = 1337;
async fn create_client(client_factory: &dyn ClientFactory) -> IggyClient {
let client = client_factory.create_client().await;
diff --git a/integration/tests/server/scenarios/stream_size_validation_scenario.rs b/integration/tests/server/scenarios/stream_size_validation_scenario.rs
index a2d489bc..27a62a61 100644
--- a/integration/tests/server/scenarios/stream_size_validation_scenario.rs
+++ b/integration/tests/server/scenarios/stream_size_validation_scenario.rs
@@ -1,12 +1,6 @@
use crate::server::scenarios::{create_client, PARTITIONS_COUNT, PARTITION_ID};
use bytes::Bytes;
-use iggy::client::{MessageClient, StreamClient, SystemClient, TopicClient};
-use iggy::clients::client::IggyClient;
-use iggy::compression::compression_algorithm::CompressionAlgorithm;
-use iggy::identifier::Identifier;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use integration::test_server::{assert_clean_system, login_root, ClientFactory};
use std::str::FromStr;
use tracing_subscriber::layer::SubscriberExt;
@@ -18,7 +12,7 @@ const T1_NAME: &str = "test-topic-1";
const S2_NAME: &str = "test-stream-2";
const T2_NAME: &str = "test-topic-2";
const MESSAGE_PAYLOAD_SIZE_BYTES: u64 = 57;
-const MSG_SIZE: u64 = 16 + 8 + 8 + 4 + 4 + 4 + 1 + MESSAGE_PAYLOAD_SIZE_BYTES; // number of bytes in a single message
+const MSG_SIZE: u64 = IGGY_MESSAGE_HEADER_SIZE as u64 + MESSAGE_PAYLOAD_SIZE_BYTES; // number of bytes in a single message
const MSGS_COUNT: u64 = 117; // number of messages in a single topic after one pass of appending
const MSGS_SIZE: u64 = MSG_SIZE * MSGS_COUNT; // number of bytes in a single topic after one pass of appending
@@ -269,18 +263,13 @@ async fn purge_stream(client: &IggyClient, stream_name: &str) {
.unwrap();
}
-fn create_messages() -> Vec<Message> {
+fn create_messages() -> Vec<IggyMessage> {
let mut messages = Vec::new();
for offset in 0..MSGS_COUNT {
let id = (offset + 1) as u128;
let payload = Bytes::from(vec![0xD; MESSAGE_PAYLOAD_SIZE_BYTES as usize]);
- let message = Message {
- id,
- length: payload.len() as u32,
- payload,
- headers: None,
- };
+ let message = IggyMessage::with_id(id, payload);
messages.push(message);
}
messages
diff --git a/integration/tests/server/scenarios/system_scenario.rs b/integration/tests/server/scenarios/system_scenario.rs
index ca0ad651..e027d0e0 100644
--- a/integration/tests/server/scenarios/system_scenario.rs
+++ b/integration/tests/server/scenarios/system_scenario.rs
@@ -1,28 +1,12 @@
-use bytes::Bytes;
-use std::str::FromStr;
-
use crate::server::scenarios::{
get_consumer_group, leave_consumer_group, CONSUMER_GROUP_ID, CONSUMER_GROUP_NAME, CONSUMER_ID,
CONSUMER_KIND, MESSAGES_COUNT, PARTITIONS_COUNT, PARTITION_ID, STREAM_ID, STREAM_NAME,
TOPIC_ID, TOPIC_NAME,
};
-use iggy::client::{
- ConsumerGroupClient, ConsumerOffsetClient, MessageClient, PartitionClient, StreamClient,
- SystemClient, TopicClient, UserClient,
-};
-use iggy::clients::client::IggyClient;
-use iggy::compression::compression_algorithm::CompressionAlgorithm;
-use iggy::consumer::Consumer;
-use iggy::error::IggyError;
-use iggy::identifier::Identifier;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::models::messages::PolledMessage;
-use iggy::users::defaults::{DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME};
-use iggy::utils::byte_size::IggyByteSize;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::topic_size::MaxTopicSize;
+use bytes::Bytes;
+use iggy::prelude::*;
use integration::test_server::{assert_clean_system, ClientFactory};
+use std::str::FromStr;
pub async fn run(client_factory: &dyn ClientFactory) {
let client = client_factory.create_client().await;
@@ -289,7 +273,7 @@ pub async fn run(client_factory: &dyn ClientFactory) {
assert_eq!(topic.name, TOPIC_NAME);
assert_eq!(topic.partitions_count, PARTITIONS_COUNT);
assert_eq!(topic.partitions.len(), PARTITIONS_COUNT as usize);
- assert_eq!(topic.size, 55914);
+ assert_eq!(topic.size, 89806);
assert_eq!(topic.messages_count, MESSAGES_COUNT as u64);
let topic_partition = topic.partitions.get((PARTITION_ID - 1) as usize).unwrap();
assert_eq!(topic_partition.id, PARTITION_ID);
@@ -403,8 +387,8 @@ pub async fn run(client_factory: &dyn ClientFactory) {
.await
.unwrap();
assert_eq!(polled_messages.messages.len() as u32, messages_count);
- let first_offset = polled_messages.messages.first().unwrap().offset;
- let last_offset = polled_messages.messages.last().unwrap().offset;
+ let first_offset = polled_messages.messages.first().unwrap().header.offset;
+ let last_offset = polled_messages.messages.last().unwrap().header.offset;
let expected_last_offset = stored_offset + messages_count as u64;
assert_eq!(first_offset, stored_offset + 1);
assert_eq!(last_offset, expected_last_offset);
@@ -785,24 +769,19 @@ pub async fn run(client_factory: &dyn ClientFactory) {
assert_clean_system(&client).await;
}
-fn assert_message(message: &PolledMessage, offset: u64) {
+fn assert_message(message: &IggyMessage, offset: u64) {
let expected_payload = create_message_payload(offset);
- assert!(message.timestamp > 0);
- assert_eq!(message.offset, offset);
+ assert!(message.header.timestamp > 0);
+ assert_eq!(message.header.offset, offset);
assert_eq!(message.payload, expected_payload);
}
-fn create_messages() -> Vec<Message> {
+fn create_messages() -> Vec<IggyMessage> {
let mut messages = Vec::new();
for offset in 0..MESSAGES_COUNT {
let id = (offset + 1) as u128;
let payload = create_message_payload(offset as u64);
- messages.push(Message {
- id,
- length: payload.len() as u32,
- payload,
- headers: None,
- });
+ messages.push(IggyMessage::with_id(id, payload));
}
messages
}
diff --git a/integration/tests/state/file.rs b/integration/tests/state/file.rs
index 39196f9f..a87e915c 100644
--- a/integration/tests/state/file.rs
+++ b/integration/tests/state/file.rs
@@ -31,7 +31,7 @@ async fn should_apply_single_entry() {
});
let command_bytes = command.to_bytes();
- state.apply(user_id, command).await.unwrap();
+ state.apply(user_id, &command).await.unwrap();
let mut entries = state.load_entries().await.unwrap();
assert_eq!(entries.len(), 1);
@@ -54,7 +54,7 @@ async fn should_apply_encrypted_entry() {
});
let command_bytes = command.to_bytes();
- state.apply(user_id, command).await.unwrap();
+ state.apply(user_id, &command).await.unwrap();
let mut entries = state.load_entries().await.unwrap();
assert_eq!(entries.len(), 1);
@@ -82,7 +82,7 @@ async fn should_apply_multiple_entries() {
});
let create_user_bytes = create_user.to_bytes();
- state.apply(first_user_id, create_user).await.unwrap();
+ state.apply(first_user_id, &create_user).await.unwrap();
assert_eq!(state.current_index(), 0);
assert_eq!(state.entries_count(), 1);
@@ -94,7 +94,7 @@ async fn should_apply_multiple_entries() {
});
let create_stream_bytes = create_stream.to_bytes();
- state.apply(second_user_id, create_stream).await.unwrap();
+ state.apply(second_user_id, &create_stream).await.unwrap();
assert_eq!(state.current_index(), 1);
assert_eq!(state.entries_count(), 2);
diff --git a/integration/tests/state/mod.rs b/integration/tests/state/mod.rs
index 94ddea2e..fe626c6a 100644
--- a/integration/tests/state/mod.rs
+++ b/integration/tests/state/mod.rs
@@ -27,7 +27,7 @@ impl StateSetup {
pub async fn create(encryption_key: Option<&[u8]>) -> StateSetup {
let directory_path = format!("state_{}", Uuid::now_v7().to_u128_le());
- let log_path = format!("{}/log", directory_path);
+ let messages_file_path = format!("{}/log", directory_path);
create_dir(&directory_path).await.unwrap();
let version = SemanticVersion::from_str("1.2.3").unwrap();
@@ -37,7 +37,12 @@ impl StateSetup {
Aes256GcmEncryptor::new(key).unwrap(),
))
});
- let state = FileState::new(&log_path, &version, Arc::new(persister), encryptor);
+ let state = FileState::new(
+ &messages_file_path,
+ &version,
+ Arc::new(persister),
+ encryptor,
+ );
Self {
directory_path,
diff --git a/integration/tests/state/system.rs b/integration/tests/state/system.rs
index 5c304ce8..7c859bda 100644
--- a/integration/tests/state/system.rs
+++ b/integration/tests/state/system.rs
@@ -105,37 +105,37 @@ async fn should_be_initialized_based_on_state_entries() {
hash: "hash".to_string(),
};
state
- .apply(user_id, EntryCommand::CreateUser(create_user))
+ .apply(user_id, &EntryCommand::CreateUser(create_user))
.await
.unwrap();
state
- .apply(user_id, EntryCommand::CreateStream(create_stream1))
+ .apply(user_id, &EntryCommand::CreateStream(create_stream1))
.await
.unwrap();
state
- .apply(user_id, EntryCommand::CreateTopic(create_topic1))
+ .apply(user_id, &EntryCommand::CreateTopic(create_topic1))
.await
.unwrap();
state
- .apply(user_id, EntryCommand::CreateStream(create_stream2))
+ .apply(user_id, &EntryCommand::CreateStream(create_stream2))
.await
.unwrap();
state
- .apply(user_id, EntryCommand::CreateTopic(create_topic2))
+ .apply(user_id, &EntryCommand::CreateTopic(create_topic2))
.await
.unwrap();
state
- .apply(user_id, EntryCommand::CreatePartitions(create_partitions))
+ .apply(user_id, &EntryCommand::CreatePartitions(create_partitions))
.await
.unwrap();
state
- .apply(user_id, EntryCommand::DeleteStream(delete_stream2))
+ .apply(user_id, &EntryCommand::DeleteStream(delete_stream2))
.await
.unwrap();
state
.apply(
user_id,
- EntryCommand::CreatePersonalAccessToken(create_personal_access_token),
+ &EntryCommand::CreatePersonalAccessToken(create_personal_access_token),
)
.await
.unwrap();
diff --git a/integration/tests/streaming/get_by_offset.rs b/integration/tests/streaming/get_by_offset.rs
index 7bafca7b..856c504e 100644
--- a/integration/tests/streaming/get_by_offset.rs
+++ b/integration/tests/streaming/get_by_offset.rs
@@ -1,16 +1,10 @@
use crate::streaming::common::test_setup::TestSetup;
use bytes::BytesMut;
-use iggy::bytes_serializable::BytesSerializable;
-use iggy::messages::send_messages::Message;
-use iggy::models::header::{HeaderKey, HeaderValue};
-use iggy::utils::byte_size::IggyByteSize;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::sizeable::Sizeable;
-use iggy::utils::timestamp::IggyTimestamp;
-use server::configs::resource_quota::MemoryResourceQuota;
-use server::configs::system::{CacheConfig, PartitionConfig, SegmentConfig, SystemConfig};
-use server::streaming::batching::appendable_batch_info::AppendableBatchInfo;
+use iggy::prelude::*;
+use server::configs::cache_indexes::CacheIndexesConfig;
+use server::configs::system::{PartitionConfig, SegmentConfig, SystemConfig};
use server::streaming::partitions::partition::Partition;
+use server::streaming::segments::IggyMessagesBatchMut;
use std::collections::HashMap;
use std::str::FromStr;
use std::sync::atomic::{AtomicU32, AtomicU64};
@@ -22,53 +16,66 @@ use test_case::test_matrix;
*/
fn msg_size(size: u64) -> IggyByteSize {
- IggyByteSize::from_str(&format!("{}b", size)).unwrap()
+ IggyByteSize::from_str(&format!("{}B", size)).unwrap()
}
fn segment_size(size: u64) -> IggyByteSize {
- IggyByteSize::from_str(&format!("{}b", size)).unwrap()
+ IggyByteSize::from_str(&format!("{}B", size)).unwrap()
}
fn msgs_req_to_save(count: u32) -> u32 {
count
}
-fn msg_cache_size(size: u64) -> Option<IggyByteSize> {
- if size == 0 {
- return None;
- }
- Some(IggyByteSize::from_str(&format!("{}b", size)).unwrap())
+fn index_cache_all() -> CacheIndexesConfig {
+ CacheIndexesConfig::All
+}
+
+fn index_cache_none() -> CacheIndexesConfig {
+ CacheIndexesConfig::None
+}
+
+fn index_cache_open_segment() -> CacheIndexesConfig {
+ CacheIndexesConfig::OpenSegment
+}
+
+fn small_batches() -> Vec<u32> {
+ vec![3, 4, 5, 6, 7]
+}
+
+fn medium_batches() -> Vec<u32> {
+ vec![10, 20, 30, 40]
}
-fn index_cache_enabled() -> bool {
- true
+fn large_batches() -> Vec<u32> {
+ vec![100, 200, 300, 400]
}
-fn index_cache_disabled() -> bool {
- false
+fn very_large_batches() -> Vec<u32> {
+ vec![500, 1000, 1500, 1000]
}
#[test_matrix(
[msg_size(50), msg_size(1000), msg_size(20000)],
- [msgs_req_to_save(10), msgs_req_to_save(24), msgs_req_to_save(1000)],
- [segment_size(500), segment_size(2000), segment_size(100000)],
- [msg_cache_size(0), msg_cache_size(5000), msg_cache_size(50000), msg_cache_size(2000000)],
- [index_cache_disabled(), index_cache_enabled()])]
+ [small_batches(), medium_batches(), large_batches(), very_large_batches()],
+ [msgs_req_to_save(1), msgs_req_to_save(24), msgs_req_to_save(1000), msgs_req_to_save(10000)],
+ [segment_size(10), segment_size(200), segment_size(10000000)],
+ [index_cache_none(), index_cache_all(), index_cache_open_segment()])]
#[tokio::test]
async fn test_get_messages_by_offset(
message_size: IggyByteSize,
+ batch_sizes: Vec<u32>,
messages_required_to_save: u32,
segment_size: IggyByteSize,
- msg_cache_size: Option<IggyByteSize>,
- index_cache_enabled: bool,
+ cache_indexes: CacheIndexesConfig,
) {
println!(
- "Running test with msg_cache_enabled: {}, messages_required_to_save: {}, segment_size: {}, message_size: {}, cache_indexes: {}",
- msg_cache_size.is_some(),
+ "Running test with message_size: {}, batches: {:?}, messages_required_to_save: {}, segment_size: {}, cache_indexes: {}",
+ message_size,
+ batch_sizes,
messages_required_to_save,
segment_size,
- message_size,
- index_cache_enabled
+ cache_indexes
);
let setup = TestSetup::init().await;
@@ -76,26 +83,17 @@ async fn test_get_messages_by_offset(
let topic_id = 1;
let partition_id = 1;
- // Define batch sizes for 5 appends
- let batch_sizes = [3, 4, 5, 6, 7];
- let total_messages: u32 = batch_sizes.iter().sum();
+ let total_messages_count = batch_sizes.iter().sum();
- let msg_cache_enabled = msg_cache_size.is_some();
- let msg_cache_size =
- MemoryResourceQuota::Bytes(msg_cache_size.unwrap_or(IggyByteSize::from_str("0").unwrap()));
let config = Arc::new(SystemConfig {
path: setup.config.path.to_string(),
- cache: CacheConfig {
- enabled: msg_cache_enabled,
- size: msg_cache_size,
- },
partition: PartitionConfig {
messages_required_to_save,
enforce_fsync: true,
..Default::default()
},
segment: SegmentConfig {
- cache_indexes: index_cache_enabled,
+ cache_indexes,
size: segment_size,
..Default::default()
},
@@ -122,8 +120,10 @@ async fn test_get_messages_by_offset(
setup.create_partitions_directory(stream_id, topic_id).await;
partition.persist().await.unwrap();
- let mut all_messages = Vec::with_capacity(total_messages as usize);
- for i in 1..=total_messages {
+ let mut all_messages = Vec::with_capacity(total_messages_count as usize);
+
+ // Generate all messages as defined in the test matrix
+ for i in 1..=total_messages_count {
let id = i as u128;
let beginning_of_payload = format!("message {}", i);
let mut payload = BytesMut::new();
@@ -144,12 +144,13 @@ async fn test_get_messages_by_offset(
HeaderKey::new("key-3").unwrap(),
HeaderValue::from_uint64(123456).unwrap(),
);
- let message = Message {
- id,
- length: payload.len() as u32,
- payload: payload.clone(),
- headers: Some(headers),
- };
+
+ let message = IggyMessage::builder()
+ .id(id)
+ .payload(payload)
+ .headers(headers)
+ .build()
+ .expect("Failed to create message with valid payload and headers");
all_messages.push(message);
}
@@ -157,130 +158,205 @@ async fn test_get_messages_by_offset(
let mut batch_offsets = Vec::with_capacity(batch_sizes.len());
let mut current_pos = 0;
- // Append all batches
- for batch_size in batch_sizes {
- let batch = all_messages[current_pos..current_pos + batch_size as usize].to_vec();
- let batch_info = AppendableBatchInfo::new(
- batch
- .iter()
- .map(|msg| msg.get_size_bytes())
- .sum::<IggyByteSize>(),
- partition.partition_id,
+ // Append all batches as defined in the test matrix
+ for (batch_idx, &batch_len) in batch_sizes.iter().enumerate() {
+ // If we've generated too many messages, skip the rest
+ if current_pos + batch_len as usize > all_messages.len() {
+ break;
+ }
+
+ println!(
+ "Appending batch {}/{} with {} messages",
+ batch_idx + 1,
+ batch_sizes.len(),
+ batch_len
);
- partition
- .append_messages(batch_info, batch.clone(), None)
- .await
- .unwrap();
+
+ let batch_end_pos = current_pos + batch_len as usize;
+ let messages_slice_to_append = &all_messages[current_pos..batch_end_pos];
+
+ let messages_size = messages_slice_to_append
+ .iter()
+ .map(|m| m.get_size_bytes().as_bytes_u64() as u32)
+ .sum();
+
+ let batch = IggyMessagesBatchMut::from_messages(messages_slice_to_append, messages_size);
+ assert_eq!(batch.count(), batch_len);
+ partition.append_messages(batch, None).await.unwrap();
batch_offsets.push(partition.current_offset);
- current_pos += batch_size as usize;
+ current_pos += batch_len as usize;
}
+ // Use the exact total messages count from the test matrix
+ let total_sent_messages = total_messages_count;
+
// Test 1: All messages from start
let all_loaded_messages = partition
- .get_messages_by_offset(0, total_messages)
+ .get_messages_by_offset(0, total_sent_messages)
.await
.unwrap();
assert_eq!(
- all_loaded_messages.len(),
- total_messages as usize,
+ all_loaded_messages.count(),
+ total_sent_messages,
"Expected {} messages from start, but got {}",
- total_messages,
- all_loaded_messages.len()
+ total_sent_messages,
+ all_loaded_messages.count()
);
// Test 2: Get messages from middle (after 3rd batch)
- let middle_offset = batch_offsets[2];
- let remaining_messages = total_messages - (batch_sizes[0] + batch_sizes[1] + batch_sizes[2]);
- let middle_messages = partition
- .get_messages_by_offset(middle_offset + 1, remaining_messages)
- .await
- .unwrap();
- assert_eq!(
- middle_messages.len(),
- remaining_messages as usize,
- "Expected {} messages from middle offset, but got {}",
- remaining_messages,
- middle_messages.len()
- );
+ if batch_offsets.len() >= 3 {
+ let middle_offset = batch_offsets[2];
+ let prior_batches_sum: u32 = batch_sizes[..3].iter().sum();
+ let remaining_messages = total_sent_messages - prior_batches_sum;
+
+ let middle_messages = partition
+ .get_messages_by_offset(middle_offset + 1, remaining_messages)
+ .await
+ .unwrap();
+
+ assert_eq!(
+ middle_messages.count(),
+ remaining_messages,
+ "Expected {} messages from middle offset, but got {}",
+ remaining_messages,
+ middle_messages.count()
+ );
+ }
// Test 3: No messages beyond final offset
- let final_offset = batch_offsets.last().unwrap();
- let no_messages = partition
- .get_messages_by_offset(final_offset + 1, 1)
- .await
- .unwrap();
- assert_eq!(
- no_messages.len(),
- 0,
- "Expected no messages beyond final offset, but got {}",
- no_messages.len()
- );
+ if !batch_offsets.is_empty() {
+ let final_offset = *batch_offsets.last().unwrap();
+ let no_messages = partition
+ .get_messages_by_offset(final_offset + 1, 1)
+ .await
+ .unwrap();
+ assert_eq!(
+ no_messages.count(),
+ 0,
+ "Expected no messages beyond final offset, but got {}",
+ no_messages.count()
+ );
+ }
// Test 4: Small subset from start
- let subset_size = 3;
+ let subset_size = std::cmp::min(3, total_sent_messages);
let subset_messages = partition
.get_messages_by_offset(0, subset_size)
.await
.unwrap();
assert_eq!(
- subset_messages.len(),
- subset_size as usize,
+ subset_messages.count(),
+ subset_size,
"Expected {} messages in subset from start, but got {}",
subset_size,
- subset_messages.len()
+ subset_messages.count()
);
// Test 5: Messages spanning multiple batches
- let span_offset = batch_offsets[1] + 1; // Start from middle of 2nd batch
- let span_size = 8; // Should span across 2nd, 3rd, and into 4th batch
- let spanning_messages = partition
- .get_messages_by_offset(span_offset, span_size)
- .await
- .unwrap();
- assert_eq!(
- spanning_messages.len(),
- span_size as usize,
- "Expected {} messages spanning multiple batches, but got {}",
- span_size,
- spanning_messages.len()
+ if batch_offsets.len() >= 4 {
+ let span_offset = batch_offsets[1] + 1; // Start from middle of 2nd batch
+ let span_size = 8; // Should span across 2nd, 3rd, and into 4th batch
+ let batches = partition
+ .get_messages_by_offset(span_offset, span_size)
+ .await
+ .unwrap();
+ assert_eq!(
+ batches.count(),
+ span_size,
+ "Expected {} messages spanning multiple batches, but got {}",
+ span_size,
+ batches.count()
+ );
+
+ // Test 6: Validate message content and ordering for all messages
+ let mut i = 0;
+
+ for batch in batches.iter() {
+ for msg in batch.iter() {
+ let expected_offset = span_offset + i as u64;
+ assert!(
+ msg.header().offset() >= expected_offset,
+ "Message offset {} at position {} should be >= expected offset {}",
+ msg.header().offset(),
+ i,
+ expected_offset
+ );
+
+ let original_offset = msg.header().offset() as usize;
+ if original_offset < all_messages.len() {
+ let original_message = &all_messages[original_offset];
+
+ let loaded_id = msg.header().id();
+ let original_id = original_message.header.id;
+ assert_eq!(
+ loaded_id,
+ original_id,
+ "Message ID mismatch at offset {}",
+ msg.header().offset(),
+ );
+
+ let loaded_payload = msg.payload();
+ let original_payload = &original_message.payload;
+ assert_eq!(
+ loaded_payload,
+ original_payload,
+ "Payload mismatch at offset {}",
+ msg.header().offset(),
+ );
+
+ let loaded_headers = msg.user_headers_map().unwrap().unwrap();
+ let original_headers = HashMap::from_bytes(
+ original_message.user_headers.as_ref().unwrap().clone(),
+ )
+ .unwrap();
+ assert_eq!(
+ loaded_headers,
+ original_headers,
+ "Headers mismatch at offset {}",
+ msg.header().offset(),
+ );
+ }
+ i += 1;
+ }
+ }
+ }
+
+ // Add sequential read test for all batch sizes
+ println!(
+ "Verifying sequential reads, expecting {} messages",
+ total_sent_messages
);
- // Test 6: Validate message content and ordering
- for (i, msg) in spanning_messages.iter().enumerate() {
- let expected_offset = span_offset + i as u64;
- assert!(
- msg.offset >= expected_offset,
- "Message offset {} at position {} should be >= expected offset {}",
- msg.offset,
- i,
- expected_offset
- );
+ let chunk_size = 500;
+ let mut verified_count = 0;
+
+ for chunk_start in (0..total_sent_messages).step_by(chunk_size as usize) {
+ let read_size = std::cmp::min(chunk_size, total_sent_messages - chunk_start);
+
+ let chunk = partition
+ .get_messages_by_offset(chunk_start as u64, read_size)
+ .await
+ .unwrap();
- // Verify message contents match original
- let original_idx = msg.offset as usize;
- let original_message = &all_messages[original_idx];
- assert_eq!(
- msg.id, original_message.id,
- "Message ID mismatch at offset {}: expected {}, got {}",
- msg.offset, original_message.id, msg.id
- );
assert_eq!(
- msg.payload, original_message.payload,
- "Payload mismatch at offset {}: expected {:?}, got {:?}",
- msg.offset, original_message.payload, msg.payload
+ chunk.count(),
+ read_size,
+ "Failed to read chunk at offset {} with size {}",
+ chunk_start,
+ read_size
);
- assert_eq!(
- msg.headers
- .as_ref()
- .map(|bytes| HashMap::from_bytes(bytes.clone()).unwrap()),
- original_message.headers,
- "Headers mismatch at offset {}: expected {:?}, got {:?}",
- msg.offset,
- original_message.headers,
- msg.headers
- .as_ref()
- .map(|bytes| HashMap::from_bytes(bytes.clone()).unwrap())
+
+ verified_count += chunk.count();
+
+ println!(
+ "Read chunk at offset {} with size {}, verified count: {}",
+ chunk_start, read_size, verified_count
);
}
+
+ assert_eq!(
+ verified_count, total_sent_messages,
+ "Sequential chunk reads didn't cover all messages"
+ );
}
diff --git a/integration/tests/streaming/get_by_timestamp.rs b/integration/tests/streaming/get_by_timestamp.rs
index 449e8155..12212550 100644
--- a/integration/tests/streaming/get_by_timestamp.rs
+++ b/integration/tests/streaming/get_by_timestamp.rs
@@ -1,20 +1,16 @@
use crate::streaming::common::test_setup::TestSetup;
use bytes::BytesMut;
-use iggy::bytes_serializable::BytesSerializable;
-use iggy::messages::send_messages::Message;
-use iggy::models::header::{HeaderKey, HeaderValue};
-use iggy::utils::byte_size::IggyByteSize;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::sizeable::Sizeable;
-use iggy::utils::timestamp::IggyTimestamp;
-use server::configs::resource_quota::MemoryResourceQuota;
-use server::configs::system::{CacheConfig, PartitionConfig, SegmentConfig, SystemConfig};
-use server::streaming::batching::appendable_batch_info::AppendableBatchInfo;
+use iggy::confirmation::Confirmation;
+use iggy::prelude::*;
+use server::configs::cache_indexes::CacheIndexesConfig;
+use server::configs::system::{PartitionConfig, SegmentConfig, SystemConfig};
use server::streaming::partitions::partition::Partition;
+use server::streaming::segments::IggyMessagesBatchMut;
use std::collections::HashMap;
use std::str::FromStr;
use std::sync::atomic::{AtomicU32, AtomicU64};
use std::sync::Arc;
+use std::thread::sleep;
use test_case::test_matrix;
/*
@@ -22,53 +18,66 @@ use test_case::test_matrix;
*/
fn msg_size(size: u64) -> IggyByteSize {
- IggyByteSize::from_str(&format!("{}b", size)).unwrap()
+ IggyByteSize::from_str(&format!("{}B", size)).unwrap()
}
fn segment_size(size: u64) -> IggyByteSize {
- IggyByteSize::from_str(&format!("{}b", size)).unwrap()
+ IggyByteSize::from_str(&format!("{}B", size)).unwrap()
}
fn msgs_req_to_save(count: u32) -> u32 {
count
}
-fn msg_cache_size(size: u64) -> Option<IggyByteSize> {
- if size == 0 {
- return None;
- }
- Some(IggyByteSize::from_str(&format!("{}b", size)).unwrap())
+fn index_cache_all() -> CacheIndexesConfig {
+ CacheIndexesConfig::All
+}
+
+fn index_cache_none() -> CacheIndexesConfig {
+ CacheIndexesConfig::None
+}
+
+fn index_cache_open_segment() -> CacheIndexesConfig {
+ CacheIndexesConfig::OpenSegment
+}
+
+fn small_batches() -> Vec<u32> {
+ vec![3, 4, 5, 6, 7]
}
-fn index_cache_enabled() -> bool {
- true
+fn medium_batches() -> Vec<u32> {
+ vec![10, 20, 30, 40]
}
-fn index_cache_disabled() -> bool {
- false
+fn large_batches() -> Vec<u32> {
+ vec![100, 200, 300, 400]
+}
+
+fn very_large_batches() -> Vec<u32> {
+ vec![500, 1000, 1500, 1000]
}
#[test_matrix(
[msg_size(50), msg_size(1000), msg_size(20000)],
- [msgs_req_to_save(3), msgs_req_to_save(10), msgs_req_to_save(1000)],
- [segment_size(500), segment_size(2000), segment_size(100000)],
- [msg_cache_size(0), msg_cache_size(5000), msg_cache_size(50000), msg_cache_size(2000000)],
- [index_cache_disabled(), index_cache_enabled()])]
+ [small_batches(), medium_batches(), large_batches(), very_large_batches()],
+ [msgs_req_to_save(1), msgs_req_to_save(24), msgs_req_to_save(1000), msgs_req_to_save(10000)],
+ [segment_size(10), segment_size(200), segment_size(10000000)],
+ [index_cache_none(), index_cache_all(), index_cache_open_segment()])]
#[tokio::test]
async fn test_get_messages_by_timestamp(
message_size: IggyByteSize,
+ batch_sizes: Vec<u32>,
messages_required_to_save: u32,
segment_size: IggyByteSize,
- msg_cache_size: Option<IggyByteSize>,
- index_cache_enabled: bool,
+ cache_indexes: CacheIndexesConfig,
) {
println!(
- "Running test with msg_cache_enabled: {}, messages_required_to_save: {}, segment_size: {}, message_size: {}, cache_indexes: {}",
- msg_cache_size.is_some(),
+ "Running test with message_size: {}, batches: {:?}, messages_required_to_save: {}, segment_size: {}, cache_indexes: {}",
+ message_size,
+ batch_sizes,
messages_required_to_save,
segment_size,
- message_size,
- index_cache_enabled
+ cache_indexes
);
let setup = TestSetup::init().await;
@@ -76,31 +85,23 @@ async fn test_get_messages_by_timestamp(
let topic_id = 1;
let partition_id = 1;
- // Define batch sizes for 5 appends
- let batch_sizes = [3, 4, 5, 6, 7];
- let total_messages: u32 = batch_sizes.iter().sum();
+ let total_messages_count = batch_sizes.iter().sum();
- let msg_cache_enabled = msg_cache_size.is_some();
- let msg_cache_size =
- MemoryResourceQuota::Bytes(msg_cache_size.unwrap_or(IggyByteSize::from_str("0").unwrap()));
let config = Arc::new(SystemConfig {
path: setup.config.path.to_string(),
- cache: CacheConfig {
- enabled: msg_cache_enabled,
- size: msg_cache_size,
- },
partition: PartitionConfig {
messages_required_to_save,
enforce_fsync: true,
..Default::default()
},
segment: SegmentConfig {
- cache_indexes: index_cache_enabled,
+ cache_indexes,
size: segment_size,
..Default::default()
},
..Default::default()
});
+
let mut partition = Partition::create(
stream_id,
topic_id,
@@ -121,8 +122,10 @@ async fn test_get_messages_by_timestamp(
setup.create_partitions_directory(stream_id, topic_id).await;
partition.persist().await.unwrap();
- let mut all_messages = Vec::with_capacity(total_messages as usize);
- for i in 1..=total_messages {
+ let mut all_messages = Vec::with_capacity(total_messages_count as usize);
+
+ // Generate all messages as defined in the test matrix
+ for i in 1..=total_messages_count {
let id = i as u128;
let beginning_of_payload = format!("message {}", i);
let mut payload = BytesMut::new();
@@ -143,149 +146,200 @@ async fn test_get_messages_by_timestamp(
HeaderKey::new("key-3").unwrap(),
HeaderValue::from_uint64(123456).unwrap(),
);
- let message = Message {
- id,
- length: payload.len() as u32,
- payload: payload.clone(),
- headers: Some(headers),
- };
+
+ let message = IggyMessage::builder()
+ .id(id)
+ .payload(payload)
+ .headers(headers)
+ .build()
+ .expect("Failed to create message with valid payload and headers");
+
all_messages.push(message);
}
+ // Timestamp tracking for messages
let initial_timestamp = IggyTimestamp::now();
let mut batch_timestamps = Vec::with_capacity(batch_sizes.len());
let mut current_pos = 0;
- // Append all batches with timestamps
- for batch_size in batch_sizes {
- let batch = all_messages[current_pos..current_pos + batch_size as usize].to_vec();
- let batch_info = AppendableBatchInfo::new(
- batch
- .iter()
- .map(|msg| msg.get_size_bytes())
- .sum::<IggyByteSize>(),
- partition.partition_id,
+ // Append all batches as defined in the test matrix with separate timestamps
+ for (batch_idx, &batch_len) in batch_sizes.iter().enumerate() {
+ // Add a small delay between batches to ensure distinct timestamps
+ sleep(std::time::Duration::from_millis(2));
+
+ // If we've generated too many messages, skip the rest
+ if current_pos + batch_len as usize > all_messages.len() {
+ break;
+ }
+
+ println!(
+ "Appending batch {}/{} with {} messages",
+ batch_idx + 1,
+ batch_sizes.len(),
+ batch_len
);
+
+ let batch_end_pos = current_pos + batch_len as usize;
+ let messages_slice_to_append = &all_messages[current_pos..batch_end_pos];
+
+ let messages_size = messages_slice_to_append
+ .iter()
+ .map(|m| m.get_size_bytes().as_bytes_u64() as u32)
+ .sum();
+
+ let batch = IggyMessagesBatchMut::from_messages(messages_slice_to_append, messages_size);
+ assert_eq!(batch.count(), batch_len);
partition
- .append_messages(batch_info, batch.clone(), None)
+ .append_messages(batch, Some(Confirmation::Wait))
.await
.unwrap();
+ // Capture the timestamp of this batch
batch_timestamps.push(IggyTimestamp::now());
- current_pos += batch_size as usize;
+ current_pos += batch_len as usize;
+
+ // Add a small delay between batches to ensure distinct timestamps
+ sleep(std::time::Duration::from_millis(2));
}
let final_timestamp = IggyTimestamp::now();
+ // Use the exact total messages count from the test matrix
+ let total_sent_messages = total_messages_count;
+
// Test 1: All messages from initial timestamp
let all_loaded_messages = partition
- .get_messages_by_timestamp(initial_timestamp, total_messages)
+ .get_messages_by_timestamp(initial_timestamp, total_sent_messages)
.await
.unwrap();
assert_eq!(
- all_loaded_messages.len(),
- total_messages as usize,
+ all_loaded_messages.count(),
+ total_sent_messages,
"Expected {} messages from initial timestamp, but got {}",
- total_messages,
- all_loaded_messages.len()
+ total_sent_messages,
+ all_loaded_messages.count()
);
// Test 2: Get messages from middle timestamp (after 3rd batch)
- let middle_timestamp = batch_timestamps[2];
- let remaining_messages = total_messages - (batch_sizes[0] + batch_sizes[1] + batch_sizes[2]);
- let middle_messages = partition
- .get_messages_by_timestamp(middle_timestamp, remaining_messages)
- .await
- .unwrap();
- assert_eq!(
- middle_messages.len(),
- remaining_messages as usize,
- "Expected {} messages from middle timestamp, but got {}",
- remaining_messages,
- middle_messages.len()
- );
+ if batch_timestamps.len() >= 3 {
+ // Use a timestamp that's just before the 3rd batch's timestamp to ensure we get messages
+ // from that batch onwards
+ let middle_timestamp = IggyTimestamp::from(batch_timestamps[2].as_micros() + 1000);
+
+ // Calculate how many messages should be in batches after the 3rd
+ let prior_batches_sum: u32 = batch_sizes[..3].iter().sum();
+ let remaining_messages = total_sent_messages - prior_batches_sum;
+
+ let middle_messages = partition
+ .get_messages_by_timestamp(middle_timestamp, remaining_messages)
+ .await
+ .unwrap();
+
+ assert_eq!(
+ middle_messages.count(),
+ remaining_messages,
+ "Expected {} messages from middle timestamp, but got {}",
+ remaining_messages,
+ middle_messages.count()
+ );
+ }
- // Test 3: No messages from final timestamp
+ // Test 3: No messages after final timestamp
let no_messages = partition
.get_messages_by_timestamp(final_timestamp, 1)
.await
.unwrap();
assert_eq!(
- no_messages.len(),
+ no_messages.count(),
0,
- "Expected no messages from final timestamp, but got {}",
- no_messages.len()
+ "Expected no messages after final timestamp, but got {}",
+ no_messages.count()
);
// Test 4: Small subset from initial timestamp
- let subset_size = 3;
+ let subset_size = std::cmp::min(3, total_sent_messages);
let subset_messages = partition
.get_messages_by_timestamp(initial_timestamp, subset_size)
.await
.unwrap();
assert_eq!(
- subset_messages.len(),
- subset_size as usize,
+ subset_messages.count(),
+ subset_size,
"Expected {} messages in subset from initial timestamp, but got {}",
subset_size,
- subset_messages.len()
+ subset_messages.count()
);
- for i in 0..subset_messages.len() {
- let loaded_message = &subset_messages[i];
- let original_message = &all_messages[i];
- assert_eq!(
- loaded_message.id, original_message.id,
- "Message ID mismatch at position {}: expected {}, got {}",
- i, original_message.id, loaded_message.id
- );
- assert_eq!(
- loaded_message.payload, original_message.payload,
- "Payload mismatch at position {}: expected {:?}, got {:?}",
- i, original_message.payload, loaded_message.payload
- );
+
+ // Test 5: Messages spanning multiple batches by timestamp
+ if batch_timestamps.len() >= 4 {
+ // Use a timestamp that's just before the 2nd batch's timestamp
+ let span_timestamp = IggyTimestamp::from(batch_timestamps[1].as_micros() + 1000);
+ let span_size = 8; // Should span across multiple batches
+
+ let spanning_messages = partition
+ .get_messages_by_timestamp(span_timestamp, span_size)
+ .await
+ .unwrap();
+
assert_eq!(
- loaded_message
- .headers
- .as_ref()
- .map(|bytes| HashMap::from_bytes(bytes.clone()).unwrap()),
- original_message.headers,
- "Headers mismatch at position {}: expected {:?}, got {:?}",
- i,
- original_message.headers,
- loaded_message
- .headers
- .as_ref()
- .map(|bytes| HashMap::from_bytes(bytes.clone()).unwrap())
- );
- assert!(
- loaded_message.timestamp >= initial_timestamp.as_micros(),
- "Message timestamp {} at position {} is less than initial timestamp {}",
- loaded_message.timestamp,
- i,
- initial_timestamp
+ spanning_messages.count(),
+ span_size,
+ "Expected {} messages spanning multiple batches, but got {}",
+ span_size,
+ spanning_messages.count()
);
- }
- // Test 5: Messages spanning multiple batches (from middle of 2nd batch timestamp)
- let span_timestamp = batch_timestamps[1];
- let span_size = 8; // Should span across 2nd, 3rd, and into 4th batch
- let spanning_messages = partition
- .get_messages_by_timestamp(span_timestamp, span_size)
- .await
- .unwrap();
- assert_eq!(
- spanning_messages.len(),
- span_size as usize,
- "Expected {} messages spanning multiple batches, but got {}",
- span_size,
- spanning_messages.len()
- );
- for msg in spanning_messages.iter() {
- assert!(
- msg.timestamp >= span_timestamp.as_micros(),
- "Message timestamp {} should be >= span timestamp {}",
- msg.timestamp,
- span_timestamp
- );
+ // Verify that all messages have timestamps >= our reference timestamp
+ let span_timestamp_micros = span_timestamp.as_micros();
+
+ // Test 6: Validate message content and ordering
+ for batch in spanning_messages.iter() {
+ for msg in batch.iter() {
+ let msg_timestamp = msg.header().timestamp();
+ assert!(
+ msg_timestamp >= span_timestamp_micros,
+ "Message timestamp {} should be >= span timestamp {}",
+ msg_timestamp,
+ span_timestamp_micros
+ );
+
+ // Verify message content
+ let loaded_id = msg.header().id();
+ let original_offset = msg.header().offset() as usize;
+
+ if original_offset < all_messages.len() {
+ let original_message = &all_messages[original_offset];
+ let original_id = original_message.header.id;
+
+ assert_eq!(
+ loaded_id,
+ original_id,
+ "Message ID mismatch at offset {}",
+ msg.header().offset(),
+ );
+
+ let loaded_payload = msg.payload();
+ let original_payload = &original_message.payload;
+ assert_eq!(
+ loaded_payload,
+ original_payload,
+ "Payload mismatch at offset {}",
+ msg.header().offset(),
+ );
+
+ let loaded_headers = msg.user_headers_map().unwrap().unwrap();
+ let original_headers = HashMap::from_bytes(
+ original_message.user_headers.as_ref().unwrap().clone(),
+ )
+ .unwrap();
+ assert_eq!(
+ loaded_headers,
+ original_headers,
+ "Headers mismatch at offset {}",
+ msg.header().offset(),
+ );
+ }
+ }
+ }
}
}
diff --git a/integration/tests/streaming/messages.rs b/integration/tests/streaming/messages.rs
index 400dd2f2..d1c4b9dd 100644
--- a/integration/tests/streaming/messages.rs
+++ b/integration/tests/streaming/messages.rs
@@ -1,16 +1,10 @@
use crate::streaming::common::test_setup::TestSetup;
use bytes::Bytes;
-use iggy::bytes_serializable::BytesSerializable;
-use iggy::messages::send_messages::Message;
-use iggy::models::header::{HeaderKey, HeaderValue};
-use iggy::utils::byte_size::IggyByteSize;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::sizeable::Sizeable;
-use iggy::utils::timestamp::IggyTimestamp;
+use iggy::prelude::*;
use server::configs::system::{PartitionConfig, SystemConfig};
use server::state::system::PartitionState;
-use server::streaming::batching::appendable_batch_info::AppendableBatchInfo;
use server::streaming::partitions::partition::Partition;
+use server::streaming::segments::IggyMessagesBatchMut;
use std::collections::HashMap;
use std::str::FromStr;
use std::sync::atomic::{AtomicU32, AtomicU64};
@@ -68,12 +62,13 @@ async fn should_persist_messages_and_then_load_them_by_timestamp() {
HeaderKey::new("key-3").unwrap(),
HeaderValue::from_uint64(123456).unwrap(),
);
- let message = Message {
- id,
- length: payload.len() as u32,
- payload: payload.clone(),
- headers: Some(headers),
- };
+ let message = IggyMessage::builder()
+ .id(id)
+ .payload(payload)
+ .headers(headers)
+ .build()
+ .expect("Failed to create message with valid payload and headers");
+
messages.push(message);
}
@@ -93,41 +88,42 @@ async fn should_persist_messages_and_then_load_them_by_timestamp() {
HeaderKey::new("key-3").unwrap(),
HeaderValue::from_uint64(123456).unwrap(),
);
- let message = Message {
- id,
- length: payload.len() as u32,
- payload: payload.clone(),
- headers: Some(headers),
- };
- appended_messages.push(message.clone());
- messages_two.push(message);
+
+ let message = IggyMessage::builder()
+ .id(id)
+ .payload(payload.clone())
+ .headers(headers.clone())
+ .build()
+ .expect("Failed to create message with valid payload and headers");
+
+ let message_clone = IggyMessage::builder()
+ .id(id)
+ .payload(payload)
+ .headers(headers)
+ .build()
+ .expect("Failed to create message with valid payload and headers");
+
+ appended_messages.push(message);
+ messages_two.push(message_clone);
}
setup.create_partitions_directory(stream_id, topic_id).await;
partition.persist().await.unwrap();
- let appendable_batch_info = AppendableBatchInfo::new(
- messages
- .iter()
- .map(|msg| msg.get_size_bytes())
- .sum::<IggyByteSize>(),
- partition.partition_id,
- );
- let appendable_batch_info_two = AppendableBatchInfo::new(
- messages_two
- .iter()
- .map(|msg| msg.get_size_bytes())
- .sum::<IggyByteSize>(),
- partition.partition_id,
- );
- partition
- .append_messages(appendable_batch_info, messages, None)
- .await
- .unwrap();
+ let messages_size = messages
+ .iter()
+ .map(|msg| msg.get_size_bytes().as_bytes_u32())
+ .sum::<u32>();
+ let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
+
+ let messages_size_two = messages_two
+ .iter()
+ .map(|msg| msg.get_size_bytes().as_bytes_u32())
+ .sum::<u32>();
+ let batch_two = IggyMessagesBatchMut::from_messages(&messages_two, messages_size_two);
+
+ partition.append_messages(batch, None).await.unwrap();
let test_timestamp = IggyTimestamp::now();
- partition
- .append_messages(appendable_batch_info_two, messages_two, None)
- .await
- .unwrap();
+ partition.append_messages(batch_two, None).await.unwrap();
let loaded_messages = partition
.get_messages_by_timestamp(test_timestamp, messages_count)
@@ -135,47 +131,36 @@ async fn should_persist_messages_and_then_load_them_by_timestamp() {
.unwrap();
assert_eq!(
- loaded_messages.len(),
- messages_count as usize,
- "Expected loaded messages count to be {}, but got {}",
+ loaded_messages.count(),
messages_count,
- loaded_messages.len()
+ "Unexpected loaded messages count"
);
- for i in 0..loaded_messages.len() {
- let loaded_message = &loaded_messages[i];
+ (0..loaded_messages.count() as usize).for_each(|i| {
+ let loaded_message = &loaded_messages.get(i).unwrap();
let appended_message = &appended_messages[i];
assert_eq!(
- loaded_message.id, appended_message.id,
- "Message ID mismatch at position {}: expected {}, got {}",
- i, appended_message.id, loaded_message.id
+ loaded_message.header().id(),
+ appended_message.header.id,
+ "Message ID mismatch at position {i}"
);
assert_eq!(
- loaded_message.payload, appended_message.payload,
- "Payload mismatch at position {}: expected {:?}, got {:?}",
- i, appended_message.payload, loaded_message.payload
+ loaded_message.payload(),
+ appended_message.payload,
+ "Payload mismatch at position {i}",
);
assert!(
- loaded_message.timestamp >= test_timestamp.as_micros(),
+ loaded_message.header().timestamp() >= test_timestamp.as_micros(),
"Message timestamp {} at position {} is less than test timestamp {}",
- loaded_message.timestamp,
+ loaded_message.header().timestamp(),
i,
test_timestamp
);
assert_eq!(
- loaded_message
- .headers
- .as_ref()
- .map(|bytes| HashMap::from_bytes(bytes.clone()).unwrap()),
- appended_message.headers,
- "Headers mismatch at position {}: expected {:?}, got {:?}",
- i,
- appended_message.headers,
- loaded_message
- .headers
- .as_ref()
- .map(|bytes| HashMap::from_bytes(bytes.clone()).unwrap())
+ loaded_message.user_headers_map().unwrap().unwrap(),
+ appended_message.user_headers_map().unwrap().unwrap(),
+ "Headers mismatch at position {i}",
);
- }
+ });
}
#[tokio::test]
@@ -229,35 +214,23 @@ async fn should_persist_messages_and_then_load_them_from_disk() {
HeaderKey::new("key-3").unwrap(),
HeaderValue::from_uint64(123456).unwrap(),
);
- let appended_message = Message {
- id,
- length: payload.len() as u32,
- payload: payload.clone(),
- headers: Some(headers.clone()),
- };
- let message = Message {
- id,
- length: payload.len() as u32,
- payload: payload.clone(),
- headers: Some(headers),
- };
+
+ let appended_message =
+ IggyMessage::with_id_and_headers(id, payload.clone(), headers.clone());
+ let message = IggyMessage::with_id_and_headers(id, payload, headers);
+
appended_messages.push(appended_message);
messages.push(message);
}
setup.create_partitions_directory(stream_id, topic_id).await;
partition.persist().await.unwrap();
- let appendable_batch_info = AppendableBatchInfo::new(
- messages
- .iter()
- .map(|msg| msg.get_size_bytes())
- .sum::<IggyByteSize>(),
- partition.partition_id,
- );
- partition
- .append_messages(appendable_batch_info, messages, None)
- .await
- .unwrap();
+ let messages_size = messages
+ .iter()
+ .map(|msg| msg.get_size_bytes().as_bytes_u32())
+ .sum::<u32>();
+ let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
+ partition.append_messages(batch, None).await.unwrap();
assert_eq!(
partition.unsaved_messages_count, 0,
"Expected unsaved messages count to be 0, but got {}",
@@ -291,39 +264,27 @@ async fn should_persist_messages_and_then_load_them_from_disk() {
.await
.unwrap();
assert_eq!(
- loaded_messages.len(),
- messages_count as usize,
- "Expected loaded messages count to be {}, but got {}",
+ loaded_messages.count(),
messages_count,
- loaded_messages.len()
+ "Unexpected loaded messages count "
);
- for i in 1..=messages_count {
- let index = i as usize - 1;
- let loaded_message = &loaded_messages[index];
- let appended_message = &appended_messages[index];
+ (0..messages_count as usize).for_each(|i| {
+ let loaded_message = &loaded_messages.get(i).unwrap();
+ let appended_message = &appended_messages[i];
assert_eq!(
- loaded_message.id, appended_message.id,
- "Message ID mismatch at position {}: expected {}, got {}",
- i, appended_message.id, loaded_message.id
+ loaded_message.header().id(),
+ appended_message.header.id,
+ "Message ID mismatch at position {i}",
);
assert_eq!(
- loaded_message.payload, appended_message.payload,
- "Payload mismatch at position {}: expected {:?}, got {:?}",
- i, appended_message.payload, loaded_message.payload
+ loaded_message.payload(),
+ appended_message.payload,
+ "Payload mismatch at position {i}",
);
assert_eq!(
- loaded_message
- .headers
- .as_ref()
- .map(|bytes| HashMap::from_bytes(bytes.clone()).unwrap()),
- appended_message.headers,
- "Headers mismatch at position {}: expected {:?}, got {:?}",
- i,
- appended_message.headers,
- loaded_message
- .headers
- .as_ref()
- .map(|bytes| HashMap::from_bytes(bytes.clone()).unwrap())
+ loaded_message.user_headers_map().unwrap().unwrap(),
+ appended_message.user_headers_map().unwrap().unwrap(),
+ "Headers mismatch at position {i}",
);
- }
+ });
}
diff --git a/integration/tests/streaming/mod.rs b/integration/tests/streaming/mod.rs
index d2ad7ec3..4e184a44 100644
--- a/integration/tests/streaming/mod.rs
+++ b/integration/tests/streaming/mod.rs
@@ -1,5 +1,5 @@
use bytes::Bytes;
-use iggy::messages::send_messages::Message;
+use iggy::prelude::IggyMessage;
mod common;
mod consumer_offset;
@@ -14,7 +14,7 @@ mod system;
mod topic;
mod topic_messages;
-fn create_messages() -> Vec<Message> {
+fn create_messages() -> Vec<IggyMessage> {
vec![
create_message(1, "message 1"),
create_message(2, "message 2"),
@@ -25,12 +25,11 @@ fn create_messages() -> Vec<Message> {
]
}
-fn create_message(id: u128, payload: &str) -> Message {
+fn create_message(id: u128, payload: &str) -> IggyMessage {
let payload = Bytes::from(payload.to_string());
- Message {
- id,
- length: payload.len() as u32,
- payload,
- headers: None,
- }
+ IggyMessage::builder()
+ .id(id)
+ .payload(payload)
+ .build()
+ .expect("Failed to create message with valid payload and headers")
}
diff --git a/integration/tests/streaming/partition.rs b/integration/tests/streaming/partition.rs
index eb27d6bf..c40410b3 100644
--- a/integration/tests/streaming/partition.rs
+++ b/integration/tests/streaming/partition.rs
@@ -1,11 +1,9 @@
use crate::streaming::common::test_setup::TestSetup;
use crate::streaming::create_messages;
-use iggy::utils::byte_size::IggyByteSize;
use iggy::utils::expiry::IggyExpiry;
use iggy::utils::sizeable::Sizeable;
use iggy::utils::timestamp::IggyTimestamp;
use server::state::system::PartitionState;
-use server::streaming::batching::appendable_batch_info::AppendableBatchInfo;
use server::streaming::partitions::partition::Partition;
use server::streaming::segments::*;
use std::sync::atomic::{AtomicU32, AtomicU64};
@@ -111,11 +109,6 @@ async fn should_load_existing_partition_from_disk() {
loaded_partition.should_increment_offset,
partition.should_increment_offset
);
- assert_eq!(loaded_partition.cache.is_some(), partition.cache.is_some());
- assert_eq!(
- loaded_partition.cache.unwrap().is_empty(),
- partition.cache.unwrap().is_empty()
- );
}
}
@@ -181,20 +174,15 @@ async fn should_purge_existing_partition_on_disk() {
partition.persist().await.unwrap();
assert_persisted_partition(&partition.partition_path, with_segment).await;
let messages = create_messages();
- let messages_count = messages.len();
- let appendable_batch_info = AppendableBatchInfo::new(
- messages
- .iter()
- .map(|msg| msg.get_size_bytes())
- .sum::<IggyByteSize>(),
- partition.partition_id,
- );
- partition
- .append_messages(appendable_batch_info, messages, None)
- .await
- .unwrap();
+ let messages_count = messages.len() as u32;
+ let messages_size: u32 = messages
+ .iter()
+ .map(|msg| msg.get_size_bytes().as_bytes_u32())
+ .sum();
+ let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
+ partition.append_messages(batch, None).await.unwrap();
let loaded_messages = partition.get_messages_by_offset(0, 100).await.unwrap();
- assert_eq!(loaded_messages.len(), messages_count);
+ assert_eq!(loaded_messages.count(), messages_count);
partition.purge().await.unwrap();
assert_eq!(partition.current_offset, 0);
assert_eq!(partition.unsaved_messages_count, 0);
@@ -210,9 +198,9 @@ async fn assert_persisted_partition(partition_path: &str, with_segment: bool) {
if with_segment {
let start_offset = 0u64;
let segment_path = format!("{}/{:0>20}", partition_path, start_offset);
- let log_path = format!("{}.{}", segment_path, LOG_EXTENSION);
+ let messages_file_path = format!("{}.{}", segment_path, LOG_EXTENSION);
let index_path = format!("{}.{}", segment_path, INDEX_EXTENSION);
- assert!(fs::metadata(&log_path).await.is_ok());
+ assert!(fs::metadata(&messages_file_path).await.is_ok());
assert!(fs::metadata(&index_path).await.is_ok());
}
}
diff --git a/integration/tests/streaming/segment.rs b/integration/tests/streaming/segment.rs
index 92d60043..42549726 100644
--- a/integration/tests/streaming/segment.rs
+++ b/integration/tests/streaming/segment.rs
@@ -1,15 +1,10 @@
use crate::streaming::common::test_setup::TestSetup;
use bytes::Bytes;
-use iggy::bytes_serializable::BytesSerializable;
use iggy::confirmation::Confirmation;
-use iggy::models::messages::{MessageState, PolledMessage};
-use iggy::utils::byte_size::IggyByteSize;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::{checksum, timestamp::IggyTimestamp};
-use server::configs::system::{SegmentConfig, SystemConfig};
-use server::streaming::local_sizeable::LocalSizeable;
-use server::streaming::models::messages::RetainedMessage;
+use iggy::prelude::*;
+use server::configs::system::{PartitionConfig, SegmentConfig, SystemConfig};
use server::streaming::segments::*;
+use std::str::FromStr;
use std::sync::atomic::AtomicU64;
use std::sync::Arc;
use std::time::Duration;
@@ -104,14 +99,19 @@ async fn should_load_existing_segment_from_disk() {
loaded_segment.load_from_disk().await.unwrap();
let loaded_messages = loaded_segment.get_messages_by_offset(0, 10).await.unwrap();
- assert_eq!(loaded_segment.partition_id, segment.partition_id);
- assert_eq!(loaded_segment.start_offset, segment.start_offset);
- assert_eq!(loaded_segment.current_offset, segment.current_offset);
- assert_eq!(loaded_segment.end_offset, segment.end_offset);
- assert_eq!(loaded_segment.size_bytes, segment.size_bytes);
- assert_eq!(loaded_segment.is_closed, segment.is_closed);
- assert_eq!(loaded_segment.log_path, segment.log_path);
- assert_eq!(loaded_segment.index_path, segment.index_path);
+ assert_eq!(loaded_segment.partition_id(), segment.partition_id());
+ assert_eq!(loaded_segment.start_offset(), segment.start_offset());
+ assert_eq!(loaded_segment.end_offset(), segment.end_offset());
+ assert_eq!(
+ loaded_segment.get_messages_size(),
+ segment.get_messages_size()
+ );
+ assert_eq!(loaded_segment.is_closed(), segment.is_closed());
+ assert_eq!(
+ loaded_segment.messages_file_path(),
+ segment.messages_file_path()
+ );
+ assert_eq!(loaded_segment.index_file_path(), segment.index_file_path());
assert!(loaded_messages.is_empty());
}
}
@@ -151,27 +151,15 @@ async fn should_persist_and_load_segment_with_messages() {
.await;
let messages_count = 10;
let mut messages = Vec::new();
- let mut batch_size = IggyByteSize::default();
+ let mut messages_size = 0;
for i in 0..messages_count {
- let message = create_message(i, "test", IggyTimestamp::now());
-
- let retained_message = Arc::new(RetainedMessage {
- id: message.id,
- offset: message.offset,
- timestamp: message.timestamp,
- checksum: message.checksum,
- message_state: message.state,
- headers: message.headers.map(|headers| headers.to_bytes()),
- payload: message.payload.clone(),
- });
- batch_size += retained_message.get_size_bytes();
- messages.push(retained_message);
+ let message = IggyMessage::with_id(i as u128, Bytes::from("test"));
+ messages_size += message.get_size_bytes().as_bytes_u32();
+ messages.push(message);
}
+ let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
- segment
- .append_batch(batch_size, messages_count as u32, &messages)
- .await
- .unwrap();
+ segment.append_batch(0, batch).await.unwrap();
segment.persist_messages(None).await.unwrap();
let mut loaded_segment = Segment::create(
stream_id,
@@ -189,10 +177,10 @@ async fn should_persist_and_load_segment_with_messages() {
);
loaded_segment.load_from_disk().await.unwrap();
let messages = loaded_segment
- .get_messages_by_offset(0, messages_count as u32)
+ .get_messages_by_offset(0, messages_count)
.await
.unwrap();
- assert_eq!(messages.len(), messages_count as usize);
+ assert_eq!(messages.count(), messages_count);
}
#[tokio::test]
@@ -237,27 +225,14 @@ async fn should_persist_and_load_segment_with_messages_with_nowait_confirmation(
.await;
let messages_count = 10;
let mut messages = Vec::new();
- let mut batch_size = IggyByteSize::default();
+ let mut messages_size = 0;
for i in 0..messages_count {
- let message = create_message(i, "test", IggyTimestamp::now());
-
- let retained_message = Arc::new(RetainedMessage {
- id: message.id,
- offset: message.offset,
- timestamp: message.timestamp,
- checksum: message.checksum,
- message_state: message.state,
- headers: message.headers.map(|headers| headers.to_bytes()),
- payload: message.payload.clone(),
- });
- batch_size += retained_message.get_size_bytes();
- messages.push(retained_message);
+ let message = IggyMessage::with_id(i as u128, Bytes::from("test"));
+ messages_size += message.get_size_bytes().as_bytes_u32();
+ messages.push(message);
}
-
- segment
- .append_batch(batch_size, messages_count as u32, &messages)
- .await
- .unwrap();
+ let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
+ segment.append_batch(0, batch).await.unwrap();
segment
.persist_messages(Some(Confirmation::NoWait))
.await
@@ -279,21 +254,32 @@ async fn should_persist_and_load_segment_with_messages_with_nowait_confirmation(
);
loaded_segment.load_from_disk().await.unwrap();
let messages = loaded_segment
- .get_messages_by_offset(0, messages_count as u32)
+ .get_messages_by_offset(0, messages_count)
.await
.unwrap();
- assert_eq!(messages.len(), messages_count as usize);
+ assert_eq!(messages.count(), messages_count);
}
#[tokio::test]
async fn given_all_expired_messages_segment_should_be_expired() {
- let setup = TestSetup::init().await;
+ let config = SystemConfig {
+ partition: PartitionConfig {
+ enforce_fsync: true,
+ ..Default::default()
+ },
+ segment: SegmentConfig {
+ size: IggyByteSize::from_str("10B").unwrap(), // small size to force expiration
+ ..Default::default()
+ },
+ ..Default::default()
+ };
+ let setup = TestSetup::init_with_config(config).await;
let stream_id = 1;
let topic_id = 2;
let partition_id = 3;
let start_offset = 0;
- let message_expiry_ms = 1000;
- let message_expiry = 1000u64.into();
+ let message_expiry_us = 100000;
+ let message_expiry = message_expiry_us.into();
let mut segment = Segment::create(
stream_id,
topic_id,
@@ -321,46 +307,43 @@ async fn given_all_expired_messages_segment_should_be_expired() {
)
.await;
let messages_count = 10;
- let now = IggyTimestamp::now();
- let mut expired_timestamp = (now.as_micros() - 2 * message_expiry_ms).into();
- let mut batch_size = IggyByteSize::default();
let mut messages = Vec::new();
+ let mut messages_size = 0;
for i in 0..messages_count {
- let message = create_message(i, "test", expired_timestamp);
- expired_timestamp = (expired_timestamp.as_micros() + 1).into();
-
- let retained_message = Arc::new(RetainedMessage {
- id: message.id,
- offset: message.offset,
- timestamp: message.timestamp,
- checksum: message.checksum,
- message_state: message.state,
- headers: message.headers.map(|headers| headers.to_bytes()),
- payload: message.payload.clone(),
- });
- batch_size += retained_message.get_size_bytes();
- messages.push(retained_message);
+ let message = IggyMessage::with_id(i as u128, Bytes::from("test"));
+ messages_size += message.get_size_bytes().as_bytes_u32();
+ messages.push(message);
}
- segment
- .append_batch(batch_size, messages_count as u32, &messages)
- .await
- .unwrap();
+ let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
+ segment.append_batch(0, batch).await.unwrap();
segment.persist_messages(None).await.unwrap();
+ let not_expired_ts = IggyTimestamp::now();
+ let expired_ts = not_expired_ts + IggyDuration::from(message_expiry_us + 1);
- segment.is_closed = true;
- let is_expired = segment.is_expired(now).await;
- assert!(is_expired);
+ assert!(segment.is_expired(expired_ts).await);
+ assert!(!segment.is_expired(not_expired_ts).await);
}
#[tokio::test]
async fn given_at_least_one_not_expired_message_segment_should_not_be_expired() {
- let setup = TestSetup::init().await;
+ let config = SystemConfig {
+ partition: PartitionConfig {
+ enforce_fsync: true,
+ ..Default::default()
+ },
+ segment: SegmentConfig {
+ size: IggyByteSize::from_str("10B").unwrap(), // small size to force expiration
+ ..Default::default()
+ },
+ ..Default::default()
+ };
+ let setup = TestSetup::init_with_config(config).await;
let stream_id = 1;
let topic_id = 2;
let partition_id = 3;
let start_offset = 0;
- let message_expiry_ms = 1000;
- let message_expiry = message_expiry_ms.into();
+ let message_expiry_us = 50000;
+ let message_expiry = message_expiry_us.into();
let mut segment = Segment::create(
stream_id,
topic_id,
@@ -387,76 +370,49 @@ async fn given_at_least_one_not_expired_message_segment_should_not_be_expired()
start_offset,
)
.await;
- let now = IggyTimestamp::now();
- let expired_timestamp = now.as_micros() - 2 * message_expiry_ms;
- let not_expired_timestamp = now.as_micros() - message_expiry_ms + 1;
- let expired_message = create_message(0, "test", expired_timestamp.into());
- let not_expired_message = create_message(1, "test", not_expired_timestamp.into());
- let expired_retained_message = Arc::new(RetainedMessage {
- id: expired_message.id,
- offset: expired_message.offset,
- timestamp: expired_message.timestamp,
- checksum: expired_message.checksum,
- message_state: expired_message.state,
- headers: expired_message.headers.map(|headers| headers.to_bytes()),
- payload: expired_message.payload.clone(),
- });
- let mut expired_messages = Vec::new();
- let expired_message_size = expired_retained_message.get_size_bytes();
- expired_messages.push(expired_retained_message);
+ let nothing_expired_ts = IggyTimestamp::now();
- let mut not_expired_messages = Vec::new();
- let not_expired_retained_message = Arc::new(RetainedMessage {
- id: not_expired_message.id,
- offset: not_expired_message.offset,
- timestamp: not_expired_message.timestamp,
- checksum: not_expired_message.checksum,
- message_state: not_expired_message.state,
- headers: not_expired_message
- .headers
- .map(|headers| headers.to_bytes()),
- payload: not_expired_message.payload.clone(),
- });
- let not_expired_message_size = not_expired_retained_message.get_size_bytes();
- not_expired_messages.push(not_expired_retained_message);
+ let first_message = vec![IggyMessage::create(Bytes::from("expired"))];
+ let first_message_size = first_message[0].get_size_bytes().as_bytes_u32();
+ let first_batch = IggyMessagesBatchMut::from_messages(&first_message, first_message_size);
+ segment.append_batch(0, first_batch).await.unwrap();
+
+ sleep(Duration::from_micros(message_expiry_us / 2)).await;
+ let first_message_expired_ts = IggyTimestamp::now();
+
+ let second_message = vec![IggyMessage::create(Bytes::from("not-expired"))];
+ let second_message_size = second_message[0].get_size_bytes().as_bytes_u32();
+ let second_batch = IggyMessagesBatchMut::from_messages(&second_message, second_message_size);
+ segment.append_batch(1, second_batch).await.unwrap();
+
+ let second_message_expired_ts =
+ IggyTimestamp::now() + IggyDuration::from(message_expiry_us * 2);
- segment
- .append_batch(expired_message_size, 1, &expired_messages)
- .await
- .unwrap();
- segment
- .append_batch(not_expired_message_size, 1, ¬_expired_messages)
- .await
- .unwrap();
segment.persist_messages(None).await.unwrap();
- let is_expired = segment.is_expired(now).await;
- assert!(!is_expired);
+ assert!(
+ !segment.is_expired(nothing_expired_ts).await,
+ "Segment should not be expired for nothing expired timestamp"
+ );
+ assert!(
+ !segment.is_expired(first_message_expired_ts).await,
+ "Segment should not be expired for first message expired timestamp"
+ );
+ assert!(
+ segment.is_expired(second_message_expired_ts).await,
+ "Segment should be expired for second message expired timestamp"
+ );
}
async fn assert_persisted_segment(partition_path: &str, start_offset: u64) {
let segment_path = format!("{}/{:0>20}", partition_path, start_offset);
- let log_path = format!("{}.{}", segment_path, LOG_EXTENSION);
+ let messages_file_path = format!("{}.{}", segment_path, LOG_EXTENSION);
let index_path = format!("{}.{}", segment_path, INDEX_EXTENSION);
- assert!(fs::metadata(&log_path).await.is_ok());
+ assert!(fs::metadata(&messages_file_path).await.is_ok());
assert!(fs::metadata(&index_path).await.is_ok());
}
-fn create_message(offset: u64, payload: &str, timestamp: IggyTimestamp) -> PolledMessage {
- let payload = Bytes::from(payload.to_string());
- let checksum = checksum::calculate(payload.as_ref());
- PolledMessage::create(
- offset,
- MessageState::Available,
- timestamp,
- 0,
- payload,
- checksum,
- None,
- )
-}
-
fn get_start_offsets() -> Vec<u64> {
vec![
0, 1, 2, 9, 10, 99, 100, 110, 200, 1000, 1234, 12345, 100000, 9999999,
diff --git a/integration/tests/streaming/snapshot.rs b/integration/tests/streaming/snapshot.rs
index 6e86abdc..1b4cd454 100644
--- a/integration/tests/streaming/snapshot.rs
+++ b/integration/tests/streaming/snapshot.rs
@@ -24,7 +24,7 @@ async fn should_create_snapshot_file() {
.get_snapshot(
&session,
SnapshotCompression::Deflated,
- vec![SystemSnapshotType::Test],
+ &vec![SystemSnapshotType::Test],
)
.await
.unwrap();
diff --git a/integration/tests/streaming/stream.rs b/integration/tests/streaming/stream.rs
index 0e65f621..61914ef5 100644
--- a/integration/tests/streaming/stream.rs
+++ b/integration/tests/streaming/stream.rs
@@ -1,16 +1,10 @@
use crate::streaming::common::test_setup::TestSetup;
use crate::streaming::create_messages;
use ahash::AHashMap;
-use iggy::identifier::Identifier;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::messages::send_messages::Partitioning;
-use iggy::utils::byte_size::IggyByteSize;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::sizeable::Sizeable;
-use iggy::utils::timestamp::IggyTimestamp;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use server::state::system::StreamState;
use server::streaming::polling_consumer::PollingConsumer;
+use server::streaming::segments::IggyMessagesBatchMut;
use server::streaming::streams::stream::Stream;
use tokio::fs;
@@ -125,19 +119,20 @@ async fn should_purge_existing_stream_on_disk() {
.unwrap();
let messages = create_messages();
- let messages_count = messages.len();
+ let messages_count = messages.len() as u32;
+ let messages_size = messages
+ .iter()
+ .map(|msg| msg.get_size_bytes().as_bytes_u32())
+ .sum::<u32>();
+ let batch = IggyMessagesBatchMut::from_messages(&messages, messages_size);
let topic = stream
.get_topic(&Identifier::numeric(topic_id).unwrap())
.unwrap();
- let batch_size = messages
- .iter()
- .map(|msg| msg.get_size_bytes())
- .sum::<IggyByteSize>();
topic
- .append_messages(batch_size, Partitioning::partition_id(1), messages, None)
+ .append_messages(&Partitioning::partition_id(1), batch, None)
.await
.unwrap();
- let loaded_messages = topic
+ let (_, loaded_messages) = topic
.get_messages(
PollingConsumer::Consumer(1, 1),
1,
@@ -147,10 +142,10 @@ async fn should_purge_existing_stream_on_disk() {
.await
.unwrap();
- assert_eq!(loaded_messages.messages.len(), messages_count);
+ assert_eq!(loaded_messages.count(), messages_count);
stream.purge().await.unwrap();
- let loaded_messages = topic
+ let (metadata, loaded_messages) = topic
.get_messages(
PollingConsumer::Consumer(1, 1),
1,
@@ -159,8 +154,8 @@ async fn should_purge_existing_stream_on_disk() {
)
.await
.unwrap();
- assert_eq!(loaded_messages.current_offset, 0);
- assert!(loaded_messages.messages.is_empty());
+ assert_eq!(metadata.current_offset, 0);
+ assert!(loaded_messages.is_empty());
}
}
diff --git a/integration/tests/streaming/topic.rs b/integration/tests/streaming/topic.rs
index 266ec149..c3442a9e 100644
--- a/integration/tests/streaming/topic.rs
+++ b/integration/tests/streaming/topic.rs
@@ -1,16 +1,10 @@
use crate::streaming::common::test_setup::TestSetup;
use crate::streaming::create_messages;
use ahash::AHashMap;
-use iggy::compression::compression_algorithm::CompressionAlgorithm;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::messages::send_messages::Partitioning;
-use iggy::utils::byte_size::IggyByteSize;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::sizeable::Sizeable;
-use iggy::utils::timestamp::IggyTimestamp;
-use iggy::utils::topic_size::MaxTopicSize;
+use iggy::prelude::*;
use server::state::system::{PartitionState, TopicState};
use server::streaming::polling_consumer::PollingConsumer;
+use server::streaming::segments::IggyMessagesBatchMut;
use server::streaming::topics::topic::Topic;
use std::default::Default;
use std::sync::atomic::{AtomicU32, AtomicU64};
@@ -208,16 +202,17 @@ async fn should_purge_existing_topic_on_disk() {
.await;
let messages = create_messages();
- let messages_count = messages.len();
+ let messages_count = messages.len() as u32;
let batch_size = messages
.iter()
- .map(|msg| msg.get_size_bytes())
- .sum::<IggyByteSize>();
+ .map(|msg| msg.get_size_bytes().as_bytes_u32())
+ .sum::<u32>();
+ let batch = IggyMessagesBatchMut::from_messages(&messages, batch_size);
topic
- .append_messages(batch_size, Partitioning::partition_id(1), messages, None)
+ .append_messages(&Partitioning::partition_id(1), batch, None)
.await
.unwrap();
- let loaded_messages = topic
+ let (_, loaded_messages) = topic
.get_messages(
PollingConsumer::Consumer(1, 1),
1,
@@ -226,10 +221,10 @@ async fn should_purge_existing_topic_on_disk() {
)
.await
.unwrap();
- assert_eq!(loaded_messages.messages.len(), messages_count);
+ assert_eq!(loaded_messages.count(), messages_count);
topic.purge().await.unwrap();
- let loaded_messages = topic
+ let (metadata, loaded_messages) = topic
.get_messages(
PollingConsumer::Consumer(1, 1),
1,
@@ -238,8 +233,8 @@ async fn should_purge_existing_topic_on_disk() {
)
.await
.unwrap();
- assert_eq!(loaded_messages.current_offset, 0);
- assert!(loaded_messages.messages.is_empty());
+ assert_eq!(metadata.current_offset, 0);
+ assert!(loaded_messages.is_empty());
}
}
diff --git a/integration/tests/streaming/topic_messages.rs b/integration/tests/streaming/topic_messages.rs
index ff85a93d..49c05915 100644
--- a/integration/tests/streaming/topic_messages.rs
+++ b/integration/tests/streaming/topic_messages.rs
@@ -1,15 +1,9 @@
use crate::streaming::common::test_setup::TestSetup;
use bytes::Bytes;
-use iggy::locking::IggySharedMutFn;
-use iggy::messages::poll_messages::PollingStrategy;
-use iggy::messages::send_messages::{Message, Partitioning};
-use iggy::utils::byte_size::IggyByteSize;
-use iggy::utils::expiry::IggyExpiry;
-use iggy::utils::sizeable::Sizeable;
-use iggy::utils::topic_size::MaxTopicSize;
-use server::configs::resource_quota::MemoryResourceQuota;
-use server::configs::system::{CacheConfig, SystemConfig};
+use iggy::prelude::*;
+use server::configs::system::SystemConfig;
use server::streaming::polling_consumer::PollingConsumer;
+use server::streaming::segments::IggyMessagesBatchMut;
use server::streaming::topics::topic::Topic;
use server::streaming::utils::hash;
use std::collections::HashMap;
@@ -18,46 +12,10 @@ use std::sync::atomic::{AtomicU32, AtomicU64};
use std::sync::Arc;
#[tokio::test]
-async fn given_disabled_cache_all_messages_should_be_polled() {
- assert_polling_messages(
- CacheConfig {
- enabled: false,
- ..Default::default()
- },
- false,
- )
- .await;
-}
-
-#[tokio::test]
-async fn given_enabled_cache_with_enough_capacity_all_messages_should_be_polled() {
- assert_polling_messages(
- CacheConfig {
- enabled: true,
- size: MemoryResourceQuota::Bytes(IggyByteSize::from(100_000_000)),
- },
- true,
- )
- .await;
-}
-
-#[tokio::test]
-async fn given_enabled_cache_without_enough_capacity_all_messages_should_be_polled() {
- assert_polling_messages(
- CacheConfig {
- enabled: true,
- size: MemoryResourceQuota::Bytes(IggyByteSize::from(100_000)),
- },
- true,
- )
- .await;
-}
-
-async fn assert_polling_messages(cache: CacheConfig, expect_enabled_cache: bool) {
+async fn assert_polling_messages() {
let messages_count = 1000;
let payload_size_bytes = 1000;
let config = SystemConfig {
- cache,
..Default::default()
};
let setup = TestSetup::init_with_config(config).await;
@@ -75,7 +33,7 @@ async fn assert_polling_messages(cache: CacheConfig, expect_enabled_cache: bool)
let mut sent_messages = Vec::new();
for message in &messages {
sent_messages.push(get_message(
- message.id,
+ message.header.id,
from_utf8(&message.payload).unwrap(),
))
}
@@ -83,13 +41,14 @@ async fn assert_polling_messages(cache: CacheConfig, expect_enabled_cache: bool)
.iter()
.map(|m| m.get_size_bytes())
.sum::<IggyByteSize>();
+ let batch = IggyMessagesBatchMut::from_messages(&messages, batch_size.as_bytes_u32());
topic
- .append_messages(batch_size, partitioning, messages, None)
+ .append_messages(&partitioning, batch, None)
.await
.unwrap();
let consumer = PollingConsumer::Consumer(1, partition_id);
- let polled_messages = topic
+ let (_, polled_messages) = topic
.get_messages(
consumer,
partition_id,
@@ -99,18 +58,16 @@ async fn assert_polling_messages(cache: CacheConfig, expect_enabled_cache: bool)
.await
.unwrap();
assert_eq!(topic.get_messages_count(), messages_count as u64);
- assert_eq!(polled_messages.messages.len(), messages_count as usize);
- let partition = topic.get_partition(partition_id).unwrap();
- let partition = partition.read().await;
- assert_eq!(partition.cache.is_some(), expect_enabled_cache);
- if expect_enabled_cache {
- assert!(partition.cache.as_ref().unwrap().current_size() > 0);
- }
- for (index, polled_message) in polled_messages.messages.iter().enumerate() {
- let sent_message = sent_messages.get(index).unwrap();
- assert_eq!(sent_message.payload, polled_message.payload);
- let polled_payload_str = from_utf8(&polled_message.payload).unwrap();
- assert!(polled_payload_str.starts_with(&format!("{}:", index + 1)));
+ assert_eq!(polled_messages.count(), messages_count);
+ let mut cnt = 0;
+ for batch in polled_messages.iter() {
+ for polled_message in batch.iter() {
+ let sent_message = sent_messages.get(cnt).unwrap();
+ assert_eq!(sent_message.payload, polled_message.payload());
+ let polled_payload_str = from_utf8(polled_message.payload()).unwrap();
+ assert!(polled_payload_str.starts_with(&format!("{}:", cnt + 1)));
+ cnt += 1;
+ }
}
}
@@ -124,13 +81,12 @@ async fn given_key_none_messages_should_be_appended_to_the_next_partition_using_
for i in 1..=partitions_count * messages_per_partition_count {
let payload = get_payload(i);
let batch_size = IggyByteSize::from(16 + 4 + payload.len() as u64);
+ let messages = IggyMessagesBatchMut::from_messages(
+ &[get_message(i as u128, &payload)],
+ batch_size.as_bytes_u32(),
+ );
topic
- .append_messages(
- batch_size,
- partitioning.clone(),
- vec![get_message(i as u128, &payload)],
- None,
- )
+ .append_messages(&partitioning, messages, None)
.await
.unwrap();
}
@@ -150,13 +106,12 @@ async fn given_key_partition_id_messages_should_be_appended_to_the_chosen_partit
for i in 1..=partitions_count * messages_per_partition_count {
let payload = get_payload(i);
let batch_size = IggyByteSize::from(16 + 4 + payload.len() as u64);
+ let messages = IggyMessagesBatchMut::from_messages(
+ &[get_message(i as u128, &payload)],
+ batch_size.as_bytes_u32(),
+ );
topic
- .append_messages(
- batch_size,
- partitioning.clone(),
- vec![get_message(i as u128, &payload)],
- None,
- )
+ .append_messages(&partitioning, messages, None)
.await
.unwrap();
}
@@ -180,13 +135,12 @@ async fn given_key_messages_key_messages_should_be_appended_to_the_calculated_pa
let payload = get_payload(entity_id);
let partitioning = Partitioning::messages_key_u32(entity_id);
let batch_size = IggyByteSize::from(16 + 4 + payload.len() as u64);
+ let messages = IggyMessagesBatchMut::from_messages(
+ &[get_message(entity_id as u128, &payload)],
+ batch_size.as_bytes_u32(),
+ );
topic
- .append_messages(
- batch_size,
- partitioning,
- vec![get_message(entity_id as u128, &payload)],
- None,
- )
+ .append_messages(&partitioning, messages, None)
.await
.unwrap();
}
@@ -217,11 +171,11 @@ fn get_payload(id: u32) -> String {
async fn assert_messages(topic: &Topic, partition_id: u32, expected_messages: u32) {
let consumer = PollingConsumer::Consumer(0, partition_id);
- let polled_messages = topic
+ let (_, polled_messages) = topic
.get_messages(consumer, partition_id, PollingStrategy::offset(0), 1000)
.await
.unwrap();
- assert_eq!(polled_messages.messages.len() as u32, expected_messages);
+ assert_eq!(polled_messages.count(), expected_messages);
}
async fn init_topic(setup: &TestSetup, partitions_count: u32) -> Topic {
@@ -250,13 +204,8 @@ async fn init_topic(setup: &TestSetup, partitions_count: u32) -> Topic {
topic
}
-fn get_message(id: u128, payload: &str) -> Message {
- Message {
- id,
- length: payload.len() as u32,
- payload: Bytes::from(payload.as_bytes().to_vec()),
- headers: None,
- }
+fn get_message(id: u128, payload: &str) -> IggyMessage {
+ IggyMessage::with_id(id, Bytes::from(payload.to_owned()))
}
fn create_payload(size: u32) -> String {
diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml
index 2e693c64..e1e0445e 100644
--- a/sdk/Cargo.toml
+++ b/sdk/Cargo.toml
@@ -25,7 +25,7 @@ byte-unit = { version = "5.1.6", default-features = false, features = [
"byte",
"std",
] }
-bytes = "1.10.0"
+bytes = { version = "1.10.1", features = ["serde"] }
chrono = { version = "0.4.39" }
clap = { version = "4.5.30", features = ["derive"] }
comfy-table = { version = "7.1.4", optional = true }
@@ -37,6 +37,8 @@ fast-async-mutex = { version = "0.6.7", optional = true }
flume = "0.11.1"
futures = "0.3.31"
futures-util = "0.3.31"
+# gxhash = { version = "3.4.1", features = ["hybrid"] }
+gxhash = "3.4.1"
humantime = "2.1.0"
keyring = { version = "3.6.1", optional = true, features = [
"sync-secret-service",
@@ -61,7 +63,7 @@ tokio-rustls = { version = "0.26.1" }
toml = "0.8.20"
tracing = { version = "0.1.41" }
trait-variant = { version = "0.1.2" }
-uuid = { version = "1.13.2", features = ["v7", "fast-rng", "zerocopy"] }
+uuid = { version = "1.14.0", features = ["v7", "fast-rng", "zerocopy"] }
webpki-roots = { version = "0.26.8" }
[build-dependencies]
diff --git a/sdk/src/binary/mapper.rs b/sdk/src/binary/mapper.rs
index 8746ded4..ecc49042 100644
--- a/sdk/src/binary/mapper.rs
+++ b/sdk/src/binary/mapper.rs
@@ -5,7 +5,6 @@ use crate::models::client_info::{ClientInfo, ClientInfoDetails, ConsumerGroupInf
use crate::models::consumer_group::{ConsumerGroup, ConsumerGroupDetails, ConsumerGroupMember};
use crate::models::consumer_offset_info::ConsumerOffsetInfo;
use crate::models::identity_info::IdentityInfo;
-use crate::models::messages::{MessageState, PolledMessage, PolledMessages};
use crate::models::partition::Partition;
use crate::models::permissions::Permissions;
use crate::models::personal_access_token::{PersonalAccessTokenInfo, RawPersonalAccessToken};
@@ -21,7 +20,6 @@ use bytes::Bytes;
use std::collections::HashMap;
use std::str::from_utf8;
-const EMPTY_MESSAGES: Vec<PolledMessage> = vec![];
const EMPTY_TOPICS: Vec<Topic> = vec![];
const EMPTY_STREAMS: Vec<Stream> = vec![];
const EMPTY_CLIENTS: Vec<ClientInfo> = vec![];
@@ -527,106 +525,6 @@ pub fn map_clients(payload: Bytes) -> Result<Vec<ClientInfo>, IggyError> {
Ok(clients)
}
-pub fn map_polled_messages(payload: Bytes) -> Result<PolledMessages, IggyError> {
- if payload.is_empty() {
- return Ok(PolledMessages {
- messages: EMPTY_MESSAGES,
- partition_id: 0,
- current_offset: 0,
- });
- }
-
- let length = payload.len();
- let partition_id = u32::from_le_bytes(
- payload[..4]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- let current_offset = u64::from_le_bytes(
- payload[4..12]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- // Currently ignored
- let _messages_count = u32::from_le_bytes(
- payload[12..16]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- let mut position = 16;
- let mut messages = Vec::new();
- while position < length {
- let offset = u64::from_le_bytes(
- payload[position..position + 8]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- let state = MessageState::from_code(payload[position + 8])?;
- let timestamp = u64::from_le_bytes(
- payload[position + 9..position + 17]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- let id = u128::from_le_bytes(
- payload[position + 17..position + 33]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- let checksum = u32::from_le_bytes(
- payload[position + 33..position + 37]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- let headers_length = u32::from_le_bytes(
- payload[position + 37..position + 41]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- let headers = if headers_length > 0 {
- let headers_payload =
- payload.slice(position + 41..position + 41 + headers_length as usize);
- Some(HashMap::from_bytes(headers_payload)?)
- } else {
- None
- };
- position += headers_length as usize;
- let message_length = u32::from_le_bytes(
- payload[position + 41..position + 45]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- let payload_range = position + 45..position + 45 + message_length as usize;
- if payload_range.start > length || payload_range.end > length {
- break;
- }
-
- let payload = payload[payload_range].to_vec();
- let total_size = 45 + message_length as usize;
- position += total_size;
- messages.push(PolledMessage {
- offset,
- timestamp,
- state,
- checksum,
- id,
- headers,
- length: IggyByteSize::from(message_length as u64),
- payload: Bytes::from(payload),
- });
-
- if position + 45 >= length {
- break;
- }
- }
-
- messages.sort_by(|x, y| x.offset.cmp(&y.offset));
- Ok(PolledMessages {
- partition_id,
- current_offset,
- messages,
- })
-}
-
pub fn map_streams(payload: Bytes) -> Result<Vec<Stream>, IggyError> {
if payload.is_empty() {
return Ok(EMPTY_STREAMS);
diff --git a/sdk/src/binary/messages.rs b/sdk/src/binary/messages.rs
index 832d4045..fc6c90f1 100644
--- a/sdk/src/binary/messages.rs
+++ b/sdk/src/binary/messages.rs
@@ -1,15 +1,13 @@
use crate::binary::binary_client::BinaryClient;
-use crate::binary::{fail_if_not_authenticated, mapper};
+use crate::binary::fail_if_not_authenticated;
use crate::client::MessageClient;
use crate::command::{POLL_MESSAGES_CODE, SEND_MESSAGES_CODE};
use crate::consumer::Consumer;
use crate::error::IggyError;
use crate::identifier::Identifier;
-use crate::messages::flush_unsaved_buffer::FlushUnsavedBuffer;
-use crate::messages::poll_messages::PollingStrategy;
-use crate::messages::send_messages::{Message, Partitioning};
-use crate::messages::{poll_messages, send_messages};
-use crate::models::messages::PolledMessages;
+use crate::messages::{FlushUnsavedBuffer, PollingStrategy};
+use crate::models::messaging::IggyMessage;
+use crate::prelude::{BytesSerializable, Partitioning, PollMessages, PolledMessages, SendMessages};
#[async_trait::async_trait]
impl<B: BinaryClient> MessageClient for B {
@@ -27,7 +25,7 @@ impl<B: BinaryClient> MessageClient for B {
let response = self
.send_raw_with_response(
POLL_MESSAGES_CODE,
- poll_messages::as_bytes(
+ PollMessages::as_bytes(
stream_id,
topic_id,
partition_id,
@@ -38,7 +36,7 @@ impl<B: BinaryClient> MessageClient for B {
),
)
.await?;
- mapper::map_polled_messages(response)
+ PolledMessages::from_bytes(response)
}
async fn send_messages(
@@ -46,12 +44,12 @@ impl<B: BinaryClient> MessageClient for B {
stream_id: &Identifier,
topic_id: &Identifier,
partitioning: &Partitioning,
- messages: &mut [Message],
+ messages: &mut [IggyMessage],
) -> Result<(), IggyError> {
fail_if_not_authenticated(self).await?;
self.send_raw_with_response(
SEND_MESSAGES_CODE,
- send_messages::as_bytes(stream_id, topic_id, partitioning, messages),
+ SendMessages::as_bytes(stream_id, topic_id, partitioning, messages),
)
.await?;
Ok(())
diff --git a/sdk/src/bytes_serializable.rs b/sdk/src/bytes_serializable.rs
index 60e1721f..e3158721 100644
--- a/sdk/src/bytes_serializable.rs
+++ b/sdk/src/bytes_serializable.rs
@@ -1,6 +1,5 @@
-use bytes::Bytes;
-
use crate::error::IggyError;
+use bytes::{Bytes, BytesMut};
/// The trait represents the logic responsible for serializing and deserializing the struct to and from bytes.
pub trait BytesSerializable {
@@ -11,4 +10,14 @@ pub trait BytesSerializable {
fn from_bytes(bytes: Bytes) -> Result<Self, IggyError>
where
Self: Sized;
+
+ /// Write the struct to a buffer.
+ fn write_to_buffer(&self, _buf: &mut BytesMut) {
+ unimplemented!();
+ }
+
+ /// Get the byte-size of the struct.
+ fn get_buffer_size(&self) -> u32 {
+ unimplemented!();
+ }
}
diff --git a/sdk/src/cli/message/poll_messages.rs b/sdk/src/cli/message/poll_messages.rs
index 4b978d6d..5577154a 100644
--- a/sdk/src/cli/message/poll_messages.rs
+++ b/sdk/src/cli/message/poll_messages.rs
@@ -3,18 +3,15 @@ use crate::cli_command::{CliCommand, PRINT_TARGET};
use crate::client::Client;
use crate::consumer::Consumer;
use crate::identifier::Identifier;
-use crate::messages::poll_messages::{PollMessages, PollingStrategy};
-use crate::messages::send_messages::Message;
-use crate::models::header::{HeaderKey, HeaderKind};
-use crate::models::messages::PolledMessages;
-use crate::utils::sizeable::Sizeable;
+use crate::messages::{PollMessages, PollingStrategy};
+use crate::models::messaging::HeaderKind;
+use crate::prelude::{HeaderKey, HeaderValue, IggyMessage, Sizeable};
use crate::utils::timestamp::IggyTimestamp;
use crate::utils::{byte_size::IggyByteSize, duration::IggyDuration};
use anyhow::Context;
use async_trait::async_trait;
use comfy_table::{Cell, CellAlignment, Row, Table};
-use std::collections::HashSet;
-use std::mem::size_of_val;
+use std::collections::{HashMap, HashSet};
use tokio::io::AsyncWriteExt;
use tracing::{event, Level};
@@ -64,19 +61,24 @@ impl PollMessagesCmd {
fn create_message_header_keys(
&self,
- polled_messages: &PolledMessages,
+ polled_messages: &[IggyMessage],
) -> HashSet<(HeaderKey, HeaderKind)> {
if !self.show_headers {
return HashSet::new();
}
polled_messages
- .messages
.iter()
- .flat_map(|m| match m.headers.as_ref() {
- Some(h) => h
- .iter()
- .map(|(k, v)| (k.clone(), v.kind))
- .collect::<Vec<_>>(),
+ .flat_map(|m| match m.user_headers.as_ref() {
+ Some(h) => match HashMap::<HeaderKey, HeaderValue>::from_bytes(h.clone()) {
+ Ok(headers) => headers
+ .iter()
+ .map(|(k, v)| (k.clone(), v.kind))
+ .collect::<Vec<_>>(),
+ Err(e) => {
+ tracing::error!("Failed to parse user headers, error: {e}");
+ vec![]
+ }
+ },
None => vec![],
})
.collect::<HashSet<_>>()
@@ -102,17 +104,17 @@ impl PollMessagesCmd {
}
fn create_table_content(
- polled_messages: &PolledMessages,
+ polled_messages: &[IggyMessage],
message_header_keys: &HashSet<(HeaderKey, HeaderKind)>,
) -> Vec<Row> {
polled_messages
- .messages
.iter()
.map(|message| {
let mut row = vec![
- format!("{}", message.offset),
- IggyTimestamp::from(message.timestamp).to_local_string("%Y-%m-%d %H:%M:%S%.6f"),
- format!("{}", message.id),
+ format!("{}", message.header.offset),
+ IggyTimestamp::from(message.header.timestamp)
+ .to_local_string("%Y-%m-%d %H:%M:%S%.6f"),
+ format!("{}", message.header.id),
format!("{}", message.payload.len()),
String::from_utf8_lossy(&message.payload).to_string(),
];
@@ -121,7 +123,8 @@ impl PollMessagesCmd {
.iter()
.map(|(key, kind)| {
message
- .headers
+ .user_headers_map()
+ .expect("Failed to parse user headers")
.as_ref()
.map(|h| {
h.get(key)
@@ -150,7 +153,7 @@ impl CliCommand for PollMessagesCmd {
async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> {
let start = std::time::Instant::now();
- let messages = client
+ let polled_messages = client
.poll_messages(
&self.poll_messages.stream_id,
&self.poll_messages.topic_id,
@@ -173,18 +176,18 @@ impl CliCommand for PollMessagesCmd {
"Polled messages from topic with ID: {} and stream with ID: {} (from partition with ID: {})",
self.poll_messages.topic_id,
self.poll_messages.stream_id,
- messages.partition_id,
+ polled_messages.partition_id,
);
let polled_size = IggyByteSize::from(
- messages
+ polled_messages
.messages
.iter()
- .map(|m| size_of_val(m) as u64)
+ .map(|m| m.get_size_bytes().as_bytes_u64())
.sum::<u64>(),
);
- let message_count_message = match messages.messages.len() {
+ let message_count_message = match polled_messages.messages.len() {
1 => "1 message".into(),
count => format!("{} messages", count),
};
@@ -201,15 +204,10 @@ impl CliCommand for PollMessagesCmd {
.await
.with_context(|| format!("Problem opening file for writing: {output_file}"))?;
- for message in messages.messages.iter() {
- let message = Message::new(
- Some(message.id),
- message.payload.clone(),
- message.headers.clone(),
- );
- saved_size += message.get_size_bytes();
-
- file.write_all(&message.to_bytes())
+ for message in polled_messages.messages.iter() {
+ let message = message.to_bytes();
+ saved_size += IggyByteSize::from(message.len() as u64);
+ file.write_all(&message)
.await
.with_context(|| format!("Problem writing message to file: {output_file}"))?;
}
@@ -217,11 +215,12 @@ impl CliCommand for PollMessagesCmd {
let saved_size_str = saved_size.as_human_string();
event!(target: PRINT_TARGET, Level::INFO, "Stored {message_count_message} of total size {saved_size_str} to {output_file} binary file");
} else {
- let message_header_keys = self.create_message_header_keys(&messages);
+ let message_header_keys = self.create_message_header_keys(&polled_messages.messages);
let mut table = Table::new();
let table_header = Self::create_table_header(&message_header_keys);
- let table_content = Self::create_table_content(&messages, &message_header_keys);
+ let table_content =
+ Self::create_table_content(&polled_messages.messages, &message_header_keys);
table.set_header(table_header);
table.add_rows(table_content);
diff --git a/sdk/src/cli/message/send_messages.rs b/sdk/src/cli/message/send_messages.rs
index 68e4d192..e6f08140 100644
--- a/sdk/src/cli/message/send_messages.rs
+++ b/sdk/src/cli/message/send_messages.rs
@@ -2,8 +2,8 @@ use crate::bytes_serializable::BytesSerializable;
use crate::cli_command::{CliCommand, PRINT_TARGET};
use crate::client::Client;
use crate::identifier::Identifier;
-use crate::messages::send_messages::{Message, Partitioning};
-use crate::models::header::{HeaderKey, HeaderValue};
+use crate::prelude::{HeaderKey, IggyMessage};
+use crate::prelude::{HeaderValue, Partitioning};
use crate::utils::sizeable::Sizeable;
use anyhow::Context;
use async_trait::async_trait;
@@ -95,13 +95,13 @@ impl CliCommand for SendMessagesCmd {
"Read {} bytes from {} file", buffer.len(), input_file,
);
- let mut messages: Vec<Message> = Vec::new();
+ let mut messages: Vec<IggyMessage> = Vec::new();
let mut bytes_read = 0usize;
let all_messages_bytes: Bytes = buffer.into();
while bytes_read < all_messages_bytes.len() {
let message_bytes = all_messages_bytes.slice(bytes_read..);
- let message = Message::from_bytes(message_bytes);
+ let message = IggyMessage::from_bytes(message_bytes);
match message {
Ok(message) => {
let message_size = message.get_size_bytes().as_bytes_usize();
@@ -122,18 +122,31 @@ impl CliCommand for SendMessagesCmd {
messages
} else {
+ let headers = self.get_headers();
match &self.messages {
Some(messages) => messages
.iter()
- .map(|s| Message::new(None, s.clone().into(), self.get_headers()))
- .collect::<Vec<_>>(),
+ .map(|s| {
+ IggyMessage::builder()
+ .payload(Bytes::from(s.clone()))
+ .headers(headers.clone().unwrap_or_default())
+ .build()
+ })
+ .collect::<Result<Vec<_>, _>>()
+ .with_context(|| "Failed to create messages from provided strings")?,
None => {
let input = self.read_message_from_stdin()?;
input
.lines()
- .map(|m| Message::new(None, String::from(m).into(), self.get_headers()))
- .collect()
+ .map(|m| {
+ IggyMessage::builder()
+ .payload(m.to_owned())
+ .headers(headers.clone().unwrap_or_default())
+ .build()
+ })
+ .collect::<Result<Vec<_>, _>>()
+ .with_context(|| "Failed to create messages from stdin")?
}
}
};
diff --git a/sdk/src/client.rs b/sdk/src/client.rs
index f08ce98e..87a45ee3 100644
--- a/sdk/src/client.rs
+++ b/sdk/src/client.rs
@@ -3,13 +3,10 @@ use crate::consumer::Consumer;
use crate::diagnostic::DiagnosticEvent;
use crate::error::IggyError;
use crate::identifier::Identifier;
-use crate::messages::poll_messages::PollingStrategy;
-use crate::messages::send_messages::{Message, Partitioning};
use crate::models::client_info::{ClientInfo, ClientInfoDetails};
use crate::models::consumer_group::{ConsumerGroup, ConsumerGroupDetails};
use crate::models::consumer_offset_info::ConsumerOffsetInfo;
use crate::models::identity_info::IdentityInfo;
-use crate::models::messages::PolledMessages;
use crate::models::permissions::Permissions;
use crate::models::personal_access_token::{PersonalAccessTokenInfo, RawPersonalAccessToken};
use crate::models::snapshot::Snapshot;
@@ -18,6 +15,7 @@ use crate::models::stream::{Stream, StreamDetails};
use crate::models::topic::{Topic, TopicDetails};
use crate::models::user_info::{UserInfo, UserInfoDetails};
use crate::models::user_status::UserStatus;
+use crate::prelude::*;
use crate::snapshot::{SnapshotCompression, SystemSnapshotType};
use crate::tcp::config::{TcpClientConfig, TcpClientReconnectionConfig};
use crate::utils::duration::IggyDuration;
@@ -330,6 +328,7 @@ pub trait MessageClient {
count: u32,
auto_commit: bool,
) -> Result<PolledMessages, IggyError>;
+
/// Send messages using specified partitioning strategy to the given stream and topic by unique IDs or names.
///
/// Authentication is required, and the permission to send the messages.
@@ -338,8 +337,9 @@ pub trait MessageClient {
stream_id: &Identifier,
topic_id: &Identifier,
partitioning: &Partitioning,
- messages: &mut [Message],
+ messages: &mut [IggyMessage],
) -> Result<(), IggyError>;
+
/// Force flush of the `unsaved_messages` buffer to disk, optionally fsyncing the data.
#[allow(clippy::too_many_arguments)]
async fn flush_unsaved_buffer(
diff --git a/sdk/src/clients/client.rs b/sdk/src/clients/client.rs
index d95a63f8..176d6b8d 100644
--- a/sdk/src/clients/client.rs
+++ b/sdk/src/clients/client.rs
@@ -12,13 +12,12 @@ use crate::error::IggyError;
use crate::identifier::Identifier;
use crate::locking::IggySharedMut;
use crate::locking::IggySharedMutFn;
-use crate::messages::poll_messages::PollingStrategy;
-use crate::messages::send_messages::{Message, Partitioning};
+use crate::messages::PollingStrategy;
use crate::models::client_info::{ClientInfo, ClientInfoDetails};
use crate::models::consumer_group::{ConsumerGroup, ConsumerGroupDetails};
use crate::models::consumer_offset_info::ConsumerOffsetInfo;
use crate::models::identity_info::IdentityInfo;
-use crate::models::messages::PolledMessages;
+use crate::models::messaging::IggyMessage;
use crate::models::permissions::Permissions;
use crate::models::personal_access_token::{PersonalAccessTokenInfo, RawPersonalAccessToken};
use crate::models::snapshot::Snapshot;
@@ -28,9 +27,9 @@ use crate::models::topic::{Topic, TopicDetails};
use crate::models::user_info::{UserInfo, UserInfoDetails};
use crate::models::user_status::UserStatus;
use crate::partitioner::Partitioner;
+use crate::prelude::{Partitioning, PolledMessages};
use crate::snapshot::{SnapshotCompression, SystemSnapshotType};
use crate::tcp::client::TcpClient;
-use crate::utils::byte_size::IggyByteSize;
use crate::utils::crypto::EncryptorKind;
use crate::utils::duration::IggyDuration;
use crate::utils::expiry::IggyExpiry;
@@ -567,7 +566,7 @@ impl MessageClient for IggyClient {
for message in &mut polled_messages.messages {
let payload = encryptor.decrypt(&message.payload)?;
message.payload = Bytes::from(payload);
- message.length = IggyByteSize::from(message.payload.len() as u64);
+ message.header.payload_length = message.payload.len() as u32;
}
}
@@ -579,7 +578,7 @@ impl MessageClient for IggyClient {
stream_id: &Identifier,
topic_id: &Identifier,
partitioning: &Partitioning,
- messages: &mut [Message],
+ messages: &mut [IggyMessage],
) -> Result<(), IggyError> {
if messages.is_empty() {
return Err(IggyError::InvalidMessagesCount);
@@ -588,7 +587,7 @@ impl MessageClient for IggyClient {
if let Some(encryptor) = &self.encryptor {
for message in &mut *messages {
message.payload = Bytes::from(encryptor.encrypt(&message.payload)?);
- message.length = message.payload.len() as u32;
+ message.header.payload_length = message.payload.len() as u32;
}
}
diff --git a/sdk/src/clients/consumer.rs b/sdk/src/clients/consumer.rs
index 85fdf457..9a621030 100644
--- a/sdk/src/clients/consumer.rs
+++ b/sdk/src/clients/consumer.rs
@@ -4,9 +4,8 @@ use crate::diagnostic::DiagnosticEvent;
use crate::error::IggyError;
use crate::identifier::{IdKind, Identifier};
use crate::locking::{IggySharedMut, IggySharedMutFn};
-use crate::messages::poll_messages::{PollingKind, PollingStrategy};
-use crate::models::messages::{PolledMessage, PolledMessages};
-use crate::utils::byte_size::IggyByteSize;
+use crate::messages::PollingStrategy;
+use crate::prelude::{IggyMessage, PolledMessages, PollingKind};
use crate::utils::crypto::EncryptorKind;
use crate::utils::duration::IggyDuration;
use crate::utils::timestamp::IggyTimestamp;
@@ -25,8 +24,6 @@ use tokio::time;
use tokio::time::sleep;
use tracing::{error, info, trace, warn};
-const EMPTY_MESSAGES: Vec<PolledMessage> = Vec::new();
-
const ORDERING: std::sync::atomic::Ordering = std::sync::atomic::Ordering::SeqCst;
type PollMessagesFuture = Pin<Box<dyn Future<Output = Result<PolledMessages, IggyError>>>>;
@@ -102,7 +99,7 @@ pub struct IggyConsumer {
last_consumed_offsets: Arc<DashMap<u32, AtomicU64>>,
current_offsets: Arc<DashMap<u32, AtomicU64>>,
poll_future: Option<PollMessagesFuture>,
- buffered_messages: VecDeque<PolledMessage>,
+ buffered_messages: VecDeque<IggyMessage>,
encryptor: Option<Arc<EncryptorKind>>,
store_offset_sender: flume::Sender<(u32, u64)>,
store_offset_after_each_message: bool,
@@ -632,13 +629,9 @@ impl IggyConsumer {
if !allow_replay && has_consumed_offset {
polled_messages
.messages
- .retain(|message| message.offset > consumed_offset);
+ .retain(|message| message.header.offset > consumed_offset);
if polled_messages.messages.is_empty() {
- return Ok(PolledMessages {
- messages: EMPTY_MESSAGES,
- current_offset: polled_messages.current_offset,
- partition_id,
- });
+ return Ok(PolledMessages::empty());
}
}
@@ -690,9 +683,10 @@ impl IggyConsumer {
}
return Ok(PolledMessages {
- messages: EMPTY_MESSAGES,
+ messages: vec![],
current_offset: polled_messages.current_offset,
partition_id,
+ count: 0,
});
}
@@ -788,13 +782,13 @@ impl IggyConsumer {
}
pub struct ReceivedMessage {
- pub message: PolledMessage,
+ pub message: IggyMessage,
pub current_offset: u64,
pub partition_id: u32,
}
impl ReceivedMessage {
- pub fn new(message: PolledMessage, current_offset: u64, partition_id: u32) -> Self {
+ pub fn new(message: IggyMessage, current_offset: u64, partition_id: u32) -> Self {
Self {
message,
current_offset,
@@ -813,27 +807,27 @@ impl Stream for IggyConsumer {
if let Some(last_consumed_offset_entry) =
self.last_consumed_offsets.get(&partition_id)
{
- last_consumed_offset_entry.store(message.offset, ORDERING);
+ last_consumed_offset_entry.store(message.header.offset, ORDERING);
} else {
self.last_consumed_offsets
- .insert(partition_id, AtomicU64::new(message.offset));
+ .insert(partition_id, AtomicU64::new(message.header.offset));
}
if (self.store_after_every_nth_message > 0
- && message.offset % self.store_after_every_nth_message == 0)
+ && message.header.offset % self.store_after_every_nth_message == 0)
|| self.store_offset_after_each_message
{
- self.send_store_offset(partition_id, message.offset);
+ self.send_store_offset(partition_id, message.header.offset);
}
}
if self.buffered_messages.is_empty() {
if self.polling_strategy.kind == PollingKind::Offset {
- self.polling_strategy = PollingStrategy::offset(message.offset + 1);
+ self.polling_strategy = PollingStrategy::offset(message.header.offset + 1);
}
if self.store_offset_after_all_messages {
- self.send_store_offset(partition_id, message.offset);
+ self.send_store_offset(partition_id, message.header.offset);
}
}
@@ -869,14 +863,14 @@ impl Stream for IggyConsumer {
let payload = encryptor.decrypt(&message.payload);
if payload.is_err() {
self.poll_future = None;
- error!("Failed to decrypt the message payload at offset: {}, partition ID: {}", message.offset, partition_id);
+ error!("Failed to decrypt the message payload at offset: {}, partition ID: {}", message.header.offset, partition_id);
let error = payload.unwrap_err();
return Poll::Ready(Some(Err(error)));
}
let payload = payload.unwrap();
message.payload = Bytes::from(payload);
- message.length = IggyByteSize::from(message.payload.len() as u64);
+ message.header.payload_length = message.payload.len() as u32;
}
}
@@ -894,25 +888,29 @@ impl Stream for IggyConsumer {
self.buffered_messages.extend(polled_messages.messages);
if self.polling_strategy.kind == PollingKind::Offset {
- self.polling_strategy = PollingStrategy::offset(message.offset + 1);
+ self.polling_strategy =
+ PollingStrategy::offset(message.header.offset + 1);
}
if let Some(last_consumed_offset_entry) =
self.last_consumed_offsets.get(&partition_id)
{
- last_consumed_offset_entry.store(message.offset, ORDERING);
+ last_consumed_offset_entry.store(message.header.offset, ORDERING);
} else {
self.last_consumed_offsets
- .insert(partition_id, AtomicU64::new(message.offset));
+ .insert(partition_id, AtomicU64::new(message.header.offset));
}
if (self.store_after_every_nth_message > 0
- && message.offset % self.store_after_every_nth_message == 0)
+ && message.header.offset % self.store_after_every_nth_message == 0)
|| self.store_offset_after_each_message
|| (self.store_offset_after_all_messages
&& self.buffered_messages.is_empty())
{
- self.send_store_offset(polled_messages.partition_id, message.offset);
+ self.send_store_offset(
+ polled_messages.partition_id,
+ message.header.offset,
+ );
}
self.poll_future = None;
diff --git a/sdk/src/clients/producer.rs b/sdk/src/clients/producer.rs
index 645ac430..f66de103 100644
--- a/sdk/src/clients/producer.rs
+++ b/sdk/src/clients/producer.rs
@@ -4,8 +4,9 @@ use crate::diagnostic::DiagnosticEvent;
use crate::error::IggyError;
use crate::identifier::{IdKind, Identifier};
use crate::locking::{IggySharedMut, IggySharedMutFn};
-use crate::messages::send_messages::{Message, Partitioning};
use crate::partitioner::Partitioner;
+use crate::prelude::IggyMessage;
+use crate::prelude::Partitioning;
use crate::utils::crypto::EncryptorKind;
use crate::utils::duration::IggyDuration;
use crate::utils::expiry::IggyExpiry;
@@ -212,7 +213,7 @@ impl IggyProducer {
});
}
- pub async fn send(&self, messages: Vec<Message>) -> Result<(), IggyError> {
+ pub async fn send(&self, messages: Vec<IggyMessage>) -> Result<(), IggyError> {
if messages.is_empty() {
trace!("No messages to send.");
return Ok(());
@@ -233,13 +234,13 @@ impl IggyProducer {
.await
}
- pub async fn send_one(&self, message: Message) -> Result<(), IggyError> {
+ pub async fn send_one(&self, message: IggyMessage) -> Result<(), IggyError> {
self.send(vec![message]).await
}
pub async fn send_with_partitioning(
&self,
- messages: Vec<Message>,
+ messages: Vec<IggyMessage>,
partitioning: Option<Arc<Partitioning>>,
) -> Result<(), IggyError> {
if messages.is_empty() {
@@ -266,7 +267,7 @@ impl IggyProducer {
&self,
stream: Arc<Identifier>,
topic: Arc<Identifier>,
- messages: Vec<Message>,
+ messages: Vec<IggyMessage>,
partitioning: Option<Arc<Partitioning>>,
) -> Result<(), IggyError> {
if messages.is_empty() {
@@ -288,7 +289,7 @@ impl IggyProducer {
&self,
stream: Arc<Identifier>,
topic: Arc<Identifier>,
- mut messages: Vec<Message>,
+ mut messages: Vec<IggyMessage>,
partitioning: Option<Arc<Partitioning>>,
) -> Result<(), IggyError> {
self.encrypt_messages(&mut messages)?;
@@ -324,7 +325,7 @@ impl IggyProducer {
&self,
stream: &Identifier,
topic: &Identifier,
- mut messages: Vec<Message>,
+ mut messages: Vec<IggyMessage>,
partitioning: Option<Arc<Partitioning>>,
) -> Result<(), IggyError> {
trace!("No batch size specified, sending messages immediately.");
@@ -365,11 +366,11 @@ impl IggyProducer {
sleep(Duration::from_micros(remaining)).await;
}
- fn encrypt_messages(&self, messages: &mut [Message]) -> Result<(), IggyError> {
+ fn encrypt_messages(&self, messages: &mut [IggyMessage]) -> Result<(), IggyError> {
if let Some(encryptor) = &self.encryptor {
for message in messages {
message.payload = Bytes::from(encryptor.encrypt(&message.payload)?);
- message.length = message.payload.len() as u32;
+ message.header.payload_length = message.payload.len() as u32;
}
}
Ok(())
@@ -380,7 +381,7 @@ impl IggyProducer {
stream: &Identifier,
topic: &Identifier,
partitioning: &Arc<Partitioning>,
- messages: &mut [Message],
+ messages: &mut [IggyMessage],
) -> Result<(), IggyError> {
let client = self.client.read().await;
let Some(max_retries) = self.send_retries_count else {
@@ -456,7 +457,7 @@ impl IggyProducer {
stream: &Identifier,
topic: &Identifier,
partitioning: &Arc<Partitioning>,
- messages: &mut [Message],
+ messages: &mut [IggyMessage],
timer: &mut Option<Interval>,
) -> Result<(), IggyError> {
let client = self.client.read().await;
@@ -498,7 +499,7 @@ impl IggyProducer {
&self,
stream: &Identifier,
topic: &Identifier,
- messages: &[Message],
+ messages: &[IggyMessage],
partitioning: Option<Arc<Partitioning>>,
) -> Result<Arc<Partitioning>, IggyError> {
if let Some(partitioner) = &self.partitioner {
diff --git a/sdk/src/consumer_ext/consumer_message_ext.rs b/sdk/src/consumer_ext/consumer_message_ext.rs
index 0c21db85..cab9e540 100644
--- a/sdk/src/consumer_ext/consumer_message_ext.rs
+++ b/sdk/src/consumer_ext/consumer_message_ext.rs
@@ -70,7 +70,7 @@ impl IggyConsumerMessageExt for IggyConsumer {
Some(Ok(received_message)) => {
let partition_id = received_message.partition_id;
let current_offset = received_message.current_offset;
- let message_offset = received_message.message.offset;
+ let message_offset = received_message.message.header.offset;
if let Err(err) = message_consumer.consume(received_message).await {
error!("Error while handling message at offset: {message_offset}/{current_offset}, partition: {partition_id} for consumer: {name} on topic: {topic} and stream: {stream} due to error: {err}",
name = self.name(), topic = self.topic(), stream = self.stream());
diff --git a/sdk/src/consumer_ext/mod.rs b/sdk/src/consumer_ext/mod.rs
index c7e686f6..ca6a9548 100644
--- a/sdk/src/consumer_ext/mod.rs
+++ b/sdk/src/consumer_ext/mod.rs
@@ -1,8 +1,7 @@
mod consumer_message_ext;
mod consumer_message_trait;
-use crate::clients::consumer::ReceivedMessage;
-use crate::error::IggyError;
+use crate::{clients::consumer::ReceivedMessage, error::IggyError};
pub use consumer_message_trait::IggyConsumerMessageExt;
/// Trait for message consumer
diff --git a/sdk/src/error.rs b/sdk/src/error.rs
index 9f7f24d5..72bd11c6 100644
--- a/sdk/src/error.rs
+++ b/sdk/src/error.rs
@@ -3,7 +3,7 @@ use crate::utils::topic_size::MaxTopicSize;
use strum::{EnumDiscriminants, FromRepr, IntoStaticStr};
use thiserror::Error;
-#[derive(Debug, Error, EnumDiscriminants, IntoStaticStr, FromRepr)]
+#[derive(Clone, Debug, Error, EnumDiscriminants, IntoStaticStr, FromRepr)]
#[repr(u32)]
#[strum(serialize_all = "snake_case")]
#[strum_discriminants(
@@ -328,6 +328,18 @@ pub enum IggyError {
InvalidKeyValueLength = 4028,
#[error("Command length error: {0}")]
CommandLengthError(String) = 4029,
+ #[error("Non-zero offset: {0} at index: {1}")]
+ NonZeroOffset(u64, u32) = 4030,
+ #[error("Non-zero timestamp: {0} at index: {1}")]
+ NonZeroTimestamp(u64, u32) = 4031,
+ #[error("Missing index: {0}")]
+ MissingIndex(u32) = 4032,
+ #[error("Invalid indexes byte size: {0}B, should be divisible by 16")]
+ InvalidIndexesByteSize(u32) = 4034,
+ #[error("Invalid indexes count: {0}, expected: {1}")]
+ InvalidIndexesCount(u32, u32) = 4035,
+ #[error("Invalid messages byte size: {0}B, expected: {1}B")]
+ InvalidMessagesSize(u32, u32) = 4036,
#[error("Cannot sed messages due to client disconnection")]
CannotSendMessagesDueToClientDisconnection = 4050,
#[error("Invalid offset: {0}")]
diff --git a/sdk/src/http/messages.rs b/sdk/src/http/messages.rs
index b14602e3..f6327b20 100644
--- a/sdk/src/http/messages.rs
+++ b/sdk/src/http/messages.rs
@@ -4,10 +4,9 @@ use crate::error::IggyError;
use crate::http::client::HttpClient;
use crate::http::HttpTransport;
use crate::identifier::Identifier;
-use crate::messages::flush_unsaved_buffer::FlushUnsavedBuffer;
-use crate::messages::poll_messages::{PollMessages, PollingStrategy};
-use crate::messages::send_messages::{Message, Partitioning, SendMessages};
-use crate::models::messages::PolledMessages;
+use crate::messages::{Partitioning, PollingStrategy};
+use crate::models::messaging::IggyMessagesBatch;
+use crate::prelude::{FlushUnsavedBuffer, IggyMessage, PollMessages, PolledMessages, SendMessages};
use async_trait::async_trait;
#[async_trait]
@@ -48,15 +47,16 @@ impl MessageClient for HttpClient {
stream_id: &Identifier,
topic_id: &Identifier,
partitioning: &Partitioning,
- messages: &mut [Message],
+ messages: &mut [IggyMessage],
) -> Result<(), IggyError> {
+ let batch = IggyMessagesBatch::from(&*messages);
self.post(
&get_path(&stream_id.as_cow_str(), &topic_id.as_cow_str()),
&SendMessages {
stream_id: stream_id.clone(),
topic_id: topic_id.clone(),
partitioning: partitioning.clone(),
- messages: messages.to_vec(),
+ batch,
},
)
.await?;
diff --git a/sdk/src/identifier.rs b/sdk/src/identifier.rs
index 804b28a9..e813c90d 100644
--- a/sdk/src/identifier.rs
+++ b/sdk/src/identifier.rs
@@ -164,6 +164,24 @@ impl Identifier {
value: value.as_bytes().to_vec(),
})
}
+
+ /// Creates identifier from raw bytes
+ pub fn from_raw_bytes(bytes: &[u8]) -> Result<Self, IggyError> {
+ let kind = IdKind::from_code(bytes[0])?;
+ let length = bytes[1];
+ let value = bytes[2..2 + length as usize].to_vec();
+ if value.len() != length as usize {
+ return Err(IggyError::InvalidIdentifier);
+ }
+
+ let identifier = Identifier {
+ kind,
+ length,
+ value,
+ };
+ identifier.validate()?;
+ Ok(identifier)
+ }
}
impl Sizeable for Identifier {
@@ -204,6 +222,16 @@ impl BytesSerializable for Identifier {
identifier.validate()?;
Ok(identifier)
}
+
+ fn write_to_buffer(&self, bytes: &mut BytesMut) {
+ bytes.put_u8(self.kind.as_code());
+ bytes.put_u8(self.length);
+ bytes.put_slice(&self.value);
+ }
+
+ fn get_buffer_size(&self) -> u32 {
+ 2 + self.length as u32
+ }
}
impl IdKind {
diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs
index 7790dc8e..62003368 100644
--- a/sdk/src/lib.rs
+++ b/sdk/src/lib.rs
@@ -28,6 +28,7 @@ pub mod models;
pub mod partitioner;
pub mod partitions;
pub mod personal_access_tokens;
+pub mod prelude;
pub mod quic;
pub mod snapshot;
pub mod stream_builder;
diff --git a/sdk/src/messages/flush_unsaved_buffer.rs b/sdk/src/messages/flush_unsaved_buffer.rs
index 80f69e03..3f6cfe7f 100644
--- a/sdk/src/messages/flush_unsaved_buffer.rs
+++ b/sdk/src/messages/flush_unsaved_buffer.rs
@@ -1,8 +1,3 @@
-use std::fmt::Display;
-
-use bytes::{BufMut, Bytes, BytesMut};
-use serde::{Deserialize, Serialize};
-
use crate::{
bytes_serializable::BytesSerializable,
command::{Command, FLUSH_UNSAVED_BUFFER_CODE},
@@ -10,6 +5,9 @@ use crate::{
identifier::Identifier,
validatable::Validatable,
};
+use bytes::{BufMut, Bytes, BytesMut};
+use serde::{Deserialize, Serialize};
+use std::fmt::Display;
/// `FlushUnsavedBuffer` command is used to force a flush of `unsaved_buffer` to disk for specific stream -> topic -> partition.
/// - `stream_id` - stream identifier
diff --git a/sdk/src/messages/mod.rs b/sdk/src/messages/mod.rs
index 8a588efc..ae88365d 100644
--- a/sdk/src/messages/mod.rs
+++ b/sdk/src/messages/mod.rs
@@ -1,6 +1,20 @@
-pub mod flush_unsaved_buffer;
-pub mod poll_messages;
-pub mod send_messages;
+mod flush_unsaved_buffer;
+mod partitioning;
+mod partitioning_kind;
+mod poll_messages;
+mod polled_messages;
+mod polling_kind;
+mod polling_strategy;
+mod send_messages;
-const MAX_HEADERS_SIZE: u32 = 100 * 1000;
+pub const MAX_USER_HEADERS_SIZE: u32 = 100 * 1000;
pub const MAX_PAYLOAD_SIZE: u32 = 10 * 1000 * 1000;
+
+pub use flush_unsaved_buffer::FlushUnsavedBuffer;
+pub use partitioning::Partitioning;
+pub use partitioning_kind::PartitioningKind;
+pub use poll_messages::PollMessages;
+pub use polled_messages::PolledMessages;
+pub use polling_kind::PollingKind;
+pub use polling_strategy::PollingStrategy;
+pub use send_messages::SendMessages;
diff --git a/sdk/src/messages/partitioning.rs b/sdk/src/messages/partitioning.rs
new file mode 100644
index 00000000..892095c0
--- /dev/null
+++ b/sdk/src/messages/partitioning.rs
@@ -0,0 +1,202 @@
+use super::PartitioningKind;
+use crate::{
+ error::IggyError,
+ prelude::{BytesSerializable, IggyByteSize, Sizeable},
+};
+use bytes::{BufMut, Bytes, BytesMut};
+use serde::{Deserialize, Serialize};
+use serde_with::base64::Base64;
+use serde_with::serde_as;
+use std::{
+ fmt::Display,
+ hash::{Hash, Hasher},
+};
+
+/// `Partitioning` is used to specify to which partition the messages should be sent.
+/// It has the following kinds:
+/// - `Balanced` - the partition ID is calculated by the server using the round-robin algorithm.
+/// - `PartitionId` - the partition ID is provided by the client.
+/// - `MessagesKey` - the partition ID is calculated by the server using the hash of the provided messages key.
+#[serde_as]
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct Partitioning {
+ /// The kind of partitioning.
+ pub kind: PartitioningKind,
+ #[serde(skip)]
+ /// The length of the value payload.
+ pub length: u8,
+ #[serde_as(as = "Base64")]
+ /// The binary value payload.
+ pub value: Vec<u8>,
+}
+
+impl Display for Partitioning {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self.kind {
+ PartitioningKind::Balanced => write!(f, "{}|0", self.kind),
+ PartitioningKind::PartitionId => write!(
+ f,
+ "{}|{}",
+ self.kind,
+ u32::from_le_bytes(self.value[..4].try_into().unwrap())
+ ),
+ PartitioningKind::MessagesKey => {
+ write!(f, "{}|{}", self.kind, String::from_utf8_lossy(&self.value))
+ }
+ }
+ }
+}
+
+impl Partitioning {
+ /// Partition the messages using the balanced round-robin algorithm on the server.
+ pub fn balanced() -> Self {
+ Partitioning {
+ kind: PartitioningKind::Balanced,
+ length: 0,
+ value: vec![],
+ }
+ }
+
+ /// Partition the messages using the provided partition ID.
+ pub fn partition_id(partition_id: u32) -> Self {
+ Partitioning {
+ kind: PartitioningKind::PartitionId,
+ length: 4,
+ value: partition_id.to_le_bytes().to_vec(),
+ }
+ }
+
+ /// Partition the messages using the provided messages key.
+ pub fn messages_key(value: &[u8]) -> Result<Self, IggyError> {
+ let length = value.len();
+ if length == 0 || length > 255 {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ Ok(Partitioning {
+ kind: PartitioningKind::MessagesKey,
+ #[allow(clippy::cast_possible_truncation)]
+ length: length as u8,
+ value: value.to_vec(),
+ })
+ }
+
+ /// Partition the messages using the provided messages key as str.
+ pub fn messages_key_str(value: &str) -> Result<Self, IggyError> {
+ Self::messages_key(value.as_bytes())
+ }
+
+ /// Partition the messages using the provided messages key as u32.
+ pub fn messages_key_u32(value: u32) -> Self {
+ Partitioning {
+ kind: PartitioningKind::MessagesKey,
+ length: 4,
+ value: value.to_le_bytes().to_vec(),
+ }
+ }
+
+ /// Partition the messages using the provided messages key as u64.
+ pub fn messages_key_u64(value: u64) -> Self {
+ Partitioning {
+ kind: PartitioningKind::MessagesKey,
+ length: 8,
+ value: value.to_le_bytes().to_vec(),
+ }
+ }
+
+ /// Partition the messages using the provided messages key as u128.
+ pub fn messages_key_u128(value: u128) -> Self {
+ Partitioning {
+ kind: PartitioningKind::MessagesKey,
+ length: 16,
+ value: value.to_le_bytes().to_vec(),
+ }
+ }
+
+ /// Create the partitioning from the provided partitioning.
+ pub fn from_partitioning(partitioning: &Partitioning) -> Self {
+ Partitioning {
+ kind: partitioning.kind,
+ length: partitioning.length,
+ value: partitioning.value.clone(),
+ }
+ }
+
+ /// Create the partitioning from BytesMut.
+ pub fn from_raw_bytes(bytes: &[u8]) -> Result<Self, IggyError> {
+ let kind = PartitioningKind::from_code(bytes[0])?;
+ let length = bytes[1];
+ let value = bytes[2..2 + length as usize].to_vec();
+ if value.len() != length as usize {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ Ok(Partitioning {
+ kind,
+ length,
+ value,
+ })
+ }
+}
+
+impl Hash for Partitioning {
+ fn hash<H: Hasher>(&self, state: &mut H) {
+ self.kind.hash(state);
+ self.length.hash(state);
+ self.value.hash(state);
+ }
+}
+
+impl Default for Partitioning {
+ fn default() -> Self {
+ Partitioning::balanced()
+ }
+}
+
+impl Sizeable for Partitioning {
+ fn get_size_bytes(&self) -> IggyByteSize {
+ IggyByteSize::from(u64::from(self.length) + 2)
+ }
+}
+
+impl BytesSerializable for Partitioning {
+ fn to_bytes(&self) -> Bytes {
+ let mut bytes = BytesMut::with_capacity(2 + self.length as usize);
+ bytes.put_u8(self.kind.as_code());
+ bytes.put_u8(self.length);
+ bytes.put_slice(&self.value);
+ bytes.freeze()
+ }
+
+ fn from_bytes(bytes: Bytes) -> Result<Self, IggyError>
+ where
+ Self: Sized,
+ {
+ if bytes.len() < 2 {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let kind = PartitioningKind::from_code(bytes[0])?;
+ let length = bytes[1];
+ let value = bytes[2..2 + length as usize].to_vec();
+ if value.len() != length as usize {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ Ok(Partitioning {
+ kind,
+ length,
+ value,
+ })
+ }
+
+ fn write_to_buffer(&self, bytes: &mut BytesMut) {
+ bytes.put_u8(self.kind.as_code());
+ bytes.put_u8(self.length);
+ bytes.put_slice(&self.value);
+ }
+
+ fn get_buffer_size(&self) -> u32 {
+ 2 + self.length as u32
+ }
+}
diff --git a/sdk/src/messages/partitioning_kind.rs b/sdk/src/messages/partitioning_kind.rs
new file mode 100644
index 00000000..fd7dbb56
--- /dev/null
+++ b/sdk/src/messages/partitioning_kind.rs
@@ -0,0 +1,56 @@
+use crate::error::IggyError;
+use serde::{Deserialize, Serialize};
+use std::{
+ fmt::Display,
+ hash::{Hash, Hasher},
+};
+
+/// `PartitioningKind` is an enum which specifies the kind of partitioning and is used by `Partitioning`.
+#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Default, Copy, Clone)]
+#[serde(rename_all = "snake_case")]
+pub enum PartitioningKind {
+ /// The partition ID is calculated by the server using the round-robin algorithm.
+ #[default]
+ Balanced,
+ /// The partition ID is provided by the client.
+ PartitionId,
+ /// The partition ID is calculated by the server using the hash of the provided messages key.
+ MessagesKey,
+}
+
+impl Hash for PartitioningKind {
+ fn hash<H: Hasher>(&self, state: &mut H) {
+ self.as_code().hash(state);
+ }
+}
+
+impl PartitioningKind {
+ /// Get the code of the partitioning kind.
+ pub fn as_code(&self) -> u8 {
+ match self {
+ PartitioningKind::Balanced => 1,
+ PartitioningKind::PartitionId => 2,
+ PartitioningKind::MessagesKey => 3,
+ }
+ }
+
+ /// Get the partitioning kind from the provided code.
+ pub fn from_code(code: u8) -> Result<Self, IggyError> {
+ match code {
+ 1 => Ok(PartitioningKind::Balanced),
+ 2 => Ok(PartitioningKind::PartitionId),
+ 3 => Ok(PartitioningKind::MessagesKey),
+ _ => Err(IggyError::InvalidCommand),
+ }
+ }
+}
+
+impl Display for PartitioningKind {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ PartitioningKind::Balanced => write!(f, "balanced"),
+ PartitioningKind::PartitionId => write!(f, "partition_id"),
+ PartitioningKind::MessagesKey => write!(f, "messages_key"),
+ }
+ }
+}
diff --git a/sdk/src/messages/poll_messages.rs b/sdk/src/messages/poll_messages.rs
index 901027f1..ded2d919 100644
--- a/sdk/src/messages/poll_messages.rs
+++ b/sdk/src/messages/poll_messages.rs
@@ -1,16 +1,18 @@
+use super::polling_kind::PollingKind;
+use super::polling_strategy::PollingStrategy;
use crate::bytes_serializable::BytesSerializable;
use crate::command::{Command, POLL_MESSAGES_CODE};
use crate::consumer::{Consumer, ConsumerKind};
use crate::error::IggyError;
use crate::identifier::Identifier;
use crate::utils::sizeable::Sizeable;
-use crate::utils::timestamp::IggyTimestamp;
use crate::validatable::Validatable;
use bytes::{BufMut, Bytes, BytesMut};
use serde::{Deserialize, Serialize};
-use serde_with::{serde_as, DisplayFromStr};
use std::fmt::Display;
-use std::str::FromStr;
+
+pub const DEFAULT_PARTITION_ID: u32 = 1;
+pub const DEFAULT_NUMBER_OF_MESSAGES_TO_POLL: u32 = 10;
/// `PollMessages` command is used to poll messages from a topic in a stream.
/// It has additional payload:
@@ -33,54 +35,65 @@ pub struct PollMessages {
#[serde(skip)]
pub topic_id: Identifier,
/// Partition ID from which messages will be polled. Has to be specified for the regular consumer. For consumer group it is ignored (use `None`).
- #[serde(default = "default_partition_id")]
+ #[serde(default = "PollMessages::default_partition_id")]
pub partition_id: Option<u32>,
/// Polling strategy which specifies from where to start polling messages.
- #[serde(default = "default_strategy", flatten)]
+ #[serde(default = "PollingStrategy::default", flatten)]
pub strategy: PollingStrategy,
- #[serde(default = "default_count")]
/// Number of messages to poll.
+ #[serde(default = "PollMessages::default_number_of_messages_to_poll")]
pub count: u32,
- #[serde(default)]
/// Whether to commit offset on the server automatically after polling the messages.
+ #[serde(default)]
pub auto_commit: bool,
}
-/// `PollingStrategy` specifies from where to start polling messages.
-/// It has the following kinds:
-/// - `Offset` - start polling from the specified offset.
-/// - `Timestamp` - start polling from the specified timestamp.
-/// - `First` - start polling from the first message in the partition.
-/// - `Last` - start polling from the last message in the partition.
-/// - `Next` - start polling from the next message after the last polled message based on the stored consumer offset.
-#[serde_as]
-#[derive(Debug, Serialize, Deserialize, PartialEq, Copy, Clone)]
-pub struct PollingStrategy {
- /// Kind of the polling strategy.
- #[serde_as(as = "DisplayFromStr")]
- #[serde(default = "default_kind")]
- pub kind: PollingKind,
- /// Value of the polling strategy.
- #[serde_as(as = "DisplayFromStr")]
- #[serde(default = "default_value")]
- pub value: u64,
-}
+impl PollMessages {
+ pub(crate) fn as_bytes(
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: Option<u32>,
+ consumer: &Consumer,
+ strategy: &PollingStrategy,
+ count: u32,
+ auto_commit: bool,
+ ) -> Bytes {
+ let consumer_bytes = consumer.to_bytes();
+ let stream_id_bytes = stream_id.to_bytes();
+ let topic_id_bytes = topic_id.to_bytes();
+ let strategy_bytes = strategy.to_bytes();
+ let mut bytes = BytesMut::with_capacity(
+ 9 + consumer_bytes.len()
+ + stream_id_bytes.len()
+ + topic_id_bytes.len()
+ + strategy_bytes.len(),
+ );
+ bytes.put_slice(&consumer_bytes);
+ bytes.put_slice(&stream_id_bytes);
+ bytes.put_slice(&topic_id_bytes);
+ if let Some(partition_id) = partition_id {
+ bytes.put_u32_le(partition_id);
+ } else {
+ bytes.put_u32_le(0);
+ }
+ bytes.put_slice(&strategy_bytes);
+ bytes.put_u32_le(count);
+ if auto_commit {
+ bytes.put_u8(1);
+ } else {
+ bytes.put_u8(0);
+ }
-/// `PollingKind` is an enum which specifies from where to start polling messages and is used by `PollingStrategy`.
-#[derive(Debug, Serialize, Deserialize, PartialEq, Default, Copy, Clone)]
-#[serde(rename_all = "snake_case")]
-pub enum PollingKind {
- #[default]
- /// Start polling from the specified offset.
- Offset,
- /// Start polling from the specified timestamp.
- Timestamp,
- /// Start polling from the first message in the partition.
- First,
- /// Start polling from the last message in the partition.
- Last,
- /// Start polling from the next message after the last polled message based on the stored consumer offset. Should be used with `auto_commit` set to `true`.
- Next,
+ bytes.freeze()
+ }
+
+ pub fn default_number_of_messages_to_poll() -> u32 {
+ DEFAULT_NUMBER_OF_MESSAGES_TO_POLL
+ }
+
+ pub fn default_partition_id() -> Option<u32> {
+ Some(DEFAULT_PARTITION_ID)
+ }
}
impl Default for PollMessages {
@@ -89,158 +102,29 @@ impl Default for PollMessages {
consumer: Consumer::default(),
stream_id: Identifier::numeric(1).unwrap(),
topic_id: Identifier::numeric(1).unwrap(),
- partition_id: default_partition_id(),
- strategy: default_strategy(),
- count: default_count(),
+ partition_id: PollMessages::default_partition_id(),
+ strategy: PollingStrategy::default(),
+ count: PollMessages::default_number_of_messages_to_poll(),
auto_commit: false,
}
}
}
-impl Default for PollingStrategy {
- fn default() -> Self {
- Self {
- kind: PollingKind::Offset,
- value: 0,
- }
- }
-}
-
impl Command for PollMessages {
fn code(&self) -> u32 {
POLL_MESSAGES_CODE
}
}
-fn default_partition_id() -> Option<u32> {
- Some(1)
-}
-
-fn default_kind() -> PollingKind {
- PollingKind::Offset
-}
-
-fn default_value() -> u64 {
- 0
-}
-
-fn default_strategy() -> PollingStrategy {
- PollingStrategy::default()
-}
-
-fn default_count() -> u32 {
- 10
-}
-
impl Validatable<IggyError> for PollMessages {
fn validate(&self) -> Result<(), IggyError> {
Ok(())
}
}
-impl PollingStrategy {
- /// Poll messages from the specified offset.
- pub fn offset(value: u64) -> Self {
- Self {
- kind: PollingKind::Offset,
- value,
- }
- }
-
- /// Poll messages from the specified timestamp.
- pub fn timestamp(value: IggyTimestamp) -> Self {
- Self {
- kind: PollingKind::Timestamp,
- value: value.into(),
- }
- }
-
- /// Poll messages from the first message in the partition.
- pub fn first() -> Self {
- Self {
- kind: PollingKind::First,
- value: 0,
- }
- }
-
- /// Poll messages from the last message in the partition.
- pub fn last() -> Self {
- Self {
- kind: PollingKind::Last,
- value: 0,
- }
- }
-
- /// Poll messages from the next message after the last polled message based on the stored consumer offset. Should be used with `auto_commit` set to `true`.
- pub fn next() -> Self {
- Self {
- kind: PollingKind::Next,
- value: 0,
- }
- }
-
- /// Change the value of the polling strategy, affects only `Offset` and `Timestamp` kinds.
- pub fn set_value(&mut self, value: u64) {
- if self.kind == PollingKind::Offset || self.kind == PollingKind::Timestamp {
- self.value = value;
- }
- }
-}
-
-impl PollingKind {
- /// Returns code of the polling kind.
- pub fn as_code(&self) -> u8 {
- match self {
- PollingKind::Offset => 1,
- PollingKind::Timestamp => 2,
- PollingKind::First => 3,
- PollingKind::Last => 4,
- PollingKind::Next => 5,
- }
- }
-
- /// Returns polling kind from the specified code.
- pub fn from_code(code: u8) -> Result<Self, IggyError> {
- match code {
- 1 => Ok(PollingKind::Offset),
- 2 => Ok(PollingKind::Timestamp),
- 3 => Ok(PollingKind::First),
- 4 => Ok(PollingKind::Last),
- 5 => Ok(PollingKind::Next),
- _ => Err(IggyError::InvalidCommand),
- }
- }
-}
-
-impl FromStr for PollingKind {
- type Err = IggyError;
- fn from_str(input: &str) -> Result<Self, Self::Err> {
- match input {
- "o" | "offset" => Ok(PollingKind::Offset),
- "t" | "timestamp" => Ok(PollingKind::Timestamp),
- "f" | "first" => Ok(PollingKind::First),
- "l" | "last" => Ok(PollingKind::Last),
- "n" | "next" => Ok(PollingKind::Next),
- _ => Err(IggyError::InvalidCommand),
- }
- }
-}
-
-impl Display for PollingKind {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self {
- PollingKind::Offset => write!(f, "offset"),
- PollingKind::Timestamp => write!(f, "timestamp"),
- PollingKind::First => write!(f, "first"),
- PollingKind::Last => write!(f, "last"),
- PollingKind::Next => write!(f, "next"),
- }
- }
-}
-
impl BytesSerializable for PollMessages {
fn to_bytes(&self) -> Bytes {
- as_bytes(
+ PollMessages::as_bytes(
&self.stream_id,
&self.topic_id,
self.partition_id,
@@ -308,45 +192,6 @@ impl BytesSerializable for PollMessages {
}
}
-// This method is used by the new version of `IggyClient` to serialize `PollMessages` without cloning the args.
-pub(crate) fn as_bytes(
- stream_id: &Identifier,
- topic_id: &Identifier,
- partition_id: Option<u32>,
- consumer: &Consumer,
- strategy: &PollingStrategy,
- count: u32,
- auto_commit: bool,
-) -> Bytes {
- let consumer_bytes = consumer.to_bytes();
- let stream_id_bytes = stream_id.to_bytes();
- let topic_id_bytes = topic_id.to_bytes();
- let strategy_bytes = strategy.to_bytes();
- let mut bytes = BytesMut::with_capacity(
- 9 + consumer_bytes.len()
- + stream_id_bytes.len()
- + topic_id_bytes.len()
- + strategy_bytes.len(),
- );
- bytes.put_slice(&consumer_bytes);
- bytes.put_slice(&stream_id_bytes);
- bytes.put_slice(&topic_id_bytes);
- if let Some(partition_id) = partition_id {
- bytes.put_u32_le(partition_id);
- } else {
- bytes.put_u32_le(0);
- }
- bytes.put_slice(&strategy_bytes);
- bytes.put_u32_le(count);
- if auto_commit {
- bytes.put_u8(1);
- } else {
- bytes.put_u8(0);
- }
-
- bytes.freeze()
-}
-
impl Display for PollMessages {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
@@ -363,12 +208,6 @@ impl Display for PollMessages {
}
}
-impl Display for PollingStrategy {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(f, "{}|{}", self.kind, self.value)
- }
-}
-
fn auto_commit_to_string(auto_commit: bool) -> &'static str {
if auto_commit {
"a"
@@ -377,30 +216,6 @@ fn auto_commit_to_string(auto_commit: bool) -> &'static str {
}
}
-impl BytesSerializable for PollingStrategy {
- fn to_bytes(&self) -> Bytes {
- let mut bytes = BytesMut::with_capacity(9);
- bytes.put_u8(self.kind.as_code());
- bytes.put_u64_le(self.value);
- bytes.freeze()
- }
-
- fn from_bytes(bytes: Bytes) -> Result<Self, IggyError> {
- if bytes.len() != 9 {
- return Err(IggyError::InvalidCommand);
- }
-
- let kind = PollingKind::from_code(bytes[0])?;
- let value = u64::from_le_bytes(
- bytes[1..9]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- let strategy = PollingStrategy { kind, value };
- Ok(strategy)
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/sdk/src/messages/polled_messages.rs b/sdk/src/messages/polled_messages.rs
new file mode 100644
index 00000000..4afb35a2
--- /dev/null
+++ b/sdk/src/messages/polled_messages.rs
@@ -0,0 +1,111 @@
+use crate::{
+ error::IggyError,
+ prelude::{BytesSerializable, IggyMessage, IggyMessageHeader, IGGY_MESSAGE_HEADER_SIZE},
+};
+use bytes::Bytes;
+use serde::{Deserialize, Serialize};
+use tracing::error;
+
+/// The wrapper on top of the collection of messages that are polled from the partition.
+/// It consists of the following fields:
+/// - `partition_id`: the identifier of the partition.
+/// - `current_offset`: the current offset of the partition.
+/// - `messages`: the collection of messages.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct PolledMessages {
+ /// The identifier of the partition. If it's '0', then there's no partition assigned to the consumer group member.
+ pub partition_id: u32,
+ /// The current offset of the partition.
+ pub current_offset: u64,
+ /// The count of messages.
+ pub count: u32,
+ /// The collection of messages.
+ pub messages: Vec<IggyMessage>,
+}
+
+impl PolledMessages {
+ pub fn empty() -> Self {
+ Self {
+ partition_id: 0,
+ current_offset: 0,
+ count: 0,
+ messages: Vec::new(),
+ }
+ }
+}
+
+impl BytesSerializable for PolledMessages {
+ fn to_bytes(&self) -> Bytes {
+ panic!("should not be used")
+ }
+
+ fn from_bytes(bytes: Bytes) -> Result<Self, IggyError> {
+ let partition_id = u32::from_le_bytes(
+ bytes[0..4]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+ let current_offset = u64::from_le_bytes(
+ bytes[4..12]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+ let count = u32::from_le_bytes(
+ bytes[12..16]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+
+ let messages = messages_from_bytes_and_count(bytes.slice(16..), count)?;
+
+ Ok(Self {
+ partition_id,
+ current_offset,
+ count,
+ messages,
+ })
+ }
+}
+
+/// Convert Bytes to messages
+fn messages_from_bytes_and_count(buffer: Bytes, count: u32) -> Result<Vec<IggyMessage>, IggyError> {
+ let mut messages = Vec::with_capacity(count as usize);
+ let mut position = 0;
+ let buf_len = buffer.len();
+ while position < buf_len {
+ if position + IGGY_MESSAGE_HEADER_SIZE as usize > buf_len {
+ break;
+ }
+ let header_bytes = buffer.slice(position..position + IGGY_MESSAGE_HEADER_SIZE as usize);
+ let header = match IggyMessageHeader::from_bytes(header_bytes) {
+ Ok(h) => h,
+ Err(e) => {
+ error!("Failed to deserialize message header: {}", e);
+ return Err(e);
+ }
+ };
+ position += IGGY_MESSAGE_HEADER_SIZE as usize;
+
+ let payload_end = position + header.payload_length as usize;
+ if payload_end > buf_len {
+ break;
+ }
+ let payload = buffer.slice(position..payload_end);
+ position = payload_end;
+
+ let headers: Option<Bytes> = if header.user_headers_length > 0 {
+ Some(buffer.slice(position..position + header.user_headers_length as usize))
+ } else {
+ None
+ };
+ position += header.user_headers_length as usize;
+
+ messages.push(IggyMessage {
+ header,
+ payload,
+ user_headers: headers,
+ });
+ }
+
+ Ok(messages)
+}
diff --git a/sdk/src/messages/polling_kind.rs b/sdk/src/messages/polling_kind.rs
new file mode 100644
index 00000000..2ee61d8a
--- /dev/null
+++ b/sdk/src/messages/polling_kind.rs
@@ -0,0 +1,72 @@
+use crate::error::IggyError;
+use serde::{Deserialize, Serialize};
+use std::fmt::Display;
+use std::str::FromStr;
+
+/// `PollingKind` is an enum which specifies from where to start polling messages and is used by `PollingStrategy`.
+#[derive(Debug, Serialize, Deserialize, PartialEq, Default, Copy, Clone)]
+#[serde(rename_all = "snake_case")]
+pub enum PollingKind {
+ #[default]
+ /// Start polling from the specified offset.
+ Offset,
+ /// Start polling from the specified timestamp.
+ Timestamp,
+ /// Start polling from the first message in the partition.
+ First,
+ /// Start polling from the last message in the partition.
+ Last,
+ /// Start polling from the next message after the last polled message based on the stored consumer offset. Should be used with `auto_commit` set to `true`.
+ Next,
+}
+
+impl PollingKind {
+ /// Returns code of the polling kind.
+ pub fn as_code(&self) -> u8 {
+ match self {
+ PollingKind::Offset => 1,
+ PollingKind::Timestamp => 2,
+ PollingKind::First => 3,
+ PollingKind::Last => 4,
+ PollingKind::Next => 5,
+ }
+ }
+
+ /// Returns polling kind from the specified code.
+ pub fn from_code(code: u8) -> Result<Self, IggyError> {
+ match code {
+ 1 => Ok(PollingKind::Offset),
+ 2 => Ok(PollingKind::Timestamp),
+ 3 => Ok(PollingKind::First),
+ 4 => Ok(PollingKind::Last),
+ 5 => Ok(PollingKind::Next),
+ _ => Err(IggyError::InvalidCommand),
+ }
+ }
+}
+
+impl FromStr for PollingKind {
+ type Err = IggyError;
+ fn from_str(input: &str) -> Result<Self, Self::Err> {
+ match input {
+ "o" | "offset" => Ok(PollingKind::Offset),
+ "t" | "timestamp" => Ok(PollingKind::Timestamp),
+ "f" | "first" => Ok(PollingKind::First),
+ "l" | "last" => Ok(PollingKind::Last),
+ "n" | "next" => Ok(PollingKind::Next),
+ _ => Err(IggyError::InvalidCommand),
+ }
+ }
+}
+
+impl Display for PollingKind {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ PollingKind::Offset => write!(f, "offset"),
+ PollingKind::Timestamp => write!(f, "timestamp"),
+ PollingKind::First => write!(f, "first"),
+ PollingKind::Last => write!(f, "last"),
+ PollingKind::Next => write!(f, "next"),
+ }
+ }
+}
diff --git a/sdk/src/messages/polling_strategy.rs b/sdk/src/messages/polling_strategy.rs
new file mode 100644
index 00000000..b77dbb55
--- /dev/null
+++ b/sdk/src/messages/polling_strategy.rs
@@ -0,0 +1,129 @@
+use super::polling_kind::PollingKind;
+use crate::bytes_serializable::BytesSerializable;
+use crate::error::IggyError;
+use crate::utils::timestamp::IggyTimestamp;
+use bytes::{BufMut, Bytes, BytesMut};
+use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
+use std::fmt::Display;
+
+/// Default value for the polling strategy.
+const DEFAULT_POLLING_STRATEGY_VALUE: u64 = 0;
+
+/// `PollingStrategy` specifies from where to start polling messages.
+/// It has the following kinds:
+/// - `Offset` - start polling from the specified offset.
+/// - `Timestamp` - start polling from the specified timestamp.
+/// - `First` - start polling from the first message in the partition.
+/// - `Last` - start polling from the last message in the partition.
+/// - `Next` - start polling from the next message after the last polled message based on the stored consumer offset.
+#[serde_as]
+#[derive(Debug, Serialize, Deserialize, PartialEq, Copy, Clone)]
+pub struct PollingStrategy {
+ /// Kind of the polling strategy.
+ #[serde_as(as = "DisplayFromStr")]
+ #[serde(default = "PollingStrategy::default_kind")]
+ pub kind: PollingKind,
+ /// Value of the polling strategy.
+ #[serde_as(as = "DisplayFromStr")]
+ #[serde(default = "PollingStrategy::default_value")]
+ pub value: u64,
+}
+
+impl Default for PollingStrategy {
+ fn default() -> Self {
+ Self {
+ kind: PollingKind::Offset,
+ value: 0,
+ }
+ }
+}
+
+impl Display for PollingStrategy {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "{}|{}", self.kind, self.value)
+ }
+}
+
+impl PollingStrategy {
+ /// Poll messages from the specified offset.
+ pub fn offset(value: u64) -> Self {
+ Self {
+ kind: PollingKind::Offset,
+ value,
+ }
+ }
+
+ /// Poll messages from the specified timestamp.
+ pub fn timestamp(value: IggyTimestamp) -> Self {
+ Self {
+ kind: PollingKind::Timestamp,
+ value: value.into(),
+ }
+ }
+
+ /// Poll messages from the first message in the partition.
+ pub fn first() -> Self {
+ Self {
+ kind: PollingKind::First,
+ value: 0,
+ }
+ }
+
+ /// Poll messages from the last message in the partition.
+ pub fn last() -> Self {
+ Self {
+ kind: PollingKind::Last,
+ value: 0,
+ }
+ }
+
+ /// Poll messages from the next message after the last polled message based on the stored consumer offset. Should be used with `auto_commit` set to `true`.
+ pub fn next() -> Self {
+ Self {
+ kind: PollingKind::Next,
+ value: 0,
+ }
+ }
+
+ /// Change the value of the polling strategy, affects only `Offset` and `Timestamp` kinds.
+ pub fn set_value(&mut self, value: u64) {
+ if self.kind == PollingKind::Offset || self.kind == PollingKind::Timestamp {
+ self.value = value;
+ }
+ }
+
+ /// Returns default value of the polling strategy.
+ pub fn default_value() -> u64 {
+ DEFAULT_POLLING_STRATEGY_VALUE
+ }
+
+ /// Returns default kind of the polling strategy.
+ pub fn default_kind() -> PollingKind {
+ PollingKind::default()
+ }
+}
+
+impl BytesSerializable for PollingStrategy {
+ fn to_bytes(&self) -> Bytes {
+ let mut bytes = BytesMut::with_capacity(9);
+ bytes.put_u8(self.kind.as_code());
+ bytes.put_u64_le(self.value);
+ bytes.freeze()
+ }
+
+ fn from_bytes(bytes: Bytes) -> Result<Self, IggyError> {
+ if bytes.len() != 9 {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let kind = PollingKind::from_code(bytes[0])?;
+ let value = u64::from_le_bytes(
+ bytes[1..9]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+ let strategy = PollingStrategy { kind, value };
+ Ok(strategy)
+ }
+}
diff --git a/sdk/src/messages/send_messages.rs b/sdk/src/messages/send_messages.rs
index af416ca4..e329555e 100644
--- a/sdk/src/messages/send_messages.rs
+++ b/sdk/src/messages/send_messages.rs
@@ -1,113 +1,102 @@
+use super::{Partitioning, PartitioningKind};
use crate::bytes_serializable::BytesSerializable;
use crate::command::{Command, SEND_MESSAGES_CODE};
use crate::error::IggyError;
use crate::identifier::Identifier;
-use crate::messages::{MAX_HEADERS_SIZE, MAX_PAYLOAD_SIZE};
-use crate::models::header;
-use crate::models::header::{HeaderKey, HeaderValue};
-use crate::utils::byte_size::IggyByteSize;
+use crate::models::messaging::{IggyMessage, IggyMessagesBatch, INDEX_SIZE};
+use crate::prelude::IggyMessageView;
use crate::utils::sizeable::Sizeable;
use crate::validatable::Validatable;
+use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use bytes::{BufMut, Bytes, BytesMut};
-use serde::{Deserialize, Serialize};
-use serde_with::base64::Base64;
-use serde_with::serde_as;
+use serde::de::{self, MapAccess, Visitor};
+use serde::ser::SerializeStruct;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use serde_json;
use std::collections::HashMap;
-use std::fmt::Display;
-use std::hash::{Hash, Hasher};
-use std::str::FromStr;
-use uuid::Uuid;
-
-const EMPTY_KEY_VALUE: Vec<u8> = vec![];
+use std::fmt::{Display, Formatter};
/// `SendMessages` command is used to send messages to a topic in a stream.
/// It has additional payload:
/// - `stream_id` - unique stream ID (numeric or name).
/// - `topic_id` - unique topic ID (numeric or name).
/// - `partitioning` - to which partition the messages should be sent - either provided by the client or calculated by the server.
-/// - `messages` - collection of messages to be sent.
-#[derive(Debug, Serialize, Deserialize, PartialEq)]
+/// - `batch` - collection of messages to be sent.
+#[derive(Debug, PartialEq)]
pub struct SendMessages {
/// Unique stream ID (numeric or name).
- #[serde(skip)]
pub stream_id: Identifier,
/// Unique topic ID (numeric or name).
- #[serde(skip)]
pub topic_id: Identifier,
/// To which partition the messages should be sent - either provided by the client or calculated by the server.
pub partitioning: Partitioning,
- /// Collection of messages to be sent.
- pub messages: Vec<Message>,
+ /// Messages collection
+ pub batch: IggyMessagesBatch,
}
-/// `Partitioning` is used to specify to which partition the messages should be sent.
-/// It has the following kinds:
-/// - `Balanced` - the partition ID is calculated by the server using the round-robin algorithm.
-/// - `PartitionId` - the partition ID is provided by the client.
-/// - `MessagesKey` - the partition ID is calculated by the server using the hash of the provided messages key.
-#[serde_as]
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
-pub struct Partitioning {
- /// The kind of partitioning.
- pub kind: PartitioningKind,
- #[serde(skip)]
- /// The length of the value payload.
- pub length: u8,
- #[serde_as(as = "Base64")]
- /// The binary value payload.
- pub value: Vec<u8>,
-}
+impl SendMessages {
+ pub fn as_bytes(
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partitioning: &Partitioning,
+ messages: &[IggyMessage],
+ ) -> Bytes {
+ let stream_id_size = stream_id.get_buffer_size();
+ let topic_id_size = topic_id.get_buffer_size();
+ let partitioning_size = partitioning.get_buffer_size();
+ let messages_count = messages.len() as u32;
+ let messages_count_size = std::mem::size_of::<u32>() as u32;
+ let indexes_size = messages_count * INDEX_SIZE as u32;
+
+ let total_size = stream_id_size
+ + topic_id_size
+ + partitioning_size
+ + messages_count_size
+ + indexes_size
+ + messages
+ .iter()
+ .map(|m| m.get_size_bytes().as_bytes_u64() as u32)
+ .sum::<u32>();
-impl Hash for Partitioning {
- fn hash<H: Hasher>(&self, state: &mut H) {
- self.kind.hash(state);
- self.length.hash(state);
- self.value.hash(state);
- }
-}
+ let mut bytes = BytesMut::with_capacity(total_size as usize);
-/// The single message to be sent. It has the following payload:
-/// - `id` - unique message ID, if not specified by the client (has value = 0), it will be generated by the server.
-/// - `length` - length of the payload.
-/// - `payload` - binary message payload.
-/// - `headers` - optional collection of headers.
-#[serde_as]
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
-pub struct Message {
- /// Unique message ID, if not specified by the client (has value = 0), it will be generated by the server.
- #[serde(default = "default_message_id")]
- pub id: u128,
- #[serde(skip)]
- /// Length of the payload.
- pub length: u32,
- #[serde_as(as = "Base64")]
- /// Binary message payload.
- pub payload: Bytes,
- /// Optional collection of headers.
- pub headers: Option<HashMap<HeaderKey, HeaderValue>>,
-}
+ stream_id.write_to_buffer(&mut bytes);
+ topic_id.write_to_buffer(&mut bytes);
+ partitioning.write_to_buffer(&mut bytes);
+ bytes.put_u32_le(messages_count);
-/// `PartitioningKind` is an enum which specifies the kind of partitioning and is used by `Partitioning`.
-#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Default, Copy, Clone)]
-#[serde(rename_all = "snake_case")]
-pub enum PartitioningKind {
- /// The partition ID is calculated by the server using the round-robin algorithm.
- #[default]
- Balanced,
- /// The partition ID is provided by the client.
- PartitionId,
- /// The partition ID is calculated by the server using the hash of the provided messages key.
- MessagesKey,
-}
+ let mut current_position = bytes.len();
+
+ bytes.put_bytes(0, indexes_size as usize);
+
+ let mut msgs_size: u32 = 0;
+ for message in messages.iter() {
+ message.write_to_buffer(&mut bytes);
+ msgs_size += message.get_size_bytes().as_bytes_u64() as u32;
+ write_value_at(&mut bytes, 0u64.to_le_bytes(), current_position);
+ write_value_at(&mut bytes, msgs_size.to_le_bytes(), current_position + 4);
+ write_value_at(&mut bytes, 0u64.to_le_bytes(), current_position + 8);
+ current_position += INDEX_SIZE;
+ }
-impl Hash for PartitioningKind {
- fn hash<H: Hasher>(&self, state: &mut H) {
- self.as_code().hash(state);
+ let out = bytes.freeze();
+
+ debug_assert_eq!(
+ total_size as usize,
+ out.len(),
+ "Calculated SendMessages command byte size doesn't match actual command size",
+ );
+
+ out
}
}
-fn default_message_id() -> u128 {
- 0
+fn write_value_at<const N: usize>(slice: &mut [u8], value: [u8; N], position: usize) {
+ let slice = &mut slice[position..position + N];
+ let ptr = slice.as_mut_ptr();
+ unsafe {
+ std::ptr::copy_nonoverlapping(value.as_ptr(), ptr, N);
+ }
}
impl Default for SendMessages {
@@ -116,99 +105,11 @@ impl Default for SendMessages {
stream_id: Identifier::default(),
topic_id: Identifier::default(),
partitioning: Partitioning::default(),
- messages: vec![Message::default()],
- }
- }
-}
-
-impl Default for Partitioning {
- fn default() -> Self {
- Partitioning::balanced()
- }
-}
-
-impl Partitioning {
- /// Partition the messages using the balanced round-robin algorithm on the server.
- pub fn balanced() -> Self {
- Partitioning {
- kind: PartitioningKind::Balanced,
- length: 0,
- value: EMPTY_KEY_VALUE,
- }
- }
-
- /// Partition the messages using the provided partition ID.
- pub fn partition_id(partition_id: u32) -> Self {
- Partitioning {
- kind: PartitioningKind::PartitionId,
- length: 4,
- value: partition_id.to_le_bytes().to_vec(),
- }
- }
-
- /// Partition the messages using the provided messages key.
- pub fn messages_key(value: &[u8]) -> Result<Self, IggyError> {
- let length = value.len();
- if length == 0 || length > 255 {
- return Err(IggyError::InvalidCommand);
- }
-
- Ok(Partitioning {
- kind: PartitioningKind::MessagesKey,
- #[allow(clippy::cast_possible_truncation)]
- length: length as u8,
- value: value.to_vec(),
- })
- }
-
- /// Partition the messages using the provided messages key as str.
- pub fn messages_key_str(value: &str) -> Result<Self, IggyError> {
- Self::messages_key(value.as_bytes())
- }
-
- /// Partition the messages using the provided messages key as u32.
- pub fn messages_key_u32(value: u32) -> Self {
- Partitioning {
- kind: PartitioningKind::MessagesKey,
- length: 4,
- value: value.to_le_bytes().to_vec(),
- }
- }
-
- /// Partition the messages using the provided messages key as u64.
- pub fn messages_key_u64(value: u64) -> Self {
- Partitioning {
- kind: PartitioningKind::MessagesKey,
- length: 8,
- value: value.to_le_bytes().to_vec(),
- }
- }
-
- /// Partition the messages using the provided messages key as u128.
- pub fn messages_key_u128(value: u128) -> Self {
- Partitioning {
- kind: PartitioningKind::MessagesKey,
- length: 16,
- value: value.to_le_bytes().to_vec(),
- }
- }
-
- /// Create the partitioning from the provided partitioning.
- pub fn from_partitioning(partitioning: &Partitioning) -> Self {
- Partitioning {
- kind: partitioning.kind,
- length: partitioning.length,
- value: partitioning.value.clone(),
+ batch: IggyMessagesBatch::empty(),
}
}
}
-impl Sizeable for Partitioning {
- fn get_size_bytes(&self) -> IggyByteSize {
- IggyByteSize::from(u64::from(self.length) + 2)
- }
-}
-
impl Command for SendMessages {
fn code(&self) -> u32 {
SEND_MESSAGES_CODE
@@ -217,347 +118,209 @@ impl Command for SendMessages {
impl Validatable<IggyError> for SendMessages {
fn validate(&self) -> Result<(), IggyError> {
- if self.messages.is_empty() {
- return Err(IggyError::InvalidMessagesCount);
- }
-
- let key_value_length = self.partitioning.value.len();
- if key_value_length > 255
- || (self.partitioning.kind != PartitioningKind::Balanced && key_value_length == 0)
+ if self.partitioning.value.len() > 255
+ || (self.partitioning.kind != PartitioningKind::Balanced
+ && self.partitioning.value.is_empty())
{
return Err(IggyError::InvalidKeyValueLength);
}
- let mut headers_size = 0;
- let mut payload_size = 0;
- for message in &self.messages {
- if let Some(headers) = &message.headers {
- for value in headers.values() {
- headers_size += value.value.len() as u32;
- if headers_size > MAX_HEADERS_SIZE {
- return Err(IggyError::TooBigHeadersPayload);
- }
- }
- }
- payload_size += message.payload.len() as u32;
- if payload_size > MAX_PAYLOAD_SIZE {
- return Err(IggyError::TooBigMessagePayload);
- }
- }
-
- if payload_size == 0 {
- return Err(IggyError::EmptyMessagePayload);
- }
+ self.batch.validate()?;
Ok(())
}
}
-impl PartitioningKind {
- /// Get the code of the partitioning kind.
- pub fn as_code(&self) -> u8 {
- match self {
- PartitioningKind::Balanced => 1,
- PartitioningKind::PartitionId => 2,
- PartitioningKind::MessagesKey => 3,
- }
- }
-
- /// Get the partitioning kind from the provided code.
- pub fn from_code(code: u8) -> Result<Self, IggyError> {
- match code {
- 1 => Ok(PartitioningKind::Balanced),
- 2 => Ok(PartitioningKind::PartitionId),
- 3 => Ok(PartitioningKind::MessagesKey),
- _ => Err(IggyError::InvalidCommand),
- }
- }
-}
-
-impl Message {
- /// Create a new message with the optional ID, payload and headers.
- pub fn new(
- id: Option<u128>,
- payload: Bytes,
- headers: Option<HashMap<HeaderKey, HeaderValue>>,
- ) -> Self {
- Message {
- id: id.unwrap_or(0),
- #[allow(clippy::cast_possible_truncation)]
- length: payload.len() as u32,
- payload,
- headers,
- }
- }
-}
-
-impl Sizeable for Message {
- fn get_size_bytes(&self) -> IggyByteSize {
- // ID + Length + Payload + Headers
- header::get_headers_size_bytes(&self.headers) + (16 + 4 + self.payload.len() as u64).into()
+impl BytesSerializable for SendMessages {
+ fn to_bytes(&self) -> Bytes {
+ panic!("should not be used")
}
-}
-impl Default for Message {
- fn default() -> Self {
- let payload = Bytes::from("hello world");
- Message {
- id: 1,
- length: payload.len() as u32,
- payload,
- headers: None,
- }
+ fn from_bytes(_bytes: Bytes) -> Result<SendMessages, IggyError> {
+ panic!("should not be used")
}
}
-impl Display for Message {
+impl Display for SendMessages {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- let len = self.payload.len();
-
- if len > 40 {
- write!(
- f,
- "{}|{}...{}",
- self.id,
- String::from_utf8_lossy(&self.payload[..20]),
- String::from_utf8_lossy(&self.payload[len - 20..])
- )
- } else {
- write!(f, "{}|{}", self.id, String::from_utf8_lossy(&self.payload))
- }
+ let messages_count = self.batch.count();
+ let messages_size = self.batch.size();
+ write!(
+ f,
+ "{}|{}|{}|messages_count:{}|messages_size:{}",
+ self.stream_id, self.topic_id, self.partitioning, messages_count, messages_size
+ )
}
}
-impl BytesSerializable for Partitioning {
- fn to_bytes(&self) -> Bytes {
- let mut bytes = BytesMut::with_capacity(2 + self.length as usize);
- bytes.put_u8(self.kind.as_code());
- bytes.put_u8(self.length);
- bytes.put_slice(&self.value);
- bytes.freeze()
- }
-
- fn from_bytes(bytes: Bytes) -> Result<Self, IggyError>
+impl Serialize for SendMessages {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
- Self: Sized,
+ S: Serializer,
{
- if bytes.len() < 3 {
- return Err(IggyError::InvalidCommand);
- }
+ // In HTTP API, we expose:
+ // - partitioning (kind, value)
+ // - messages as an array of {id, payload, headers}
+ // We don't expose stream_id and topic_id via JSON as they're in URL path
- let kind = PartitioningKind::from_code(bytes[0])?;
- let length = bytes[1];
- let value = bytes[2..2 + length as usize].to_vec();
- if value.len() != length as usize {
- return Err(IggyError::InvalidCommand);
- }
+ let messages: Vec<HashMap<&str, serde_json::Value>> = self
+ .batch
+ .iter()
+ .map(|msg_view: IggyMessageView<'_>| {
+ let mut map = HashMap::with_capacity(self.batch.count() as usize);
+ map.insert("id", serde_json::to_value(msg_view.header().id()).unwrap());
- Ok(Partitioning {
- kind,
- length,
- value,
- })
- }
-}
+ let payload_base64 = BASE64.encode(msg_view.payload());
+ map.insert("payload", serde_json::to_value(payload_base64).unwrap());
-impl BytesSerializable for Message {
- fn to_bytes(&self) -> Bytes {
- let mut bytes = BytesMut::with_capacity(self.get_size_bytes().as_bytes_usize());
- bytes.put_u128_le(self.id);
- if let Some(headers) = &self.headers {
- let headers_bytes = headers.to_bytes();
- bytes.put_u32_le(headers_bytes.len() as u32);
- bytes.put_slice(&headers_bytes);
- } else {
- bytes.put_u32_le(0);
- }
- bytes.put_u32_le(self.length);
- bytes.put_slice(&self.payload);
- bytes.freeze()
- }
+ if let Ok(Some(headers)) = msg_view.user_headers_map() {
+ map.insert("headers", serde_json::to_value(&headers).unwrap());
+ }
- fn from_bytes(bytes: Bytes) -> Result<Self, IggyError> {
- if bytes.len() < 24 {
- return Err(IggyError::InvalidCommand);
- }
+ map
+ })
+ .collect();
- let mut id = u128::from_le_bytes(
- bytes[..16]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- if id == 0 {
- id = Uuid::now_v7().to_u128_le();
- }
- let headers_length = u32::from_le_bytes(
- bytes[16..20]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- let headers = if headers_length > 0 {
- Some(HashMap::from_bytes(
- bytes.slice(20..20 + headers_length as usize),
- )?)
- } else {
- None
- };
-
- let payload_length = u32::from_le_bytes(
- bytes[20 + headers_length as usize..24 + headers_length as usize]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- );
- if payload_length == 0 {
- return Err(IggyError::EmptyMessagePayload);
- }
+ let mut state = serializer.serialize_struct("SendMessages", 2)?;
+ state.serialize_field("partitioning", &self.partitioning)?;
+ state.serialize_field("messages", &messages)?;
+ state.end()
+ }
+}
- let payload = bytes.slice(
- 24 + headers_length as usize..24 + headers_length as usize + payload_length as usize,
- );
- if payload.len() != payload_length as usize {
- return Err(IggyError::InvalidMessagePayloadLength);
+impl<'de> Deserialize<'de> for SendMessages {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ enum Field {
+ Partitioning,
+ Messages,
}
- Ok(Message {
- id,
- length: payload_length,
- payload,
- headers,
- })
- }
-}
+ impl<'de> Deserialize<'de> for Field {
+ fn deserialize<D>(deserializer: D) -> Result<Field, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ struct FieldVisitor;
-// This method is used by the new version of `IggyClient` to serialize `SendMessages` without copying the messages.
-pub(crate) fn as_bytes(
- stream_id: &Identifier,
- topic_id: &Identifier,
- partitioning: &Partitioning,
- messages: &[Message],
-) -> Bytes {
- let messages_size = messages
- .iter()
- .map(Message::get_size_bytes)
- .sum::<IggyByteSize>();
- let key_bytes = partitioning.to_bytes();
- let stream_id_bytes = stream_id.to_bytes();
- let topic_id_bytes = topic_id.to_bytes();
- let mut bytes = BytesMut::with_capacity(
- stream_id_bytes.len()
- + topic_id_bytes.len()
- + key_bytes.len()
- + messages_size.as_bytes_usize(),
- );
- bytes.put_slice(&stream_id_bytes);
- bytes.put_slice(&topic_id_bytes);
- bytes.put_slice(&key_bytes);
- for message in messages {
- bytes.put_slice(&message.to_bytes());
- }
+ impl Visitor<'_> for FieldVisitor {
+ type Value = Field;
- bytes.freeze()
-}
+ fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
+ formatter.write_str("`partitioning` or `messages`")
+ }
-impl FromStr for Message {
- type Err = IggyError;
- fn from_str(input: &str) -> Result<Self, Self::Err> {
- let id = default_message_id();
- let payload = Bytes::from(input.as_bytes().to_vec());
- let length = payload.len() as u32;
- if length == 0 {
- return Err(IggyError::EmptyMessagePayload);
+ fn visit_str<E>(self, value: &str) -> Result<Field, E>
+ where
+ E: de::Error,
+ {
+ match value {
+ "partitioning" => Ok(Field::Partitioning),
+ "messages" => Ok(Field::Messages),
+ _ => Err(de::Error::unknown_field(
+ value,
+ &["partitioning", "messages"],
+ )),
+ }
+ }
+ }
+
+ deserializer.deserialize_identifier(FieldVisitor)
+ }
}
- Ok(Message {
- id,
- length,
- payload,
- headers: None,
- })
- }
-}
+ struct SendMessagesVisitor;
-impl BytesSerializable for SendMessages {
- fn to_bytes(&self) -> Bytes {
- as_bytes(
- &self.stream_id,
- &self.topic_id,
- &self.partitioning,
- &self.messages,
- )
- }
+ impl<'de> Visitor<'de> for SendMessagesVisitor {
+ type Value = SendMessages;
- fn from_bytes(bytes: Bytes) -> Result<SendMessages, IggyError> {
- if bytes.len() < 11 {
- return Err(IggyError::InvalidCommand);
- }
+ fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
+ formatter.write_str("struct SendMessages")
+ }
- let mut position = 0;
- let stream_id = Identifier::from_bytes(bytes.clone())?;
- position += stream_id.get_size_bytes().as_bytes_usize();
- let topic_id = Identifier::from_bytes(bytes.slice(position..))?;
- position += topic_id.get_size_bytes().as_bytes_usize();
- let key = Partitioning::from_bytes(bytes.slice(position..))?;
- position += key.get_size_bytes().as_bytes_usize();
- let messages_payloads = bytes.slice(position..);
- position = 0;
- let mut messages = Vec::new();
- while position < messages_payloads.len() {
- let message = Message::from_bytes(messages_payloads.slice(position..))?;
- position += message.get_size_bytes().as_bytes_usize();
- messages.push(message);
- }
+ fn visit_map<V>(self, mut map: V) -> Result<SendMessages, V::Error>
+ where
+ V: MapAccess<'de>,
+ {
+ let mut partitioning = None;
+ let mut messages = None;
+
+ while let Some(key) = map.next_key()? {
+ match key {
+ Field::Partitioning => {
+ if partitioning.is_some() {
+ return Err(de::Error::duplicate_field("partitioning"));
+ }
+ partitioning = Some(map.next_value()?);
+ }
+ Field::Messages => {
+ if messages.is_some() {
+ return Err(de::Error::duplicate_field("messages"));
+ }
+
+ let message_data: Vec<serde_json::Value> = map.next_value()?;
+ let mut iggy_messages = Vec::new();
+
+ for msg in message_data {
+ let id =
+ msg.get("id").and_then(|v| v.as_u64()).unwrap_or(0) as u128;
+
+ let payload = msg
+ .get("payload")
+ .and_then(|v| v.as_str())
+ .ok_or_else(|| de::Error::missing_field("payload"))?;
+ let payload_bytes = BASE64
+ .decode(payload)
+ .map_err(|_| de::Error::custom("Invalid base64 payload"))?;
+
+ let headers_map = if let Some(headers) = msg.get("headers") {
+ if headers.is_null() {
+ None
+ } else {
+ Some(serde_json::from_value(headers.clone()).map_err(
+ |_| de::Error::custom("Invalid headers format"),
+ )?)
+ }
+ } else {
+ None
+ };
+
+ let iggy_message = if let Some(headers) = headers_map {
+ IggyMessage::with_id_and_headers(id, payload_bytes, headers)
+ } else {
+ IggyMessage::with_id(id, payload_bytes)
+ };
+
+ iggy_messages.push(iggy_message);
+ }
+
+ messages = Some(iggy_messages);
+ }
+ }
+ }
- let command = SendMessages {
- stream_id,
- topic_id,
- partitioning: key,
- messages,
- };
- Ok(command)
- }
-}
+ let partitioning =
+ partitioning.ok_or_else(|| de::Error::missing_field("partitioning"))?;
+ let messages = messages.ok_or_else(|| de::Error::missing_field("messages"))?;
-impl Display for SendMessages {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(
- f,
- "{}|{}|{}|batch_len:{}|batch_size:{}",
- self.stream_id,
- self.topic_id,
- self.partitioning,
- self.messages.len(),
- self.messages
- .iter()
- .map(Message::get_size_bytes)
- .sum::<IggyByteSize>(),
- )
- }
-}
+ let batch = IggyMessagesBatch::from(&messages);
-impl Display for Partitioning {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self.kind {
- PartitioningKind::Balanced => write!(f, "{}|0", self.kind),
- PartitioningKind::PartitionId => write!(
- f,
- "{}|{}",
- self.kind,
- u32::from_le_bytes(self.value[..4].try_into().unwrap())
- ),
- PartitioningKind::MessagesKey => {
- write!(f, "{}|{}", self.kind, String::from_utf8_lossy(&self.value))
+ Ok(SendMessages {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partitioning,
+ batch,
+ })
}
}
- }
-}
-impl Display for PartitioningKind {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self {
- PartitioningKind::Balanced => write!(f, "balanced"),
- PartitioningKind::PartitionId => write!(f, "partition_id"),
- PartitioningKind::MessagesKey => write!(f, "messages_key"),
- }
+ deserializer.deserialize_struct(
+ "SendMessages",
+ &["partitioning", "messages"],
+ SendMessagesVisitor,
+ )
}
}
@@ -565,101 +328,12 @@ impl Display for PartitioningKind {
mod tests {
use super::*;
- #[test]
- fn should_be_serialized_as_bytes() {
- let message_1 = Message::from_str("hello 1").unwrap();
- let message_2 = Message::new(Some(2), "hello 2".into(), None);
- let message_3 = Message::new(Some(3), "hello 3".into(), None);
- let messages = vec![message_1, message_2, message_3];
- let command = SendMessages {
- stream_id: Identifier::numeric(1).unwrap(),
- topic_id: Identifier::numeric(2).unwrap(),
- partitioning: Partitioning::partition_id(4),
- messages,
- };
-
- let bytes = command.to_bytes();
-
- let mut position = 0;
- let stream_id = Identifier::from_bytes(bytes.clone()).unwrap();
- position += stream_id.get_size_bytes().as_bytes_usize();
- let topic_id = Identifier::from_bytes(bytes.slice(position..)).unwrap();
- position += topic_id.get_size_bytes().as_bytes_usize();
- let key = Partitioning::from_bytes(bytes.slice(position..)).unwrap();
- position += key.get_size_bytes().as_bytes_usize();
- let messages = bytes.slice(position..);
- let command_messages = command
- .messages
- .iter()
- .fold(BytesMut::new(), |mut bytes_mut, message| {
- bytes_mut.put(message.to_bytes());
- bytes_mut
- })
- .freeze();
-
- assert!(!bytes.is_empty());
- assert_eq!(stream_id, command.stream_id);
- assert_eq!(topic_id, command.topic_id);
- assert_eq!(key, command.partitioning);
- assert_eq!(messages, command_messages);
- }
-
- #[test]
- fn should_be_deserialized_from_bytes() {
- let stream_id = Identifier::numeric(1).unwrap();
- let topic_id = Identifier::numeric(2).unwrap();
- let key = Partitioning::partition_id(4);
-
- let message_1 = Message::from_str("hello 1").unwrap();
- let message_2 = Message::new(Some(2), "hello 2".into(), None);
- let message_3 = Message::new(Some(3), "hello 3".into(), None);
- let messages = [
- message_1.to_bytes(),
- message_2.to_bytes(),
- message_3.to_bytes(),
- ]
- .concat();
-
- let key_bytes = key.to_bytes();
- let stream_id_bytes = stream_id.to_bytes();
- let topic_id_bytes = topic_id.to_bytes();
- let current_position = stream_id_bytes.len() + topic_id_bytes.len() + key_bytes.len();
- let mut bytes = BytesMut::with_capacity(current_position);
- bytes.put_slice(&stream_id_bytes);
- bytes.put_slice(&topic_id_bytes);
- bytes.put_slice(&key_bytes);
- bytes.put_slice(&messages);
- let bytes = bytes.freeze();
- let command = SendMessages::from_bytes(bytes.clone());
- assert!(command.is_ok());
-
- let messages_payloads = bytes.slice(current_position..);
- let mut position = 0;
- let mut messages = Vec::new();
- while position < messages_payloads.len() {
- let message = Message::from_bytes(messages_payloads.slice(position..)).unwrap();
- position += message.get_size_bytes().as_bytes_usize();
- messages.push(message);
- }
-
- let command = command.unwrap();
- assert_eq!(command.stream_id, stream_id);
- assert_eq!(command.topic_id, topic_id);
- assert_eq!(command.partitioning, key);
- for (index, message) in command.messages.iter().enumerate() {
- let command_message = &command.messages[index];
- assert_eq!(command_message.id, message.id);
- assert_eq!(command_message.length, message.length);
- assert_eq!(command_message.payload, message.payload);
- }
- }
-
#[test]
fn key_of_type_balanced_should_have_empty_value() {
let key = Partitioning::balanced();
assert_eq!(key.kind, PartitioningKind::Balanced);
assert_eq!(key.length, 0);
- assert_eq!(key.value, EMPTY_KEY_VALUE);
+ assert!(key.value.is_empty());
assert_eq!(
PartitioningKind::from_code(1).unwrap(),
PartitioningKind::Balanced
diff --git a/sdk/src/models/messages.rs b/sdk/src/models/messages.rs
deleted file mode 100644
index 74548f0c..00000000
--- a/sdk/src/models/messages.rs
+++ /dev/null
@@ -1,182 +0,0 @@
-use crate::bytes_serializable::BytesSerializable;
-use crate::error::IggyError;
-use crate::models::header;
-use crate::models::header::{HeaderKey, HeaderValue};
-use crate::utils::byte_size::IggyByteSize;
-use crate::utils::sizeable::Sizeable;
-use crate::utils::timestamp::IggyTimestamp;
-use bytes::{BufMut, Bytes, BytesMut};
-use serde::{Deserialize, Serialize};
-use serde_with::base64::Base64;
-use serde_with::serde_as;
-use std::collections::HashMap;
-use std::fmt::{Display, Formatter};
-use std::str::FromStr;
-
-pub const POLLED_MESSAGE_METADATA: u32 = 8 + 1 + 8 + 4;
-
-/// The wrapper on top of the collection of messages that are polled from the partition.
-/// It consists of the following fields:
-/// - `partition_id`: the identifier of the partition.
-/// - `current_offset`: the current offset of the partition.
-/// - `messages`: the collection of messages.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct PolledMessages {
- /// The identifier of the partition. If it's '0', then there's no partition assigned to the consumer group member.
- pub partition_id: u32,
- /// The current offset of the partition.
- pub current_offset: u64,
- /// The collection of messages.
- pub messages: Vec<PolledMessage>,
-}
-
-/// The single message that is polled from the partition.
-/// It consists of the following fields:
-/// - `offset`: the offset of the message.
-/// - `state`: the state of the message.
-/// - `timestamp`: the timestamp of the message.
-/// - `id`: the identifier of the message.
-/// - `checksum`: the checksum of the message, can be used to verify the integrity of the message.
-/// - `headers`: the optional headers of the message.
-/// - `length`: the length of the payload.
-/// - `payload`: the binary payload of the message.
-#[serde_as]
-#[derive(Debug, Serialize, Deserialize)]
-pub struct PolledMessage {
- /// The offset of the message.
- pub offset: u64,
- /// The state of the message.
- pub state: MessageState,
- /// The timestamp of the message.
- pub timestamp: u64,
- /// The identifier of the message.
- pub id: u128,
- /// The checksum of the message, can be used to verify the integrity of the message.
- pub checksum: u32,
- /// The optional headers of the message.
- pub headers: Option<HashMap<HeaderKey, HeaderValue>>,
- /// The length of the payload.
- #[serde(skip)]
- pub length: IggyByteSize,
- /// The binary payload of the message.
- #[serde_as(as = "Base64")]
- pub payload: Bytes,
-}
-
-/// The state of the message, currently only the `Available` state is used.
-#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum MessageState {
- /// The message is available.
- Available,
- /// The message is unavailable.
- Unavailable,
- /// The message is poisoned.
- Poisoned,
- /// The message is marked for deletion.
- MarkedForDeletion,
-}
-
-impl MessageState {
- /// Returns the code of the message state.
- pub fn as_code(&self) -> u8 {
- match self {
- MessageState::Available => 1,
- MessageState::Unavailable => 10,
- MessageState::Poisoned => 20,
- MessageState::MarkedForDeletion => 30,
- }
- }
-
- /// Returns the message state from the code.
- pub fn from_code(code: u8) -> Result<Self, IggyError> {
- match code {
- 1 => Ok(MessageState::Available),
- 10 => Ok(MessageState::Unavailable),
- 20 => Ok(MessageState::Poisoned),
- 30 => Ok(MessageState::MarkedForDeletion),
- _ => Err(IggyError::InvalidCommand),
- }
- }
-}
-
-impl Display for MessageState {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- match self {
- MessageState::Available => write!(f, "available"),
- MessageState::Unavailable => write!(f, "unavailable"),
- MessageState::Poisoned => write!(f, "poisoned"),
- MessageState::MarkedForDeletion => write!(f, "marked_for_deletion"),
- }
- }
-}
-
-impl FromStr for MessageState {
- type Err = IggyError;
- fn from_str(s: &str) -> Result<Self, Self::Err> {
- match s {
- "available" => Ok(MessageState::Available),
- "unavailable" => Ok(MessageState::Unavailable),
- "poisoned" => Ok(MessageState::Poisoned),
- "marked_for_deletion" => Ok(MessageState::MarkedForDeletion),
- _ => Err(IggyError::InvalidCommand),
- }
- }
-}
-
-impl PolledMessage {
- /// Creates a new message with a specified offset.
- pub fn create(
- offset: u64,
- state: MessageState,
- timestamp: IggyTimestamp,
- id: u128,
- payload: Bytes,
- checksum: u32,
- headers: Option<HashMap<HeaderKey, HeaderValue>>,
- ) -> Self {
- PolledMessage {
- offset,
- state,
- timestamp: timestamp.as_micros(),
- id,
- checksum,
- length: IggyByteSize::from(payload.len() as u64),
- payload,
- headers,
- }
- }
-
- /// Returns the timestamp of the message as `IggyTimestamp`.
- pub fn timestamp(&self) -> IggyTimestamp {
- self.timestamp.into()
- }
-
- /// Extends the provided bytes with the message.
- pub fn extend(&self, bytes: &mut BytesMut) {
- bytes.put_u64_le(self.offset);
- bytes.put_u8(self.state.as_code());
- bytes.put_u64_le(self.timestamp);
- bytes.put_u128_le(self.id);
- bytes.put_u32_le(self.checksum);
- if let Some(headers) = &self.headers {
- let headers_bytes = headers.to_bytes();
- #[allow(clippy::cast_possible_truncation)]
- bytes.put_u32_le(headers_bytes.len() as u32);
- bytes.put_slice(&headers_bytes);
- } else {
- bytes.put_u32_le(0u32);
- }
- bytes.put_u32_le(self.length.as_bytes_u64() as u32);
- bytes.put_slice(&self.payload);
- }
-}
-
-impl Sizeable for PolledMessage {
- fn get_size_bytes(&self) -> IggyByteSize {
- // Offset + State + Timestamp + ID + Checksum + Length + Payload + Headers
- header::get_headers_size_bytes(&self.headers)
- + self.length
- + IggyByteSize::from(8 + 1 + 8 + 16 + 4 + 4)
- }
-}
diff --git a/sdk/src/models/messaging/index.rs b/sdk/src/models/messaging/index.rs
new file mode 100644
index 00000000..55cac36c
--- /dev/null
+++ b/sdk/src/models/messaging/index.rs
@@ -0,0 +1,6 @@
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)]
+pub struct IggyIndex {
+ pub offset: u32,
+ pub position: u32,
+ pub timestamp: u64,
+}
diff --git a/sdk/src/models/messaging/index_view.rs b/sdk/src/models/messaging/index_view.rs
new file mode 100644
index 00000000..8443a1d0
--- /dev/null
+++ b/sdk/src/models/messaging/index_view.rs
@@ -0,0 +1,62 @@
+use crate::models::messaging::INDEX_SIZE;
+use bytes::Buf;
+
+use super::IggyIndex;
+
+/// View into a single index entry in a binary buffer.
+/// Provides zero-copy access to index data.
+#[derive(Debug, Clone, Copy)]
+pub struct IggyIndexView<'a> {
+ data: &'a [u8],
+}
+
+impl<'a> IggyIndexView<'a> {
+ /// Creates a new index view from a byte slice
+ /// Slice must be exactly INDEX_SIZE (16 bytes) long
+ pub fn new(data: &'a [u8]) -> Self {
+ debug_assert!(
+ data.len() == INDEX_SIZE,
+ "Index data must be exactly {INDEX_SIZE} bytes"
+ );
+ Self { data }
+ }
+
+ /// Gets the offset value from the view
+ pub fn offset(&self) -> u32 {
+ let mut buf = &self.data[0..4];
+ buf.get_u32_le()
+ }
+
+ /// Gets the position value from the view
+ pub fn position(&self) -> u32 {
+ let mut buf = &self.data[4..8];
+ buf.get_u32_le()
+ }
+
+ /// Gets the timestamp value from the view
+ pub fn timestamp(&self) -> u64 {
+ let mut buf = &self.data[8..16];
+ buf.get_u64_le()
+ }
+
+ /// Converts the view into an `IggyIndex`
+ pub fn to_index(&self) -> IggyIndex {
+ IggyIndex {
+ offset: self.offset(),
+ position: self.position(),
+ timestamp: self.timestamp(),
+ }
+ }
+}
+
+impl std::fmt::Display for IggyIndexView<'_> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(
+ f,
+ "offset: {}, position: {}, timestamp: {}",
+ self.offset(),
+ self.position(),
+ self.timestamp()
+ )
+ }
+}
diff --git a/sdk/src/models/messaging/indexes.rs b/sdk/src/models/messaging/indexes.rs
new file mode 100644
index 00000000..0ac98254
--- /dev/null
+++ b/sdk/src/models/messaging/indexes.rs
@@ -0,0 +1,185 @@
+use super::{index_view::IggyIndexView, INDEX_SIZE};
+use bytes::Bytes;
+use serde::{Deserialize, Serialize};
+use serde_with::base64::Base64;
+use serde_with::serde_as;
+use std::ops::{Deref, Index as StdIndex};
+
+/// A container for binary-encoded index data.
+/// Optimized for efficient storage and I/O operations.
+#[serde_as]
+#[derive(Default, Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct IggyIndexes {
+ #[serde(skip)]
+ base_position: u32,
+ #[serde_as(as = "Base64")]
+ buffer: Bytes,
+}
+
+impl IggyIndexes {
+ /// Creates a new empty container
+ pub fn new(indexes: Bytes, base_position: u32) -> Self {
+ Self {
+ buffer: indexes,
+ base_position,
+ }
+ }
+
+ /// Creates a new empty container
+ pub fn empty() -> Self {
+ Self {
+ buffer: Bytes::new(),
+ base_position: 0,
+ }
+ }
+
+ /// Gets the number of indexes in the container
+ pub fn count(&self) -> u32 {
+ self.buffer.len() as u32 / INDEX_SIZE as u32
+ }
+
+ /// Gets the size of the buffer in bytes
+ pub fn size(&self) -> u32 {
+ self.buffer.len() as u32
+ }
+
+ /// Gets the size of all indexes messages
+ pub fn messages_size(&self) -> u32 {
+ self.last_position() - self.base_position
+ }
+
+ /// Gets a view of the index at the specified position
+ pub fn get(&self, position: u32) -> Option<IggyIndexView> {
+ if position >= self.count() {
+ return None;
+ }
+
+ let start = position as usize * INDEX_SIZE;
+ let end = start + INDEX_SIZE;
+
+ if end <= self.buffer.len() {
+ Some(IggyIndexView::new(&self.buffer[start..end]))
+ } else {
+ None
+ }
+ }
+
+ /// Gets a slice of the container
+ pub fn slice_by_offset(&self, relative_start_offset: u32, count: u32) -> Option<IggyIndexes> {
+ if self.count() == 0 || relative_start_offset >= self.count() {
+ return None;
+ }
+
+ let available_count = self.count().saturating_sub(relative_start_offset);
+ let actual_count = std::cmp::min(count, available_count);
+
+ if actual_count == 0 {
+ return None;
+ }
+
+ let end_pos = relative_start_offset + actual_count;
+ let start_byte = relative_start_offset as usize * INDEX_SIZE;
+ let end_byte = end_pos as usize * INDEX_SIZE;
+
+ if end_byte > self.buffer.len() {
+ return None;
+ }
+
+ let slice = self.buffer.slice(start_byte..end_byte);
+
+ let base_position = if relative_start_offset > 0 {
+ match self.get(relative_start_offset - 1) {
+ Some(index) => index.position(),
+ None => self.base_position,
+ }
+ } else {
+ self.base_position
+ };
+
+ Some(IggyIndexes::new(slice, base_position))
+ }
+
+ /// Finds an index by timestamp using binary search
+ /// If an exact match isn't found, returns the index with the nearest timestamp
+ /// that is greater than or equal to the requested timestamp
+ pub fn find_by_timestamp(&self, timestamp: u64) -> Option<IggyIndexView> {
+ if self.count() == 0 {
+ return None;
+ }
+
+ let first_idx = self.get(0)?;
+ if timestamp <= first_idx.timestamp() {
+ tracing::trace!(
+ "Requested timestamp {} is less than any available",
+ timestamp
+ );
+ return Some(first_idx);
+ }
+
+ let last_saved_idx = self.get(self.count() - 1)?;
+ if timestamp > last_saved_idx.timestamp() {
+ return None;
+ }
+
+ let mut left = 0;
+ let mut right = self.count() as isize - 1;
+ let mut result: Option<IggyIndexView> = None;
+
+ while left <= right {
+ let mid = left + (right - left) / 2;
+ let view = self.get(mid as u32).unwrap();
+ let current_timestamp = view.timestamp();
+
+ match current_timestamp.cmp(×tamp) {
+ std::cmp::Ordering::Equal => {
+ result = Some(view);
+ right = mid - 1;
+ }
+ std::cmp::Ordering::Less => {
+ left = mid + 1;
+ }
+ std::cmp::Ordering::Greater => {
+ result = Some(view);
+ right = mid - 1;
+ }
+ }
+ }
+
+ result
+ }
+
+ /// Gets the base position of the container
+ pub fn base_position(&self) -> u32 {
+ self.base_position
+ }
+
+ /// Helper method to get the last index position
+ pub fn last_position(&self) -> u32 {
+ self.get(self.count() - 1)
+ .map(|idx| idx.position())
+ .unwrap_or(0)
+ }
+
+ /// Decompose the container into its components
+ pub fn decompose(self) -> (u32, Bytes) {
+ (self.base_position, self.buffer)
+ }
+}
+
+impl StdIndex<usize> for IggyIndexes {
+ type Output = [u8];
+
+ fn index(&self, index: usize) -> &Self::Output {
+ let start = index * INDEX_SIZE;
+ let end = start + INDEX_SIZE;
+ &self.buffer[start..end]
+ }
+}
+
+impl Deref for IggyIndexes {
+ type Target = [u8];
+
+ fn deref(&self) -> &Self::Target {
+ &self.buffer
+ }
+}
diff --git a/sdk/src/models/messaging/message.rs b/sdk/src/models/messaging/message.rs
new file mode 100644
index 00000000..789a9af9
--- /dev/null
+++ b/sdk/src/models/messaging/message.rs
@@ -0,0 +1,574 @@
+use super::message_header::{IggyMessageHeader, IGGY_MESSAGE_HEADER_SIZE};
+use super::user_headers::get_headers_size_bytes;
+use crate::bytes_serializable::BytesSerializable;
+use crate::error::IggyError;
+use crate::models::messaging::user_headers::{HeaderKey, HeaderValue};
+use crate::utils::byte_size::IggyByteSize;
+use crate::utils::sizeable::Sizeable;
+use crate::utils::timestamp::IggyTimestamp;
+use bytes::{BufMut, Bytes, BytesMut};
+use serde::{Deserialize, Serialize};
+use serde_with::base64::Base64;
+use serde_with::serde_as;
+use std::collections::HashMap;
+use std::convert::TryFrom;
+use std::str::FromStr;
+use tracing::warn;
+
+/// A message stored in the Iggy messaging system.
+///
+/// `IggyMessage` represents a single message that can be sent to or received from
+/// a stream. Each message consists of:
+/// * A header with message metadata
+/// * A payload (the actual content)
+/// * Optional user-defined headers for additional context
+///
+/// # Examples
+///
+/// ```
+/// // Create a simple text message
+/// let message = IggyMessage::create("Hello world!");
+///
+/// // Create a message with custom ID
+/// let message = IggyMessage::with_id(42, "Custom message".into());
+///
+/// // Create a message with headers
+/// let mut headers = HashMap::new();
+/// headers.insert(HeaderKey::new("content-type")?, HeaderValue::from_str("text/plain")?);
+/// let message = IggyMessage::with_headers("Message with metadata".into(), headers);
+/// ```
+#[serde_as]
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+pub struct IggyMessage {
+ /// Message metadata
+ pub header: IggyMessageHeader,
+
+ /// Message content
+ #[serde_as(as = "Base64")]
+ pub payload: Bytes,
+
+ /// Optional user-defined headers
+ pub user_headers: Option<Bytes>,
+}
+
+impl IggyMessage {
+ /// Creates a new message with the given payload.
+ ///
+ /// This is the simplest way to create a message when you don't need
+ /// custom IDs or headers.
+ ///
+ /// # Arguments
+ ///
+ /// * `payload` - The message content
+ ///
+ /// # Returns
+ ///
+ /// A new `IggyMessage` with the provided payload
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let message = IggyMessage::create("Hello world!");
+ /// ```
+ pub fn create<T: Into<Bytes>>(payload: T) -> Self {
+ Self::builder()
+ .payload(payload)
+ .build()
+ .expect("Failed to create message with valid payload")
+ }
+
+ /// Creates a new message with a specific ID and payload.
+ ///
+ /// Use this when you need to control the message ID.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - Custom message ID
+ /// * `payload` - The message content
+ ///
+ /// # Returns
+ ///
+ /// A new `IggyMessage` with the provided ID and payload
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let message = IggyMessage::with_id(42, "My message".into());
+ /// ```
+ pub fn with_id<T: Into<Bytes>>(id: u128, payload: T) -> Self {
+ Self::builder()
+ .id(id)
+ .payload(payload)
+ .build()
+ .expect("Failed to create message with valid payload")
+ }
+
+ /// Creates a new message with payload and user-defined headers.
+ ///
+ /// # Arguments
+ ///
+ /// * `payload` - The message content
+ /// * `headers` - Key-value headers to attach to the message
+ ///
+ /// # Returns
+ ///
+ /// A new `IggyMessage` with the provided payload and headers
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut headers = HashMap::new();
+ /// headers.insert(HeaderKey::new("content-type")?, HeaderValue::from_str("text/plain")?);
+ ///
+ /// let message = IggyMessage::with_headers("My message".into(), headers);
+ /// ```
+ pub fn with_headers<T: Into<Bytes>>(
+ payload: T,
+ headers: HashMap<HeaderKey, HeaderValue>,
+ ) -> Self {
+ Self::builder()
+ .payload(payload)
+ .headers(headers)
+ .build()
+ .expect("Failed to create message with valid payload and headers")
+ }
+
+ /// Creates a new message with a specific ID, payload, and user-defined headers.
+ ///
+ /// This is the most flexible way to create a message with full control.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - Custom message ID
+ /// * `payload` - The message content
+ /// * `headers` - Key-value headers to attach to the message
+ ///
+ /// # Returns
+ ///
+ /// A new `IggyMessage` with all provided parameters
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut headers = HashMap::new();
+ /// headers.insert(HeaderKey::new("content-type")?, HeaderValue::from_str("text/plain")?);
+ ///
+ /// let message = IggyMessage::with_id_and_headers(42, "My message".into(), headers);
+ /// ```
+ pub fn with_id_and_headers<T: Into<Bytes>>(
+ id: u128,
+ payload: T,
+ headers: HashMap<HeaderKey, HeaderValue>,
+ ) -> Self {
+ Self::builder()
+ .id(id)
+ .payload(payload)
+ .headers(headers)
+ .build()
+ .expect("Failed to create message with valid payload and headers")
+ }
+
+ /// Creates a message builder for advanced configuration.
+ ///
+ /// Use the builder when you need fine-grained control over message creation.
+ ///
+ /// # Returns
+ ///
+ /// A new `IggyMessageBuilder` instance
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let message = IggyMessage::builder()
+ /// .id(123)
+ /// .payload("Hello")
+ /// .header("content-type", "text/plain")
+ /// .build()?;
+ /// ```
+ pub fn builder() -> IggyMessageBuilder {
+ IggyMessageBuilder::new()
+ }
+
+ /// Gets the user headers as a typed HashMap.
+ ///
+ /// This method parses the binary header data into a typed HashMap for easy access.
+ ///
+ /// # Returns
+ ///
+ /// * `Ok(Some(HashMap))` - Successfully parsed headers
+ /// * `Ok(None)` - No headers present
+ /// * `Err(IggyError)` - Error parsing headers
+ pub fn user_headers_map(&self) -> Result<Option<HashMap<HeaderKey, HeaderValue>>, IggyError> {
+ match &self.user_headers {
+ Some(headers) => {
+ let headers_bytes = Bytes::copy_from_slice(headers);
+ match HashMap::<HeaderKey, HeaderValue>::from_bytes(headers_bytes) {
+ Ok(h) => Ok(Some(h)),
+ Err(e) => {
+ warn!(
+ "Failed to deserialize user headers: {e}, user_headers_length: {}, skipping field...",
+ self.header.user_headers_length
+ );
+ Ok(None)
+ }
+ }
+ }
+ None => Ok(None),
+ }
+ }
+
+ /// Retrieves a specific user header value by key.
+ ///
+ /// This is a convenience method to get a specific header without handling the full map.
+ ///
+ /// # Arguments
+ ///
+ /// * `key` - The header key to look up
+ ///
+ /// # Returns
+ ///
+ /// * `Ok(Some(HeaderValue))` - Header found with its value
+ /// * `Ok(None)` - Header not found or headers couldn't be parsed
+ /// * `Err(IggyError)` - Error accessing headers
+ pub fn get_header(&self, key: &HeaderKey) -> Result<Option<HeaderValue>, IggyError> {
+ Ok(self
+ .user_headers_map()?
+ .and_then(|map| map.get(key).cloned()))
+ }
+
+ /// Checks if this message contains a specific header key.
+ ///
+ /// # Arguments
+ ///
+ /// * `key` - The header key to check for
+ ///
+ /// # Returns
+ ///
+ /// * `Ok(true)` - Header exists
+ /// * `Ok(false)` - Header doesn't exist or headers couldn't be parsed
+ /// * `Err(IggyError)` - Error accessing headers
+ pub fn has_header(&self, key: &HeaderKey) -> Result<bool, IggyError> {
+ Ok(self
+ .user_headers_map()?
+ .is_some_and(|map| map.contains_key(key)))
+ }
+
+ /// Gets the payload as a UTF-8 string, if valid.
+ ///
+ /// # Returns
+ ///
+ /// * `Ok(String)` - Successfully converted payload to string
+ /// * `Err(IggyError)` - Payload is not valid UTF-8
+ pub fn payload_as_string(&self) -> Result<String, IggyError> {
+ String::from_utf8(self.payload.to_vec()).map_err(|_| IggyError::InvalidUtf8)
+ }
+}
+
+impl Default for IggyMessage {
+ fn default() -> Self {
+ Self::create("hello world")
+ }
+}
+
+impl FromStr for IggyMessage {
+ type Err = IggyError;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ Ok(Self::create(Bytes::from(s.to_owned())))
+ }
+}
+
+impl std::fmt::Display for IggyMessage {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match String::from_utf8(self.payload.to_vec()) {
+ Ok(payload) => {
+ let preview = if payload.len() > 50 {
+ format!("{}... ({}B)", &payload[..47], self.payload.len())
+ } else {
+ payload
+ };
+ write!(
+ f,
+ "[{}] ID:{} '{}'",
+ self.header.offset, self.header.id, preview
+ )
+ }
+ Err(_) => {
+ write!(
+ f,
+ "[{}] ID:{} <binary {}B>",
+ self.header.offset,
+ self.header.id,
+ self.payload.len()
+ )
+ }
+ }
+ }
+}
+
+impl Sizeable for IggyMessage {
+ fn get_size_bytes(&self) -> IggyByteSize {
+ let payload_len = self.payload.len() as u64;
+ let headers_len = self.user_headers.as_ref().map_or(0, |h| h.len() as u64);
+ let message_header_len = IGGY_MESSAGE_HEADER_SIZE as u64;
+
+ IggyByteSize::from(payload_len + headers_len + message_header_len)
+ }
+}
+
+impl BytesSerializable for IggyMessage {
+ fn to_bytes(&self) -> Bytes {
+ let mut bytes = BytesMut::with_capacity(self.get_size_bytes().as_bytes_usize());
+ let message_header = self.header.to_bytes();
+ bytes.put_slice(&message_header);
+ bytes.put_slice(&self.payload);
+ if let Some(headers) = &self.user_headers {
+ bytes.put_slice(headers);
+ }
+ bytes.freeze()
+ }
+
+ fn from_bytes(bytes: Bytes) -> Result<Self, IggyError> {
+ if bytes.len() < IGGY_MESSAGE_HEADER_SIZE as usize {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let mut position = 0;
+ let header =
+ IggyMessageHeader::from_bytes(bytes.slice(0..IGGY_MESSAGE_HEADER_SIZE as usize))?;
+
+ position += IGGY_MESSAGE_HEADER_SIZE as usize;
+ let payload_end = position + header.payload_length as usize;
+
+ if payload_end > bytes.len() {
+ return Err(IggyError::InvalidMessagePayloadLength);
+ }
+
+ let payload = bytes.slice(position..payload_end);
+ position = payload_end;
+
+ let user_headers = if header.user_headers_length > 0 {
+ let headers_end = position + header.user_headers_length as usize;
+ if headers_end > bytes.len() {
+ return Err(IggyError::InvalidHeaderValue);
+ }
+ Some(bytes.slice(position..headers_end))
+ } else {
+ None
+ };
+
+ Ok(IggyMessage {
+ header,
+ payload,
+ user_headers,
+ })
+ }
+
+ fn write_to_buffer(&self, buf: &mut BytesMut) {
+ buf.put_slice(&self.header.to_bytes());
+ buf.put_slice(&self.payload);
+ if let Some(headers) = &self.user_headers {
+ buf.put_slice(headers);
+ }
+ }
+}
+
+/// Builder for creating `IggyMessage` instances with flexible configuration.
+///
+/// The builder pattern allows for clear, step-by-step construction of complex
+/// message configurations, with better error handling than chained constructors.
+#[derive(Debug, Default)]
+pub struct IggyMessageBuilder {
+ id: Option<u128>,
+ payload: Option<Bytes>,
+ headers: Option<HashMap<HeaderKey, HeaderValue>>,
+}
+
+impl IggyMessageBuilder {
+ /// Creates a new empty message builder.
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Sets the message ID.
+ ///
+ /// If not specified, a default ID (0) will be used.
+ pub fn id(mut self, id: u128) -> Self {
+ self.id = Some(id);
+ self
+ }
+
+ /// Sets the message payload.
+ ///
+ /// This method accepts any type that can be converted into `Bytes`,
+ /// including strings, byte slices, and vectors.
+ pub fn payload<T: Into<Bytes>>(mut self, payload: T) -> Self {
+ self.payload = Some(payload.into());
+ self
+ }
+
+ /// Adds a single header key-value pair to the message.
+ ///
+ /// Multiple calls will add multiple headers.
+ pub fn header(mut self, key: HeaderKey, value: HeaderValue) -> Self {
+ let headers = self.headers.get_or_insert_with(HashMap::new);
+ headers.insert(key, value);
+ self
+ }
+
+ /// Adds a string header with the given key and value.
+ ///
+ /// This is a convenience method for adding string headers without
+ /// manually creating HeaderValue objects.
+ pub fn string_header(mut self, key: &str, value: &str) -> Result<Self, IggyError> {
+ let key = HeaderKey::new(key)?;
+ let value = HeaderValue::from_str(value)?;
+ let headers = self.headers.get_or_insert_with(HashMap::new);
+ headers.insert(key, value);
+ Ok(self)
+ }
+
+ /// Sets all headers at once from a HashMap.
+ ///
+ /// This replaces any headers previously added.
+ pub fn headers(mut self, headers: HashMap<HeaderKey, HeaderValue>) -> Self {
+ self.headers = Some(headers);
+ self
+ }
+
+ /// Builds the final IggyMessage from the configured parameters.
+ ///
+ /// # Returns
+ ///
+ /// * `Ok(IggyMessage)` - Successfully built message
+ /// * `Err(IggyError)` - Error during message construction
+ pub fn build(self) -> Result<IggyMessage, IggyError> {
+ let payload = self.payload.unwrap_or_default();
+ let id = self.id.unwrap_or(0);
+ let headers_length = get_headers_size_bytes(&self.headers).as_bytes_u64() as u32;
+
+ let header = IggyMessageHeader {
+ checksum: 0, // Checksum is calculated on server side
+ id,
+ offset: 0,
+ timestamp: 0,
+ origin_timestamp: IggyTimestamp::now().as_micros(),
+ user_headers_length: headers_length,
+ payload_length: payload.len() as u32,
+ };
+
+ let user_headers = self.headers.map(|headers| headers.to_bytes());
+
+ Ok(IggyMessage {
+ header,
+ payload,
+ user_headers,
+ })
+ }
+}
+
+// Clean implementations of conversion traits
+
+impl From<IggyMessage> for Bytes {
+ fn from(message: IggyMessage) -> Self {
+ message.to_bytes()
+ }
+}
+
+impl From<String> for IggyMessage {
+ fn from(s: String) -> Self {
+ Self::create(s)
+ }
+}
+
+impl From<&str> for IggyMessage {
+ fn from(s: &str) -> Self {
+ Self::create(Bytes::from(s.to_owned()))
+ }
+}
+
+impl From<Vec<u8>> for IggyMessage {
+ fn from(v: Vec<u8>) -> Self {
+ Self::create(v)
+ }
+}
+
+impl From<&[u8]> for IggyMessage {
+ fn from(bytes: &[u8]) -> Self {
+ Self::create(Bytes::copy_from_slice(bytes))
+ }
+}
+
+impl TryFrom<Bytes> for IggyMessage {
+ type Error = IggyError;
+
+ fn try_from(bytes: Bytes) -> Result<Self, Self::Error> {
+ Self::from_bytes(bytes)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_create_simple_message() {
+ let message = IggyMessage::create("test message");
+ assert_eq!(message.payload, Bytes::from("test message"));
+ assert_eq!(message.header.id, 0);
+ assert!(message.user_headers.is_none());
+ }
+
+ #[test]
+ fn test_create_with_id() {
+ let message = IggyMessage::with_id(42, "test with id");
+ assert_eq!(message.payload, Bytes::from("test with id"));
+ assert_eq!(message.header.id, 42);
+ }
+
+ #[test]
+ fn test_create_with_headers() {
+ let mut headers = HashMap::new();
+ headers.insert(
+ HeaderKey::new("content-type").unwrap(),
+ HeaderValue::from_str("text/plain").unwrap(),
+ );
+
+ let message = IggyMessage::with_headers("test with headers", headers);
+ assert_eq!(message.payload, Bytes::from("test with headers"));
+ assert!(message.user_headers.is_some());
+
+ let headers_map = message.user_headers_map().unwrap().unwrap();
+ assert_eq!(headers_map.len(), 1);
+ assert!(headers_map.contains_key(&HeaderKey::new("content-type").unwrap()));
+ }
+
+ #[test]
+ fn test_empty_payload() {
+ let message = IggyMessage::create(Bytes::new());
+ assert_eq!(message.payload.len(), 0);
+ assert_eq!(message.header.payload_length, 0);
+ }
+
+ #[test]
+ fn test_from_string() {
+ let message: IggyMessage = "simple message".into();
+ assert_eq!(message.payload, Bytes::from("simple message"));
+ }
+
+ #[test]
+ fn test_payload_as_string() {
+ let message = IggyMessage::create("test message");
+ assert_eq!(message.payload_as_string().unwrap(), "test message");
+ }
+
+ #[test]
+ fn test_serialization_roundtrip() {
+ let original = IggyMessage::with_id(123, "serialization test");
+ let bytes = original.to_bytes();
+ let decoded = IggyMessage::from_bytes(bytes).unwrap();
+
+ assert_eq!(original.header.id, decoded.header.id);
+ assert_eq!(original.payload, decoded.payload);
+ }
+}
diff --git a/sdk/src/models/messaging/message_header.rs b/sdk/src/models/messaging/message_header.rs
new file mode 100644
index 00000000..085edbae
--- /dev/null
+++ b/sdk/src/models/messaging/message_header.rs
@@ -0,0 +1,154 @@
+use crate::{
+ bytes_serializable::BytesSerializable,
+ error::IggyError,
+ utils::{byte_size::IggyByteSize, sizeable::Sizeable},
+};
+use bytes::{BufMut, Bytes, BytesMut};
+use serde::{Deserialize, Serialize};
+use std::ops::Range;
+
+pub const IGGY_MESSAGE_HEADER_SIZE: u32 = 8 + 16 + 8 + 8 + 8 + 4 + 4;
+pub const IGGY_MESSAGE_HEADER_RANGE: Range<usize> = 0..(IGGY_MESSAGE_HEADER_SIZE as usize);
+
+pub const IGGY_MESSAGE_CHECKSUM_OFFSET_RANGE: Range<usize> = 0..8;
+pub const IGGY_MESSAGE_ID_OFFSET_RANGE: Range<usize> = 8..24;
+pub const IGGY_MESSAGE_OFFSET_OFFSET_RANGE: Range<usize> = 24..32;
+pub const IGGY_MESSAGE_TIMESTAMP_OFFSET_RANGE: Range<usize> = 32..40;
+pub const IGGY_MESSAGE_ORIGIN_TIMESTAMP_OFFSET_RANGE: Range<usize> = 40..48;
+pub const IGGY_MESSAGE_HEADERS_LENGTH_OFFSET_RANGE: Range<usize> = 48..52;
+pub const IGGY_MESSAGE_PAYLOAD_LENGTH_OFFSET_RANGE: Range<usize> = 52..56;
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Default)]
+pub struct IggyMessageHeader {
+ pub checksum: u64,
+ pub id: u128,
+ pub offset: u64,
+ pub timestamp: u64,
+ pub origin_timestamp: u64,
+ pub user_headers_length: u32,
+ pub payload_length: u32,
+}
+
+impl Sizeable for IggyMessageHeader {
+ fn get_size_bytes(&self) -> IggyByteSize {
+ IggyByteSize::from(IGGY_MESSAGE_HEADER_SIZE as u64)
+ }
+}
+
+impl IggyMessageHeader {
+ pub fn from_raw_bytes(bytes: &[u8]) -> Result<Self, IggyError> {
+ if bytes.len() != IGGY_MESSAGE_HEADER_SIZE as usize {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ Ok(IggyMessageHeader {
+ checksum: u64::from_le_bytes(
+ bytes[IGGY_MESSAGE_CHECKSUM_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ),
+ id: u128::from_le_bytes(
+ bytes[IGGY_MESSAGE_ID_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ),
+ offset: u64::from_le_bytes(
+ bytes[IGGY_MESSAGE_OFFSET_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ),
+ timestamp: u64::from_le_bytes(
+ bytes[IGGY_MESSAGE_TIMESTAMP_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ),
+ origin_timestamp: u64::from_le_bytes(
+ bytes[IGGY_MESSAGE_ORIGIN_TIMESTAMP_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ),
+ user_headers_length: u32::from_le_bytes(
+ bytes[IGGY_MESSAGE_HEADERS_LENGTH_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ),
+ payload_length: u32::from_le_bytes(
+ bytes[IGGY_MESSAGE_PAYLOAD_LENGTH_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ),
+ })
+ }
+}
+
+impl BytesSerializable for IggyMessageHeader {
+ fn to_bytes(&self) -> Bytes {
+ let mut bytes = BytesMut::with_capacity(self.get_size_bytes().as_bytes_usize());
+ bytes.put_u64_le(self.checksum);
+ bytes.put_u128_le(self.id);
+ bytes.put_u64_le(self.offset);
+ bytes.put_u64_le(self.timestamp);
+ bytes.put_u64_le(self.origin_timestamp);
+ bytes.put_u32_le(self.user_headers_length);
+ bytes.put_u32_le(self.payload_length);
+ bytes.freeze()
+ }
+
+ fn from_bytes(bytes: Bytes) -> Result<Self, IggyError> {
+ if bytes.len() != IGGY_MESSAGE_HEADER_SIZE as usize {
+ return Err(IggyError::InvalidCommand);
+ }
+
+ let checksum = u64::from_le_bytes(
+ bytes[IGGY_MESSAGE_CHECKSUM_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+
+ let id = u128::from_le_bytes(
+ bytes[IGGY_MESSAGE_ID_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+
+ let offset = u64::from_le_bytes(
+ bytes[IGGY_MESSAGE_OFFSET_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+
+ let timestamp = u64::from_le_bytes(
+ bytes[IGGY_MESSAGE_TIMESTAMP_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+
+ let origin_timestamp = u64::from_le_bytes(
+ bytes[IGGY_MESSAGE_ORIGIN_TIMESTAMP_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+
+ let headers_length = u32::from_le_bytes(
+ bytes[IGGY_MESSAGE_HEADERS_LENGTH_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+
+ let payload_length = u32::from_le_bytes(
+ bytes[IGGY_MESSAGE_PAYLOAD_LENGTH_OFFSET_RANGE]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ );
+
+ Ok(IggyMessageHeader {
+ checksum,
+ id,
+ offset,
+ timestamp,
+ origin_timestamp,
+ user_headers_length: headers_length,
+ payload_length,
+ })
+ }
+}
diff --git a/sdk/src/models/messaging/message_header_view.rs b/sdk/src/models/messaging/message_header_view.rs
new file mode 100644
index 00000000..1c35eb60
--- /dev/null
+++ b/sdk/src/models/messaging/message_header_view.rs
@@ -0,0 +1,99 @@
+use bytes::{Bytes, BytesMut};
+
+use crate::{error::IggyError, prelude::BytesSerializable};
+
+use super::message_header::*;
+
+/// A read-only, typed view into a message header in a raw buffer.
+///
+/// This wraps a `&[u8]` slice of at least `IGGY_MESSAGE_HEADER_SIZE` bytes.
+/// All accessor methods decode fields from the underlying buffer.
+#[derive(Debug)]
+pub struct IggyMessageHeaderView<'a> {
+ data: &'a [u8],
+}
+
+impl<'a> IggyMessageHeaderView<'a> {
+ /// Creates a new `IggyMessageHeaderView` over `data`.
+ pub fn new(data: &'a [u8]) -> Self {
+ debug_assert!(
+ data.len() >= IGGY_MESSAGE_HEADER_SIZE as usize,
+ "Header view requires at least {} bytes",
+ IGGY_MESSAGE_HEADER_SIZE
+ );
+ Self { data }
+ }
+
+ /// The stored checksum at the start of the header
+ pub fn checksum(&self) -> u64 {
+ let bytes = &self.data[IGGY_MESSAGE_CHECKSUM_OFFSET_RANGE];
+ u64::from_le_bytes(bytes.try_into().unwrap())
+ }
+
+ /// The 128-bit ID (16 bytes)
+ pub fn id(&self) -> u128 {
+ let bytes = &self.data[IGGY_MESSAGE_ID_OFFSET_RANGE];
+ u128::from_le_bytes(bytes.try_into().unwrap())
+ }
+
+ /// The `offset` field (8 bytes)
+ pub fn offset(&self) -> u64 {
+ let bytes = &self.data[IGGY_MESSAGE_OFFSET_OFFSET_RANGE];
+ u64::from_le_bytes(bytes.try_into().unwrap())
+ }
+
+ /// The `timestamp` field (8 bytes)
+ pub fn timestamp(&self) -> u64 {
+ let bytes = &self.data[IGGY_MESSAGE_TIMESTAMP_OFFSET_RANGE];
+ u64::from_le_bytes(bytes.try_into().unwrap())
+ }
+
+ /// The `origin_timestamp` field (8 bytes)
+ pub fn origin_timestamp(&self) -> u64 {
+ let bytes = &self.data[IGGY_MESSAGE_ORIGIN_TIMESTAMP_OFFSET_RANGE];
+ u64::from_le_bytes(bytes.try_into().unwrap())
+ }
+
+ /// The size in bytes of the user headers
+ pub fn user_headers_length(&self) -> u32 {
+ let bytes = &self.data[IGGY_MESSAGE_HEADERS_LENGTH_OFFSET_RANGE];
+ u32::from_le_bytes(bytes.try_into().unwrap())
+ }
+
+ /// The size in bytes of the message payload
+ pub fn payload_length(&self) -> u32 {
+ let bytes = &self.data[IGGY_MESSAGE_PAYLOAD_LENGTH_OFFSET_RANGE];
+ u32::from_le_bytes(bytes.try_into().unwrap())
+ }
+
+ /// Convert this view to a full IggyMessageHeader struct
+ pub fn to_header(&self) -> IggyMessageHeader {
+ IggyMessageHeader {
+ checksum: self.checksum(),
+ id: self.id(),
+ offset: self.offset(),
+ timestamp: self.timestamp(),
+ origin_timestamp: self.origin_timestamp(),
+ user_headers_length: self.user_headers_length(),
+ payload_length: self.payload_length(),
+ }
+ }
+}
+
+impl BytesSerializable for IggyMessageHeaderView<'_> {
+ fn to_bytes(&self) -> Bytes {
+ panic!("should not be used")
+ }
+
+ fn from_bytes(_bytes: Bytes) -> Result<Self, IggyError> {
+ panic!("should not be used")
+ }
+
+ fn write_to_buffer(&self, buf: &mut BytesMut) {
+ buf.extend_from_slice(self.data);
+ }
+
+ fn get_buffer_size(&self) -> u32 {
+ IGGY_MESSAGE_HEADER_SIZE
+ }
+}
diff --git a/sdk/src/models/messaging/message_view.rs b/sdk/src/models/messaging/message_view.rs
new file mode 100644
index 00000000..742cc015
--- /dev/null
+++ b/sdk/src/models/messaging/message_view.rs
@@ -0,0 +1,157 @@
+use super::message_header::*;
+use super::HeaderValue;
+use super::{message_header_view::IggyMessageHeaderView, HeaderKey};
+use crate::error::IggyError;
+use crate::prelude::BytesSerializable;
+use bytes::{Bytes, BytesMut};
+use std::{collections::HashMap, iter::Iterator};
+
+/// A immutable view of a message.
+#[derive(Debug)]
+pub struct IggyMessageView<'a> {
+ buffer: &'a [u8],
+ payload_offset: usize,
+ user_headers_offset: usize,
+}
+
+impl<'a> IggyMessageView<'a> {
+ /// Creates a new immutable message view from a buffer.
+ pub fn new(buffer: &'a [u8]) -> Self {
+ let header_view = IggyMessageHeaderView::new(&buffer[IGGY_MESSAGE_HEADER_RANGE]);
+ let payload_len = header_view.payload_length() as usize;
+ let payload_offset = IGGY_MESSAGE_HEADER_SIZE as usize;
+ let headers_offset = payload_offset + payload_len;
+
+ Self {
+ buffer,
+ payload_offset,
+ user_headers_offset: headers_offset,
+ }
+ }
+
+ /// Returns an immutable header view.
+ pub fn header(&self) -> IggyMessageHeaderView<'_> {
+ IggyMessageHeaderView::new(&self.buffer[0..IGGY_MESSAGE_HEADER_SIZE as usize])
+ }
+
+ /// Returns an immutable slice of the user headers.
+ pub fn user_headers(&self) -> Option<&[u8]> {
+ if self.header().user_headers_length() > 0 {
+ let header_length = self.header().user_headers_length() as usize;
+ let end_offset = self.user_headers_offset + header_length;
+
+ if end_offset <= self.buffer.len() {
+ Some(&self.buffer[self.user_headers_offset..end_offset])
+ } else {
+ tracing::error!(
+ "Header length in message exceeds buffer bounds: length={}, buffer_remaining={}",
+ header_length,
+ self.buffer.len() - self.user_headers_offset
+ );
+ None
+ }
+ } else {
+ None
+ }
+ }
+
+ /// Return instantiated user headers map
+ pub fn user_headers_map(&self) -> Result<Option<HashMap<HeaderKey, HeaderValue>>, IggyError> {
+ if let Some(headers) = self.user_headers() {
+ let headers_bytes = Bytes::copy_from_slice(headers);
+
+ match HashMap::<HeaderKey, HeaderValue>::from_bytes(headers_bytes) {
+ Ok(h) => Ok(Some(h)),
+ Err(e) => {
+ tracing::error!(
+ "Error parsing headers: {}, header_length={}",
+ e,
+ self.header().user_headers_length()
+ );
+
+ Ok(None)
+ }
+ }
+ } else {
+ Ok(None)
+ }
+ }
+
+ /// Returns the size of the entire message.
+ pub fn size(&self) -> u32 {
+ let header_view = self.header();
+ IGGY_MESSAGE_HEADER_SIZE + header_view.payload_length() + header_view.user_headers_length()
+ }
+
+ /// Returns a reference to the payload portion.
+ pub fn payload(&self) -> &[u8] {
+ let header_view = self.header();
+ let payload_len = header_view.payload_length() as usize;
+ &self.buffer[self.payload_offset..self.payload_offset + payload_len]
+ }
+
+ /// Validates that the message view is properly formatted and has valid data.
+ pub fn validate(&self) -> Result<(), IggyError> {
+ if self.buffer.len() < IGGY_MESSAGE_HEADER_SIZE as usize {
+ return Err(IggyError::InvalidMessagePayloadLength);
+ }
+
+ let header = self.header();
+ let payload_len = header.payload_length() as usize;
+ let user_headers_len = header.user_headers_length() as usize;
+ let total_size = IGGY_MESSAGE_HEADER_SIZE as usize + payload_len + user_headers_len;
+
+ if self.buffer.len() < total_size {
+ return Err(IggyError::InvalidMessagePayloadLength);
+ }
+ Ok(())
+ }
+}
+
+impl BytesSerializable for IggyMessageView<'_> {
+ fn to_bytes(&self) -> Bytes {
+ panic!("should not be used")
+ }
+
+ fn from_bytes(_bytes: Bytes) -> Result<Self, IggyError> {
+ panic!("should not be used")
+ }
+
+ fn write_to_buffer(&self, buf: &mut BytesMut) {
+ buf.extend_from_slice(self.buffer);
+ }
+
+ fn get_buffer_size(&self) -> u32 {
+ self.buffer.len() as u32
+ }
+}
+
+/// Iterator over immutable message views in a buffer.
+pub struct IggyMessageViewIterator<'a> {
+ buffer: &'a [u8],
+ position: usize,
+}
+
+impl<'a> IggyMessageViewIterator<'a> {
+ pub fn new(buffer: &'a [u8]) -> Self {
+ Self {
+ buffer,
+ position: 0,
+ }
+ }
+}
+
+impl<'a> Iterator for IggyMessageViewIterator<'a> {
+ type Item = IggyMessageView<'a>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.position >= self.buffer.len() {
+ return None;
+ }
+
+ let remaining = &self.buffer[self.position..];
+ let view = IggyMessageView::new(remaining);
+ self.position += view.size() as usize;
+ Some(view)
+ }
+}
diff --git a/sdk/src/models/messaging/messages_batch.rs b/sdk/src/models/messaging/messages_batch.rs
new file mode 100644
index 00000000..3ff535bc
--- /dev/null
+++ b/sdk/src/models/messaging/messages_batch.rs
@@ -0,0 +1,586 @@
+use super::{IggyIndexes, IggyMessage, IggyMessageView, IggyMessageViewIterator};
+use crate::{
+ error::IggyError,
+ messages::MAX_PAYLOAD_SIZE,
+ models::messaging::INDEX_SIZE,
+ prelude::{BytesSerializable, IggyByteSize, Sizeable, Validatable},
+};
+use bytes::{BufMut, Bytes, BytesMut};
+use std::ops::{Deref, Index};
+
+/// An immutable messages container that holds a buffer of messages
+#[derive(Clone, Debug, PartialEq)]
+pub struct IggyMessagesBatch {
+ /// The number of messages in the batch
+ count: u32,
+ /// The byte-indexes of messages in the buffer, represented as array of u32's. Offsets are relative.
+ /// Each index consists of offset, position (byte offset in the buffer) and timestamp.
+ indexes: IggyIndexes,
+ /// The buffer containing the messages
+ messages: Bytes,
+}
+
+impl IggyMessagesBatch {
+ /// Create a batch from indexes buffer and messages buffer
+ pub fn new(indexes: IggyIndexes, messages: Bytes, count: u32) -> Self {
+ #[cfg(debug_assertions)]
+ {
+ debug_assert_eq!(indexes.len() % INDEX_SIZE, 0);
+ let indexes_count = indexes.len() / INDEX_SIZE;
+ debug_assert_eq!(indexes_count, count as usize);
+
+ if count > 0 {
+ let first_position = indexes.get(0).unwrap().position();
+
+ let iter = IggyMessageViewIterator::new(&messages);
+ let mut current_position = first_position;
+
+ for (i, message) in iter.enumerate().skip(1) {
+ if i < count as usize {
+ current_position += message.size();
+ let expected_position = indexes.get(i as u32).unwrap().position();
+ debug_assert_eq!(
+ current_position, expected_position,
+ "Message {} position mismatch: {} != {}",
+ i, current_position, expected_position
+ );
+ }
+ }
+ }
+ }
+
+ Self {
+ count,
+ indexes,
+ messages,
+ }
+ }
+
+ /// Creates a empty messages batch
+ pub fn empty() -> Self {
+ Self::new(IggyIndexes::empty(), BytesMut::new().freeze(), 0)
+ }
+
+ /// Create iterator over messages
+ pub fn iter(&self) -> IggyMessageViewIterator {
+ IggyMessageViewIterator::new(&self.messages)
+ }
+
+ /// Get the number of messages
+ pub fn count(&self) -> u32 {
+ self.count
+ }
+
+ /// Check if the batch is empty
+ pub fn is_empty(&self) -> bool {
+ self.count() == 0
+ }
+
+ /// Get the total size of all messages in bytes
+ pub fn size(&self) -> u32 {
+ self.messages.len() as u32
+ }
+
+ /// Get access to the underlying buffer
+ pub fn buffer(&self) -> &[u8] {
+ &self.messages
+ }
+
+ pub fn indexes_slice(&self) -> &[u8] {
+ &self.indexes
+ }
+
+ /// Decompose the batch into its components
+ pub fn decompose(self) -> (u32, IggyIndexes, Bytes) {
+ (self.count, self.indexes, self.messages)
+ }
+
+ /// Get index of first message
+ pub fn first_offset(&self) -> u64 {
+ self.iter()
+ .next()
+ .map(|msg| msg.header().offset())
+ .unwrap_or(0)
+ }
+
+ /// Get timestamp of first message
+ pub fn first_timestamp(&self) -> u64 {
+ self.iter()
+ .next()
+ .map(|msg| msg.header().timestamp())
+ .unwrap_or(0)
+ }
+
+ /// Get offset of last message
+ pub fn last_offset(&self) -> u64 {
+ self.iter()
+ .last()
+ .map(|msg| msg.header().offset())
+ .unwrap_or(0)
+ }
+
+ /// Get timestamp of last message
+ pub fn last_timestamp(&self) -> u64 {
+ self.iter()
+ .last()
+ .map(|msg| msg.header().timestamp())
+ .unwrap_or(0)
+ }
+
+ /// Helper method to read a position (u32) from the byte array at the given index
+ fn position_at(&self, position_index: u32) -> u32 {
+ if let Some(index) = self.indexes.get(position_index) {
+ index.position()
+ } else {
+ 0
+ }
+ }
+
+ /// Returns a contiguous slice (as a new `IggyMessagesBatch`) of up to `count` messages
+ /// whose message headers have an offset greater than or equal to the provided `start_offset`.
+ pub fn slice_by_offset(&self, start_offset: u64, count: u32) -> Option<Self> {
+ if self.is_empty() || count == 0 {
+ return None;
+ }
+
+ // TODO(hubcio): this can be optimized via binary search
+ let first_message_index = self
+ .iter()
+ .position(|msg| msg.header().offset() >= start_offset);
+
+ first_message_index?;
+
+ let first_message_index = first_message_index.unwrap();
+ let last_message_index =
+ std::cmp::min(first_message_index + count as usize, self.count() as usize);
+
+ let sub_indexes = self.indexes.slice_by_offset(
+ first_message_index as u32,
+ (last_message_index - first_message_index) as u32,
+ )?;
+
+ let first_message_position = if first_message_index == 0 {
+ 0
+ } else {
+ self.position_at(first_message_index as u32 - 1) as usize
+ - self.indexes.base_position() as usize
+ };
+
+ let last_message_position = if last_message_index >= self.count() as usize {
+ self.messages.len()
+ } else {
+ self.position_at(last_message_index as u32 - 1) as usize
+ - self.indexes.base_position() as usize
+ };
+
+ debug_assert!(
+ first_message_position <= self.messages.len(),
+ "First message position {} exceeds buffer length {}",
+ first_message_position,
+ self.messages.len()
+ );
+ debug_assert!(
+ last_message_position <= self.messages.len(),
+ "Last message position {} exceeds buffer length {}",
+ last_message_position,
+ self.messages.len()
+ );
+
+ let sub_buffer = self
+ .messages
+ .slice(first_message_position..last_message_position);
+
+ Some(IggyMessagesBatch {
+ count: (last_message_index - first_message_index) as u32,
+ indexes: sub_indexes,
+ messages: sub_buffer,
+ })
+ }
+
+ /// Returns a contiguous slice (as a new `IggyMessagesBatch`) of up to `count` messages
+ /// whose message headers have a timestamp greater than or equal to the provided `timestamp`.
+ ///
+ /// If no messages meet the criteria, returns `None`.
+ pub fn slice_by_timestamp(&self, timestamp: u64, count: u32) -> Option<Self> {
+ if self.is_empty() || count == 0 {
+ return None;
+ }
+
+ // TODO(hubcio): this can be optimized via binary search
+ let first_message_index = self
+ .iter()
+ .position(|msg| msg.header().timestamp() >= timestamp);
+
+ first_message_index?;
+
+ let first_message_index = first_message_index.unwrap();
+ let last_message_index =
+ std::cmp::min(first_message_index + count as usize, self.count() as usize);
+
+ let sub_indexes = self.indexes.slice_by_offset(
+ first_message_index as u32,
+ (last_message_index - first_message_index) as u32,
+ )?;
+
+ let first_message_position = if first_message_index == 0 {
+ 0
+ } else {
+ self.position_at(first_message_index as u32 - 1) as usize
+ - self.indexes.base_position() as usize
+ };
+
+ let last_message_position = if last_message_index >= self.count() as usize {
+ self.messages.len()
+ } else {
+ self.position_at(last_message_index as u32 - 1) as usize
+ - self.indexes.base_position() as usize
+ };
+
+ debug_assert!(
+ first_message_position <= self.messages.len(),
+ "First message position {} exceeds buffer length {}",
+ first_message_position,
+ self.messages.len()
+ );
+ debug_assert!(
+ last_message_position <= self.messages.len(),
+ "Last message position {} exceeds buffer length {}",
+ last_message_position,
+ self.messages.len()
+ );
+
+ let sub_buffer = self
+ .messages
+ .slice(first_message_position..last_message_position);
+
+ Some(IggyMessagesBatch {
+ count: (last_message_index - first_message_index) as u32,
+ indexes: sub_indexes,
+ messages: sub_buffer,
+ })
+ }
+
+ /// Get the message at the specified index.
+ /// Returns None if the index is out of bounds.
+ pub fn get(&self, index: usize) -> Option<IggyMessageView> {
+ if index >= self.count as usize {
+ return None;
+ }
+
+ // TODO: refactor this
+ let start_position = if index == 0 {
+ 0
+ } else {
+ self.position_at(index as u32 - 1) as usize - self.indexes.base_position() as usize
+ };
+
+ let end_position = if index == self.count as usize - 1 {
+ self.messages.len()
+ } else {
+ self.position_at(index as u32) as usize - self.indexes.base_position() as usize
+ };
+
+ debug_assert!(
+ start_position <= self.messages.len(),
+ "Start message position {} exceeds buffer length {}, index: {}, self.count: {}, self.base_position: {}",
+ start_position,
+ self.messages.len(),
+ index,
+ self.count,
+ self.indexes.base_position()
+ );
+ debug_assert!(
+ end_position <= self.messages.len(),
+ "End message position {} exceeds buffer length {}, index: {}, self.count: {}, self.base_position: {}",
+ end_position,
+ self.messages.len(),
+ index,
+ self.count,
+ self.indexes.base_position()
+ );
+
+ Some(IggyMessageView::new(
+ &self.messages[start_position..end_position],
+ ))
+ }
+
+ /// Consumes the batch and returns a vector of messages.
+ pub fn into_messages(self) -> Vec<IggyMessage> {
+ if self.is_empty() {
+ return Vec::new();
+ }
+
+ let (count, indexes, buffer) = self.decompose();
+ let mut result = Vec::with_capacity(count as usize);
+ let base_position = indexes.base_position();
+
+ for i in 0..count {
+ let start_position = if i == 0 {
+ 0
+ } else {
+ let prev_idx = indexes.get(i - 1).map(|idx| idx.position()).unwrap_or(0);
+ prev_idx as usize - base_position as usize
+ };
+
+ let end_position = if i == count - 1 {
+ buffer.len()
+ } else {
+ let curr_idx = indexes.get(i).map(|idx| idx.position()).unwrap_or(0);
+ curr_idx as usize - base_position as usize
+ };
+
+ if start_position < buffer.len()
+ && end_position <= buffer.len()
+ && start_position < end_position
+ {
+ if let Ok(message) =
+ IggyMessage::from_bytes(buffer.slice(start_position..end_position))
+ {
+ result.push(message);
+ }
+ }
+ }
+
+ result
+ }
+}
+
+impl IntoIterator for IggyMessagesBatch {
+ type Item = IggyMessage;
+ type IntoIter = std::vec::IntoIter<IggyMessage>;
+
+ fn into_iter(self) -> Self::IntoIter {
+ self.into_messages().into_iter()
+ }
+}
+
+impl Index<usize> for IggyMessagesBatch {
+ type Output = [u8];
+
+ /// Get the message bytes at the specified index
+ ///
+ /// # Panics
+ ///
+ /// Panics if the index is out of bounds (>= count)
+ fn index(&self, index: usize) -> &Self::Output {
+ if index >= self.count as usize {
+ panic!(
+ "Index out of bounds: the len is {} but the index is {}",
+ self.count, index
+ );
+ }
+
+ let start_position = if index == 0 {
+ 0
+ } else {
+ self.position_at(index as u32 - 1) as usize
+ };
+
+ let end_position = if index == self.count as usize - 1 {
+ self.messages.len()
+ } else {
+ self.position_at(index as u32) as usize
+ };
+
+ &self.messages[start_position..end_position]
+ }
+}
+
+impl BytesSerializable for IggyMessagesBatch {
+ fn to_bytes(&self) -> Bytes {
+ panic!("should not be used");
+ }
+
+ fn from_bytes(_bytes: Bytes) -> Result<Self, IggyError> {
+ panic!("don't use");
+ }
+
+ fn write_to_buffer(&self, buf: &mut BytesMut) {
+ buf.put_u32_le(self.count);
+ buf.put_slice(&self.indexes);
+ buf.put_slice(&self.messages);
+ }
+
+ fn get_buffer_size(&self) -> u32 {
+ 4 + self.indexes.len() as u32 + self.messages.len() as u32
+ }
+}
+
+impl Validatable<IggyError> for IggyMessagesBatch {
+ fn validate(&self) -> Result<(), IggyError> {
+ if self.is_empty() {
+ return Err(IggyError::InvalidMessagesCount);
+ }
+
+ let indexes_count = self.indexes.count();
+ let indexes_size = self.indexes.size();
+
+ if indexes_size % INDEX_SIZE as u32 != 0 {
+ tracing::error!(
+ "Indexes size {} is not a multiple of index size {}",
+ indexes_size,
+ INDEX_SIZE
+ );
+ return Err(IggyError::InvalidIndexesByteSize(indexes_size));
+ }
+
+ if indexes_count != self.count() {
+ tracing::error!(
+ "Indexes count {} does not match messages count {}",
+ indexes_count,
+ self.count()
+ );
+ return Err(IggyError::InvalidIndexesCount(indexes_count, self.count()));
+ }
+
+ let mut messages_count = 0;
+ let mut messages_size = 0;
+
+ for i in 0..self.count() {
+ if let Some(index_view) = self.indexes.get(i) {
+ if index_view.offset() != 0 {
+ tracing::error!("Non-zero offset {} at index: {}", index_view.offset(), i);
+ return Err(IggyError::NonZeroOffset(index_view.offset() as u64, i));
+ }
+ if index_view.timestamp() != 0 {
+ tracing::error!(
+ "Non-zero timestamp {} at index: {}",
+ index_view.timestamp(),
+ i
+ );
+ return Err(IggyError::NonZeroTimestamp(index_view.timestamp(), i));
+ }
+ } else {
+ tracing::error!("Index {} is missing", i);
+ return Err(IggyError::MissingIndex(i));
... 15024 lines suppressed ...