You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/01/14 19:34:57 UTC
(iggy) branch global-metadata-leftright updated (9ab177967 -> dc82ca865)
This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a change to branch global-metadata-leftright
in repository https://gitbox.apache.org/repos/asf/iggy.git
omit 9ab177967 refactor(server): WIP, don't merge: replace slab ECS with LeftRight-based global metadata
new dc82ca865 refactor(server): WIP, don't merge: replace slab ECS with LeftRight-based global metadata
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (9ab177967)
\
N -- N -- N refs/heads/global-metadata-leftright (dc82ca865)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
core/server/src/shard/system/utils.rs | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
(iggy) 01/01: refactor(server): WIP, don't merge: replace slab ECS with LeftRight-based global metadata
Posted by hg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a commit to branch global-metadata-leftright
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit dc82ca86530b35844c3ba34e2b5b680ca1053a75
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Mon Jan 5 14:14:16 2026 +0100
refactor(server): WIP, don't merge: replace slab ECS with LeftRight-based global metadata
---
Cargo.lock | 637 ++++----
Cargo.toml | 4 +-
DEPENDENCIES.md | 144 +-
core/integration/Cargo.toml | 1 +
core/integration/src/test_server.rs | 109 +-
.../data_integrity/verify_after_server_restart.rs | 205 ++-
core/integration/tests/server/mod.rs | 85 +-
.../tests/server/scenarios/concurrent_scenario.rs | 112 +-
core/integration/tests/streaming/get_by_offset.rs | 2 +-
.../tests/streaming/get_by_timestamp.rs | 2 +-
core/integration/tests/streaming/mod.rs | 585 +++++++-
core/server/Cargo.toml | 7 +-
core/server/src/binary/command.rs | 265 ----
.../cluster/get_cluster_metadata_handler.rs | 2 -
.../create_consumer_group_handler.rs | 85 +-
.../delete_consumer_group_handler.rs | 109 +-
.../consumer_groups/get_consumer_group_handler.rs | 37 +-
.../consumer_groups/get_consumer_groups_handler.rs | 45 +-
.../consumer_groups/join_consumer_group_handler.rs | 66 +-
.../leave_consumer_group_handler.rs | 67 +-
.../delete_consumer_offset_handler.rs | 1 -
.../get_consumer_offset_handler.rs | 1 -
.../store_consumer_offset_handler.rs | 1 -
.../messages/flush_unsaved_buffer_handler.rs | 1 -
.../handlers/messages/poll_messages_handler.rs | 1 -
.../handlers/messages/send_messages_handler.rs | 66 +-
.../partitions/create_partitions_handler.rs | 136 +-
.../partitions/delete_partitions_handler.rs | 142 +-
.../create_personal_access_token_handler.rs | 64 +-
.../delete_personal_access_token_handler.rs | 54 +-
.../login_with_personal_access_token_handler.rs | 1 -
.../handlers/segments/delete_segments_handler.rs | 23 +-
.../handlers/streams/create_stream_handler.rs | 25 +-
.../handlers/streams/delete_stream_handler.rs | 19 +-
.../binary/handlers/streams/get_stream_handler.rs | 22 +-
.../binary/handlers/streams/get_streams_handler.rs | 9 +-
.../handlers/streams/purge_stream_handler.rs | 2 -
.../handlers/streams/update_stream_handler.rs | 82 +-
.../src/binary/handlers/system/ping_handler.rs | 1 -
.../binary/handlers/topics/create_topic_handler.rs | 71 +-
.../binary/handlers/topics/delete_topic_handler.rs | 46 +-
.../binary/handlers/topics/get_topic_handler.rs | 40 +-
.../binary/handlers/topics/get_topics_handler.rs | 21 +-
.../binary/handlers/topics/purge_topic_handler.rs | 1 -
.../binary/handlers/topics/update_topic_handler.rs | 52 +-
.../handlers/users/change_password_handler.rs | 143 +-
.../binary/handlers/users/create_user_handler.rs | 12 -
.../binary/handlers/users/delete_user_handler.rs | 8 +-
.../binary/handlers/users/login_user_handler.rs | 1 -
.../binary/handlers/users/logout_user_handler.rs | 1 -
.../handlers/users/update_permissions_handler.rs | 103 +-
.../binary/handlers/users/update_user_handler.rs | 130 +-
core/server/src/binary/macros.rs | 2 +-
core/server/src/binary/mapper.rs | 152 +-
core/server/src/bootstrap.rs | 437 +++---
core/server/src/configs/system.rs | 12 +-
core/server/src/http/consumer_groups.rs | 228 ++-
core/server/src/http/consumer_offsets.rs | 6 +-
core/server/src/http/http_shard_wrapper.rs | 101 +-
core/server/src/http/mapper.rs | 383 ++---
core/server/src/http/partitions.rs | 4 +-
core/server/src/http/streams.rs | 117 +-
core/server/src/http/topics.rs | 217 ++-
core/server/src/http/users.rs | 89 --
core/server/src/lib.rs | 1 -
core/server/src/main.rs | 111 +-
core/server/src/metadata/mod.rs | 4 +-
core/server/src/metadata/ops.rs | 3 +-
core/server/src/metadata/partition.rs | 9 +-
core/server/src/metadata/reader.rs | 3 +-
core/server/src/metadata/writer.rs | 3 +-
core/server/src/quic/listener.rs | 2 +-
core/server/src/shard/builder.rs | 51 +-
core/server/src/shard/handlers.rs | 750 +++++++---
core/server/src/shard/mod.rs | 133 +-
core/server/src/shard/namespace.rs | 53 -
core/server/src/shard/system/clients.rs | 54 +-
core/server/src/shard/system/consumer_groups.rs | 389 +++--
core/server/src/shard/system/consumer_offsets.rs | 422 +++---
core/server/src/shard/system/messages.rs | 579 ++++++--
core/server/src/shard/system/partitions.rs | 526 ++++---
.../src/shard/system/personal_access_tokens.rs | 155 +-
core/server/src/shard/system/segments.rs | 220 ++-
core/server/src/shard/system/stats.rs | 58 +-
core/server/src/shard/system/streams.rs | 351 +++--
core/server/src/shard/system/topics.rs | 547 ++++---
core/server/src/shard/system/users.rs | 266 ++--
core/server/src/shard/system/utils.rs | 206 +--
.../src/shard/tasks/periodic/heartbeat_verifier.rs | 2 +-
.../src/shard/tasks/periodic/message_cleaner.rs | 182 +--
.../src/shard/tasks/periodic/message_saver.rs | 80 +-
.../periodic/personal_access_token_cleaner.rs | 40 +-
core/server/src/shard/transmission/event.rs | 115 +-
core/server/src/shard/transmission/frame.rs | 26 +-
core/server/src/shard/transmission/message.rs | 79 +-
core/server/src/slab/consumer_groups.rs | 200 ---
core/server/src/slab/helpers.rs | 70 -
core/server/src/slab/mod.rs | 43 -
core/server/src/slab/partitions.rs | 281 ----
core/server/src/slab/streams.rs | 1522 --------------------
core/server/src/slab/topics.rs | 307 ----
core/server/src/slab/traits_ext.rs | 117 --
core/server/src/slab/users.rs | 207 ---
.../streaming/partitions/consumer_group_offsets.rs | 51 +
.../src/streaming/partitions/consumer_offset.rs | 3 +-
.../src/streaming/partitions/consumer_offsets.rs | 50 +
core/server/src/streaming/partitions/helpers.rs | 512 +------
.../src/streaming/partitions/local_partition.rs | 98 ++
.../src/streaming/partitions/local_partitions.rs | 213 +++
core/server/src/streaming/partitions/mod.rs | 6 +-
core/server/src/streaming/partitions/ops.rs | 671 +++++++++
core/server/src/streaming/partitions/partition.rs | 454 ------
core/server/src/streaming/stats/mod.rs | 15 +
core/server/src/streaming/streams/helpers.rs | 107 --
core/server/src/streaming/streams/mod.rs | 2 -
core/server/src/streaming/streams/storage.rs | 27 +-
core/server/src/streaming/streams/stream.rs | 225 ---
core/server/src/streaming/topics/consumer_group.rs | 243 ----
core/server/src/streaming/topics/helpers.rs | 305 +---
core/server/src/streaming/topics/mod.rs | 2 -
core/server/src/streaming/topics/storage.rs | 43 +-
core/server/src/streaming/topics/topic.rs | 379 -----
core/server/src/streaming/users/permissioner.rs | 112 +-
.../streaming/users/permissioner_rules/messages.rs | 110 +-
.../streaming/users/permissioner_rules/streams.rs | 32 +-
.../streaming/users/permissioner_rules/system.rs | 6 +-
.../streaming/users/permissioner_rules/topics.rs | 80 +-
.../streaming/users/permissioner_rules/users.rs | 24 +-
core/server/src/tcp/tcp_listener.rs | 2 +-
core/server/src/tcp/tcp_tls_listener.rs | 2 +-
core/server/src/websocket/websocket_listener.rs | 2 +-
.../server/src/websocket/websocket_tls_listener.rs | 2 +-
132 files changed, 7644 insertions(+), 9898 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index a9e03e889..2bdf3f869 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -103,7 +103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
dependencies = [
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -220,7 +220,7 @@ dependencies = [
"actix-router",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -276,7 +276,7 @@ version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"once_cell",
"version_check",
]
@@ -415,15 +415,6 @@ dependencies = [
"uuid",
]
-[[package]]
-name = "arbitrary"
-version = "1.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
-dependencies = [
- "derive_arbitrary",
-]
-
[[package]]
name = "arc-swap"
version = "1.8.0"
@@ -433,12 +424,6 @@ dependencies = [
"rustversion",
]
-[[package]]
-name = "arcshift"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "809ce7e2c99d17c161c0ca1673665bab3a3e1aa277b0f793abffc66b57c32d9f"
-
[[package]]
name = "argon2"
version = "0.5.3"
@@ -568,7 +553,7 @@ dependencies = [
"arrow-schema",
"chrono",
"half",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"lexical-core",
"memchr",
"num",
@@ -657,7 +642,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"synstructure",
]
@@ -669,14 +654,14 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "assert_cmd"
-version = "2.1.1"
+version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcbb6924530aa9e0432442af08bbcafdad182db80d2e560da42a6d442535bf85"
+checksum = "9c5bcfa8749ac45dd12cb11055aeeb6b27a3895560d60d71e3c23bf979e60514"
dependencies = [
"anstyle",
"bstr",
@@ -729,13 +714,12 @@ dependencies = [
[[package]]
name = "async-compression"
-version = "0.4.36"
+version = "0.4.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98ec5f6c2f8bc326c994cb9e241cc257ddaba9afa8555a43cffbb5dd86efaa37"
+checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40"
dependencies = [
"compression-codecs",
"compression-core",
- "futures-core",
"futures-io",
"pin-project-lite",
"tokio",
@@ -763,7 +747,7 @@ dependencies = [
"async-trait",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"tokio",
]
@@ -821,7 +805,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -838,7 +822,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -987,7 +971,7 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -1043,9 +1027,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "base64ct"
-version = "1.8.1"
+version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a"
+checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bdd"
@@ -1207,15 +1191,16 @@ dependencies = [
[[package]]
name = "blake3"
-version = "1.8.2"
+version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0"
+checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq",
+ "cpufeatures",
]
[[package]]
@@ -1315,9 +1300,9 @@ dependencies = [
[[package]]
name = "bon"
-version = "3.8.1"
+version = "3.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1"
+checksum = "234655ec178edd82b891e262ea7cf71f6584bcd09eff94db786be23f1821825c"
dependencies = [
"bon-macros",
"rustversion",
@@ -1325,17 +1310,17 @@ dependencies = [
[[package]]
name = "bon-macros"
-version = "3.8.1"
+version = "3.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645"
+checksum = "89ec27229c38ed0eb3c0feee3d2c1d6a4379ae44f418a29a658890e062d8f365"
dependencies = [
- "darling 0.21.3",
+ "darling 0.23.0",
"ident_case",
"prettyplease",
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -1358,7 +1343,7 @@ dependencies = [
"proc-macro-crate 3.4.0",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -1533,9 +1518,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.51"
+version = "1.2.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203"
+checksum = "cd4932aefd12402b36c60956a4fe0035421f544799057659ff86f923657aada3"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1586,7 +1571,7 @@ checksum = "5ec2398273c047c67d69794a924b1a2a5c14a5fab6bcbe8b24e86a0df9328e5e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -1615,9 +1600,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.53"
+version = "4.5.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8"
+checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1625,9 +1610,9 @@ dependencies = [
[[package]]
name = "clap_builder"
-version = "4.5.53"
+version = "4.5.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00"
+checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00"
dependencies = [
"anstream",
"anstyle",
@@ -1638,9 +1623,9 @@ dependencies = [
[[package]]
name = "clap_complete"
-version = "4.5.63"
+version = "4.5.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90ef1fcbbf16b486489d0df91725ccc653c07115dd61f46363162535b74c6bc3"
+checksum = "430b4dc2b5e3861848de79627b2bedc9f3342c7da5173a14eaa5d0f8dc18ae5d"
dependencies = [
"clap",
]
@@ -1654,14 +1639,14 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "clap_lex"
-version = "0.7.6"
+version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
+checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32"
[[package]]
name = "clock"
@@ -1715,9 +1700,9 @@ dependencies = [
[[package]]
name = "comfy-table"
-version = "7.2.1"
+version = "7.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b"
+checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47"
dependencies = [
"crossterm",
"unicode-segmentation",
@@ -1825,7 +1810,7 @@ dependencies = [
"proc-macro-crate 3.4.0",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -1921,9 +1906,9 @@ dependencies = [
[[package]]
name = "compression-codecs"
-version = "0.4.35"
+version = "0.4.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0f7ac3e5b97fdce45e8922fb05cae2c37f7bbd63d30dd94821dacfd8f3f2bf2"
+checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a"
dependencies = [
"bzip2",
"compression-core",
@@ -2004,16 +1989,16 @@ version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"once_cell",
"tiny-keccak",
]
[[package]]
name = "constant_time_eq"
-version = "0.3.1"
+version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
+checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
[[package]]
name = "convert_case"
@@ -2091,9 +2076,9 @@ dependencies = [
[[package]]
name = "crc"
-version = "3.3.0"
+version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675"
+checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
dependencies = [
"crc-catalog",
]
@@ -2311,7 +2296,7 @@ dependencies = [
"proc-macro2",
"quote",
"regex",
- "syn 2.0.111",
+ "syn 2.0.114",
"synthez",
]
@@ -2353,7 +2338,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2457,7 +2442,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2471,7 +2456,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2484,7 +2469,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2495,7 +2480,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
dependencies = [
"darling_core 0.20.11",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2506,7 +2491,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81"
dependencies = [
"darling_core 0.21.3",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2517,7 +2502,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
dependencies = [
"darling_core 0.23.0",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2542,9 +2527,9 @@ dependencies = [
[[package]]
name = "data-encoding"
-version = "2.9.0"
+version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
+checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea"
[[package]]
name = "dbus"
@@ -2568,12 +2553,6 @@ dependencies = [
"zeroize",
]
-[[package]]
-name = "deflate64"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204"
-
[[package]]
name = "der"
version = "0.7.10"
@@ -2617,18 +2596,7 @@ checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
-]
-
-[[package]]
-name = "derive_arbitrary"
-version = "1.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2649,7 +2617,7 @@ dependencies = [
"darling 0.20.11",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2659,7 +2627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
dependencies = [
"derive_builder_core",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2681,7 +2649,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustc_version",
- "syn 2.0.111",
+ "syn 2.0.114",
"unicode-xid",
]
@@ -2755,7 +2723,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2784,7 +2752,7 @@ checksum = "0fbbb781877580993a8707ec48672673ec7b81eeba04cfd2310bd28c08e47c8f"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2988,7 +2956,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -3057,7 +3025,7 @@ dependencies = [
"proc-macro2",
"quote",
"regex",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -3149,9 +3117,9 @@ checksum = "b877ceff2e3d2922823bab7960826ce198181f9c25f75d67b913fde9237e2da6"
[[package]]
name = "fastbloom"
-version = "0.14.0"
+version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18c1ddb9231d8554c2d6bdf4cfaabf0c59251658c68b6c95cd52dd0c513a912a"
+checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4"
dependencies = [
"getrandom 0.3.4",
"libm",
@@ -3165,14 +3133,14 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
]
[[package]]
name = "ferroid"
-version = "0.8.8"
+version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce161062fb044bd629c2393590efd47cab8d0241faf15704ffb0d47b7b4e4a35"
+checksum = "bb330bbd4cb7a5b9f559427f06f98a4f853a137c8298f3bd3f8ca57663e21986"
dependencies = [
"portable-atomic",
"rand 0.9.2",
@@ -3218,9 +3186,9 @@ dependencies = [
[[package]]
name = "file-operation"
-version = "0.8.8"
+version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34664772b984b86dbccb14ab56722778cca458b38f6f324db3331141ef441e05"
+checksum = "42c2d0511147e9c5aebb94162f9c90aaeb1c232dcd8f864654d30edfbf0385bc"
dependencies = [
"tokio",
]
@@ -3239,9 +3207,9 @@ dependencies = [
[[package]]
name = "find-msvc-tools"
-version = "0.1.6"
+version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff"
+checksum = "f449e6c6c08c865631d4890cfacf252b3d396c9bcc83adb6623cdb02a8336c41"
[[package]]
name = "flatbuffers"
@@ -3255,13 +3223,13 @@ dependencies = [
[[package]]
name = "flate2"
-version = "1.1.5"
+version = "1.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb"
+checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369"
dependencies = [
"crc32fast",
- "libz-rs-sys",
"miniz_oxide",
+ "zlib-rs",
]
[[package]]
@@ -3455,7 +3423,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -3533,9 +3501,9 @@ dependencies = [
[[package]]
name = "getrandom"
-version = "0.2.16"
+version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
"js-sys",
@@ -3579,7 +3547,7 @@ dependencies = [
"quote",
"serde",
"serde_json",
- "syn 2.0.111",
+ "syn 2.0.114",
"textwrap",
"thiserror 2.0.17",
"typed-builder 0.23.2",
@@ -3693,7 +3661,7 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "903f432be5ba34427eac5e16048ef65604a82061fe93789f2212afc73d8617d6"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"gloo-events",
"gloo-utils",
"serde",
@@ -3803,7 +3771,7 @@ dependencies = [
"proc-macro-crate 1.3.1",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -3852,7 +3820,7 @@ dependencies = [
"futures-sink",
"futures-util",
"http 0.2.12",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"slab",
"tokio",
"tokio-util",
@@ -3861,9 +3829,9 @@ dependencies = [
[[package]]
name = "h2"
-version = "0.4.12"
+version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386"
+checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54"
dependencies = [
"atomic-waker",
"bytes",
@@ -3871,7 +3839,7 @@ dependencies = [
"futures-core",
"futures-sink",
"http 1.4.0",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"slab",
"tokio",
"tokio-util",
@@ -3902,9 +3870,9 @@ dependencies = [
[[package]]
name = "handlebars"
-version = "6.3.2"
+version = "6.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "759e2d5aea3287cb1190c8ec394f42866cb5bf74fcbf213f354e3c856ea26098"
+checksum = "9b3f9296c208515b87bd915a2f5d1163d4b3f863ba83337d7713cf478055948e"
dependencies = [
"derive_builder",
"log",
@@ -4165,7 +4133,7 @@ dependencies = [
"bytes",
"futures-channel",
"futures-core",
- "h2 0.4.12",
+ "h2 0.4.13",
"http 1.4.0",
"http-body",
"httparse",
@@ -4209,7 +4177,7 @@ dependencies = [
"tokio",
"tokio-rustls",
"tower-service",
- "webpki-roots 1.0.4",
+ "webpki-roots 1.0.5",
]
[[package]]
@@ -4502,7 +4470,7 @@ dependencies = [
"tracing",
"trait-variant",
"tungstenite",
- "webpki-roots 1.0.4",
+ "webpki-roots 1.0.5",
]
[[package]]
@@ -4528,7 +4496,7 @@ dependencies = [
"serde",
"sysinfo 0.37.2",
"tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
"tracing",
"tracing-appender",
"tracing-subscriber",
@@ -4615,7 +4583,7 @@ dependencies = [
"strum 0.27.2",
"thiserror 2.0.17",
"tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
"tower-http",
"tracing",
"tracing-subscriber",
@@ -4660,7 +4628,7 @@ dependencies = [
"serde",
"serde_json",
"tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
"tracing",
]
@@ -4702,7 +4670,7 @@ dependencies = [
"strum 0.27.2",
"thiserror 2.0.17",
"tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
"tracing",
"tungstenite",
"twox-hash",
@@ -4922,7 +4890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1689b939ee35e3a075b0834b5672efd43aec8a6e81a1c6002b76a5ca2f211ae0"
dependencies = [
"implicit-clone-derive",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
]
[[package]]
@@ -4932,7 +4900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "699c1b6d335e63d0ba5c1e1c7f647371ce989c3bcbe1f7ed2b85fa56e3bd1a21"
dependencies = [
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -4948,9 +4916,9 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "2.12.1"
+version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2"
+checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
dependencies = [
"equivalent",
"hashbrown 0.16.1",
@@ -5035,6 +5003,7 @@ dependencies = [
"serde_json",
"serial_test",
"server",
+ "slab",
"strum 0.27.2",
"strum_macros 0.27.2",
"tempfile",
@@ -5125,9 +5094,9 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
[[package]]
name = "jiff"
-version = "0.2.17"
+version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a87d9b8105c23642f50cbbae03d1f75d8422c5cb98ce7ee9271f7ff7505be6b8"
+checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50"
dependencies = [
"jiff-static",
"log",
@@ -5138,13 +5107,13 @@ dependencies = [
[[package]]
name = "jiff-static"
-version = "0.2.17"
+version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b787bebb543f8969132630c51fd0afab173a86c6abae56ff3b9e5e3e3f9f6e58"
+checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -5201,7 +5170,7 @@ checksum = "c76e1c7d7df3e34443b3621b459b066a7b79644f059fc8b2db7070c825fd417e"
dependencies = [
"base64 0.22.1",
"ed25519-dalek",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"hmac",
"js-sys",
"p256",
@@ -5383,9 +5352,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
[[package]]
name = "libc"
-version = "0.2.178"
+version = "0.2.180"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"
+checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc"
[[package]]
name = "libdbus-sys"
@@ -5490,15 +5459,6 @@ dependencies = [
"vcpkg",
]
-[[package]]
-name = "libz-rs-sys"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415"
-dependencies = [
- "zlib-rs",
-]
-
[[package]]
name = "libz-sys"
version = "1.1.23"
@@ -5589,7 +5549,7 @@ dependencies = [
"quote",
"regex-syntax",
"rustc_version",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -5629,16 +5589,6 @@ dependencies = [
"twox-hash",
]
-[[package]]
-name = "lzma-rust2"
-version = "0.15.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48172246aa7c3ea28e423295dd1ca2589a24617cc4e588bb8cfe177cb2c54d95"
-dependencies = [
- "crc",
- "sha2",
-]
-
[[package]]
name = "macro_rules_attribute"
version = "0.1.3"
@@ -5727,7 +5677,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -5806,7 +5756,7 @@ dependencies = [
"cfg-if",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6131,9 +6081,9 @@ dependencies = [
[[package]]
name = "octocrab"
-version = "0.49.4"
+version = "0.49.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "add74bcdaf5b0a0c3edb0f8c0a952d7a52d9e9a76ac9c690a3e477c4e8c343b9"
+checksum = "89f6f72d7084a80bf261bb6b6f83bd633323d5633d5ec7988c6c95b20448b2b5"
dependencies = [
"arc-swap",
"async-trait",
@@ -6145,7 +6095,7 @@ dependencies = [
"either",
"futures",
"futures-util",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"http 1.4.0",
"http-body",
"http-body-util",
@@ -6211,7 +6161,7 @@ dependencies = [
"chrono",
"crc32c",
"futures",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"http 1.4.0",
"http-body",
"log",
@@ -6249,7 +6199,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6538,7 +6488,7 @@ dependencies = [
"regex",
"regex-syntax",
"structmeta",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6581,16 +6531,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b867cad97c0791bbd3aaa6472142568c6c9e8f71937e98379f584cfb0cf35bec"
-[[package]]
-name = "pbkdf2"
-version = "0.12.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
-dependencies = [
- "digest",
- "hmac",
-]
-
[[package]]
name = "pear"
version = "0.2.9"
@@ -6611,7 +6551,7 @@ dependencies = [
"proc-macro2",
"proc-macro2-diagnostics",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6668,9 +6608,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "pest"
-version = "2.8.4"
+version = "2.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22"
+checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7"
dependencies = [
"memchr",
"ucd-trie",
@@ -6678,9 +6618,9 @@ dependencies = [
[[package]]
name = "pest_derive"
-version = "2.8.4"
+version = "2.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51f72981ade67b1ca6adc26ec221be9f463f2b5839c7508998daa17c23d94d7f"
+checksum = "68f9dbced329c441fa79d80472764b1a2c7e57123553b8519b36663a2fb234ed"
dependencies = [
"pest",
"pest_generator",
@@ -6688,22 +6628,22 @@ dependencies = [
[[package]]
name = "pest_generator"
-version = "2.8.4"
+version = "2.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dee9efd8cdb50d719a80088b76f81aec7c41ed6d522ee750178f83883d271625"
+checksum = "3bb96d5051a78f44f43c8f712d8e810adb0ebf923fc9ed2655a7f66f63ba8ee5"
dependencies = [
"pest",
"pest_meta",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "pest_meta"
-version = "2.8.4"
+version = "2.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf1d70880e76bdc13ba52eafa6239ce793d85c8e43896507e43dd8984ff05b82"
+checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365"
dependencies = [
"pest",
"sha2",
@@ -6726,7 +6666,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6854,12 +6794,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
-[[package]]
-name = "ppmd-rust"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d558c559f0450f16f2a27a1f017ef38468c1090c9ce63c8e51366232d53717b4"
-
[[package]]
name = "ppv-lite86"
version = "0.2.21"
@@ -6906,7 +6840,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6963,9 +6897,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
-version = "1.0.104"
+version = "1.0.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0"
+checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7"
dependencies = [
"unicode-ident",
]
@@ -6978,7 +6912,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"version_check",
"yansi",
]
@@ -7003,14 +6937,14 @@ checksum = "9adf1691c04c0a5ff46ff8f262b58beb07b0dbb61f96f9f54f6cbd82106ed87f"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "prost"
-version = "0.14.1"
+version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d"
+checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
dependencies = [
"bytes",
"prost-derive",
@@ -7018,15 +6952,15 @@ dependencies = [
[[package]]
name = "prost-derive"
-version = "0.14.1"
+version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
+checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
"itertools 0.14.0",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -7043,9 +6977,9 @@ dependencies = [
[[package]]
name = "prost-types"
-version = "0.14.1"
+version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72"
+checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7"
dependencies = [
"prost",
]
@@ -7197,9 +7131,9 @@ dependencies = [
[[package]]
name = "quote"
-version = "1.0.42"
+version = "1.0.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f"
+checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a"
dependencies = [
"proc-macro2",
]
@@ -7234,7 +7168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
dependencies = [
"rand_chacha 0.9.0",
- "rand_core 0.9.3",
+ "rand_core 0.9.5",
]
[[package]]
@@ -7254,7 +7188,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
- "rand_core 0.9.3",
+ "rand_core 0.9.5",
]
[[package]]
@@ -7263,14 +7197,14 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
]
[[package]]
name = "rand_core"
-version = "0.9.3"
+version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
dependencies = [
"getrandom 0.3.4",
]
@@ -7281,7 +7215,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"
dependencies = [
- "rand_core 0.9.3",
+ "rand_core 0.9.5",
]
[[package]]
@@ -7351,7 +7285,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"libredox",
"thiserror 2.0.17",
]
@@ -7373,7 +7307,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -7431,7 +7365,7 @@ dependencies = [
"base64 0.22.1",
"chrono",
"form_urlencoded",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"hex",
"hmac",
"home",
@@ -7488,7 +7422,7 @@ dependencies = [
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
- "webpki-roots 1.0.4",
+ "webpki-roots 1.0.5",
]
[[package]]
@@ -7515,7 +7449,7 @@ dependencies = [
"anyhow",
"async-trait",
"futures",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"http 1.4.0",
"hyper",
"reqwest",
@@ -7535,7 +7469,7 @@ checksum = "d70ea85f131b2ee9874f0b160ac5976f8af75f3c9badfe0d955880257d10bd83"
dependencies = [
"anyhow",
"async-trait",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"http 1.4.0",
"matchit",
"reqwest",
@@ -7570,7 +7504,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
dependencies = [
"cc",
"cfg-if",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"libc",
"untrusted",
"windows-sys 0.52.0",
@@ -7584,9 +7518,9 @@ checksum = "57b0b88a509053cbfd535726dcaaceee631313cef981266119527a1d110f6d2b"
[[package]]
name = "rkyv"
-version = "0.7.45"
+version = "0.7.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b"
+checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1"
dependencies = [
"bitvec",
"bytecheck",
@@ -7602,9 +7536,9 @@ dependencies = [
[[package]]
name = "rkyv_derive"
-version = "0.7.45"
+version = "0.7.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0"
+checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5"
dependencies = [
"proc-macro2",
"quote",
@@ -7659,7 +7593,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_json",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -7699,9 +7633,9 @@ checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746"
[[package]]
name = "rsa"
-version = "0.9.9"
+version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88"
+checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d"
dependencies = [
"const-oid",
"digest",
@@ -7719,9 +7653,9 @@ dependencies = [
[[package]]
name = "rust-embed"
-version = "8.9.0"
+version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "947d7f3fad52b283d261c4c99a084937e2fe492248cb9a68a8435a861b8798ca"
+checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27"
dependencies = [
"rust-embed-impl",
"rust-embed-utils",
@@ -7730,22 +7664,22 @@ dependencies = [
[[package]]
name = "rust-embed-impl"
-version = "8.9.0"
+version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fa2c8c9e8711e10f9c4fd2d64317ef13feaab820a4c51541f1a8c8e2e851ab2"
+checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa"
dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
- "syn 2.0.111",
+ "syn 2.0.114",
"walkdir",
]
[[package]]
name = "rust-embed-utils"
-version = "8.9.0"
+version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60b161f275cb337fe0a44d924a5f4df0ed69c2c39519858f931ce61c779d3475"
+checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1"
dependencies = [
"sha2",
"walkdir",
@@ -7763,9 +7697,9 @@ dependencies = [
[[package]]
name = "rust_decimal"
-version = "1.39.0"
+version = "1.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282"
+checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0"
dependencies = [
"arrayvec",
"borsh",
@@ -7816,9 +7750,9 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.23.35"
+version = "0.23.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
+checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b"
dependencies = [
"aws-lc-rs",
"log",
@@ -7974,7 +7908,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8009,7 +7943,7 @@ checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8141,7 +8075,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8152,14 +8086,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "serde_json"
-version = "1.0.148"
+version = "1.0.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3084b546a1dd6289475996f182a22aba973866ea8e8b02c51d9f46b1336a22da"
+checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
dependencies = [
"itoa",
"memchr",
@@ -8187,7 +8121,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8230,7 +8164,7 @@ dependencies = [
"chrono",
"hex",
"indexmap 1.9.3",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"schemars 0.9.0",
"schemars 1.2.0",
"serde_core",
@@ -8248,7 +8182,7 @@ dependencies = [
"darling 0.21.3",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8257,7 +8191,7 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f"
dependencies = [
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"itoa",
"ryu",
"serde",
@@ -8266,11 +8200,12 @@ dependencies = [
[[package]]
name = "serial_test"
-version = "3.2.0"
+version = "3.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9"
+checksum = "0d0b343e184fc3b7bb44dff0705fffcf4b3756ba6aff420dddd8b24ca145e555"
dependencies = [
- "futures",
+ "futures-executor",
+ "futures-util",
"log",
"once_cell",
"parking_lot",
@@ -8280,13 +8215,13 @@ dependencies = [
[[package]]
name = "serial_test_derive"
-version = "3.2.0"
+version = "3.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef"
+checksum = "6f50427f258fb77356e4cd4aa0e87e2bd2c66dbcee41dc405282cae2bfc26c83"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8295,7 +8230,6 @@ version = "0.6.1-edge.4"
dependencies = [
"ahash 0.8.12",
"anyhow",
- "arcshift",
"argon2",
"async-channel",
"async_zip",
@@ -8358,7 +8292,7 @@ dependencies = [
"tempfile",
"thiserror 2.0.17",
"tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
"tower-http",
"tracing",
"tracing-appender",
@@ -8504,7 +8438,7 @@ checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8531,7 +8465,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8620,7 +8554,7 @@ dependencies = [
"futures-util",
"hashbrown 0.15.5",
"hashlink",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"log",
"memchr",
"once_cell",
@@ -8649,7 +8583,7 @@ dependencies = [
"quote",
"sqlx-core",
"sqlx-macros-core",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8672,7 +8606,7 @@ dependencies = [
"sqlx-mysql",
"sqlx-postgres",
"sqlx-sqlite",
- "syn 2.0.111",
+ "syn 2.0.114",
"tokio",
"url",
]
@@ -8815,7 +8749,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"toml 0.8.23",
]
@@ -8845,7 +8779,7 @@ dependencies = [
"proc-macro2",
"quote",
"structmeta-derive",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8856,7 +8790,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8884,7 +8818,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8896,7 +8830,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8918,9 +8852,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.111"
+version = "2.0.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87"
+checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a"
dependencies = [
"proc-macro2",
"quote",
@@ -8944,7 +8878,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8953,7 +8887,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d8a928f38f1bc873f28e0d2ba8298ad65374a6ac2241dabd297271531a736cd"
dependencies = [
- "syn 2.0.111",
+ "syn 2.0.114",
"synthez-codegen",
"synthez-core",
]
@@ -8964,7 +8898,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fb83b8df4238e11746984dfb3819b155cd270de0e25847f45abad56b3671047"
dependencies = [
- "syn 2.0.111",
+ "syn 2.0.114",
"synthez-core",
]
@@ -8977,7 +8911,7 @@ dependencies = [
"proc-macro2",
"quote",
"sealed",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9077,7 +9011,7 @@ dependencies = [
"cfg-if",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9088,7 +9022,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"test-case-core",
]
@@ -9168,7 +9102,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9179,7 +9113,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9215,9 +9149,9 @@ dependencies = [
[[package]]
name = "time"
-version = "0.3.44"
+version = "0.3.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d"
+checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd"
dependencies = [
"deranged",
"itoa",
@@ -9225,22 +9159,22 @@ dependencies = [
"num-conv",
"num_threads",
"powerfmt",
- "serde",
+ "serde_core",
"time-core",
"time-macros",
]
[[package]]
name = "time-core"
-version = "0.1.6"
+version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b"
+checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca"
[[package]]
name = "time-macros"
-version = "0.2.24"
+version = "0.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3"
+checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd"
dependencies = [
"num-conv",
"time-core",
@@ -9282,9 +9216,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.48.0"
+version = "1.49.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408"
+checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86"
dependencies = [
"bytes",
"libc",
@@ -9305,7 +9239,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9320,9 +9254,9 @@ dependencies = [
[[package]]
name = "tokio-stream"
-version = "0.1.17"
+version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
+checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
dependencies = [
"futures-core",
"pin-project-lite",
@@ -9347,9 +9281,9 @@ dependencies = [
[[package]]
name = "tokio-util"
-version = "0.7.17"
+version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594"
+checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
dependencies = [
"bytes",
"futures-core",
@@ -9390,11 +9324,11 @@ dependencies = [
[[package]]
name = "toml"
-version = "0.9.10+spec-1.1.0"
+version = "0.9.11+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0825052159284a1a8b4d6c0c86cbc801f2da5afd2b225fa548c72f2e74002f48"
+checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46"
dependencies = [
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"serde_core",
"serde_spanned 1.0.4",
"toml_datetime 0.7.5+spec-1.1.0",
@@ -9427,7 +9361,7 @@ version = "0.19.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
dependencies = [
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"toml_datetime 0.6.11",
"winnow 0.5.40",
]
@@ -9438,7 +9372,7 @@ version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"serde",
"serde_spanned 0.6.9",
"toml_datetime 0.6.11",
@@ -9452,7 +9386,7 @@ version = "0.23.10+spec-1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269"
dependencies = [
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"toml_datetime 0.7.5+spec-1.1.0",
"toml_parser",
"winnow 0.7.14",
@@ -9489,7 +9423,7 @@ dependencies = [
"axum",
"base64 0.22.1",
"bytes",
- "h2 0.4.12",
+ "h2 0.4.13",
"http 1.4.0",
"http-body",
"http-body-util",
@@ -9534,13 +9468,13 @@ dependencies = [
[[package]]
name = "tower"
-version = "0.5.2"
+version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
+checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
dependencies = [
"futures-core",
"futures-util",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"pin-project-lite",
"slab",
"sync_wrapper",
@@ -9619,7 +9553,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9645,16 +9579,13 @@ dependencies = [
[[package]]
name = "tracing-opentelemetry"
-version = "0.32.0"
+version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e6e5658463dd88089aba75c7791e1d3120633b1bfde22478b28f625a9bb1b8e"
+checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc"
dependencies = [
"js-sys",
"opentelemetry",
- "opentelemetry_sdk",
- "rustversion",
"smallvec",
- "thiserror 2.0.17",
"tracing",
"tracing-core",
"tracing-log",
@@ -9688,7 +9619,7 @@ checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9760,7 +9691,7 @@ checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9771,7 +9702,7 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9782,7 +9713,7 @@ checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9818,9 +9749,9 @@ dependencies = [
[[package]]
name = "unicase"
-version = "2.8.1"
+version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
+checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-bidi"
@@ -9915,7 +9846,7 @@ dependencies = [
"rustls-pki-types",
"ureq-proto",
"utf-8",
- "webpki-roots 1.0.4",
+ "webpki-roots 1.0.5",
]
[[package]]
@@ -9932,14 +9863,15 @@ dependencies = [
[[package]]
name = "url"
-version = "2.5.7"
+version = "2.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b"
+checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
dependencies = [
"form_urlencoded",
"idna",
"percent-encoding",
"serde",
+ "serde_derive",
]
[[package]]
@@ -10165,7 +10097,7 @@ dependencies = [
"bumpalo",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"wasm-bindgen-shared",
]
@@ -10228,9 +10160,9 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
-version = "1.0.4"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b"
+checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc"
dependencies = [
"rustls-pki-types",
]
@@ -10241,14 +10173,14 @@ version = "0.26.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
dependencies = [
- "webpki-roots 1.0.4",
+ "webpki-roots 1.0.5",
]
[[package]]
name = "webpki-roots"
-version = "1.0.4"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e"
+checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c"
dependencies = [
"rustls-pki-types",
]
@@ -10389,7 +10321,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -10400,7 +10332,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -10411,7 +10343,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -10422,7 +10354,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -10890,7 +10822,7 @@ dependencies = [
"futures",
"gloo",
"implicit-clone",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"js-sys",
"rustversion",
"serde",
@@ -10917,7 +10849,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -10947,7 +10879,7 @@ checksum = "9e87a3ce33434ab66a700edbaf2cc8a417d9b89f00a6fd8216fd6ac83b0e7b1c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -10969,28 +10901,28 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"synstructure",
]
[[package]]
name = "zerocopy"
-version = "0.8.31"
+version = "0.8.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3"
+checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.8.31"
+version = "0.8.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a"
+checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -11010,7 +10942,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"synstructure",
]
@@ -11031,7 +10963,7 @@ checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -11064,35 +10996,20 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "zip"
-version = "7.0.0"
+version = "7.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdd8a47718a4ee5fe78e07667cd36f3de80e7c2bfe727c7074245ffc7303c037"
+checksum = "9013f1222db8a6d680f13a7ccdc60a781199cd09c2fa4eff58e728bb181757fc"
dependencies = [
- "aes",
- "arbitrary",
- "bzip2",
- "constant_time_eq",
"crc32fast",
- "deflate64",
"flate2",
- "generic-array",
- "getrandom 0.3.4",
- "hmac",
- "indexmap 2.12.1",
- "lzma-rust2",
+ "indexmap 2.13.0",
"memchr",
- "pbkdf2",
- "ppmd-rust",
- "sha1",
- "time",
- "zeroize",
"zopfli",
- "zstd",
]
[[package]]
@@ -11103,9 +11020,9 @@ checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3"
[[package]]
name = "zmij"
-version = "1.0.3"
+version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9747e91771f56fd7893e1164abd78febd14a670ceec257caad15e051de35f06"
+checksum = "bd8f3f50b848df28f887acb68e41201b5aea6bc8a8dacc00fb40635ff9a72fea"
[[package]]
name = "zopfli"
diff --git a/Cargo.toml b/Cargo.toml
index 3b68471ce..469e788bd 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -121,7 +121,6 @@ derive_more = { version = "2.1.1", features = ["full"] }
dirs = "6.0.0"
dlopen2 = "0.8.2"
dotenvy = "0.15.7"
-enum_dispatch = "0.3.13"
env_logger = "0.11.8"
err_trail = { version = "0.11.0", features = ["tracing"] }
error_set = "0.9.1"
@@ -176,6 +175,7 @@ serde_yaml_ng = "0.10.0"
serial_test = "3.2.0"
server = { path = "core/server" }
simd-json = { version = "0.17.0", features = ["serde_impl"] }
+slab = "0.4"
strum = { version = "0.27.2", features = ["derive"] }
strum_macros = "0.27.2"
sysinfo = "0.37.2"
@@ -210,7 +210,7 @@ uuid = { version = "1.19.0", features = [
"zerocopy",
] }
webpki-roots = "1.0.4"
-zip = "7.0.0"
+zip = { version = "7.1.0", default-features = false, features = ["deflate"] }
[profile.release]
lto = true
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index ac097d204..5eda74ac3 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -31,9 +31,7 @@ anstyle-query: 1.1.5, "Apache-2.0 OR MIT",
anstyle-wincon: 3.0.11, "Apache-2.0 OR MIT",
anyhow: 1.0.100, "Apache-2.0 OR MIT",
apache-avro: 0.17.0, "Apache-2.0",
-arbitrary: 1.4.2, "Apache-2.0 OR MIT",
arc-swap: 1.8.0, "Apache-2.0 OR MIT",
-arcshift: 0.4.2, "Apache-2.0 OR MIT",
argon2: 0.5.3, "Apache-2.0 OR MIT",
array-init: 2.1.0, "Apache-2.0 OR MIT",
arrayref: 0.3.9, "BSD-2-Clause",
@@ -53,11 +51,11 @@ as-any: 0.3.2, "Apache-2.0 OR MIT",
asn1-rs: 0.7.1, "Apache-2.0 OR MIT",
asn1-rs-derive: 0.6.0, "Apache-2.0 OR MIT",
asn1-rs-impl: 0.2.0, "Apache-2.0 OR MIT",
-assert_cmd: 2.1.1, "Apache-2.0 OR MIT",
+assert_cmd: 2.1.2, "Apache-2.0 OR MIT",
astral-tokio-tar: 0.5.6, "Apache-2.0 OR MIT",
async-broadcast: 0.7.2, "Apache-2.0 OR MIT",
async-channel: 2.5.0, "Apache-2.0 OR MIT",
-async-compression: 0.4.36, "Apache-2.0 OR MIT",
+async-compression: 0.4.37, "Apache-2.0 OR MIT",
async-dropper: 0.3.1, "MIT",
async-dropper-derive: 0.3.1, "MIT",
async-dropper-simple: 0.2.6, "MIT",
@@ -84,7 +82,7 @@ backon: 1.6.0, "Apache-2.0",
base16ct: 0.2.0, "Apache-2.0 OR MIT",
base64: 0.21.7, "Apache-2.0 OR MIT",
base64: 0.22.1, "Apache-2.0 OR MIT",
-base64ct: 1.8.1, "Apache-2.0 OR MIT",
+base64ct: 1.8.3, "Apache-2.0 OR MIT",
bdd: 0.0.1, "Apache-2.0",
beef: 0.5.2, "Apache-2.0 OR MIT",
bench-dashboard-frontend: 0.4.1, "Apache-2.0",
@@ -100,14 +98,14 @@ bitflags: 1.3.2, "Apache-2.0 OR MIT",
bitflags: 2.10.0, "Apache-2.0 OR MIT",
bitvec: 1.0.1, "MIT",
blake2: 0.10.6, "Apache-2.0 OR MIT",
-blake3: 1.8.2, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR CC0-1.0",
+blake3: 1.8.3, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR CC0-1.0",
block-buffer: 0.10.4, "Apache-2.0 OR MIT",
block2: 0.6.2, "MIT",
bollard: 0.19.4, "Apache-2.0",
bollard-buildkit-proto: 0.7.0, "Apache-2.0",
bollard-stubs: 1.49.1-rc.28.4.0, "Apache-2.0",
-bon: 3.8.1, "Apache-2.0 OR MIT",
-bon-macros: 3.8.1, "Apache-2.0 OR MIT",
+bon: 3.8.2, "Apache-2.0 OR MIT",
+bon-macros: 3.8.2, "Apache-2.0 OR MIT",
borsh: 1.6.0, "Apache-2.0 OR MIT",
borsh-derive: 1.6.0, "Apache-2.0",
brotli: 8.0.2, "BSD-3-Clause AND MIT",
@@ -128,7 +126,7 @@ cargo-platform: 0.1.9, "Apache-2.0 OR MIT",
cargo-platform: 0.3.2, "Apache-2.0 OR MIT",
cargo_metadata: 0.19.2, "MIT",
cargo_metadata: 0.23.1, "MIT",
-cc: 1.2.51, "Apache-2.0 OR MIT",
+cc: 1.2.52, "Apache-2.0 OR MIT",
cesu8: 1.1.0, "Apache-2.0 OR MIT",
cfg-if: 1.0.4, "Apache-2.0 OR MIT",
cfg_aliases: 0.2.1, "MIT",
@@ -136,18 +134,18 @@ charming: 0.6.0, "Apache-2.0 OR MIT",
charming_macros: 0.1.0, "Apache-2.0 OR MIT",
chrono: 0.4.42, "Apache-2.0 OR MIT",
cipher: 0.4.4, "Apache-2.0 OR MIT",
-clap: 4.5.53, "Apache-2.0 OR MIT",
-clap_builder: 4.5.53, "Apache-2.0 OR MIT",
-clap_complete: 4.5.63, "Apache-2.0 OR MIT",
+clap: 4.5.54, "Apache-2.0 OR MIT",
+clap_builder: 4.5.54, "Apache-2.0 OR MIT",
+clap_complete: 4.5.65, "Apache-2.0 OR MIT",
clap_derive: 4.5.49, "Apache-2.0 OR MIT",
-clap_lex: 0.7.6, "Apache-2.0 OR MIT",
+clap_lex: 0.7.7, "Apache-2.0 OR MIT",
clock: 0.1.0, "N/A",
cmake: 0.1.57, "Apache-2.0 OR MIT",
cobs: 0.3.0, "Apache-2.0 OR MIT",
colorchoice: 1.0.4, "Apache-2.0 OR MIT",
colored: 3.0.0, "MPL-2.0",
combine: 4.6.7, "MIT",
-comfy-table: 7.2.1, "MIT",
+comfy-table: 7.2.2, "MIT",
compio: 0.17.0, "MIT",
compio-buf: 0.7.2, "MIT",
compio-driver: 0.10.0, "MIT",
@@ -160,7 +158,7 @@ compio-quic: 0.6.0, "MIT",
compio-runtime: 0.10.1, "MIT",
compio-tls: 0.8.0, "MIT",
compio-ws: 0.2.0, "MIT",
-compression-codecs: 0.4.35, "Apache-2.0 OR MIT",
+compression-codecs: 0.4.36, "Apache-2.0 OR MIT",
compression-core: 0.4.31, "Apache-2.0 OR MIT",
concurrent-queue: 2.5.0, "Apache-2.0 OR MIT",
consensus: 0.1.0, "Apache-2.0",
@@ -169,7 +167,7 @@ console_error_panic_hook: 0.1.7, "Apache-2.0 OR MIT",
const-oid: 0.9.6, "Apache-2.0 OR MIT",
const-random: 0.1.18, "Apache-2.0 OR MIT",
const-random-macro: 0.1.16, "Apache-2.0 OR MIT",
-constant_time_eq: 0.3.1, "Apache-2.0 OR CC0-1.0 OR MIT-0",
+constant_time_eq: 0.4.2, "Apache-2.0 OR CC0-1.0 OR MIT-0",
convert_case: 0.6.0, "MIT",
convert_case: 0.10.0, "MIT",
cookie: 0.16.2, "Apache-2.0 OR MIT",
@@ -178,7 +176,7 @@ core-foundation-sys: 0.8.7, "Apache-2.0 OR MIT",
core2: 0.4.0, "Apache-2.0 OR MIT",
core_affinity: 0.8.3, "Apache-2.0 OR MIT",
cpufeatures: 0.2.17, "Apache-2.0 OR MIT",
-crc: 3.3.0, "Apache-2.0 OR MIT",
+crc: 3.4.0, "Apache-2.0 OR MIT",
crc-catalog: 2.4.0, "Apache-2.0 OR MIT",
crc32c: 0.6.8, "Apache-2.0 OR MIT",
crc32fast: 1.5.0, "Apache-2.0 OR MIT",
@@ -217,15 +215,13 @@ darling_macro: 0.21.3, "MIT",
darling_macro: 0.23.0, "MIT",
dary_heap: 0.3.8, "Apache-2.0 OR MIT",
dashmap: 6.1.0, "MIT",
-data-encoding: 2.9.0, "MIT",
+data-encoding: 2.10.0, "MIT",
dbus: 0.9.10, "Apache-2.0 OR MIT",
dbus-secret-service: 4.1.0, "Apache-2.0 OR MIT",
-deflate64: 0.1.10, "MIT",
der: 0.7.10, "Apache-2.0 OR MIT",
der-parser: 10.0.0, "Apache-2.0 OR MIT",
deranged: 0.5.5, "Apache-2.0 OR MIT",
derive-new: 0.7.0, "MIT",
-derive_arbitrary: 1.4.2, "Apache-2.0 OR MIT",
derive_builder: 0.20.2, "Apache-2.0 OR MIT",
derive_builder_core: 0.20.2, "Apache-2.0 OR MIT",
derive_builder_macro: 0.20.2, "Apache-2.0 OR MIT",
@@ -278,18 +274,18 @@ ext-trait: 1.0.1, "Apache-2.0 OR MIT OR Zlib",
ext-trait-proc_macros: 1.0.1, "Apache-2.0 OR MIT OR Zlib",
extension-traits: 1.0.1, "Apache-2.0 OR MIT OR Zlib",
fast-async-mutex: 0.6.7, "Apache-2.0 OR MIT",
-fastbloom: 0.14.0, "Apache-2.0 OR MIT",
+fastbloom: 0.14.1, "Apache-2.0 OR MIT",
fastrand: 2.3.0, "Apache-2.0 OR MIT",
-ferroid: 0.8.8, "Apache-2.0 OR MIT",
+ferroid: 0.8.9, "Apache-2.0 OR MIT",
ff: 0.13.1, "Apache-2.0 OR MIT",
fiat-crypto: 0.2.9, "Apache-2.0 OR BSD-1-Clause OR MIT",
figlet-rs: 0.1.5, "Apache-2.0",
figment: 0.10.19, "Apache-2.0 OR MIT",
-file-operation: 0.8.8, "MIT",
+file-operation: 0.8.11, "MIT",
filetime: 0.2.26, "Apache-2.0 OR MIT",
-find-msvc-tools: 0.1.6, "Apache-2.0 OR MIT",
+find-msvc-tools: 0.1.7, "Apache-2.0 OR MIT",
flatbuffers: 25.12.19, "Apache-2.0",
-flate2: 1.1.5, "Apache-2.0 OR MIT",
+flate2: 1.1.8, "Apache-2.0 OR MIT",
float-cmp: 0.10.0, "MIT",
flume: 0.11.1, "Apache-2.0 OR MIT",
flume: 0.12.0, "Apache-2.0 OR MIT",
@@ -319,7 +315,7 @@ futures-timer: 3.0.3, "Apache-2.0 OR MIT",
futures-util: 0.3.31, "Apache-2.0 OR MIT",
generator: 0.8.8, "Apache-2.0 OR MIT",
generic-array: 0.14.7, "MIT",
-getrandom: 0.2.16, "Apache-2.0 OR MIT",
+getrandom: 0.2.17, "Apache-2.0 OR MIT",
getrandom: 0.3.4, "Apache-2.0 OR MIT",
ghash: 0.5.1, "Apache-2.0 OR MIT",
gherkin: 0.15.0, "Apache-2.0 OR MIT",
@@ -342,10 +338,10 @@ gloo-worker-macros: 0.1.0, "Apache-2.0 OR MIT",
governor: 0.10.4, "MIT",
group: 0.13.0, "Apache-2.0 OR MIT",
h2: 0.3.27, "MIT",
-h2: 0.4.12, "MIT",
+h2: 0.4.13, "MIT",
half: 2.7.1, "Apache-2.0 OR MIT",
halfbrown: 0.4.0, "Apache-2.0 OR MIT",
-handlebars: 6.3.2, "MIT",
+handlebars: 6.4.0, "MIT",
hash32: 0.2.1, "Apache-2.0 OR MIT",
hash32: 1.0.0, "Apache-2.0 OR MIT",
hashbrown: 0.12.3, "Apache-2.0 OR MIT",
@@ -415,7 +411,7 @@ impl-more: 0.1.9, "Apache-2.0 OR MIT",
implicit-clone: 0.6.0, "Apache-2.0 OR MIT",
implicit-clone-derive: 0.1.2, "Apache-2.0 OR MIT",
indexmap: 1.9.3, "Apache-2.0 OR MIT",
-indexmap: 2.12.1, "Apache-2.0 OR MIT",
+indexmap: 2.13.0, "Apache-2.0 OR MIT",
inflections: 1.1.1, "MIT",
inlinable_string: 0.1.15, "Apache-2.0 OR MIT",
inotify: 0.11.0, "ISC",
@@ -432,8 +428,8 @@ is_terminal_polyfill: 1.70.2, "Apache-2.0 OR MIT",
itertools: 0.13.0, "Apache-2.0 OR MIT",
itertools: 0.14.0, "Apache-2.0 OR MIT",
itoa: 1.0.17, "Apache-2.0 OR MIT",
-jiff: 0.2.17, "MIT OR Unlicense",
-jiff-static: 0.2.17, "MIT OR Unlicense",
+jiff: 0.2.18, "MIT OR Unlicense",
+jiff-static: 0.2.18, "MIT OR Unlicense",
jni: 0.21.1, "Apache-2.0 OR MIT",
jni-sys: 0.3.0, "Apache-2.0 OR MIT",
jobserver: 0.1.34, "Apache-2.0 OR MIT",
@@ -457,7 +453,7 @@ lexical-util: 1.0.7, "Apache-2.0 OR MIT",
lexical-write-float: 1.0.6, "Apache-2.0 OR MIT",
lexical-write-integer: 1.0.6, "Apache-2.0 OR MIT",
libbz2-rs-sys: 0.2.2, "bzip2-1.0.6",
-libc: 0.2.178, "Apache-2.0 OR MIT",
+libc: 0.2.180, "Apache-2.0 OR MIT",
libdbus-sys: 0.2.7, "Apache-2.0 OR MIT",
libflate: 2.2.1, "MIT",
libflate_lz77: 2.2.0, "MIT",
@@ -468,7 +464,6 @@ libm: 0.2.15, "MIT",
libmimalloc-sys: 0.1.44, "MIT",
libredox: 0.1.12, "MIT",
libsqlite3-sys: 0.30.1, "MIT",
-libz-rs-sys: 0.5.5, "Zlib",
libz-sys: 1.1.23, "Apache-2.0 OR MIT",
linked-hash-map: 0.5.6, "Apache-2.0 OR MIT",
linux-raw-sys: 0.11.0, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
@@ -484,7 +479,6 @@ logos-derive: 0.15.1, "Apache-2.0 OR MIT",
loom: 0.7.2, "MIT",
lru-slab: 0.1.2, "Apache-2.0 OR MIT OR Zlib",
lz4_flex: 0.11.5, "MIT",
-lzma-rust2: 0.15.4, "Apache-2.0",
macro_rules_attribute: 0.1.3, "MIT",
macro_rules_attribute-proc_macro: 0.1.3, "MIT",
matchers: 0.2.0, "MIT",
@@ -536,7 +530,7 @@ objc2: 0.6.3, "MIT",
objc2-core-foundation: 0.3.2, "Apache-2.0 OR MIT OR Zlib",
objc2-encode: 4.1.0, "MIT",
objc2-io-kit: 0.3.2, "Apache-2.0 OR MIT OR Zlib",
-octocrab: 0.49.4, "Apache-2.0 OR MIT",
+octocrab: 0.49.5, "Apache-2.0 OR MIT",
oid-registry: 0.8.1, "Apache-2.0 OR MIT",
once_cell: 1.21.3, "Apache-2.0 OR MIT",
once_cell_polyfill: 1.70.2, "Apache-2.0 OR MIT",
@@ -573,7 +567,6 @@ passterm: 2.0.1, "BSD-3-Clause",
password-hash: 0.5.0, "Apache-2.0 OR MIT",
paste: 1.0.15, "Apache-2.0 OR MIT",
pastey: 0.2.1, "Apache-2.0 OR MIT",
-pbkdf2: 0.12.2, "Apache-2.0 OR MIT",
pear: 0.2.9, "Apache-2.0 OR MIT",
pear_codegen: 0.2.9, "Apache-2.0 OR MIT",
peg: 0.6.3, "MIT",
@@ -582,10 +575,10 @@ peg-runtime: 0.6.3, "MIT",
pem: 3.0.6, "MIT",
pem-rfc7468: 0.7.0, "Apache-2.0 OR MIT",
percent-encoding: 2.3.2, "Apache-2.0 OR MIT",
-pest: 2.8.4, "Apache-2.0 OR MIT",
-pest_derive: 2.8.4, "Apache-2.0 OR MIT",
-pest_generator: 2.8.4, "Apache-2.0 OR MIT",
-pest_meta: 2.8.4, "Apache-2.0 OR MIT",
+pest: 2.8.5, "Apache-2.0 OR MIT",
+pest_derive: 2.8.5, "Apache-2.0 OR MIT",
+pest_generator: 2.8.5, "Apache-2.0 OR MIT",
+pest_meta: 2.8.5, "Apache-2.0 OR MIT",
pin-project: 1.1.10, "Apache-2.0 OR MIT",
pin-project-internal: 1.1.10, "Apache-2.0 OR MIT",
pin-project-lite: 0.2.16, "Apache-2.0 OR MIT",
@@ -602,7 +595,6 @@ portable-atomic-util: 0.2.4, "Apache-2.0 OR MIT",
postcard: 1.1.3, "Apache-2.0 OR MIT",
potential_utf: 0.1.4, "Unicode-3.0",
powerfmt: 0.2.0, "Apache-2.0 OR MIT",
-ppmd-rust: 1.3.0, "CC0-1.0",
ppv-lite86: 0.2.21, "Apache-2.0 OR MIT",
predicates: 3.1.3, "Apache-2.0 OR MIT",
predicates-core: 1.0.9, "Apache-2.0 OR MIT",
@@ -613,14 +605,14 @@ proc-macro-crate: 1.3.1, "Apache-2.0 OR MIT",
proc-macro-crate: 3.4.0, "Apache-2.0 OR MIT",
proc-macro-error: 1.0.4, "Apache-2.0 OR MIT",
proc-macro-error-attr: 1.0.4, "Apache-2.0 OR MIT",
-proc-macro2: 1.0.104, "Apache-2.0 OR MIT",
+proc-macro2: 1.0.105, "Apache-2.0 OR MIT",
proc-macro2-diagnostics: 0.10.1, "Apache-2.0 OR MIT",
prometheus-client: 0.24.0, "Apache-2.0 OR MIT",
prometheus-client-derive-encode: 0.5.0, "Apache-2.0 OR MIT",
-prost: 0.14.1, "Apache-2.0",
-prost-derive: 0.14.1, "Apache-2.0",
+prost: 0.14.3, "Apache-2.0",
+prost-derive: 0.14.3, "Apache-2.0",
prost-reflect: 0.16.3, "Apache-2.0 OR MIT",
-prost-types: 0.14.1, "Apache-2.0",
+prost-types: 0.14.3, "Apache-2.0",
protox: 0.9.1, "Apache-2.0 OR MIT",
protox-parse: 0.9.0, "Apache-2.0 OR MIT",
ptr_meta: 0.1.4, "MIT",
@@ -632,7 +624,7 @@ quick-xml: 0.38.4, "MIT",
quinn: 0.11.9, "Apache-2.0 OR MIT",
quinn-proto: 0.11.13, "Apache-2.0 OR MIT",
quinn-udp: 0.5.14, "Apache-2.0 OR MIT",
-quote: 1.0.42, "Apache-2.0 OR MIT",
+quote: 1.0.43, "Apache-2.0 OR MIT",
r-efi: 5.3.0, "Apache-2.0 OR LGPL-2.1-or-later OR MIT",
radium: 0.7.0, "MIT",
rand: 0.8.5, "Apache-2.0 OR MIT",
@@ -640,7 +632,7 @@ rand: 0.9.2, "Apache-2.0 OR MIT",
rand_chacha: 0.3.1, "Apache-2.0 OR MIT",
rand_chacha: 0.9.0, "Apache-2.0 OR MIT",
rand_core: 0.6.4, "Apache-2.0 OR MIT",
-rand_core: 0.9.3, "Apache-2.0 OR MIT",
+rand_core: 0.9.5, "Apache-2.0 OR MIT",
rand_xoshiro: 0.7.0, "Apache-2.0 OR MIT",
raw-cpuid: 11.6.0, "MIT",
rayon: 1.11.0, "Apache-2.0 OR MIT",
@@ -665,8 +657,8 @@ retry-policies: 0.5.1, "Apache-2.0 OR MIT",
rfc6979: 0.4.0, "Apache-2.0 OR MIT",
ring: 0.17.14, "Apache-2.0 AND ISC",
ringbuffer: 0.16.0, "MIT",
-rkyv: 0.7.45, "MIT",
-rkyv_derive: 0.7.45, "MIT",
+rkyv: 0.7.46, "MIT",
+rkyv_derive: 0.7.46, "MIT",
rle-decode-fast: 1.0.3, "Apache-2.0 OR MIT",
rmcp: 0.12.0, "MIT",
rmcp-macros: 0.12.0, "MIT",
@@ -674,17 +666,17 @@ rmp: 0.8.15, "MIT",
rmp-serde: 1.3.1, "MIT",
roaring: 0.10.12, "Apache-2.0 OR MIT",
route-recognizer: 0.3.1, "MIT",
-rsa: 0.9.9, "Apache-2.0 OR MIT",
-rust-embed: 8.9.0, "MIT",
-rust-embed-impl: 8.9.0, "MIT",
-rust-embed-utils: 8.9.0, "MIT",
+rsa: 0.9.10, "Apache-2.0 OR MIT",
+rust-embed: 8.11.0, "MIT",
+rust-embed-impl: 8.11.0, "MIT",
+rust-embed-utils: 8.11.0, "MIT",
rust-ini: 0.21.3, "MIT",
-rust_decimal: 1.39.0, "MIT",
+rust_decimal: 1.40.0, "MIT",
rustc-hash: 2.1.1, "Apache-2.0 OR MIT",
rustc_version: 0.4.1, "Apache-2.0 OR MIT",
rusticata-macros: 4.1.0, "Apache-2.0 OR MIT",
rustix: 1.1.3, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
-rustls: 0.23.35, "Apache-2.0 OR ISC OR MIT",
+rustls: 0.23.36, "Apache-2.0 OR ISC OR MIT",
rustls-native-certs: 0.8.3, "Apache-2.0 OR ISC OR MIT",
rustls-pemfile: 2.2.0, "Apache-2.0 OR ISC OR MIT",
rustls-pki-types: 1.13.2, "Apache-2.0 OR MIT",
@@ -718,7 +710,7 @@ serde_bytes: 0.11.19, "Apache-2.0 OR MIT",
serde_core: 1.0.228, "Apache-2.0 OR MIT",
serde_derive: 1.0.228, "Apache-2.0 OR MIT",
serde_derive_internals: 0.29.1, "Apache-2.0 OR MIT",
-serde_json: 1.0.148, "Apache-2.0 OR MIT",
+serde_json: 1.0.149, "Apache-2.0 OR MIT",
serde_path_to_error: 0.1.20, "Apache-2.0 OR MIT",
serde_repr: 0.1.20, "Apache-2.0 OR MIT",
serde_spanned: 0.6.9, "Apache-2.0 OR MIT",
@@ -727,8 +719,8 @@ serde_urlencoded: 0.7.1, "Apache-2.0 OR MIT",
serde_with: 3.16.1, "Apache-2.0 OR MIT",
serde_with_macros: 3.16.1, "Apache-2.0 OR MIT",
serde_yaml_ng: 0.10.0, "MIT",
-serial_test: 3.2.0, "MIT",
-serial_test_derive: 3.2.0, "MIT",
+serial_test: 3.3.1, "MIT",
+serial_test_derive: 3.3.1, "MIT",
server: 0.6.1-edge.4, "Apache-2.0",
sha1: 0.10.6, "Apache-2.0 OR MIT",
sha2: 0.10.9, "Apache-2.0 OR MIT",
@@ -774,7 +766,7 @@ strum_macros: 0.26.4, "MIT",
strum_macros: 0.27.2, "MIT",
subtle: 2.6.1, "BSD-3-Clause",
syn: 1.0.109, "Apache-2.0 OR MIT",
-syn: 2.0.111, "Apache-2.0 OR MIT",
+syn: 2.0.114, "Apache-2.0 OR MIT",
sync_wrapper: 1.0.2, "Apache-2.0",
synstructure: 0.13.2, "MIT",
synthez: 0.4.0, "BlueOak-1.0.0",
@@ -801,22 +793,22 @@ thiserror-impl: 2.0.17, "Apache-2.0 OR MIT",
thread_local: 1.1.9, "Apache-2.0 OR MIT",
threadpool: 1.8.1, "Apache-2.0 OR MIT",
thrift: 0.17.0, "Apache-2.0",
-time: 0.3.44, "Apache-2.0 OR MIT",
-time-core: 0.1.6, "Apache-2.0 OR MIT",
-time-macros: 0.2.24, "Apache-2.0 OR MIT",
+time: 0.3.45, "Apache-2.0 OR MIT",
+time-core: 0.1.7, "Apache-2.0 OR MIT",
+time-macros: 0.2.25, "Apache-2.0 OR MIT",
tiny-keccak: 2.0.2, "CC0-1.0",
tinystr: 0.8.2, "Unicode-3.0",
tinyvec: 1.10.0, "Apache-2.0 OR MIT OR Zlib",
tinyvec_macros: 0.1.1, "Apache-2.0 OR MIT OR Zlib",
-tokio: 1.48.0, "MIT",
+tokio: 1.49.0, "MIT",
tokio-macros: 2.6.0, "MIT",
tokio-rustls: 0.26.4, "Apache-2.0 OR MIT",
-tokio-stream: 0.1.17, "MIT",
+tokio-stream: 0.1.18, "MIT",
tokio-tungstenite: 0.28.0, "MIT",
-tokio-util: 0.7.17, "MIT",
+tokio-util: 0.7.18, "MIT",
tokise: 0.2.0, "Apache-2.0 OR MIT",
toml: 0.8.23, "Apache-2.0 OR MIT",
-toml: 0.9.10+spec-1.1.0, "Apache-2.0 OR MIT",
+toml: 0.9.11+spec-1.1.0, "Apache-2.0 OR MIT",
toml_datetime: 0.6.11, "Apache-2.0 OR MIT",
toml_datetime: 0.7.5+spec-1.1.0, "Apache-2.0 OR MIT",
toml_edit: 0.19.15, "Apache-2.0 OR MIT",
@@ -828,7 +820,7 @@ toml_writer: 1.0.6+spec-1.1.0, "Apache-2.0 OR MIT",
tonic: 0.14.2, "MIT",
tonic-prost: 0.14.2, "MIT",
tools: 0.1.0, "Apache-2.0",
-tower: 0.5.2, "MIT",
+tower: 0.5.3, "MIT",
tower-http: 0.6.8, "MIT",
tower-layer: 0.3.3, "MIT",
tower-service: 0.3.3, "MIT",
@@ -837,7 +829,7 @@ tracing-appender: 0.2.4, "MIT",
tracing-attributes: 0.1.31, "MIT",
tracing-core: 0.1.36, "MIT",
tracing-log: 0.2.0, "MIT",
-tracing-opentelemetry: 0.32.0, "MIT",
+tracing-opentelemetry: 0.32.1, "MIT",
tracing-subscriber: 0.3.22, "MIT",
trait-variant: 0.1.2, "Apache-2.0 OR MIT",
try-lock: 0.2.5, "MIT",
@@ -853,7 +845,7 @@ typenum: 1.19.0, "Apache-2.0 OR MIT",
ucd-trie: 0.1.7, "Apache-2.0 OR MIT",
ulid: 1.2.1, "MIT",
uncased: 0.9.10, "Apache-2.0 OR MIT",
-unicase: 2.8.1, "Apache-2.0 OR MIT",
+unicase: 2.9.0, "Apache-2.0 OR MIT",
unicode-bidi: 0.3.18, "Apache-2.0 OR MIT",
unicode-ident: 1.0.22, "(Apache-2.0 OR MIT) AND Unicode-3.0",
unicode-linebreak: 0.1.5, "Apache-2.0",
@@ -868,7 +860,7 @@ unsafe-libyaml: 0.2.11, "MIT",
untrusted: 0.9.0, "ISC",
ureq: 3.1.4, "Apache-2.0 OR MIT",
ureq-proto: 0.5.3, "Apache-2.0 OR MIT",
-url: 2.5.7, "Apache-2.0 OR MIT",
+url: 2.5.8, "Apache-2.0 OR MIT",
urlencoding: 2.1.3, "MIT",
utf-8: 0.7.6, "Apache-2.0 OR MIT",
utf8-width: 0.1.8, "MIT",
@@ -899,9 +891,9 @@ wasm-streams: 0.4.2, "Apache-2.0 OR MIT",
wasmtimer: 0.4.3, "MIT",
web-sys: 0.3.83, "Apache-2.0 OR MIT",
web-time: 1.1.0, "Apache-2.0 OR MIT",
-webpki-root-certs: 1.0.4, "CDLA-Permissive-2.0",
+webpki-root-certs: 1.0.5, "CDLA-Permissive-2.0",
webpki-roots: 0.26.11, "CDLA-Permissive-2.0",
-webpki-roots: 1.0.4, "CDLA-Permissive-2.0",
+webpki-roots: 1.0.5, "CDLA-Permissive-2.0",
whoami: 1.6.1, "Apache-2.0 OR BSL-1.0 OR MIT",
widestring: 1.2.1, "Apache-2.0 OR MIT",
winapi: 0.3.9, "Apache-2.0 OR MIT",
@@ -983,8 +975,8 @@ yew-router: 0.19.0, "Apache-2.0 OR MIT",
yew-router-macro: 0.19.0, "Apache-2.0 OR MIT",
yoke: 0.8.1, "Unicode-3.0",
yoke-derive: 0.8.1, "Unicode-3.0",
-zerocopy: 0.8.31, "Apache-2.0 OR BSD-2-Clause OR MIT",
-zerocopy-derive: 0.8.31, "Apache-2.0 OR BSD-2-Clause OR MIT",
+zerocopy: 0.8.33, "Apache-2.0 OR BSD-2-Clause OR MIT",
+zerocopy-derive: 0.8.33, "Apache-2.0 OR BSD-2-Clause OR MIT",
zerofrom: 0.1.6, "Unicode-3.0",
zerofrom-derive: 0.1.6, "Unicode-3.0",
zeroize: 1.8.2, "Apache-2.0 OR MIT",
@@ -992,9 +984,9 @@ zeroize_derive: 1.4.3, "Apache-2.0 OR MIT",
zerotrie: 0.2.3, "Unicode-3.0",
zerovec: 0.11.5, "Unicode-3.0",
zerovec-derive: 0.11.2, "Unicode-3.0",
-zip: 7.0.0, "MIT",
+zip: 7.1.0, "MIT",
zlib-rs: 0.5.5, "Zlib",
-zmij: 1.0.3, "MIT",
+zmij: 1.0.14, "MIT",
zopfli: 0.8.3, "Apache-2.0",
zstd: 0.13.3, "MIT",
zstd-safe: 7.2.4, "Apache-2.0 OR MIT",
diff --git a/core/integration/Cargo.toml b/core/integration/Cargo.toml
index e30ef758c..5d9e58409 100644
--- a/core/integration/Cargo.toml
+++ b/core/integration/Cargo.toml
@@ -58,6 +58,7 @@ rmcp = { version = "0.12.0", features = [
serde_json = { workspace = true }
serial_test = { workspace = true }
server = { workspace = true }
+slab = { workspace = true }
strum = { workspace = true }
strum_macros = { workspace = true }
tempfile = { workspace = true }
diff --git a/core/integration/src/test_server.rs b/core/integration/src/test_server.rs
index fd718178e..ac406452d 100644
--- a/core/integration/src/test_server.rs
+++ b/core/integration/src/test_server.rs
@@ -33,7 +33,9 @@ use std::io::Write;
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
-use std::thread::{available_parallelism, panicking, sleep};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::thread::{self, JoinHandle, available_parallelism, panicking, sleep};
use std::time::Duration;
use uuid::Uuid;
@@ -75,7 +77,6 @@ enum ServerProtocolAddr {
WebSocket(SocketAddr),
}
-#[derive(Debug)]
pub struct TestServer {
local_data_path: String,
envs: HashMap<String, String>,
@@ -85,6 +86,18 @@ pub struct TestServer {
stderr_file_path: Option<PathBuf>,
cleanup: bool,
server_executable_path: Option<String>,
+ watchdog_handle: Option<JoinHandle<()>>,
+ watchdog_stop: Arc<AtomicBool>,
+}
+
+impl std::fmt::Debug for TestServer {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("TestServer")
+ .field("local_data_path", &self.local_data_path)
+ .field("server_addrs", &self.server_addrs)
+ .field("cleanup", &self.cleanup)
+ .finish_non_exhaustive()
+ }
}
impl TestServer {
@@ -194,6 +207,8 @@ impl TestServer {
stderr_file_path: None,
cleanup,
server_executable_path,
+ watchdog_handle: None,
+ watchdog_stop: Arc::new(AtomicBool::new(false)),
}
}
@@ -234,11 +249,101 @@ impl TestServer {
}
let child = command.spawn().unwrap();
+ let pid = child.id();
self.child_handle = Some(child);
self.wait_until_server_has_bound();
+
+ let watchdog_stop = self.watchdog_stop.clone();
+ let stdout_path = self.stdout_file_path.clone();
+ let stderr_path = self.stderr_file_path.clone();
+ let watchdog_handle = thread::Builder::new()
+ .name("test-server-watchdog".to_string())
+ .spawn(move || {
+ Self::watchdog_loop(pid, watchdog_stop, stdout_path, stderr_path);
+ })
+ .expect("Failed to spawn watchdog thread");
+ self.watchdog_handle = Some(watchdog_handle);
+ }
+
+ /// Watchdog loop that monitors the server process.
+ /// Panics if the server exits while the watchdog is still running (i.e., not gracefully stopped).
+ fn watchdog_loop(
+ pid: u32,
+ stop_signal: Arc<AtomicBool>,
+ stdout_path: Option<PathBuf>,
+ stderr_path: Option<PathBuf>,
+ ) {
+ const CHECK_INTERVAL: Duration = Duration::from_millis(100);
+
+ loop {
+ if stop_signal.load(Ordering::SeqCst) {
+ return;
+ }
+
+ // Check if process is alive AND not a zombie via /proc/{pid}/stat
+ // A zombie process still exists in the process table (kill returns 0)
+ // but has state 'Z' in /proc/pid/stat
+ let process_alive = Self::is_process_alive(pid);
+
+ if !process_alive {
+ // Server process has died unexpectedly!
+ // We print to stderr and abort because panic in a spawned thread
+ // doesn't propagate to the test thread.
+ let stdout_content = stdout_path
+ .as_ref()
+ .and_then(|p| fs::read_to_string(p).ok())
+ .unwrap_or_else(|| "[No stdout log]".to_string());
+
+ let stderr_content = stderr_path
+ .as_ref()
+ .and_then(|p| fs::read_to_string(p).ok())
+ .unwrap_or_else(|| "[No stderr log]".to_string());
+
+ eprintln!(
+ "\n\n=== SERVER CRASHED ===\n\
+ The iggy-server process (PID {}) has died unexpectedly!\n\
+ This usually indicates a bug in the server.\n\n\
+ === STDOUT ===\n{}\n\n\
+ === STDERR ===\n{}\n",
+ pid, stdout_content, stderr_content
+ );
+ std::process::abort();
+ }
+
+ thread::sleep(CHECK_INTERVAL);
+ }
+ }
+
+ /// Check if a process is alive (exists and is not a zombie).
+ fn is_process_alive(pid: u32) -> bool {
+ // Read /proc/{pid}/stat to get process state
+ let stat_path = format!("/proc/{}/stat", pid);
+ match fs::read_to_string(&stat_path) {
+ Ok(content) => {
+ // Format: pid (comm) state ...
+ // State is the third field after the command name in parentheses
+ // States: R=running, S=sleeping, D=disk sleep, Z=zombie, T=stopped, etc.
+ if let Some(state_start) = content.rfind(')') {
+ let after_comm = &content[state_start + 1..];
+ let state = after_comm.trim().chars().next();
+ // Process is dead if state is 'Z' (zombie) or 'X' (dead)
+ !matches!(state, Some('Z') | Some('X'))
+ } else {
+ false // Can't parse, assume dead
+ }
+ }
+ Err(_) => false, // Can't read /proc/{pid}/stat, process doesn't exist
+ }
}
pub fn stop(&mut self) {
+ // Signal the watchdog to stop FIRST, before we terminate the server.
+ // This prevents the watchdog from panicking when we gracefully stop.
+ self.watchdog_stop.store(true, Ordering::SeqCst);
+ if let Some(watchdog) = self.watchdog_handle.take() {
+ let _ = watchdog.join();
+ }
+
#[allow(unused_mut)]
if let Some(mut child_handle) = self.child_handle.take() {
#[cfg(unix)]
diff --git a/core/integration/tests/data_integrity/verify_after_server_restart.rs b/core/integration/tests/data_integrity/verify_after_server_restart.rs
index f632abd92..ac63d6e71 100644
--- a/core/integration/tests/data_integrity/verify_after_server_restart.rs
+++ b/core/integration/tests/data_integrity/verify_after_server_restart.rs
@@ -326,6 +326,10 @@ async fn should_fill_data_and_verify_after_restart(cache_setting: &'static str)
/// - Topics per stream: 0, 2 (1 deleted)
/// - Partitions per topic: 0, 2 (1 deleted)
/// - Consumer groups per topic: 0, 2 (1 deleted)
+///
+/// Also tests ID reuse: after deletion, creating new resources should reuse the freed IDs.
+/// This catches the bug where `streams_count()`/`topics_count()` (slab length) was used
+/// instead of `vacant_key()` to predict assigned IDs, causing mismatch when IDs are reused.
#[tokio::test]
#[parallel]
async fn should_handle_resource_deletion_and_restart() {
@@ -424,11 +428,55 @@ async fn should_handle_resource_deletion_and_restart() {
.await
.unwrap();
+ // TEST ID REUSE: Create a new stream after deletion - it should reuse ID 1
+ // This is the critical test that catches the bug where streams_count() (slab.len())
+ // was used instead of vacant_key() to predict the assigned ID.
+ // After deletion: streams are [0, 2], len=2, but vacant_key=1
+ // Bug: code used len (2) to create directory, but metadata assigned ID 1
+ let stream_reused = client.create_stream("stream-reused").await.unwrap();
+ assert_eq!(
+ stream_reused.id, 1,
+ "New stream should reuse deleted ID 1, got {}",
+ stream_reused.id
+ );
+
+ // Also test topic ID reuse within a stream
+ // Delete topic 0 in stream 0, then create new topic - should reuse ID 0... wait, topic 1 was already deleted
+ // Actually let's delete topic 0 and create new one, it should get ID 0 or 1 depending on vacant_key
+ let stream_0_ident = Identifier::numeric(0).unwrap();
+ client
+ .delete_topic(&stream_0_ident, &Identifier::numeric(0).unwrap())
+ .await
+ .unwrap();
+
+ // Now stream 0 has only topic 2. Creating new topic should reuse ID 0 (first vacant slot)
+ let topic_reused = client
+ .create_topic(
+ &stream_0_ident,
+ "topic-reused",
+ 1,
+ CompressionAlgorithm::None,
+ None,
+ IggyExpiry::NeverExpire,
+ MaxTopicSize::Unlimited,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ topic_reused.id, 0,
+ "New topic should reuse deleted ID 0, got {}",
+ topic_reused.id
+ );
+
// Verify state before restart
let streams = client.get_streams().await.unwrap();
- assert_eq!(streams.len(), 2);
+ assert_eq!(streams.len(), 3); // 0, 1 (reused), 2
let stream_ids: Vec<u32> = streams.iter().map(|s| s.id).collect();
- assert!(stream_ids.contains(&0) && stream_ids.contains(&2));
+ assert!(
+ stream_ids.contains(&0) && stream_ids.contains(&1) && stream_ids.contains(&2),
+ "Expected streams 0, 1, 2 after ID reuse, got: {:?}",
+ stream_ids
+ );
drop(client);
test_server.stop();
@@ -451,72 +499,117 @@ async fn should_handle_resource_deletion_and_restart() {
);
login_root(&client).await;
- // Verify streams after restart
+ // Verify streams after restart - should have 3 streams: 0, 1 (reused), 2
let streams = client.get_streams().await.unwrap();
- assert_eq!(streams.len(), 2, "Expected 2 streams after restart");
+ assert_eq!(streams.len(), 3, "Expected 3 streams after restart");
let stream_ids: Vec<u32> = streams.iter().map(|s| s.id).collect();
assert!(
- stream_ids.contains(&0) && stream_ids.contains(&2),
- "Expected streams 0 and 2, got: {:?}",
+ stream_ids.contains(&0) && stream_ids.contains(&1) && stream_ids.contains(&2),
+ "Expected streams 0, 1, 2 after restart, got: {:?}",
stream_ids
);
- // Verify topics, partitions, and consumer groups for each stream
- for stream_id in [0u32, 2u32] {
- let stream_ident = Identifier::numeric(stream_id).unwrap();
- let stream = client.get_stream(&stream_ident).await.unwrap().unwrap();
+ // Verify the reused stream 1 has the correct name
+ let stream_1 = client
+ .get_stream(&Identifier::numeric(1).unwrap())
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ stream_1.name, "stream-reused",
+ "Stream 1 should be the reused stream"
+ );
+
+ // Verify stream 0 topics after restart
+ // Stream 0 should have: topic 0 (reused, "topic-reused") and topic 2
+ let stream_0_ident = Identifier::numeric(0).unwrap();
+ let topics_stream_0 = client.get_topics(&stream_0_ident).await.unwrap();
+ let topic_ids_0: Vec<u32> = topics_stream_0.iter().map(|t| t.id).collect();
+ assert!(
+ topic_ids_0.contains(&0) && topic_ids_0.contains(&2),
+ "Stream 0 should have topics 0 and 2, got: {:?}",
+ topic_ids_0
+ );
+
+ // Verify the reused topic has correct name
+ let topic_0 = client
+ .get_topic(&stream_0_ident, &Identifier::numeric(0).unwrap())
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ topic_0.name, "topic-reused",
+ "Topic 0 in stream 0 should be the reused topic"
+ );
+
+ // Verify topic 2 in stream 0 still has correct structure
+ let topic_2_ident = Identifier::numeric(2).unwrap();
+ let topic_2 = client
+ .get_topic(&stream_0_ident, &topic_2_ident)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ topic_2.partitions_count, 2,
+ "Topic 2 in stream 0 should have 2 partitions"
+ );
+ let cgs_topic_2 = client
+ .get_consumer_groups(&stream_0_ident, &topic_2_ident)
+ .await
+ .unwrap();
+ assert_eq!(
+ cgs_topic_2.len(),
+ 2,
+ "Topic 2 in stream 0 should have 2 consumer groups"
+ );
+ let cg_ids: Vec<u32> = cgs_topic_2.iter().map(|c| c.id).collect();
+ assert!(
+ cg_ids.contains(&0) && cg_ids.contains(&2),
+ "Topic 2 in stream 0 should have consumer groups 0 and 2, got: {:?}",
+ cg_ids
+ );
+
+ // Verify stream 2 structure (unchanged from original test)
+ let stream_2_ident = Identifier::numeric(2).unwrap();
+ let topics_stream_2 = client.get_topics(&stream_2_ident).await.unwrap();
+ let topic_ids_2: Vec<u32> = topics_stream_2.iter().map(|t| t.id).collect();
+ assert!(
+ topic_ids_2.contains(&0) && topic_ids_2.contains(&2),
+ "Stream 2 should have topics 0 and 2, got: {:?}",
+ topic_ids_2
+ );
+
+ for topic_id in [0u32, 2u32] {
+ let topic_ident = Identifier::numeric(topic_id).unwrap();
+ let topic = client
+ .get_topic(&stream_2_ident, &topic_ident)
+ .await
+ .unwrap()
+ .unwrap();
assert_eq!(
- stream.topics_count, 2,
- "Stream {} should have 2 topics",
- stream_id
+ topic.partitions_count, 2,
+ "Topic {} in stream 2 should have 2 partitions",
+ topic_id
);
- // Verify topics have correct IDs (0 and 2, not 0 and 1)
- let topics = client.get_topics(&stream_ident).await.unwrap();
- let topic_ids: Vec<u32> = topics.iter().map(|t| t.id).collect();
+ let cgs = client
+ .get_consumer_groups(&stream_2_ident, &topic_ident)
+ .await
+ .unwrap();
+ assert_eq!(
+ cgs.len(),
+ 2,
+ "Topic {} in stream 2 should have 2 consumer groups",
+ topic_id
+ );
+ let cg_ids: Vec<u32> = cgs.iter().map(|c| c.id).collect();
assert!(
- topic_ids.contains(&0) && topic_ids.contains(&2),
- "Stream {} should have topics 0 and 2, got: {:?}",
- stream_id,
- topic_ids
+ cg_ids.contains(&0) && cg_ids.contains(&2),
+ "Topic {} in stream 2 should have consumer groups 0 and 2, got: {:?}",
+ topic_id,
+ cg_ids
);
-
- for topic_id in [0u32, 2u32] {
- let topic_ident = Identifier::numeric(topic_id).unwrap();
- let topic = client
- .get_topic(&stream_ident, &topic_ident)
- .await
- .unwrap()
- .unwrap();
-
- assert_eq!(
- topic.partitions_count, 2,
- "Topic {} in stream {} should have 2 partitions",
- topic_id, stream_id
- );
-
- // Verify consumer groups have correct IDs (0 and 2)
- let cgs = client
- .get_consumer_groups(&stream_ident, &topic_ident)
- .await
- .unwrap();
- assert_eq!(
- cgs.len(),
- 2,
- "Topic {} in stream {} should have 2 consumer groups",
- topic_id,
- stream_id
- );
- let cg_ids: Vec<u32> = cgs.iter().map(|c| c.id).collect();
- assert!(
- cg_ids.contains(&0) && cg_ids.contains(&2),
- "Topic {} in stream {} should have consumer groups 0 and 2, got: {:?}",
- topic_id,
- stream_id,
- cg_ids
- );
- }
}
drop(client);
diff --git a/core/integration/tests/server/mod.rs b/core/integration/tests/server/mod.rs
index 0b8fd502b..60a7031d7 100644
--- a/core/integration/tests/server/mod.rs
+++ b/core/integration/tests/server/mod.rs
@@ -22,7 +22,6 @@ mod general;
mod scenarios;
mod specific;
-use futures::FutureExt;
use iggy_common::TransportProtocol;
use integration::{
http_client::HttpClientFactory,
@@ -40,8 +39,6 @@ use scenarios::{
stream_size_validation_scenario, system_scenario, user_scenario,
};
use std::pin::Pin;
-use std::sync::{Arc, Mutex};
-use std::time::Duration;
use std::{collections::HashMap, future::Future};
type ScenarioFn = fn(&dyn ClientFactory) -> Pin<Box<dyn Future<Output = ()> + '_>>;
@@ -116,67 +113,31 @@ async fn run_scenario(transport: TransportProtocol, scenario: ScenarioFn) {
"IGGY_QUIC_KEEP_ALIVE_INTERVAL".to_string(),
"15s".to_string(),
);
- let test_server = TestServer::new(Some(extra_envs), true, None, IpAddrKind::V4);
- let test_server = Arc::new(Mutex::new(test_server));
-
- test_server.lock().unwrap().start();
-
- let client_factory: Box<dyn ClientFactory> = {
- let server = test_server.lock().unwrap();
- match transport {
- TransportProtocol::Tcp => {
- let server_addr = server.get_raw_tcp_addr().unwrap();
- Box::new(TcpClientFactory {
- server_addr,
- ..Default::default()
- })
- }
- TransportProtocol::Quic => {
- let server_addr = server.get_quic_udp_addr().unwrap();
- Box::new(QuicClientFactory { server_addr })
- }
- TransportProtocol::Http => {
- let server_addr = server.get_http_api_addr().unwrap();
- Box::new(HttpClientFactory { server_addr })
- }
- TransportProtocol::WebSocket => {
- let server_addr = server.get_websocket_addr().unwrap();
- Box::new(WebSocketClientFactory { server_addr })
- }
+ let mut test_server = TestServer::new(Some(extra_envs), true, None, IpAddrKind::V4);
+ test_server.start();
+
+ // TestServer's watchdog thread will panic the test if the server crashes unexpectedly
+ let client_factory: Box<dyn ClientFactory> = match transport {
+ TransportProtocol::Tcp => {
+ let server_addr = test_server.get_raw_tcp_addr().unwrap();
+ Box::new(TcpClientFactory {
+ server_addr,
+ ..Default::default()
+ })
}
- };
-
- let monitor_server = test_server.clone();
- let crash_monitor = async move {
- loop {
- tokio::time::sleep(Duration::from_millis(100)).await;
- let mut server = monitor_server.lock().unwrap();
- if !server.is_running() {
- let (stdout, stderr) = server.collect_logs();
- return (stdout, stderr);
- }
+ TransportProtocol::Quic => {
+ let server_addr = test_server.get_quic_udp_addr().unwrap();
+ Box::new(QuicClientFactory { server_addr })
}
- };
-
- tokio::select! {
- biased;
-
- (stdout, stderr) = crash_monitor => {
- panic!(
- "Server crashed during test!\n\n\
- === STDOUT ===\n{}\n\n\
- === STDERR ===\n{}",
- stdout, stderr
- );
+ TransportProtocol::Http => {
+ let server_addr = test_server.get_http_api_addr().unwrap();
+ Box::new(HttpClientFactory { server_addr })
}
-
- result = std::panic::AssertUnwindSafe(scenario(&*client_factory)).catch_unwind() => {
- test_server.lock().unwrap().assert_running();
-
- // Re-raise any panic from the scenario
- if let Err(panic_payload) = result {
- std::panic::resume_unwind(panic_payload);
- }
+ TransportProtocol::WebSocket => {
+ let server_addr = test_server.get_websocket_addr().unwrap();
+ Box::new(WebSocketClientFactory { server_addr })
}
- }
+ };
+
+ scenario(&*client_factory).await;
}
diff --git a/core/integration/tests/server/scenarios/concurrent_scenario.rs b/core/integration/tests/server/scenarios/concurrent_scenario.rs
index 535d6ac76..2c8767270 100644
--- a/core/integration/tests/server/scenarios/concurrent_scenario.rs
+++ b/core/integration/tests/server/scenarios/concurrent_scenario.rs
@@ -29,15 +29,17 @@ const MULTIPLE_CLIENT_COUNT: usize = 10;
const OPERATIONS_PER_CLIENT: usize = OPERATIONS_COUNT / MULTIPLE_CLIENT_COUNT;
const USER_PASSWORD: &str = "secret";
const TEST_STREAM_NAME: &str = "race-test-stream";
+const TEST_TOPIC_NAME: &str = "race-test-topic";
const PARTITIONS_COUNT: u32 = 1;
+const PARTITIONS_TO_ADD: u32 = 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResourceType {
User,
Stream,
Topic,
+ Partition,
// TODO(hubcio): add ConsumerGroup
- // TODO(hubcio): add Partition
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -70,6 +72,24 @@ pub async fn run(
root_client.create_stream(TEST_STREAM_NAME).await.unwrap();
}
+ // For partition tests, create parent stream and topic first
+ if resource_type == ResourceType::Partition {
+ root_client.create_stream(TEST_STREAM_NAME).await.unwrap();
+ let stream_id = Identifier::named(TEST_STREAM_NAME).unwrap();
+ root_client
+ .create_topic(
+ &stream_id,
+ TEST_TOPIC_NAME,
+ PARTITIONS_COUNT,
+ CompressionAlgorithm::default(),
+ None,
+ IggyExpiry::NeverExpire,
+ MaxTopicSize::ServerDefault,
+ )
+ .await
+ .unwrap();
+ }
+
let results = match (resource_type, scenario_type) {
(ResourceType::User, ScenarioType::Cold) => {
execute_multiple_clients_users_cold(client_factory, use_barrier).await
@@ -86,6 +106,11 @@ pub async fn run(
(ResourceType::Topic, ScenarioType::Cold) => {
execute_multiple_clients_topics_cold(client_factory, use_barrier).await
}
+ (ResourceType::Partition, ScenarioType::Hot) => {
+ execute_multiple_clients_partitions_hot(client_factory, use_barrier).await
+ }
+ // Partitions don't have names, so Cold scenario doesn't apply
+ (ResourceType::Partition, ScenarioType::Cold) => vec![],
_ => vec![],
// TODO: Figure out why those tests timeout in CI.
/*
@@ -374,6 +399,51 @@ async fn execute_multiple_clients_topics_cold(
.collect()
}
+async fn execute_multiple_clients_partitions_hot(
+ client_factory: &dyn ClientFactory,
+ use_barrier: bool,
+) -> Vec<OperationResult> {
+ let mut handles = Vec::with_capacity(MULTIPLE_CLIENT_COUNT);
+ let barrier = if use_barrier {
+ Some(Arc::new(Barrier::new(MULTIPLE_CLIENT_COUNT)))
+ } else {
+ None
+ };
+
+ for _ in 0..MULTIPLE_CLIENT_COUNT {
+ let client = create_client(client_factory).await;
+ login_root(&client).await;
+
+ let barrier_clone = barrier.clone();
+ let handle = tokio::spawn(async move {
+ if let Some(b) = barrier_clone {
+ b.wait().await;
+ }
+
+ let mut results = Vec::with_capacity(OPERATIONS_PER_CLIENT);
+ let stream_id = Identifier::named(TEST_STREAM_NAME).unwrap();
+ let topic_id = Identifier::named(TEST_TOPIC_NAME).unwrap();
+
+ for _ in 0..OPERATIONS_PER_CLIENT {
+ let result = client
+ .create_partitions(&stream_id, &topic_id, PARTITIONS_TO_ADD)
+ .await
+ .map(|_| ());
+ results.push(result);
+ }
+ results
+ });
+
+ handles.push(handle);
+ }
+
+ let all_results = join_all(handles).await;
+ all_results
+ .into_iter()
+ .flat_map(|r| r.expect("Tokio task panicked"))
+ .collect()
+}
+
fn validate_results(results: &[OperationResult], scenario_type: ScenarioType) {
let success_count = results.iter().filter(|r| r.is_ok()).count();
let error_count = results.iter().filter(|r| r.is_err()).count();
@@ -542,6 +612,22 @@ async fn validate_server_state(
);
}
}
+ ResourceType::Partition => {
+ let mut partition_counts = Vec::new();
+ for client in &clients {
+ let count = validate_partitions_state(client).await;
+ partition_counts.push(count);
+ }
+
+ let first_count = partition_counts[0];
+ for (i, count) in partition_counts.iter().enumerate().skip(1) {
+ assert_eq!(
+ *count, first_count,
+ "Client {} sees different partition count ({}) than client 0 ({})",
+ i, count, first_count
+ );
+ }
+ }
}
}
@@ -735,6 +821,26 @@ async fn validate_topics_state(client: &IggyClient, scenario_type: ScenarioType)
test_topics
}
+async fn validate_partitions_state(client: &IggyClient) -> u32 {
+ let stream_id = Identifier::named(TEST_STREAM_NAME).unwrap();
+ let topic_id = Identifier::named(TEST_TOPIC_NAME).unwrap();
+ let topic = client
+ .get_topic(&stream_id, &topic_id)
+ .await
+ .expect("Failed to get test topic")
+ .expect("Test topic not found");
+
+ // Expected: initial partition + OPERATIONS_COUNT added partitions
+ let expected_partitions = PARTITIONS_COUNT + (OPERATIONS_COUNT as u32 * PARTITIONS_TO_ADD);
+ assert_eq!(
+ topic.partitions_count, expected_partitions,
+ "Hot path: Expected {} partitions (1 initial + {} added), but found {}",
+ expected_partitions, OPERATIONS_COUNT, topic.partitions_count
+ );
+
+ topic.partitions_count
+}
+
async fn cleanup_resources(client: &IggyClient, resource_type: ResourceType) {
match resource_type {
ResourceType::User => {
@@ -755,8 +861,8 @@ async fn cleanup_resources(client: &IggyClient, resource_type: ResourceType) {
.await;
}
}
- // Just delete test stream
- ResourceType::Topic => {
+ // Just delete test stream (also cleans up topics and partitions)
+ ResourceType::Topic | ResourceType::Partition => {
let _ = client
.delete_stream(&Identifier::named(TEST_STREAM_NAME).unwrap())
.await;
diff --git a/core/integration/tests/streaming/get_by_offset.rs b/core/integration/tests/streaming/get_by_offset.rs
index ca5438529..159edba70 100644
--- a/core/integration/tests/streaming/get_by_offset.rs
+++ b/core/integration/tests/streaming/get_by_offset.rs
@@ -20,12 +20,12 @@
*/
use super::bootstrap_test_environment;
+use crate::streaming::IggyFullNamespace;
use crate::streaming::common::test_setup::TestSetup;
use bytes::BytesMut;
use iggy::prelude::*;
use server::configs::cache_indexes::CacheIndexesConfig;
use server::configs::system::{PartitionConfig, SegmentConfig, SystemConfig};
-use server::shard::namespace::IggyFullNamespace;
use server::shard::system::messages::PollingArgs;
use server::streaming::polling_consumer::PollingConsumer;
use server::streaming::segments::IggyMessagesBatchMut;
diff --git a/core/integration/tests/streaming/get_by_timestamp.rs b/core/integration/tests/streaming/get_by_timestamp.rs
index 90afc5967..64c2e78f8 100644
--- a/core/integration/tests/streaming/get_by_timestamp.rs
+++ b/core/integration/tests/streaming/get_by_timestamp.rs
@@ -17,12 +17,12 @@
*/
use super::bootstrap_test_environment;
+use crate::streaming::IggyFullNamespace;
use crate::streaming::common::test_setup::TestSetup;
use bytes::BytesMut;
use iggy::prelude::*;
use server::configs::cache_indexes::CacheIndexesConfig;
use server::configs::system::{PartitionConfig, SegmentConfig, SystemConfig};
-use server::shard::namespace::IggyFullNamespace;
use server::shard::system::messages::PollingArgs;
use server::streaming::polling_consumer::PollingConsumer;
use server::streaming::segments::IggyMessagesBatchMut;
diff --git a/core/integration/tests/streaming/mod.rs b/core/integration/tests/streaming/mod.rs
index 6ad773647..14823ccd9 100644
--- a/core/integration/tests/streaming/mod.rs
+++ b/core/integration/tests/streaming/mod.rs
@@ -16,117 +16,570 @@
* under the License.
*/
-use iggy_common::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize};
+use iggy_common::sharding::IggyNamespace;
+use iggy_common::{
+ CompressionAlgorithm, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize,
+};
use server::{
configs::system::SystemConfig,
- shard::{task_registry::TaskRegistry, transmission::connector::ShardConnector},
- slab::{streams::Streams, traits_ext::EntityMarker},
+ metadata::{Metadata, create_metadata_handles},
+ shard::{
+ system::messages::PollingArgs, task_registry::TaskRegistry,
+ transmission::connector::ShardConnector,
+ },
streaming::{
- self,
- partitions::{partition, storage::create_partition_file_hierarchy},
- segments::{Segment, storage::create_segment_storage},
- streams::{storage::create_stream_file_hierarchy, stream},
- topics::{storage::create_topic_file_hierarchy, topic},
+ partitions::{
+ consumer_group_offsets::ConsumerGroupOffsets, consumer_offsets::ConsumerOffsets,
+ helpers::create_message_deduplicator, journal::Journal,
+ local_partition::LocalPartition, local_partitions::LocalPartitions,
+ storage::create_partition_file_hierarchy,
+ },
+ polling_consumer::PollingConsumer,
+ segments::{
+ IggyMessagesBatchMut, IggyMessagesBatchSet, Segment, storage::create_segment_storage,
+ },
+ streams::storage::create_stream_file_hierarchy,
+ topics::storage::create_topic_file_hierarchy,
+ traits::MainOps,
},
};
+use std::cell::RefCell;
use std::rc::Rc;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
+
+/// Namespace with Identifier-based stream/topic IDs (used for test API).
+#[derive(Debug)]
+pub struct IggyFullNamespace {
+ stream: Identifier,
+ topic: Identifier,
+ partition: usize,
+}
+
+impl IggyFullNamespace {
+ pub fn new(stream: Identifier, topic: Identifier, partition: usize) -> Self {
+ Self {
+ stream,
+ topic,
+ partition,
+ }
+ }
+
+ pub fn stream_id(&self) -> &Identifier {
+ &self.stream
+ }
+
+ pub fn topic_id(&self) -> &Identifier {
+ &self.topic
+ }
+
+ pub fn partition_id(&self) -> usize {
+ self.partition
+ }
+}
mod common;
mod get_by_offset;
mod get_by_timestamp;
mod snapshot;
+/// Test harness that uses Metadata + LocalPartitions
+pub struct TestStreams {
+ pub shared_metadata: &'static Metadata,
+ pub local_partitions: RefCell<LocalPartitions>,
+}
+
+impl TestStreams {
+ pub fn new(shared_metadata: &'static Metadata) -> Self {
+ Self {
+ shared_metadata,
+ local_partitions: RefCell::new(LocalPartitions::new()),
+ }
+ }
+
+ /// Access partition data by Identifier-based stream/topic IDs.
+ /// Converts Identifiers to numeric IDs using SharedMetadata, then accesses local_partitions.
+ pub fn with_partition_by_id<F, R>(
+ &self,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ partition_id: usize,
+ f: F,
+ ) -> R
+ where
+ F: FnOnce(
+ (
+ &LocalPartition,
+ usize,
+ usize,
+ &Arc<AtomicU64>,
+ &Arc<ConsumerOffsets>,
+ &Arc<ConsumerGroupOffsets>,
+ ),
+ ) -> R,
+ {
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(stream_id)
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, topic_id)
+ .expect("Topic must exist");
+
+ let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+ let partitions = self.local_partitions.borrow();
+ let partition = partitions.get(&namespace).expect("Partition must exist");
+
+ f((
+ partition,
+ numeric_stream_id,
+ numeric_topic_id,
+ &partition.offset,
+ &partition.consumer_offsets,
+ &partition.consumer_group_offsets,
+ ))
+ }
+}
+
+impl MainOps for TestStreams {
+ type Namespace = IggyFullNamespace;
+ type PollingArgs = PollingArgs;
+ type Consumer = PollingConsumer;
+ type In = IggyMessagesBatchMut;
+ type Out = (
+ server::binary::handlers::messages::poll_messages_handler::IggyPollMetadata,
+ IggyMessagesBatchSet,
+ );
+ type Error = IggyError;
+
+ async fn append_messages(
+ &self,
+ config: &SystemConfig,
+ _registry: &Rc<TaskRegistry>,
+ ns: &Self::Namespace,
+ mut batch: Self::In,
+ ) -> Result<(), Self::Error> {
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(ns.stream_id())
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, ns.topic_id())
+ .expect("Topic must exist");
+ let partition_id = ns.partition_id();
+
+ let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+
+ if batch.count() == 0 {
+ return Ok(());
+ }
+
+ // Get necessary data from local_partitions
+ let (current_offset, current_position, segment_start_offset, message_deduplicator) = {
+ let partitions = self.local_partitions.borrow();
+ let partition = partitions
+ .get(&namespace)
+ .expect("local_partitions: partition must exist");
+
+ let current_offset = if partition.should_increment_offset {
+ partition.offset.load(Ordering::Relaxed) + 1
+ } else {
+ 0
+ };
+
+ let segment = partition.log.active_segment();
+ let current_position = segment.current_position;
+ let segment_start_offset = segment.start_offset;
+ let message_deduplicator = partition.message_deduplicator.clone();
+
+ (
+ current_offset,
+ current_position,
+ segment_start_offset,
+ message_deduplicator,
+ )
+ };
+
+ // Prepare batch for persistence (outside the borrow)
+ batch
+ .prepare_for_persistence(
+ segment_start_offset,
+ current_offset,
+ current_position,
+ message_deduplicator.as_ref(),
+ )
+ .await;
+
+ // Append to journal
+ let (journal_messages_count, journal_size) = {
+ let mut store = self.local_partitions.borrow_mut();
+ let partition_data = store
+ .get_mut(&namespace)
+ .expect("local_partitions: partition must exist");
+
+ let segment = partition_data.log.active_segment_mut();
+
+ if segment.end_offset == 0 {
+ segment.start_timestamp = batch.first_timestamp().unwrap();
+ }
+
+ let batch_messages_size = batch.size();
+ let batch_messages_count = batch.count();
+
+ partition_data
+ .stats
+ .increment_size_bytes(batch_messages_size as u64);
+ partition_data
+ .stats
+ .increment_messages_count(batch_messages_count as u64);
+
+ segment.end_timestamp = batch.last_timestamp().unwrap();
+ segment.end_offset = batch.last_offset().unwrap();
+
+ let (journal_messages_count, journal_size) =
+ partition_data.log.journal_mut().append(batch)?;
+
+ let last_offset = if batch_messages_count == 0 {
+ current_offset
+ } else {
+ current_offset + batch_messages_count as u64 - 1
+ };
+
+ if partition_data.should_increment_offset {
+ partition_data.offset.store(last_offset, Ordering::Relaxed);
+ } else {
+ partition_data.should_increment_offset = true;
+ partition_data.offset.store(last_offset, Ordering::Relaxed);
+ }
+ partition_data.log.active_segment_mut().current_position += batch_messages_size;
+
+ (journal_messages_count, journal_size)
+ };
+
+ // Check if journal should be persisted
+ let unsaved_messages_count_exceeded =
+ journal_messages_count >= config.partition.messages_required_to_save;
+ let unsaved_messages_size_exceeded = journal_size
+ >= config
+ .partition
+ .size_of_messages_required_to_save
+ .as_bytes_u64() as u32;
+
+ let is_full = {
+ let partitions = self.local_partitions.borrow();
+ let partition = partitions
+ .get(&namespace)
+ .expect("local_partitions: partition must exist");
+ partition.log.active_segment().is_full()
+ };
+
+ // Persist if needed
+ if is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded {
+ // Commit journal and persist to disk
+ let batches = {
+ let mut store = self.local_partitions.borrow_mut();
+ let partition_data = store
+ .get_mut(&namespace)
+ .expect("local_partitions: partition must exist");
+ let batches = partition_data.log.journal_mut().commit();
+ partition_data.log.ensure_indexes();
+ batches.append_indexes_to(partition_data.log.active_indexes_mut().unwrap());
+ batches
+ };
+
+ self.persist_messages_to_disk(&namespace, batches).await?;
+ }
+
+ Ok(())
+ }
+
+ async fn poll_messages(
+ &self,
+ ns: &Self::Namespace,
+ consumer: Self::Consumer,
+ args: Self::PollingArgs,
+ ) -> Result<Self::Out, Self::Error> {
+ // Resolve Identifier to numeric IDs
+ let numeric_stream_id = self
+ .shared_metadata
+ .get_stream_id(ns.stream_id())
+ .expect("Stream must exist");
+ let numeric_topic_id = self
+ .shared_metadata
+ .get_topic_id(numeric_stream_id, ns.topic_id())
+ .expect("Topic must exist");
+ let partition_id = ns.partition_id();
+
+ let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
+
+ server::streaming::partitions::ops::poll_messages(
+ &self.local_partitions,
+ &namespace,
+ consumer,
+ args,
+ )
+ .await
+ }
+}
+
+impl TestStreams {
+ /// Persist messages to disk using local_partitions's storage.
+ async fn persist_messages_to_disk(
+ &self,
+ namespace: &IggyNamespace,
+ batches: IggyMessagesBatchSet,
+ ) -> Result<u32, IggyError> {
+ let batch_count = batches.count();
+
+ if batch_count == 0 {
+ return Ok(0);
+ }
+
+ // Extract storage from local_partitions
+ let (messages_writer, index_writer, has_segments) = {
+ let partitions = self.local_partitions.borrow();
+ let partition = partitions
+ .get(namespace)
+ .expect("local_partitions: partition must exist");
+
+ if !partition.log.has_segments() {
+ return Ok(0);
+ }
+
+ let messages_writer = partition
+ .log
+ .active_storage()
+ .messages_writer
+ .as_ref()
+ .expect("Messages writer not initialized")
+ .clone();
+ let index_writer = partition
+ .log
+ .active_storage()
+ .index_writer
+ .as_ref()
+ .expect("Index writer not initialized")
+ .clone();
+ (messages_writer, index_writer, true)
+ };
+
+ if !has_segments {
+ return Ok(0);
+ }
+
+ // Lock and save messages
+ let guard = messages_writer.lock.lock().await;
+ let saved = messages_writer.as_ref().save_batch_set(batches).await?;
+
+ // Get unsaved indexes from local_partitions
+ let unsaved_indexes_slice = {
+ let partitions = self.local_partitions.borrow();
+ let partition = partitions
+ .get(namespace)
+ .expect("local_partitions: partition must exist");
+ partition.log.active_indexes().unwrap().unsaved_slice()
+ };
+
+ // Save indexes
+ index_writer
+ .as_ref()
+ .save_indexes(unsaved_indexes_slice)
+ .await?;
+
+ // Update index and increment segment stats
+ {
+ let mut store = self.local_partitions.borrow_mut();
+ let partition_data = store
+ .get_mut(namespace)
+ .expect("local_partitions: partition must exist");
+
+ let indexes = partition_data.log.active_indexes_mut().unwrap();
+ indexes.mark_saved();
+
+ let segment = partition_data.log.active_segment_mut();
+ segment.size =
+ iggy_common::IggyByteSize::from(segment.size.as_bytes_u64() + saved.as_bytes_u64());
+ }
+
+ drop(guard);
+ Ok(batch_count)
+ }
+}
+
struct BootstrapResult {
- streams: Streams,
+ streams: TestStreams,
stream_id: Identifier,
topic_id: Identifier,
partition_id: usize,
+ #[allow(dead_code)]
+ namespace: IggyNamespace,
task_registry: Rc<TaskRegistry>,
+ /// Keep the metadata writer alive to prevent the ReadHandle from becoming invalid.
+ /// When WriteHandle is dropped, the ReadHandle may no longer work.
+ #[allow(dead_code)]
+ metadata_writer: server::metadata::MetadataWriter,
}
async fn bootstrap_test_environment(
shard_id: u16,
config: &SystemConfig,
) -> Result<BootstrapResult, IggyError> {
+ use server::metadata::{InnerMetadata, PartitionMeta, StreamMeta, TopicMeta};
+ use server::streaming::stats::{PartitionStats, StreamStats, TopicStats};
+ use slab::Slab;
+ use std::sync::atomic::AtomicUsize;
+
let stream_name = "stream-1".to_owned();
let topic_name = "topic-1".to_owned();
let topic_expiry = IggyExpiry::NeverExpire;
let topic_size = MaxTopicSize::Unlimited;
- let partitions_count = 1;
-
- let streams = Streams::default();
- // Create stream together with its dirs
- let stream = stream::create_and_insert_stream_mem(&streams, stream_name);
- create_stream_file_hierarchy(stream.id(), config).await?;
- // Create topic together with its dirs
- let stream_id = Identifier::numeric(stream.id() as u32).unwrap();
- let parent_stats = streams.with_stream_by_id(&stream_id, |(_, stats)| stats.clone());
+
+ // Create left-right handles for metadata
+ let (mut metadata_writer, metadata_reader) = create_metadata_handles();
+
+ // Build IDs and stats
+ let stream_id_num: usize = 1;
+ let topic_id_num: usize = 1;
+ let partition_id: usize = 0;
+
+ let stream_stats = Arc::new(StreamStats::default());
+ let topic_stats = Arc::new(TopicStats::new(stream_stats.clone()));
+ let partition_stats = Arc::new(PartitionStats::new(topic_stats.clone()));
+
let message_expiry = config.resolve_message_expiry(topic_expiry);
let max_topic_size = config.resolve_max_topic_size(topic_size)?;
- let topic = topic::create_and_insert_topics_mem(
- &streams,
- &stream_id,
- topic_name,
- 1,
+ // Build partition metadata
+ let partition_meta = PartitionMeta {
+ id: partition_id,
+ created_at: IggyTimestamp::now(),
+ revision_id: 0,
+ stats: partition_stats.clone(),
+ consumer_offsets: None,
+ consumer_group_offsets: None,
+ };
+
+ // Build topic metadata with partition
+ let topic_name_arc: Arc<str> = Arc::from(topic_name.as_str());
+ let topic_meta = TopicMeta {
+ id: topic_id_num,
+ name: topic_name_arc.clone(),
+ created_at: IggyTimestamp::now(),
message_expiry,
- CompressionAlgorithm::default(),
+ compression_algorithm: CompressionAlgorithm::default(),
max_topic_size,
- parent_stats,
- );
- create_topic_file_hierarchy(stream.id(), topic.id(), config).await?;
- // Create partition together with its dirs
- let topic_id = Identifier::numeric(topic.id() as u32).unwrap();
- let parent_stats = streams.with_topic_by_id(
- &stream_id,
- &topic_id,
- streaming::topics::helpers::get_stats(),
+ replication_factor: 1,
+ stats: topic_stats.clone(),
+ partitions: vec![partition_meta],
+ consumer_groups: Slab::new(),
+ consumer_group_index: ahash::AHashMap::default(),
+ round_robin_counter: Arc::new(AtomicUsize::new(0)),
+ };
+
+ // Build stream metadata with topic
+ let stream_name_arc: Arc<str> = Arc::from(stream_name.as_str());
+ let mut topic_index = ahash::AHashMap::default();
+ topic_index.insert(topic_name_arc, topic_id_num);
+
+ let stream_meta = StreamMeta {
+ id: stream_id_num,
+ name: stream_name_arc.clone(),
+ created_at: IggyTimestamp::now(),
+ stats: stream_stats,
+ topics: vec![(topic_id_num, topic_meta)].into_iter().collect(),
+ topic_index,
+ };
+
+ // Build InnerMetadata
+ let mut stream_index = ahash::AHashMap::default();
+ stream_index.insert(stream_name_arc, stream_id_num);
+
+ let inner = InnerMetadata {
+ streams: vec![(stream_id_num, stream_meta)].into_iter().collect(),
+ users: Slab::new(),
+ stream_index,
+ user_index: ahash::AHashMap::default(),
+ personal_access_tokens: ahash::AHashMap::default(),
+ };
+
+ // Initialize metadata atomically
+ metadata_writer.initialize(inner);
+
+ // Create file hierarchies
+ create_stream_file_hierarchy(stream_id_num, config).await?;
+ create_topic_file_hierarchy(stream_id_num, topic_id_num, config).await?;
+ create_partition_file_hierarchy(stream_id_num, topic_id_num, partition_id, config).await?;
+
+ let namespace = IggyNamespace::new(stream_id_num, topic_id_num, partition_id);
+
+ // Leak Metadata for 'static lifetime in tests
+ let shared_metadata: &'static Metadata = Box::leak(Box::new(Metadata::new(metadata_reader)));
+
+ // Create TestStreams with partition data
+ let streams = TestStreams::new(shared_metadata);
+
+ // Initialize partition data in local_partitions
+ let start_offset = 0u64;
+ let segment = Segment::new(
+ start_offset,
+ config.segment.size,
+ config.segment.message_expiry,
);
- let partitions = partition::create_and_insert_partitions_mem(
- &streams,
- &stream_id,
- &topic_id,
- parent_stats,
- partitions_count,
+ let messages_size = 0;
+ let indexes_size = 0;
+ let storage = create_segment_storage(
config,
+ stream_id_num,
+ topic_id_num,
+ partition_id,
+ messages_size,
+ indexes_size,
+ start_offset,
+ )
+ .await?;
+
+ // Create partition data with the log
+ let consumer_offsets = Arc::new(ConsumerOffsets::with_capacity(0));
+ let consumer_group_offsets = Arc::new(ConsumerGroupOffsets::with_capacity(0));
+ let message_deduplicator = create_message_deduplicator(config).map(Arc::new);
+ let current_offset = Arc::new(AtomicU64::new(0));
+
+ let mut partition_data = LocalPartition::new(
+ partition_stats,
+ current_offset,
+ consumer_offsets,
+ consumer_group_offsets,
+ message_deduplicator,
+ IggyTimestamp::now(),
+ 0, // revision_id
+ false, // should_increment_offset
);
- for partition in partitions {
- create_partition_file_hierarchy(stream.id(), topic.id(), partition.id(), config).await?;
-
- // Open the log
- let start_offset = 0;
- let segment = Segment::new(
- start_offset,
- config.segment.size,
- config.segment.message_expiry,
- );
- let messages_size = 0;
- let indexes_size = 0;
- let storage = create_segment_storage(
- config,
- stream.id(),
- topic.id(),
- partition.id(),
- messages_size,
- indexes_size,
- start_offset,
- )
- .await?;
- streams.with_partition_by_id_mut(&stream_id, &topic_id, partition.id(), |(.., log)| {
- log.add_persisted_segment(segment, storage);
- });
- }
+ // Add the segment to the log
+ partition_data.log.add_persisted_segment(segment, storage);
+
+ // Insert into partition store
+ streams
+ .local_partitions
+ .borrow_mut()
+ .insert(namespace, partition_data);
// Create a test task registry with dummy stop sender from ShardConnector
let connector: ShardConnector<()> = ShardConnector::new(shard_id);
let task_registry = Rc::new(TaskRegistry::new(shard_id, vec![connector.stop_sender]));
+ let stream_id = Identifier::numeric(stream_id_num as u32).unwrap();
+ let topic_id = Identifier::numeric(topic_id_num as u32).unwrap();
+
Ok(BootstrapResult {
streams,
stream_id,
topic_id,
- partition_id: 0,
+ partition_id,
+ namespace,
task_registry,
+ metadata_writer,
})
}
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index fd72144be..2eaba66df 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -40,7 +40,6 @@ iggy-web = ["dep:rust-embed", "dep:mime_guess"]
[dependencies]
ahash = { workspace = true }
anyhow = { workspace = true }
-arcshift = "0.4.2"
argon2 = { workspace = true }
async-channel = { workspace = true }
async_zip = { workspace = true }
@@ -60,7 +59,7 @@ cyper-axum = { workspace = true }
dashmap = { workspace = true }
derive_more = { workspace = true }
dotenvy = { workspace = true }
-enum_dispatch = { workspace = true }
+enum_dispatch = "0.3"
err_trail = { workspace = true }
error_set = { workspace = true }
figlet-rs = { workspace = true }
@@ -107,7 +106,7 @@ rustls-pemfile = "2.2.0"
send_wrapper = "0.6.0"
serde = { workspace = true }
serde_with = { workspace = true }
-slab = "0.4.11"
+slab = { workspace = true }
socket2 = "0.6.1"
static-toml = "1.3.0"
strum = { workspace = true }
@@ -139,3 +138,5 @@ vergen-git2 = { version = "1.0.7", features = [
"rustc",
"si",
] }
+
+[dev-dependencies]
diff --git a/core/server/src/binary/command.rs b/core/server/src/binary/command.rs
index 78a2b4417..e6f64b081 100644
--- a/core/server/src/binary/command.rs
+++ b/core/server/src/binary/command.rs
@@ -165,268 +165,3 @@ fn as_bytes<T: Command>(command: &T) -> Bytes {
bytes.put_slice(&payload);
bytes.freeze()
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn should_be_serialized_as_bytes_and_deserialized_from_bytes() {
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::Ping(Ping::default()),
- PING_CODE,
- &Ping::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetStats(GetStats::default()),
- GET_STATS_CODE,
- &GetStats::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetClusterMetadata(GetClusterMetadata::default()),
- GET_CLUSTER_METADATA_CODE,
- &GetClusterMetadata::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetMe(GetMe::default()),
- GET_ME_CODE,
- &GetMe::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetClient(GetClient::default()),
- GET_CLIENT_CODE,
- &GetClient::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetClients(GetClients::default()),
- GET_CLIENTS_CODE,
- &GetClients::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetUser(GetUser::default()),
- GET_USER_CODE,
- &GetUser::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetUsers(GetUsers::default()),
- GET_USERS_CODE,
- &GetUsers::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::CreateUser(CreateUser::default()),
- CREATE_USER_CODE,
- &CreateUser::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::DeleteUser(DeleteUser::default()),
- DELETE_USER_CODE,
- &DeleteUser::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::UpdateUser(UpdateUser::default()),
- UPDATE_USER_CODE,
- &UpdateUser::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::UpdatePermissions(UpdatePermissions::default()),
- UPDATE_PERMISSIONS_CODE,
- &UpdatePermissions::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::ChangePassword(ChangePassword::default()),
- CHANGE_PASSWORD_CODE,
- &ChangePassword::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::LoginUser(LoginUser::default()),
- LOGIN_USER_CODE,
- &LoginUser::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::LogoutUser(LogoutUser::default()),
- LOGOUT_USER_CODE,
- &LogoutUser::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetPersonalAccessTokens(GetPersonalAccessTokens::default()),
- GET_PERSONAL_ACCESS_TOKENS_CODE,
- &GetPersonalAccessTokens::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::CreatePersonalAccessToken(CreatePersonalAccessToken::default()),
- CREATE_PERSONAL_ACCESS_TOKEN_CODE,
- &CreatePersonalAccessToken::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::DeletePersonalAccessToken(DeletePersonalAccessToken::default()),
- DELETE_PERSONAL_ACCESS_TOKEN_CODE,
- &DeletePersonalAccessToken::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::LoginWithPersonalAccessToken(LoginWithPersonalAccessToken::default()),
- LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE,
- &LoginWithPersonalAccessToken::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::PollMessages(PollMessages::default()),
- POLL_MESSAGES_CODE,
- &PollMessages::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::StoreConsumerOffset(StoreConsumerOffset::default()),
- STORE_CONSUMER_OFFSET_CODE,
- &StoreConsumerOffset::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetConsumerOffset(GetConsumerOffset::default()),
- GET_CONSUMER_OFFSET_CODE,
- &GetConsumerOffset::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetStream(GetStream::default()),
- GET_STREAM_CODE,
- &GetStream::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetStreams(GetStreams::default()),
- GET_STREAMS_CODE,
- &GetStreams::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::CreateStream(CreateStream::default()),
- CREATE_STREAM_CODE,
- &CreateStream::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::DeleteStream(DeleteStream::default()),
- DELETE_STREAM_CODE,
- &DeleteStream::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::UpdateStream(UpdateStream::default()),
- UPDATE_STREAM_CODE,
- &UpdateStream::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::PurgeStream(PurgeStream::default()),
- PURGE_STREAM_CODE,
- &PurgeStream::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetTopic(GetTopic::default()),
- GET_TOPIC_CODE,
- &GetTopic::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetTopics(GetTopics::default()),
- GET_TOPICS_CODE,
- &GetTopics::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::CreateTopic(CreateTopic::default()),
- CREATE_TOPIC_CODE,
- &CreateTopic::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::DeleteTopic(DeleteTopic::default()),
- DELETE_TOPIC_CODE,
- &DeleteTopic::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::UpdateTopic(UpdateTopic::default()),
- UPDATE_TOPIC_CODE,
- &UpdateTopic::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::PurgeTopic(PurgeTopic::default()),
- PURGE_TOPIC_CODE,
- &PurgeTopic::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::CreatePartitions(CreatePartitions::default()),
- CREATE_PARTITIONS_CODE,
- &CreatePartitions::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::DeletePartitions(DeletePartitions::default()),
- DELETE_PARTITIONS_CODE,
- &DeletePartitions::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::DeleteSegments(DeleteSegments::default()),
- DELETE_SEGMENTS_CODE,
- &DeleteSegments::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetConsumerGroup(GetConsumerGroup::default()),
- GET_CONSUMER_GROUP_CODE,
- &GetConsumerGroup::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::GetConsumerGroups(GetConsumerGroups::default()),
- GET_CONSUMER_GROUPS_CODE,
- &GetConsumerGroups::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::CreateConsumerGroup(CreateConsumerGroup::default()),
- CREATE_CONSUMER_GROUP_CODE,
- &CreateConsumerGroup::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::DeleteConsumerGroup(DeleteConsumerGroup::default()),
- DELETE_CONSUMER_GROUP_CODE,
- &DeleteConsumerGroup::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::JoinConsumerGroup(JoinConsumerGroup::default()),
- JOIN_CONSUMER_GROUP_CODE,
- &JoinConsumerGroup::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::LeaveConsumerGroup(LeaveConsumerGroup::default()),
- LEAVE_CONSUMER_GROUP_CODE,
- &LeaveConsumerGroup::default(),
- );
- assert_serialized_as_bytes_and_deserialized_from_bytes(
- &ServerCommand::FlushUnsavedBuffer(FlushUnsavedBuffer::default()),
- FLUSH_UNSAVED_BUFFER_CODE,
- &FlushUnsavedBuffer::default(),
- );
- }
-
- fn assert_serialized_as_bytes_and_deserialized_from_bytes(
- command: &ServerCommand,
- code: u32,
- payload: &dyn Command,
- ) {
- assert_serialized_as_bytes(command, code, payload);
- assert_deserialized_from_bytes(command, code, payload);
- }
-
- fn assert_serialized_as_bytes(
- server_command: &ServerCommand,
- code: u32,
- command: &dyn Command,
- ) {
- let payload = command.to_bytes();
- let mut bytes = BytesMut::with_capacity(4 + payload.len());
- bytes.put_u32_le(code);
- bytes.put_slice(&payload);
- assert_eq!(server_command.to_bytes(), bytes);
- }
-
- fn assert_deserialized_from_bytes(
- command: &ServerCommand,
- command_id: u32,
- payload: &dyn Command,
- ) {
- let payload = payload.to_bytes();
- let mut bytes = BytesMut::with_capacity(payload.len());
- bytes.put_slice(&payload);
- let bytes = Bytes::from(bytes);
- assert_eq!(
- &ServerCommand::from_code_and_payload(command_id, bytes).unwrap(),
- command
- );
- }
-}
diff --git a/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs b/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs
index 501c697ba..694d5b161 100644
--- a/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs
+++ b/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs
@@ -24,8 +24,6 @@ use crate::binary::command::{
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-//use crate::streaming::systems::system::SharedSystem;
-use anyhow::Result;
use iggy_common::get_cluster_metadata::GetClusterMetadata;
use iggy_common::{BytesSerializable, IggyError, SenderKind};
use tracing::{debug, instrument};
diff --git a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
index fe1350a9f..55d5d1671 100644
--- a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
@@ -23,12 +23,13 @@ use crate::binary::handlers::consumer_groups::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
-use crate::slab::traits_ext::EntityMarker;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::state::models::CreateConsumerGroupWithId;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::create_consumer_group::CreateConsumerGroup;
use iggy_common::{Identifier, IggyError, SenderKind};
@@ -50,44 +51,76 @@ impl ServerCommandHandler for CreateConsumerGroup {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let cg = shard.create_consumer_group(
- session,
- &self.stream_id,
- &self.topic_id,
- self.name.clone(),
- )?;
- let cg_id = cg.id();
- let event = ShardEvent::CreatedConsumerGroup {
+ let request = ShardRequest {
stream_id: self.stream_id.clone(),
topic_id: self.topic_id.clone(),
- cg,
+ partition_id: 0,
+ payload: ShardRequestPayload::CreateConsumerGroup {
+ user_id: session.get_user_id(),
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ name: self.name.clone(),
+ },
+ };
+
+ let message = ShardMessage::Request(request);
+ let cg_id = match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::CreateConsumerGroup {
+ stream_id,
+ topic_id,
+ name,
+ ..
+ } = payload
+ {
+ shard.create_consumer_group(session, &stream_id, &topic_id, name)?
+ } else {
+ unreachable!(
+ "Expected a CreateConsumerGroup request inside of CreateConsumerGroup handler"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::CreateConsumerGroupResponse(cg_id) => cg_id,
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!(
+ "Expected a CreateConsumerGroupResponse inside of CreateConsumerGroup handler"
+ ),
+ },
};
- shard.broadcast_event_to_all_shards(event).await?;
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
shard
.state
- .apply(
- session.get_user_id(),
- &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId {
- group_id: cg_id as u32,
- command: self
- }),
- )
+ .apply(
+ session.get_user_id(),
+ &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId {
+ group_id: cg_id as u32,
+ command: self,
+ }),
+ )
.await
.error(|e: &IggyError| {
format!(
"{COMPONENT} (error: {e}) - failed to apply create consumer group for stream_id: {stream_id}, topic_id: {topic_id}, group_id: {cg_id}, session: {session}"
)
})?;
- let response = shard.streams.with_consumer_group_by_id(
- &stream_id,
- &topic_id,
- &Identifier::numeric(cg_id as u32).unwrap(),
- |(root, members)| mapper::map_consumer_group(root, members),
- );
+
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&stream_id, &topic_id)?;
+
+ let cg_identifier = Identifier::numeric(cg_id as u32).unwrap();
+ let metadata = shard.metadata.load();
+ let response = metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(numeric_topic_id))
+ .and_then(|t| t.consumer_groups.get(cg_id))
+ .map(mapper::map_consumer_group_from_meta)
+ .ok_or_else(|| IggyError::ConsumerGroupIdNotFound(cg_identifier, topic_id.clone()))?;
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
index 07751b591..b39f387c8 100644
--- a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs
@@ -21,14 +21,15 @@ use crate::binary::command::{
};
use crate::binary::handlers::consumer_groups::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
+use crate::metadata::ConsumerGroupMeta;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
-use crate::slab::traits_ext::EntityMarker;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::polling_consumer::ConsumerGroupId;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::delete_consumer_group::DeleteConsumerGroup;
use iggy_common::{IggyError, SenderKind};
@@ -50,33 +51,77 @@ impl ServerCommandHandler for DeleteConsumerGroup {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let cg = shard.delete_consumer_group(session, &self.stream_id, &self.topic_id, &self.group_id).error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to delete consumer group with ID: {} for topic with ID: {} in stream with ID: {} for session: {}",
- self.group_id, self.topic_id, self.stream_id, session
- )
- })?;
- let cg_id = cg.id();
- let partition_ids = cg.partitions();
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- // Remove all consumer group members from ClientManager using helper functions to resolve identifiers
- let stream_id_usize = shard.streams.with_stream_by_id(
- &self.stream_id,
- crate::streaming::streams::helpers::get_stream_id(),
- );
- let topic_id_usize = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- crate::streaming::topics::helpers::get_topic_id(),
- );
+ let request = ShardRequest {
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ partition_id: 0,
+ payload: ShardRequestPayload::DeleteConsumerGroup {
+ user_id: session.get_user_id(),
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ group_id: self.group_id.clone(),
+ },
+ };
+
+ let message = ShardMessage::Request(request);
+ let cg_meta: ConsumerGroupMeta = match shard
+ .send_request_to_shard_or_recoil(None, message)
+ .await?
+ {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::DeleteConsumerGroup {
+ stream_id,
+ topic_id,
+ group_id,
+ ..
+ } = payload
+ {
+ shard.delete_consumer_group(session, &stream_id, &topic_id, &group_id).error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to delete consumer group with ID: {} for topic with ID: {} in stream with ID: {} for session: {}",
+ group_id, topic_id, stream_id, session
+ )
+ })?
+ } else {
+ unreachable!(
+ "Expected a DeleteConsumerGroup request inside of DeleteConsumerGroup handler"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::DeleteConsumerGroupResponse => {
+ let stream_id = self.stream_id.clone();
+ let topic_id = self.topic_id.clone();
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::DeleteConsumerGroup(self))
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply delete consumer group for stream_id: {stream_id}, topic_id: {topic_id}, session: {session}"
+ )
+ })?;
+ sender.send_empty_ok_response().await?;
+ return Ok(HandlerResult::Finished);
+ }
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!(
+ "Expected a DeleteConsumerGroupResponse inside of DeleteConsumerGroup handler"
+ ),
+ },
+ };
- // Get members from the deleted consumer group and make them leave
- let slab = cg.members().inner().shared_get();
- for (_, member) in slab.iter() {
+ let cg_id = cg_meta.id;
+
+ for (_, member) in cg_meta.members.iter() {
if let Err(err) = shard.client_manager.leave_consumer_group(
member.client_id,
- stream_id_usize,
- topic_id_usize,
+ numeric_stream_id,
+ numeric_topic_id,
cg_id,
) {
tracing::warn!(
@@ -88,12 +133,11 @@ impl ServerCommandHandler for DeleteConsumerGroup {
}
let cg_id_spez = ConsumerGroupId(cg_id);
- // Delete all consumer group offsets for this group using the specialized method
shard.delete_consumer_group_offsets(
cg_id_spez,
&self.stream_id,
&self.topic_id,
- partition_ids,
+ &cg_meta.partitions,
).await.error(|e: &IggyError| {
format!(
"{COMPONENT} (error: {e}) - failed to delete consumer group offsets for group ID: {} in stream: {}, topic: {}",
@@ -103,13 +147,6 @@ impl ServerCommandHandler for DeleteConsumerGroup {
)
})?;
- let event = ShardEvent::DeletedConsumerGroup {
- id: cg_id,
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- group_id: self.group_id.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
shard
diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
index 7156a4d53..fcc7d3288 100644
--- a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs
@@ -23,8 +23,6 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
-use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::SenderKind;
use iggy_common::get_consumer_group::GetConsumerGroup;
@@ -52,17 +50,11 @@ impl ServerCommandHandler for GetConsumerGroup {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
}
- let numeric_topic_id = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- topics::helpers::get_topic_id(),
- );
- let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
+
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
let has_permission = shard
.permissioner
- .borrow()
.get_consumer_group(session.get_user_id(), numeric_stream_id, numeric_topic_id)
.is_ok();
if !has_permission {
@@ -70,12 +62,23 @@ impl ServerCommandHandler for GetConsumerGroup {
return Ok(HandlerResult::Finished);
}
- let consumer_group = shard.streams.with_consumer_group_by_id(
- &self.stream_id,
- &self.topic_id,
- &self.group_id,
- |(root, members)| mapper::map_consumer_group(root, members),
- );
+ let numeric_group_id = shard
+ .metadata
+ .get_consumer_group_id(numeric_stream_id, numeric_topic_id, &self.group_id)
+ .ok_or_else(|| {
+ IggyError::ConsumerGroupIdNotFound(self.group_id.clone(), self.topic_id.clone())
+ })?;
+
+ let metadata = shard.metadata.load();
+ let consumer_group = metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(numeric_topic_id))
+ .and_then(|t| t.consumer_groups.get(numeric_group_id))
+ .map(mapper::map_consumer_group_from_meta)
+ .ok_or_else(|| {
+ IggyError::ConsumerGroupIdNotFound(self.group_id.clone(), self.topic_id.clone())
+ })?;
sender.send_ok_response(&consumer_group).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
index d59f33c97..7f9949892 100644
--- a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs
@@ -20,12 +20,8 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
-use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::SenderKind;
use iggy_common::get_consumer_groups::GetConsumerGroups;
@@ -46,30 +42,33 @@ impl ServerCommandHandler for GetConsumerGroups {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- shard.ensure_topic_exists(&self.stream_id, &self.topic_id)?;
- let numeric_topic_id = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- topics::helpers::get_topic_id(),
- );
- let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
- shard.permissioner.borrow().get_consumer_groups(
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
+ shard.permissioner.get_consumer_groups(
session.get_user_id(),
numeric_stream_id,
numeric_topic_id,
)?;
- let consumer_groups =
- shard
- .streams
- .with_consumer_groups(&self.stream_id, &self.topic_id, |cgs| {
- cgs.with_components(|cgs| {
- let (roots, members) = cgs.into_components();
- mapper::map_consumer_groups(roots, members)
- })
- });
+ let metadata = shard.metadata.load();
+
+ let consumer_groups = metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(numeric_topic_id))
+ .map(|topic| {
+ use bytes::{BufMut, BytesMut};
+ let mut bytes = BytesMut::new();
+ for (_, cg_meta) in topic.consumer_groups.iter() {
+ bytes.put_u32_le(cg_meta.id as u32);
+ bytes.put_u32_le(cg_meta.partitions.len() as u32);
+ bytes.put_u32_le(cg_meta.members.len() as u32);
+ bytes.put_u8(cg_meta.name.len() as u8);
+ bytes.put_slice(cg_meta.name.as_bytes());
+ }
+ bytes.freeze()
+ })
+ .unwrap_or_default();
sender.send_ok_response(&consumer_groups).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs
index 8aab2dca5..b3f48efa5 100644
--- a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs
@@ -21,10 +21,12 @@ use crate::binary::command::{
};
use crate::binary::handlers::consumer_groups::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::join_consumer_group::JoinConsumerGroup;
use iggy_common::{IggyError, SenderKind};
@@ -46,19 +48,53 @@ impl ServerCommandHandler for JoinConsumerGroup {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- shard
- .join_consumer_group(
- session,
- &self.stream_id,
- &self.topic_id,
- &self.group_id,
- )
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to join consumer group for stream_id: {}, topic_id: {}, group_id: {}, session: {}",
- self.stream_id, self.topic_id, self.group_id, session
- )
- })?;
+
+ let request = ShardRequest {
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ partition_id: 0,
+ payload: ShardRequestPayload::JoinConsumerGroup {
+ user_id: session.get_user_id(),
+ client_id: session.client_id,
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ group_id: self.group_id.clone(),
+ },
+ };
+
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::JoinConsumerGroup {
+ stream_id,
+ topic_id,
+ group_id,
+ ..
+ } = payload
+ {
+ shard
+ .join_consumer_group(session, &stream_id, &topic_id, &group_id)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to join consumer group for stream_id: {}, topic_id: {}, group_id: {}, session: {}",
+ stream_id, topic_id, group_id, session
+ )
+ })?;
+ } else {
+ unreachable!(
+ "Expected a JoinConsumerGroup request inside of JoinConsumerGroup handler"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::JoinConsumerGroupResponse => {}
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!(
+ "Expected a JoinConsumerGroupResponse inside of JoinConsumerGroup handler"
+ ),
+ },
+ }
sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
index 32f1c4d5a..591a38ff9 100644
--- a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs
@@ -21,13 +21,15 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
-use iggy_common::SenderKind;
-
use crate::shard::IggyShard;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::IggyError;
+use iggy_common::SenderKind;
use iggy_common::leave_consumer_group::LeaveConsumerGroup;
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -48,19 +50,52 @@ impl ServerCommandHandler for LeaveConsumerGroup {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- shard
- .leave_consumer_group(
- session,
- &self.stream_id,
- &self.topic_id,
- &self.group_id,
- )
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to leave consumer group for stream_id: {}, topic_id: {}, group_id: {}, session: {}",
- self.stream_id, self.topic_id, self.group_id, session
- )
- })?;
+ let request = ShardRequest {
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ partition_id: 0,
+ payload: ShardRequestPayload::LeaveConsumerGroup {
+ user_id: session.get_user_id(),
+ client_id: session.client_id,
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ group_id: self.group_id.clone(),
+ },
+ };
+
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::LeaveConsumerGroup {
+ stream_id,
+ topic_id,
+ group_id,
+ ..
+ } = payload
+ {
+ shard
+ .leave_consumer_group(session, &stream_id, &topic_id, &group_id)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to leave consumer group for stream_id: {}, topic_id: {}, group_id: {}, session: {}",
+ stream_id, topic_id, group_id, session
+ )
+ })?;
+ } else {
+ unreachable!(
+ "Expected a LeaveConsumerGroup request inside of LeaveConsumerGroup handler"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::LeaveConsumerGroupResponse => {}
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!(
+ "Expected a LeaveConsumerGroupResponse inside of LeaveConsumerGroup handler"
+ ),
+ },
+ }
sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
diff --git a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs
index 64d01255e..7e7eed644 100644
--- a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs
+++ b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs
@@ -23,7 +23,6 @@ use crate::binary::handlers::consumer_offsets::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::delete_consumer_offset::DeleteConsumerOffset;
use iggy_common::{IggyError, SenderKind};
diff --git a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs
index 7b8991981..1c8b76e0d 100644
--- a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs
+++ b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs
@@ -23,7 +23,6 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::SenderKind;
use iggy_common::get_consumer_offset::GetConsumerOffset;
diff --git a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs
index c603f4cb8..e703f1370 100644
--- a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs
+++ b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs
@@ -25,7 +25,6 @@ use crate::binary::handlers::consumer_offsets::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::IggyError;
use iggy_common::SenderKind;
diff --git a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs
index 008d1c600..28100048f 100644
--- a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs
+++ b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs
@@ -23,7 +23,6 @@ use crate::binary::handlers::messages::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::{FlushUnsavedBuffer, IggyError, SenderKind};
use std::rc::Rc;
diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
index ea8ccbb8c..d3c12957f 100644
--- a/core/server/src/binary/handlers/messages/poll_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/poll_messages_handler.rs
@@ -23,7 +23,6 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::shard::system::messages::PollingArgs;
use crate::streaming::session::Session;
-use anyhow::Result;
use iggy_common::SenderKind;
use iggy_common::{IggyError, PollMessages, PooledBuffer};
use std::rc::Rc;
diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs
index 6aa6786ac..bdd9a9197 100644
--- a/core/server/src/binary/handlers/messages/send_messages_handler.rs
+++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs
@@ -21,8 +21,7 @@ use crate::shard::IggyShard;
use crate::shard::transmission::message::{ShardMessage, ShardRequest, ShardRequestPayload};
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
-use anyhow::Result;
+use crate::streaming::topics;
use compio::buf::{IntoInner as _, IoBuf};
use iggy_common::Identifier;
use iggy_common::PooledBuffer;
@@ -50,6 +49,7 @@ impl ServerCommandHandler for SendMessages {
mut self,
sender: &mut SenderKind,
length: u32,
+
session: &Session,
shard: &Rc<IggyShard>,
) -> Result<HandlerResult, IggyError> {
@@ -111,44 +111,32 @@ impl ServerCommandHandler for SendMessages {
);
batch.validate()?;
- shard.ensure_topic_exists(&self.stream_id, &self.topic_id)?;
-
- let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&self.stream_id, &self.topic_id)?;
- let numeric_topic_id = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- topics::helpers::get_topic_id(),
- );
-
- // TODO(tungtose): dry this code && get partition_id below have a side effect
- let partition_id = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- |(root, auxilary, ..)| match self.partitioning.kind {
- PartitioningKind::Balanced => {
- let upperbound = root.partitions().len();
- let pid = auxilary.get_next_partition_id(upperbound);
- Ok(pid)
- }
- PartitioningKind::PartitionId => Ok(u32::from_le_bytes(
- self.partitioning.value[..self.partitioning.length as usize]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- ) as usize),
- PartitioningKind::MessagesKey => {
- let upperbound = root.partitions().len();
- Ok(
- topics::helpers::calculate_partition_id_by_messages_key_hash(
- upperbound,
- &self.partitioning.value,
- ),
- )
- }
- },
- )?;
+ let partition_id = match self.partitioning.kind {
+ PartitioningKind::Balanced => shard
+ .metadata
+ .get_next_partition_id(numeric_stream_id, numeric_topic_id)
+ .ok_or(IggyError::TopicIdNotFound(
+ self.stream_id.clone(),
+ self.topic_id.clone(),
+ ))?,
+ PartitioningKind::PartitionId => u32::from_le_bytes(
+ self.partitioning.value[..self.partitioning.length as usize]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ) as usize,
+ PartitioningKind::MessagesKey => {
+ let partitions_count = shard
+ .metadata
+ .partitions_count(numeric_stream_id, numeric_topic_id);
+ topics::helpers::calculate_partition_id_by_messages_key_hash(
+ partitions_count,
+ &self.partitioning.value,
+ )
+ }
+ };
let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
let user_id = session.get_user_id();
diff --git a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
index 85b4e3030..2c6760759 100644
--- a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
+++ b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs
@@ -21,17 +21,16 @@ use crate::binary::command::{
};
use crate::binary::handlers::partitions::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
-use crate::slab::traits_ext::EntityMarker;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::create_partitions::CreatePartitions;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -51,52 +50,91 @@ impl ServerCommandHandler for CreatePartitions {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- // Acquire partition lock to serialize filesystem operations
- let _partition_guard = shard.fs_locks.partition_lock.lock().await;
-
- let partitions = shard
- .create_partitions(
- session,
- &self.stream_id,
- &self.topic_id,
- self.partitions_count,
- )
- .await?;
- let partition_ids = partitions.iter().map(|p| p.id()).collect::<Vec<_>>();
- let event = ShardEvent::CreatedPartitions {
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- partitions,
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::CreatePartitions {
+ user_id: session.get_user_id(),
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ partitions_count: self.partitions_count,
+ },
};
- shard.broadcast_event_to_all_shards(event).await?;
- shard.streams.with_topic_by_id_mut(
- &self.stream_id,
- &self.topic_id,
- topics::helpers::rebalance_consumer_groups(&partition_ids),
- );
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::CreatePartitions {
+ stream_id,
+ topic_id,
+ partitions_count,
+ ..
+ } = payload
+ {
+ let _partition_guard = shard.fs_locks.partition_lock.lock().await;
+
+ // Get numeric IDs BEFORE create (for rebalance operation)
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&stream_id, &topic_id)?;
+
+ let partition_infos = shard
+ .create_partitions(session, &stream_id, &topic_id, partitions_count)
+ .await?;
+ let partition_ids = partition_infos.iter().map(|p| p.id).collect::<Vec<_>>();
+
+ let event = crate::shard::transmission::event::ShardEvent::CreatedPartitions {
+ stream_id: stream_id.clone(),
+ topic_id: topic_id.clone(),
+ partitions: partition_infos,
+ };
+ shard.broadcast_event_to_all_shards(event).await?;
+
+ shard.writer().rebalance_consumer_groups_for_topic(
+ numeric_stream_id,
+ numeric_topic_id,
+ partition_ids.len() as u32,
+ );
+
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::CreatePartitions(self))
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply create partitions for stream_id: {numeric_stream_id}, topic_id: {numeric_topic_id}, session: {session}"
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ } else {
+ unreachable!(
+ "Expected a CreatePartitions request inside of CreatePartitions handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::CreatePartitionsResponse(_partitions) => {
+ // Consumer group rebalancing is handled via the CreatedPartitions event
+ // broadcast in the shard-0 handler, so we don't need to do it here.
+ // Apply state for WAL replay.
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::CreatePartitions(self))
+ .await?;
+
+ sender.send_empty_ok_response().await?;
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!(
+ "Expected a CreatePartitionsResponse inside of CreatePartitions handler, impossible state"
+ ),
+ },
+ }
- let stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
- let topic_id = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- topics::helpers::get_topic_id(),
- );
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::CreatePartitions(self),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply create partitions for stream_id: {stream_id}, topic_id: {topic_id}, session: {session}"
- )
- })?;
- sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
index f4bed945f..8ec4533f4 100644
--- a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
+++ b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs
@@ -21,15 +21,16 @@ use crate::binary::command::{
};
use crate::binary::handlers::partitions::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::delete_partitions::DeletePartitions;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -48,52 +49,101 @@ impl ServerCommandHandler for DeletePartitions {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let stream_id = self.stream_id.clone();
- let topic_id = self.topic_id.clone();
-
- // Acquire partition lock to serialize filesystem operations
- let _partition_guard = shard.fs_locks.partition_lock.lock().await;
- let deleted_partition_ids = shard
- .delete_partitions(
- session,
- &self.stream_id,
- &self.topic_id,
- self.partitions_count,
- )
- .await?;
- let event = ShardEvent::DeletedPartitions {
- stream_id: self.stream_id.clone(),
- topic_id: self.topic_id.clone(),
- partitions_count: self.partitions_count,
- partition_ids: deleted_partition_ids,
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::DeletePartitions {
+ user_id: session.get_user_id(),
+ stream_id: self.stream_id.clone(),
+ topic_id: self.topic_id.clone(),
+ partitions_count: self.partitions_count,
+ },
};
- shard.broadcast_event_to_all_shards(event).await?;
- let remaining_partition_ids = shard.streams.with_topic_by_id(
- &self.stream_id,
- &self.topic_id,
- crate::streaming::topics::helpers::get_partition_ids(),
- );
- shard.streams.with_topic_by_id_mut(
- &self.stream_id,
- &self.topic_id,
- crate::streaming::topics::helpers::rebalance_consumer_groups(&remaining_partition_ids),
- );
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::DeletePartitions {
+ stream_id,
+ topic_id,
+ partitions_count,
+ ..
+ } = payload
+ {
+ let _partition_guard = shard.fs_locks.partition_lock.lock().await;
+
+ // Get numeric IDs BEFORE delete
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&stream_id, &topic_id)?;
+
+ let deleted_partition_ids = shard
+ .delete_partitions(session, &stream_id, &topic_id, partitions_count)
+ .await?;
+
+ let event = crate::shard::transmission::event::ShardEvent::DeletedPartitions {
+ stream_id: stream_id.clone(),
+ topic_id: topic_id.clone(),
+ partitions_count,
+ partition_ids: deleted_partition_ids,
+ };
+ shard.broadcast_event_to_all_shards(event).await?;
+
+ let remaining_partition_count: u32 = shard
+ .metadata
+ .load()
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(numeric_topic_id))
+ .map(|t| t.partitions.len() as u32)
+ .unwrap_or(0);
+ shard.writer().rebalance_consumer_groups_for_topic(
+ numeric_stream_id,
+ numeric_topic_id,
+ remaining_partition_count,
+ );
+
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::DeletePartitions(self))
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply delete partitions for stream_id: {}, topic_id: {}, session: {session}",
+ stream_id, topic_id
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ } else {
+ unreachable!(
+ "Expected a DeletePartitions request inside of DeletePartitions handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::DeletePartitionsResponse(_partition_ids) => {
+ // Consumer group rebalancing is handled via the DeletedPartitions event
+ // broadcast in the shard-0 handler, so we don't need to do it here.
+ // Apply state for WAL replay.
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::DeletePartitions(self))
+ .await?;
+
+ sender.send_empty_ok_response().await?;
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!(
+ "Expected a DeletePartitionsResponse inside of DeletePartitions handler, impossible state"
+ ),
+ },
+ }
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::DeletePartitions(self),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply delete partitions for stream_id: {stream_id}, topic_id: {topic_id}, session: {session}"
- )
- })?;
- sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
index 314f1821d..32784d05c 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs
@@ -22,16 +22,17 @@ use crate::binary::command::{
use crate::binary::handlers::personal_access_tokens::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
-
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::state::models::CreatePersonalAccessTokenWithHash;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::create_personal_access_token::CreatePersonalAccessToken;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -51,20 +52,51 @@ impl ServerCommandHandler for CreatePersonalAccessToken {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let (personal_access_token, token) = shard
- .create_personal_access_token(session, &self.name, self.expiry)
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to create personal access token with name: {}, session: {session}",
- self.name
- )
- })?;
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::CreatePersonalAccessToken {
+ user_id: session.get_user_id(),
+ name: self.name.clone(),
+ expiry: self.expiry,
+ },
+ };
+
+ let message = ShardMessage::Request(request);
+ let (personal_access_token, token) = match shard
+ .send_request_to_shard_or_recoil(None, message)
+ .await?
+ {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::CreatePersonalAccessToken { name, expiry, .. } =
+ payload
+ {
+ shard
+ .create_personal_access_token(session, &name, expiry)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to create personal access token with name: {name}, session: {session}"
+ )
+ })?
+ } else {
+ unreachable!(
+ "Expected a CreatePersonalAccessToken request inside of CreatePersonalAccessToken handler"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::CreatePersonalAccessTokenResponse(pat, token) => (pat, token),
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!(
+ "Expected a CreatePersonalAccessTokenResponse inside of CreatePersonalAccessToken handler"
+ ),
+ },
+ };
+
let bytes = mapper::map_raw_pat(&token);
let hash = personal_access_token.token.to_string();
- let event = ShardEvent::CreatedPersonalAccessToken {
- personal_access_token: personal_access_token.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
shard
.state
diff --git a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
index edb1e8e9a..814470b2c 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs
@@ -22,12 +22,15 @@ use crate::binary::command::{
use crate::binary::handlers::personal_access_tokens::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::delete_personal_access_token::DeletePersonalAccessToken;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -48,18 +51,43 @@ impl ServerCommandHandler for DeletePersonalAccessToken {
shard.ensure_authenticated(session)?;
let token_name = self.name.clone();
- shard
- .delete_personal_access_token(session, &self.name)
- .error(|e: &IggyError| {format!(
- "{COMPONENT} (error: {e}) - failed to delete personal access token with name: {token_name}, session: {session}"
- )})?;
-
- // Broadcast the event to other shards
- let event = crate::shard::transmission::event::ShardEvent::DeletedPersonalAccessToken {
- user_id: session.get_user_id(),
- name: self.name.clone(),
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::DeletePersonalAccessToken {
+ user_id: session.get_user_id(),
+ name: self.name.clone(),
+ },
};
- shard.broadcast_event_to_all_shards(event).await?;
+
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::DeletePersonalAccessToken { name, .. } = payload
+ {
+ shard.delete_personal_access_token(session, &name).error(
+ |e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to delete personal access token with name: {name}, session: {session}"
+ )
+ },
+ )?;
+ } else {
+ unreachable!(
+ "Expected a DeletePersonalAccessToken request inside of DeletePersonalAccessToken handler"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::DeletePersonalAccessTokenResponse => {}
+ ShardResponse::ErrorResponse(err) => return Err(err),
+ _ => unreachable!(
+ "Expected a DeletePersonalAccessTokenResponse inside of DeletePersonalAccessToken handler"
+ ),
+ },
+ }
shard
.state
diff --git a/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs
index 1603aaa6a..db8a94f51 100644
--- a/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs
+++ b/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs
@@ -25,7 +25,6 @@ use crate::binary::handlers::personal_access_tokens::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::login_with_personal_access_token::LoginWithPersonalAccessToken;
use iggy_common::{IggyError, SenderKind};
diff --git a/core/server/src/binary/handlers/segments/delete_segments_handler.rs b/core/server/src/binary/handlers/segments/delete_segments_handler.rs
index a4eb20654..371871965 100644
--- a/core/server/src/binary/handlers/segments/delete_segments_handler.rs
+++ b/core/server/src/binary/handlers/segments/delete_segments_handler.rs
@@ -27,9 +27,7 @@ use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
use crate::state::command::EntryCommand;
-use crate::streaming;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::delete_segments::DeleteSegments;
use iggy_common::sharding::IggyNamespace;
@@ -51,29 +49,17 @@ impl ServerCommandHandler for DeleteSegments {
shard: &Rc<IggyShard>,
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
+ shard.ensure_authenticated(session)?;
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
let partition_id = self.partition_id as usize;
let segments_count = self.segments_count;
-
- // Ensure authentication and topic exists
- shard.ensure_authenticated(session)?;
- shard.ensure_topic_exists(&stream_id, &topic_id)?;
shard.ensure_partition_exists(&stream_id, &topic_id, partition_id)?;
- // Get numeric IDs for namespace
- let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&stream_id, streaming::streams::helpers::get_stream_id());
-
- let numeric_topic_id = shard.streams.with_topic_by_id(
- &stream_id,
- &topic_id,
- streaming::topics::helpers::get_topic_id(),
- );
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&stream_id, &topic_id)?;
- // Route request to the correct shard
let namespace = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id);
let payload = ShardRequestPayload::DeleteSegments { segments_count };
let request = ShardRequest::new(stream_id.clone(), topic_id.clone(), partition_id, payload);
@@ -92,8 +78,9 @@ impl ServerCommandHandler for DeleteSegments {
}) = message
&& let ShardRequestPayload::DeleteSegments { segments_count } = payload
{
+ let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?;
shard
- .delete_segments_base(&stream_id, &topic_id, partition_id, segments_count)
+ .delete_segments_base(stream, topic, partition_id, segments_count)
.await
.error(|e: &IggyError| {
format!(
diff --git a/core/server/src/binary/handlers/streams/create_stream_handler.rs b/core/server/src/binary/handlers/streams/create_stream_handler.rs
index 4d9e85d5d..5de9b4ba1 100644
--- a/core/server/src/binary/handlers/streams/create_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/create_stream_handler.rs
@@ -21,19 +21,14 @@ use crate::binary::command::{
};
use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
-
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
-use crate::slab::traits_ext::{EntityComponentSystem, EntityMarker};
use crate::state::command::EntryCommand;
use crate::state::models::CreateStreamWithId;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::create_stream::CreateStream;
use iggy_common::{Identifier, IggyError, SenderKind};
@@ -75,20 +70,9 @@ impl ServerCommandHandler for CreateStream {
// Acquire stream lock to serialize filesystem operations
let _stream_guard = shard.fs_locks.stream_lock.lock().await;
- let stream = shard.create_stream(session, name).await?;
- let created_stream_id = stream.id();
-
- let event = ShardEvent::CreatedStream {
- id: created_stream_id,
- stream: stream.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
+ let created_stream_id = shard.create_stream(session, name).await?;
- let response = shard
- .streams
- .with_components_by_id(created_stream_id, |(root, stats)| {
- mapper::map_stream(&root, &stats)
- });
+ let response = shard.get_stream_from_shared_metadata(created_stream_id);
shard
.state
@@ -111,9 +95,8 @@ impl ServerCommandHandler for CreateStream {
}
}
ShardSendRequestResult::Response(response) => match response {
- ShardResponse::CreateStreamResponse(stream) => {
- let created_stream_id = stream.id();
- let response = mapper::map_stream(stream.root(), stream.stats());
+ ShardResponse::CreateStreamResponse(created_stream_id) => {
+ let response = shard.get_stream_from_shared_metadata(created_stream_id);
shard
.state
diff --git a/core/server/src/binary/handlers/streams/delete_stream_handler.rs b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
index addc25af3..e3c919790 100644
--- a/core/server/src/binary/handlers/streams/delete_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/delete_stream_handler.rs
@@ -22,15 +22,13 @@ use crate::binary::command::{
use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
-use crate::slab::traits_ext::EntityMarker;
+
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::delete_stream::DeleteStream;
use iggy_common::{Identifier, IggyError, SenderKind};
@@ -72,7 +70,7 @@ impl ServerCommandHandler for DeleteStream {
{
// Acquire stream lock to serialize filesystem operations
let _stream_guard = shard.fs_locks.stream_lock.lock().await;
- let stream = shard
+ let stream_info = shard
.delete_stream(session, &stream_id)
.await
.error(|e: &IggyError| {
@@ -80,16 +78,9 @@ impl ServerCommandHandler for DeleteStream {
})?;
info!(
"Deleted stream with name: {}, ID: {}",
- stream.root().name(),
- stream.id()
+ stream_info.name, stream_info.id
);
- let event = ShardEvent::DeletedStream {
- id: stream.id(),
- stream_id: stream_id.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
shard
.state
.apply(session.get_user_id(), &EntryCommand::DeleteStream(DeleteStream { stream_id: stream_id.clone() }))
@@ -101,10 +92,10 @@ impl ServerCommandHandler for DeleteStream {
}
}
ShardSendRequestResult::Response(response) => match response {
- ShardResponse::DeleteStreamResponse(stream) => {
+ ShardResponse::DeleteStreamResponse(stream_id_num) => {
shard
.state
- .apply(session.get_user_id(), &EntryCommand::DeleteStream(DeleteStream { stream_id: (stream.id() as u32).try_into().unwrap() }))
+ .apply(session.get_user_id(), &EntryCommand::DeleteStream(DeleteStream { stream_id: (stream_id_num as u32).try_into().unwrap() }))
.await
.error(|e: &IggyError| {
format!("{COMPONENT} (error: {e}) - failed to apply delete stream with ID: {stream_id}, session: {session}")
diff --git a/core/server/src/binary/handlers/streams/get_stream_handler.rs b/core/server/src/binary/handlers/streams/get_stream_handler.rs
index fe9cd1f83..d7cdeb36f 100644
--- a/core/server/src/binary/handlers/streams/get_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/get_stream_handler.rs
@@ -20,12 +20,8 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::slab::traits_ext::EntityComponentSystem;
use crate::streaming::session::Session;
-use crate::streaming::streams;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::IggyError;
use iggy_common::SenderKind;
@@ -47,18 +43,15 @@ impl ServerCommandHandler for GetStream {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let exists = shard.ensure_stream_exists(&self.stream_id).is_ok();
- if !exists {
+
+ let Some(numeric_stream_id) = shard.metadata.get_stream_id(&self.stream_id) else {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
- }
- let stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
+ };
+
let has_permission = shard
.permissioner
- .borrow()
- .get_stream(session.get_user_id(), stream_id)
+ .get_stream(session.get_user_id(), numeric_stream_id)
.error(|e: &IggyError| {
format!(
"permission denied to get stream with ID: {} for user with ID: {}, error: {e}",
@@ -71,9 +64,8 @@ impl ServerCommandHandler for GetStream {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
}
- let response = shard
- .streams
- .with_components_by_id(stream_id, |(root, stats)| mapper::map_stream(&root, &stats));
+
+ let response = shard.get_stream_from_shared_metadata(numeric_stream_id);
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/streams/get_streams_handler.rs b/core/server/src/binary/handlers/streams/get_streams_handler.rs
index f7bf06f2d..f97a63069 100644
--- a/core/server/src/binary/handlers/streams/get_streams_handler.rs
+++ b/core/server/src/binary/handlers/streams/get_streams_handler.rs
@@ -21,11 +21,8 @@ use crate::binary::command::{
};
use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::IggyError;
use iggy_common::SenderKind;
@@ -49,7 +46,6 @@ impl ServerCommandHandler for GetStreams {
shard.ensure_authenticated(session)?;
shard
.permissioner
- .borrow()
.get_streams(session.get_user_id())
.error(|e: &IggyError| {
format!(
@@ -58,10 +54,7 @@ impl ServerCommandHandler for GetStreams {
)
})?;
- let response = shard.streams.with_components(|stream_ref| {
- let (roots, stats) = stream_ref.into_components();
- mapper::map_streams(&roots, &stats)
- });
+ let response = shard.get_streams_from_shared_metadata();
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/streams/purge_stream_handler.rs b/core/server/src/binary/handlers/streams/purge_stream_handler.rs
index c688ecd96..28c82f7fe 100644
--- a/core/server/src/binary/handlers/streams/purge_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/purge_stream_handler.rs
@@ -21,12 +21,10 @@ use crate::binary::command::{
};
use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
use crate::shard::transmission::event::ShardEvent;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::purge_stream::PurgeStream;
use iggy_common::{IggyError, SenderKind};
diff --git a/core/server/src/binary/handlers/streams/update_stream_handler.rs b/core/server/src/binary/handlers/streams/update_stream_handler.rs
index 3986f589c..21747d2d9 100644
--- a/core/server/src/binary/handlers/streams/update_stream_handler.rs
+++ b/core/server/src/binary/handlers/streams/update_stream_handler.rs
@@ -21,15 +21,16 @@ use crate::binary::command::{
};
use crate::binary::handlers::streams::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::update_stream::UpdateStream;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use std::rc::Rc;
use tracing::{debug, instrument};
@@ -48,26 +49,63 @@ impl ServerCommandHandler for UpdateStream {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let stream_id = self.stream_id.clone();
- shard
- .update_stream(session, &self.stream_id, self.name.clone())
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to update stream with id: {stream_id}, session: {session}")
- })?;
- let event = ShardEvent::UpdatedStream {
- stream_id: self.stream_id.clone(),
- name: self.name.clone(),
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::UpdateStream {
+ user_id: session.get_user_id(),
+ stream_id: self.stream_id.clone(),
+ name: self.name.clone(),
+ },
};
- shard.broadcast_event_to_all_shards(event).await?;
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::UpdateStream(self))
- .await
- .error(|e: &IggyError| {
- format!("{COMPONENT} (error: {e}) - failed to apply update stream with id: {stream_id}, session: {session}")
- })?;
- sender.send_empty_ok_response().await?;
+
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::UpdateStream {
+ stream_id, name, ..
+ } = payload
+ {
+ shard.update_stream(session, &stream_id, name.clone())?;
+
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::UpdateStream(self))
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply update stream with id: {stream_id}, session: {session}"
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ } else {
+ unreachable!(
+ "Expected an UpdateStream request inside of UpdateStream handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::UpdateStreamResponse => {
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::UpdateStream(self))
+ .await?;
+
+ sender.send_empty_ok_response().await?;
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!(
+ "Expected an UpdateStreamResponse inside of UpdateStream handler, impossible state"
+ ),
+ },
+ }
+
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/system/ping_handler.rs b/core/server/src/binary/handlers/system/ping_handler.rs
index e22fa7ba8..746de588f 100644
--- a/core/server/src/binary/handlers/system/ping_handler.rs
+++ b/core/server/src/binary/handlers/system/ping_handler.rs
@@ -19,7 +19,6 @@
use crate::binary::command::{BinaryServerCommand, HandlerResult, ServerCommandHandler};
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::IggyTimestamp;
use iggy_common::SenderKind;
diff --git a/core/server/src/binary/handlers/topics/create_topic_handler.rs b/core/server/src/binary/handlers/topics/create_topic_handler.rs
index 95cbbe4c5..f5396e799 100644
--- a/core/server/src/binary/handlers/topics/create_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/create_topic_handler.rs
@@ -21,20 +21,16 @@ use crate::binary::command::{
};
use crate::binary::handlers::topics::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
-
use crate::shard::IggyShard;
use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
-use crate::slab::traits_ext::EntityMarker;
+
use crate::state::command::EntryCommand;
use crate::state::models::CreateTopicWithId;
use crate::streaming::session::Session;
-use crate::streaming::streams;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::create_topic::CreateTopic;
use iggy_common::{Identifier, IggyError, SenderKind};
@@ -91,7 +87,7 @@ impl ServerCommandHandler for CreateTopic {
// Acquire topic lock to serialize filesystem operations
let _topic_guard = shard.fs_locks.topic_lock.lock().await;
- let topic = shard
+ let topic_id = shard
.create_topic(
session,
&stream_id,
@@ -102,21 +98,24 @@ impl ServerCommandHandler for CreateTopic {
replication_factor,
)
.await?;
- self.message_expiry = topic.root().message_expiry();
- self.max_topic_size = topic.root().max_topic_size();
let stream_id_num = shard
- .streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
- let topic_id = topic.id();
-
- let event = ShardEvent::CreatedTopic {
- stream_id: stream_id.clone(),
- topic,
- };
+ .metadata
+ .get_stream_id(&stream_id)
+ .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?;
- shard.broadcast_event_to_all_shards(event).await?;
- let partitions = shard
+ // Get topic metadata to update command fields
+ let metadata = shard.metadata.load();
+ if let Some(topic_meta) = metadata
+ .streams
+ .get(stream_id_num)
+ .and_then(|s| s.topics.get(topic_id))
+ {
+ self.message_expiry = topic_meta.message_expiry;
+ self.max_topic_size = topic_meta.max_topic_size;
+ }
+
+ let partition_infos = shard
.create_partitions(
session,
&stream_id,
@@ -127,15 +126,11 @@ impl ServerCommandHandler for CreateTopic {
let event = ShardEvent::CreatedPartitions {
stream_id: stream_id.clone(),
topic_id: Identifier::numeric(topic_id as u32).unwrap(),
- partitions,
+ partitions: partition_infos.clone(),
};
shard.broadcast_event_to_all_shards(event).await?;
- let response = shard.streams.with_topic_by_id(
- &stream_id,
- &Identifier::numeric(topic_id as u32).unwrap(),
- |(root, _, stats)| mapper::map_topic(&root, &stats),
- );
+ let response = shard.get_topic_from_shared_metadata(stream_id_num, topic_id);
shard
.state
.apply(session.get_user_id(), &EntryCommand::CreateTopic(CreateTopicWithId {
@@ -156,20 +151,24 @@ impl ServerCommandHandler for CreateTopic {
}
}
ShardSendRequestResult::Response(response) => match response {
- ShardResponse::CreateTopicResponse(topic) => {
- let topic_id = topic.id();
- self.message_expiry = topic.root().message_expiry();
- self.max_topic_size = topic.root().max_topic_size();
+ ShardResponse::CreateTopicResponse(topic_id) => {
+ let stream_id_num = shard
+ .metadata
+ .get_stream_id(&self.stream_id)
+ .ok_or_else(|| IggyError::StreamIdNotFound(self.stream_id.clone()))?;
- let stream_id = shard
+ // Get topic metadata to update command fields
+ let metadata = shard.metadata.load();
+ if let Some(topic_meta) = metadata
.streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
+ .get(stream_id_num)
+ .and_then(|s| s.topics.get(topic_id))
+ {
+ self.message_expiry = topic_meta.message_expiry;
+ self.max_topic_size = topic_meta.max_topic_size;
+ }
- let response = shard.streams.with_topic_by_id(
- &self.stream_id,
- &Identifier::numeric(topic_id as u32).unwrap(),
- |(root, _, stats)| mapper::map_topic(&root, &stats),
- );
+ let response = shard.get_topic_from_shared_metadata(stream_id_num, topic_id);
shard
.state
@@ -180,7 +179,7 @@ impl ServerCommandHandler for CreateTopic {
.await
.error(|e: &IggyError| {
format!(
- "{COMPONENT} (error: {e}) - failed to apply create topic for stream_id: {stream_id}, topic_id: {topic_id:?}"
+ "{COMPONENT} (error: {e}) - failed to apply create topic for stream_id: {stream_id_num}, topic_id: {topic_id:?}"
)
})?;
diff --git a/core/server/src/binary/handlers/topics/delete_topic_handler.rs b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
index 85db4d075..72687edfd 100644
--- a/core/server/src/binary/handlers/topics/delete_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/delete_topic_handler.rs
@@ -28,11 +28,9 @@ use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
-use crate::slab::traits_ext::EntityMarker;
+
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::streams;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::delete_topic::DeleteTopic;
use iggy_common::{Identifier, IggyError, SenderKind};
@@ -77,25 +75,38 @@ impl ServerCommandHandler for DeleteTopic {
..
} = payload
{
+ // Capture numeric IDs and partition_ids BEFORE deletion for broadcast.
+ // After deletion, the topic won't exist in metadata.
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&stream_id, &topic_id)?;
+ let partition_ids: Vec<usize> = {
+ let metadata = shard.metadata.load();
+ metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(numeric_topic_id))
+ .map(|t| t.partitions.iter().enumerate().map(|(k, _)| k).collect())
+ .unwrap_or_default()
+ };
+
// Acquire topic lock to serialize filesystem operations
let _topic_guard = shard.fs_locks.topic_lock.lock().await;
- let topic = shard.delete_topic(session, &stream_id, &topic_id).await?;
- let stream_id_num = shard
- .streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
- let topic_id_num = topic.root().id();
+ let topic_info = shard.delete_topic(session, &stream_id, &topic_id).await?;
+ let topic_id_num = topic_info.id;
+ let stream_id_num = topic_info.stream_id;
info!(
"Deleted topic with name: {}, ID: {} in stream with ID: {}",
- topic.root().name(),
- topic_id_num,
- stream_id_num
+ topic_info.name, topic_id_num, stream_id_num
);
- let event = ShardEvent::DeletedTopic {
- id: topic_id_num,
- stream_id: stream_id.clone(),
- topic_id: topic_id.clone(),
+ // Broadcast to all shards to clean up their local_partitions entries.
+ // Use numeric Identifiers since the topic is already deleted from metadata.
+ let event = ShardEvent::DeletedPartitions {
+ stream_id: Identifier::numeric(numeric_stream_id as u32).unwrap(),
+ topic_id: Identifier::numeric(numeric_topic_id as u32).unwrap(),
+ partitions_count: partition_ids.len() as u32,
+ partition_ids,
};
shard.broadcast_event_to_all_shards(event).await?;
@@ -114,14 +125,13 @@ impl ServerCommandHandler for DeleteTopic {
}
}
ShardSendRequestResult::Response(response) => match response {
- ShardResponse::DeleteTopicResponse(topic) => {
+ ShardResponse::DeleteTopicResponse(topic_id_num) => {
shard
.state
.apply(session.get_user_id(), &EntryCommand::DeleteTopic(self))
.await
.error(|e: &IggyError| format!(
- "{COMPONENT} (error: {e}) - failed to apply delete topic with ID: {}, session: {session}",
- topic.id()
+ "{COMPONENT} (error: {e}) - failed to apply delete topic with ID: {topic_id_num}, session: {session}"
))?;
sender.send_empty_ok_response().await?;
diff --git a/core/server/src/binary/handlers/topics/get_topic_handler.rs b/core/server/src/binary/handlers/topics/get_topic_handler.rs
index fcc42ac0b..ca1219dc6 100644
--- a/core/server/src/binary/handlers/topics/get_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/get_topic_handler.rs
@@ -20,11 +20,8 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::streams;
-use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::SenderKind;
use iggy_common::get_topic::GetTopic;
@@ -45,41 +42,30 @@ impl ServerCommandHandler for GetTopic {
) -> Result<HandlerResult, IggyError> {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let exists = shard
- .ensure_topic_exists(&self.stream_id, &self.topic_id)
- .is_ok();
- if !exists {
+
+ let Some(numeric_stream_id) = shard.metadata.get_stream_id(&self.stream_id) else {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
- }
+ };
+
+ let Some(numeric_topic_id) = shard
+ .metadata
+ .get_topic_id(numeric_stream_id, &self.topic_id)
+ else {
+ sender.send_empty_ok_response().await?;
+ return Ok(HandlerResult::Finished);
+ };
- let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
let has_permission = shard
.permissioner
- .borrow()
- .get_topic(
- session.get_user_id(),
- numeric_stream_id,
- self.topic_id
- .get_u32_value()
- .unwrap_or(0)
- .try_into()
- .unwrap(),
- )
+ .get_topic(session.get_user_id(), numeric_stream_id, numeric_topic_id)
.is_ok();
if !has_permission {
sender.send_empty_ok_response().await?;
return Ok(HandlerResult::Finished);
}
- let response =
- shard
- .streams
- .with_topic_by_id(&self.stream_id, &self.topic_id, |(root, _, stats)| {
- mapper::map_topic(&root, &stats)
- });
+ let response = shard.get_topic_from_shared_metadata(numeric_stream_id, numeric_topic_id);
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/topics/get_topics_handler.rs b/core/server/src/binary/handlers/topics/get_topics_handler.rs
index 96c3f9325..b6f2ec766 100644
--- a/core/server/src/binary/handlers/topics/get_topics_handler.rs
+++ b/core/server/src/binary/handlers/topics/get_topics_handler.rs
@@ -20,12 +20,8 @@ use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::utils::receive_and_validate;
-use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
use crate::streaming::session::Session;
-use crate::streaming::streams;
-use anyhow::Result;
use iggy_common::IggyError;
use iggy_common::SenderKind;
use iggy_common::get_topics::GetTopics;
@@ -47,20 +43,17 @@ impl ServerCommandHandler for GetTopics {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
shard.ensure_stream_exists(&self.stream_id)?;
- let numeric_stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
+
+ let Some(numeric_stream_id) = shard.metadata.get_stream_id(&self.stream_id) else {
+ sender.send_empty_ok_response().await?;
+ return Ok(HandlerResult::Finished);
+ };
+
shard
.permissioner
- .borrow()
.get_topics(session.get_user_id(), numeric_stream_id)?;
- let response = shard.streams.with_topics(&self.stream_id, |topics| {
- topics.with_components(|topics| {
- let (roots, _, stats) = topics.into_components();
- mapper::map_topics(&roots, &stats)
- })
- });
+ let response = shard.get_topics_from_shared_metadata(numeric_stream_id);
sender.send_ok_response(&response).await?;
Ok(HandlerResult::Finished)
}
diff --git a/core/server/src/binary/handlers/topics/purge_topic_handler.rs b/core/server/src/binary/handlers/topics/purge_topic_handler.rs
index 012795da6..ef26e5807 100644
--- a/core/server/src/binary/handlers/topics/purge_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/purge_topic_handler.rs
@@ -24,7 +24,6 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::purge_topic::PurgeTopic;
use iggy_common::{IggyError, SenderKind};
diff --git a/core/server/src/binary/handlers/topics/update_topic_handler.rs b/core/server/src/binary/handlers/topics/update_topic_handler.rs
index aa8912fa7..d76beb042 100644
--- a/core/server/src/binary/handlers/topics/update_topic_handler.rs
+++ b/core/server/src/binary/handlers/topics/update_topic_handler.rs
@@ -23,15 +23,12 @@ use crate::binary::handlers::topics::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::update_topic::UpdateTopic;
use iggy_common::{Identifier, IggyError, SenderKind};
@@ -85,6 +82,10 @@ impl ServerCommandHandler for UpdateTopic {
..
} = payload
{
+ // Get numeric IDs BEFORE update (name might change during update)
+ let (stream_id_num, topic_id_num) =
+ shard.resolve_topic_id(&stream_id, &topic_id)?;
+
shard.update_topic(
session,
&stream_id,
@@ -96,38 +97,16 @@ impl ServerCommandHandler for UpdateTopic {
replication_factor,
)?;
- let name_changed = !name.is_empty();
- let lookup_topic_id = if name_changed {
- Identifier::named(&name).unwrap()
- } else {
- topic_id.clone()
- };
-
- self.message_expiry = shard.streams.with_topic_by_id(
- &stream_id,
- &lookup_topic_id,
- topics::helpers::get_message_expiry(),
- );
- self.max_topic_size = shard.streams.with_topic_by_id(
- &stream_id,
- &lookup_topic_id,
- topics::helpers::get_max_topic_size(),
- );
-
- let stream_id_num = shard
+ let metadata = shard.metadata.load();
+ let topic_meta = metadata
.streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
-
- let event = ShardEvent::UpdatedTopic {
- stream_id: stream_id.clone(),
- topic_id: topic_id.clone(),
- name: name.clone(),
- message_expiry: self.message_expiry,
- compression_algorithm: self.compression_algorithm,
- max_topic_size: self.max_topic_size,
- replication_factor: self.replication_factor,
- };
- shard.broadcast_event_to_all_shards(event).await?;
+ .get(stream_id_num)
+ .and_then(|s| s.topics.get(topic_id_num))
+ .ok_or_else(|| {
+ IggyError::TopicIdNotFound(topic_id.clone(), stream_id.clone())
+ })?;
+ self.message_expiry = topic_meta.message_expiry;
+ self.max_topic_size = topic_meta.max_topic_size;
shard
.state
@@ -146,8 +125,9 @@ impl ServerCommandHandler for UpdateTopic {
ShardSendRequestResult::Response(response) => match response {
ShardResponse::UpdateTopicResponse => {
let stream_id = shard
- .streams
- .with_stream_by_id(&self.stream_id, streams::helpers::get_stream_id());
+ .metadata
+ .get_stream_id(&self.stream_id)
+ .unwrap_or_default();
shard
.state
diff --git a/core/server/src/binary/handlers/users/change_password_handler.rs b/core/server/src/binary/handlers/users/change_password_handler.rs
index 9b11969cc..a6dc1cfb7 100644
--- a/core/server/src/binary/handlers/users/change_password_handler.rs
+++ b/core/server/src/binary/handlers/users/change_password_handler.rs
@@ -21,16 +21,17 @@ use crate::binary::command::{
};
use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
use crate::streaming::utils::crypto;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::change_password::ChangePassword;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use std::rc::Rc;
use tracing::info;
use tracing::{debug, instrument};
@@ -51,49 +52,103 @@ impl ServerCommandHandler for ChangePassword {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- info!("Changing password for user with ID: {}...", self.user_id);
- shard
- .change_password(
- session,
- &self.user_id,
- &self.current_password,
- &self.new_password,
- )
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to change password for user_id: {}, session: {session}",
- self.user_id
- )
- })?;
+ let user_id_for_log = self.user_id.clone();
+ let new_password_hash = crypto::hash_password(&self.new_password);
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::ChangePassword {
+ session_user_id: session.get_user_id(),
+ user_id: self.user_id.clone(),
+ current_password: self.current_password.clone(),
+ new_password: self.new_password.clone(),
+ },
+ };
- info!("Changed password for user with ID: {}.", self.user_id);
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::ChangePassword {
+ user_id,
+ current_password,
+ new_password,
+ ..
+ } = payload
+ {
+ info!("Changing password for user with ID: {}...", user_id);
- let event = ShardEvent::ChangedPassword {
- user_id: self.user_id.clone(),
- current_password: self.current_password.clone(),
- new_password: self.new_password.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
+ let _user_guard = shard.fs_locks.user_lock.lock().await;
+ shard
+ .change_password(session, &user_id, ¤t_password, &new_password)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to change password for user_id: {}, session: {session}",
+ user_id
+ )
+ })?;
+
+ info!("Changed password for user with ID: {}.", user_id_for_log);
+
+ shard
+ .state
+ .apply(
+ session.get_user_id(),
+ &EntryCommand::ChangePassword(ChangePassword {
+ user_id: user_id_for_log.clone(),
+ current_password: "".into(),
+ new_password: new_password_hash.clone(),
+ }),
+ )
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply change password for user_id: {}, session: {session}",
+ user_id_for_log
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ } else {
+ unreachable!(
+ "Expected a ChangePassword request inside of ChangePassword handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::ChangePasswordResponse => {
+ info!("Changed password for user with ID: {}.", user_id_for_log);
+
+ shard
+ .state
+ .apply(
+ session.get_user_id(),
+ &EntryCommand::ChangePassword(ChangePassword {
+ user_id: user_id_for_log.clone(),
+ current_password: "".into(),
+ new_password: new_password_hash.clone(),
+ }),
+ )
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply change password for user_id: {}, session: {session}",
+ user_id_for_log
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!(
+ "Expected a ChangePasswordResponse inside of ChangePassword handler, impossible state"
+ ),
+ },
+ }
- // For the security of the system, we hash the password before storing it in metadata.
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::ChangePassword(ChangePassword {
- user_id: self.user_id.to_owned(),
- current_password: "".into(),
- new_password: crypto::hash_password(&self.new_password),
- }),
- )
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply change password for user_id: {}, session: {session}",
- self.user_id
- )
- })?;
- sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/users/create_user_handler.rs b/core/server/src/binary/handlers/users/create_user_handler.rs
index 57501efc0..0288b22ec 100644
--- a/core/server/src/binary/handlers/users/create_user_handler.rs
+++ b/core/server/src/binary/handlers/users/create_user_handler.rs
@@ -23,7 +23,6 @@ use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
@@ -32,7 +31,6 @@ use crate::state::command::EntryCommand;
use crate::state::models::CreateUserWithId;
use crate::streaming::session::Session;
use crate::streaming::utils::crypto;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::create_user::CreateUser;
use iggy_common::{Identifier, IggyError, SenderKind};
@@ -92,16 +90,6 @@ impl ServerCommandHandler for CreateUser {
})?;
let user_id = user.id;
-
- let event = ShardEvent::CreatedUser {
- user_id,
- username: username.clone(),
- password: password.clone(),
- status,
- permissions: permissions.clone(),
- };
- shard.broadcast_event_to_all_shards(event).await?;
-
let response = mapper::map_user(&user);
shard
diff --git a/core/server/src/binary/handlers/users/delete_user_handler.rs b/core/server/src/binary/handlers/users/delete_user_handler.rs
index 0c9151f91..3194ad33d 100644
--- a/core/server/src/binary/handlers/users/delete_user_handler.rs
+++ b/core/server/src/binary/handlers/users/delete_user_handler.rs
@@ -16,26 +16,22 @@
* under the License.
*/
-use std::rc::Rc;
-
use crate::binary::command::{
BinaryServerCommand, HandlerResult, ServerCommand, ServerCommandHandler,
};
use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
use crate::shard::transmission::frame::ShardResponse;
use crate::shard::transmission::message::{
ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::delete_user::DeleteUser;
use iggy_common::{Identifier, IggyError, SenderKind};
+use std::rc::Rc;
use tracing::info;
use tracing::{debug, instrument};
@@ -82,8 +78,6 @@ impl ServerCommandHandler for DeleteUser {
})?;
info!("Deleted user: {} with ID: {}.", user.username, user.id);
- let event = ShardEvent::DeletedUser { user_id };
- shard.broadcast_event_to_all_shards(event).await?;
shard
.state
diff --git a/core/server/src/binary/handlers/users/login_user_handler.rs b/core/server/src/binary/handlers/users/login_user_handler.rs
index 550be02b7..89a981c2a 100644
--- a/core/server/src/binary/handlers/users/login_user_handler.rs
+++ b/core/server/src/binary/handlers/users/login_user_handler.rs
@@ -24,7 +24,6 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::login_user::LoginUser;
use iggy_common::{IggyError, SenderKind};
diff --git a/core/server/src/binary/handlers/users/logout_user_handler.rs b/core/server/src/binary/handlers/users/logout_user_handler.rs
index f09a20b36..39689cfa6 100644
--- a/core/server/src/binary/handlers/users/logout_user_handler.rs
+++ b/core/server/src/binary/handlers/users/logout_user_handler.rs
@@ -26,7 +26,6 @@ use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::logout_user::LogoutUser;
use iggy_common::{IggyError, SenderKind};
diff --git a/core/server/src/binary/handlers/users/update_permissions_handler.rs b/core/server/src/binary/handlers/users/update_permissions_handler.rs
index 5e46b8a9b..fcb19b219 100644
--- a/core/server/src/binary/handlers/users/update_permissions_handler.rs
+++ b/core/server/src/binary/handlers/users/update_permissions_handler.rs
@@ -23,15 +23,16 @@ use crate::binary::command::{
};
use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
-
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::update_permissions::UpdatePermissions;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use tracing::info;
use tracing::{debug, instrument};
@@ -51,26 +52,84 @@ impl ServerCommandHandler for UpdatePermissions {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- shard
- .update_permissions(session, &self.user_id, self.permissions.clone())
- .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to update permissions for user_id: {}, session: {session}",
- self.user_id
- ))?;
- info!("Updated permissions for user with ID: {}.", self.user_id);
- let event = ShardEvent::UpdatedPermissions {
- user_id: self.user_id.clone(),
- permissions: self.permissions.clone(),
+ let user_id_for_log = self.user_id.clone();
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::UpdatePermissions {
+ session_user_id: session.get_user_id(),
+ user_id: self.user_id.clone(),
+ permissions: self.permissions.clone(),
+ },
};
- shard.broadcast_event_to_all_shards(event).await?;
- shard
- .state
- .apply(
- session.get_user_id(),
- &EntryCommand::UpdatePermissions(self),
- )
- .await?;
- sender.send_empty_ok_response().await?;
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::UpdatePermissions {
+ user_id,
+ permissions,
+ ..
+ } = payload
+ {
+ let _user_guard = shard.fs_locks.user_lock.lock().await;
+ shard
+ .update_permissions(session, &user_id, permissions)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to update permissions for user_id: {}, session: {session}",
+ user_id
+ )
+ })?;
+
+ info!("Updated permissions for user with ID: {}.", user_id_for_log);
+
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::UpdatePermissions(self))
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply update permissions for user_id: {}, session: {session}",
+ user_id_for_log
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ } else {
+ unreachable!(
+ "Expected an UpdatePermissions request inside of UpdatePermissions handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::UpdatePermissionsResponse => {
+ info!("Updated permissions for user with ID: {}.", user_id_for_log);
+
+ shard
+ .state
+ .apply(session.get_user_id(), &EntryCommand::UpdatePermissions(self))
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply update permissions for user_id: {}, session: {session}",
+ user_id_for_log
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!(
+ "Expected an UpdatePermissionsResponse inside of UpdatePermissions handler, impossible state"
+ ),
+ },
+ }
+
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/handlers/users/update_user_handler.rs b/core/server/src/binary/handlers/users/update_user_handler.rs
index 433e024af..c6e08a373 100644
--- a/core/server/src/binary/handlers/users/update_user_handler.rs
+++ b/core/server/src/binary/handlers/users/update_user_handler.rs
@@ -25,13 +25,15 @@ use crate::binary::handlers::users::COMPONENT;
use crate::binary::handlers::utils::receive_and_validate;
use crate::shard::IggyShard;
-use crate::shard::transmission::event::ShardEvent;
+use crate::shard::transmission::frame::ShardResponse;
+use crate::shard::transmission::message::{
+ ShardMessage, ShardRequest, ShardRequestPayload, ShardSendRequestResult,
+};
use crate::state::command::EntryCommand;
use crate::streaming::session::Session;
-use anyhow::Result;
use err_trail::ErrContext;
use iggy_common::update_user::UpdateUser;
-use iggy_common::{IggyError, SenderKind};
+use iggy_common::{Identifier, IggyError, SenderKind};
use tracing::info;
use tracing::{debug, instrument};
@@ -51,40 +53,100 @@ impl ServerCommandHandler for UpdateUser {
debug!("session: {session}, command: {self}");
shard.ensure_authenticated(session)?;
- let user =shard
- .update_user(
- session,
- &self.user_id,
- self.username.clone(),
- self.status,
- )
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to update user with user_id: {}, session: {session}",
- self.user_id
- )
- })?;
+ let user_id_for_log = self.user_id.clone();
+ let request = ShardRequest {
+ stream_id: Identifier::default(),
+ topic_id: Identifier::default(),
+ partition_id: 0,
+ payload: ShardRequestPayload::UpdateUser {
+ session_user_id: session.get_user_id(),
+ user_id: self.user_id.clone(),
+ username: self.username.clone(),
+ status: self.status,
+ },
+ };
+
+ let message = ShardMessage::Request(request);
+ match shard.send_request_to_shard_or_recoil(None, message).await? {
+ ShardSendRequestResult::Recoil(message) => {
+ if let ShardMessage::Request(ShardRequest { payload, .. }) = message
+ && let ShardRequestPayload::UpdateUser {
+ user_id,
+ username,
+ status,
+ ..
+ } = payload
+ {
+ let _user_guard = shard.fs_locks.user_lock.lock().await;
+ let user = shard
+ .update_user(session, &user_id, username.clone(), status)
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to update user with user_id: {}, session: {session}",
+ user_id
+ )
+ })?;
- info!("Updated user: {} with ID: {}.", user.username, user.id);
+ info!("Updated user: {} with ID: {}.", user.username, user.id);
- let event = ShardEvent::UpdatedUser {
- user_id: self.user_id.clone(),
- username: self.username.clone(),
- status: self.status,
- };
- shard.broadcast_event_to_all_shards(event).await?;
+ shard
+ .state
+ .apply(
+ session.get_user_id(),
+ &EntryCommand::UpdateUser(UpdateUser {
+ user_id: user_id_for_log.clone(),
+ username,
+ status,
+ }),
+ )
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply update user with user_id: {}, session: {session}",
+ user_id_for_log
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ } else {
+ unreachable!(
+ "Expected an UpdateUser request inside of UpdateUser handler, impossible state"
+ );
+ }
+ }
+ ShardSendRequestResult::Response(response) => match response {
+ ShardResponse::UpdateUserResponse(user) => {
+ info!("Updated user: {} with ID: {}.", user.username, user.id);
+
+ shard
+ .state
+ .apply(
+ session.get_user_id(),
+ &EntryCommand::UpdateUser(UpdateUser {
+ user_id: user_id_for_log.clone(),
+ username: self.username.clone(),
+ status: self.status,
+ }),
+ )
+ .await
+ .error(|e: &IggyError| {
+ format!(
+ "{COMPONENT} (error: {e}) - failed to apply update user with user_id: {}, session: {session}",
+ user_id_for_log
+ )
+ })?;
+
+ sender.send_empty_ok_response().await?;
+ }
+ ShardResponse::ErrorResponse(err) => {
+ return Err(err);
+ }
+ _ => unreachable!(
+ "Expected an UpdateUserResponse inside of UpdateUser handler, impossible state"
+ ),
+ },
+ }
- let user_id = self.user_id.clone();
- shard
- .state
- .apply(session.get_user_id(), &EntryCommand::UpdateUser(self))
- .await
- .error(|e: &IggyError| {
- format!(
- "{COMPONENT} (error: {e}) - failed to apply update user with user_id: {user_id}, session: {session}"
- )
- })?;
- sender.send_empty_ok_response().await?;
Ok(HandlerResult::Finished)
}
}
diff --git a/core/server/src/binary/macros.rs b/core/server/src/binary/macros.rs
index f69632c4c..22e5a6677 100644
--- a/core/server/src/binary/macros.rs
+++ b/core/server/src/binary/macros.rs
@@ -81,7 +81,7 @@ macro_rules! define_server_command_enum {
}
}
- /// Validate the command by delegating to the inner command’s implementation.
+ /// Validate the command by delegating to the inner command's implementation.
pub fn validate(&self) -> Result<(), IggyError> {
match self {
$(
diff --git a/core/server/src/binary/mapper.rs b/core/server/src/binary/mapper.rs
index 6e98081d6..89dac029b 100644
--- a/core/server/src/binary/mapper.rs
+++ b/core/server/src/binary/mapper.rs
@@ -16,23 +16,13 @@
* under the License.
*/
-use std::sync::{Arc, atomic::AtomicU64};
-
-use crate::slab::Keyed;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
+use crate::metadata::ConsumerGroupMeta;
use crate::streaming::clients::client_manager::Client;
-use crate::streaming::partitions::partition::PartitionRoot;
-use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats};
-use crate::streaming::streams::stream;
-use crate::streaming::topics::consumer_group::{ConsumerGroupMembers, ConsumerGroupRoot, Member};
-use crate::streaming::topics::topic::{self, TopicRoot};
use crate::streaming::users::user::User;
-use arcshift::SharedGetGuard;
use bytes::{BufMut, Bytes, BytesMut};
use iggy_common::{
BytesSerializable, ConsumerOffsetInfo, PersonalAccessToken, Stats, TransportProtocol, UserId,
};
-use slab::Slab;
pub fn map_stats(stats: &Stats) -> Bytes {
let mut bytes = BytesMut::with_capacity(104);
@@ -153,145 +143,29 @@ pub fn map_personal_access_tokens(personal_access_tokens: Vec<PersonalAccessToke
bytes.freeze()
}
-pub fn map_streams(roots: &Slab<stream::StreamRoot>, stats: &Slab<Arc<StreamStats>>) -> Bytes {
+/// Map consumer group from SharedMetadata format.
+pub fn map_consumer_group_from_meta(meta: &ConsumerGroupMeta) -> Bytes {
let mut bytes = BytesMut::new();
- for (root, stat) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(stats.iter().map(|(_, val)| val))
- {
- extend_stream(root, stat, &mut bytes);
- }
- bytes.freeze()
-}
-pub fn map_stream(root: &stream::StreamRoot, stats: &StreamStats) -> Bytes {
- let mut bytes = BytesMut::new();
- extend_stream(root, stats, &mut bytes);
- root.topics().with_components(|topics| {
- let (roots, _, stats, ..) = topics.into_components();
- for (root, stat) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(stats.iter().map(|(_, val)| val))
- {
- extend_topic(root, stat, &mut bytes);
- }
- });
- bytes.freeze()
-}
+ // Header: id, partitions_count, members_count, name_len, name
+ bytes.put_u32_le(meta.id as u32);
+ bytes.put_u32_le(meta.partitions.len() as u32);
+ bytes.put_u32_le(meta.members.len() as u32);
+ bytes.put_u8(meta.name.len() as u8);
+ bytes.put_slice(meta.name.as_bytes());
-pub fn map_topics(roots: &Slab<TopicRoot>, stats: &Slab<Arc<TopicStats>>) -> Bytes {
- let mut bytes = BytesMut::new();
- for (root, stat) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(stats.iter().map(|(_, val)| val))
- {
- extend_topic(root, stat, &mut bytes);
- }
- bytes.freeze()
-}
-
-pub fn map_topic(root: &topic::TopicRoot, stats: &TopicStats) -> Bytes {
- let mut bytes = BytesMut::new();
- extend_topic(root, stats, &mut bytes);
- root.partitions().with_components(|partitions| {
- let (roots, stats, _, offsets, _, _, _) = partitions.into_components();
- for (root, stat, offset) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(stats.iter().map(|(_, val)| val))
- .zip(offsets.iter().map(|(_, val)| val))
- .map(|((root, stat), offset)| (root, stat, offset))
- {
- extend_partition(root, stat, offset, &mut bytes);
- }
- });
-
- bytes.freeze()
-}
-
-pub fn map_consumer_group(root: &ConsumerGroupRoot, members: &ConsumerGroupMembers) -> Bytes {
- let mut bytes = BytesMut::new();
- let members = members.inner().shared_get();
- extend_consumer_group(root, &members, &mut bytes);
-
- for (_, member) in members.iter() {
+ // Members
+ for (_, member) in meta.members.iter() {
bytes.put_u32_le(member.id as u32);
bytes.put_u32_le(member.partitions.len() as u32);
- for partition in &member.partitions {
- bytes.put_u32_le(*partition as u32);
+ for &partition_id in &member.partitions {
+ bytes.put_u32_le(partition_id as u32);
}
}
- bytes.freeze()
-}
-pub fn map_consumer_groups(
- roots: &Slab<ConsumerGroupRoot>,
- members: &Slab<ConsumerGroupMembers>,
-) -> Bytes {
- let mut bytes = BytesMut::new();
- for (root, member) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(members.iter().map(|(_, val)| val.inner().shared_get()))
- {
- extend_consumer_group(root, &member, &mut bytes);
- }
bytes.freeze()
}
-fn extend_stream(root: &stream::StreamRoot, stats: &StreamStats, bytes: &mut BytesMut) {
- bytes.put_u32_le(root.id() as u32);
- bytes.put_u64_le(root.created_at().into());
- bytes.put_u32_le(root.topics_count() as u32);
- bytes.put_u64_le(stats.size_bytes_inconsistent());
- bytes.put_u64_le(stats.messages_count_inconsistent());
- bytes.put_u8(root.name().len() as u8);
- bytes.put_slice(root.name().as_bytes());
-}
-
-fn extend_topic(root: &TopicRoot, stats: &TopicStats, bytes: &mut BytesMut) {
- bytes.put_u32_le(root.id() as u32);
- bytes.put_u64_le(root.created_at().into());
- bytes.put_u32_le(root.partitions().len() as u32);
- bytes.put_u64_le(root.message_expiry().into());
- bytes.put_u8(root.compression_algorithm().as_code());
- bytes.put_u64_le(root.max_topic_size().into());
- bytes.put_u8(root.replication_factor());
- bytes.put_u64_le(stats.size_bytes_inconsistent());
- bytes.put_u64_le(stats.messages_count_inconsistent());
- bytes.put_u8(root.name().len() as u8);
- bytes.put_slice(root.name().as_bytes());
-}
-
-fn extend_partition(
- root: &PartitionRoot,
- stats: &PartitionStats,
- offset: &Arc<AtomicU64>,
- bytes: &mut BytesMut,
-) {
- bytes.put_u32_le(root.id() as u32);
- bytes.put_u64_le(root.created_at().into());
- bytes.put_u32_le(stats.segments_count_inconsistent());
- bytes.put_u64_le(offset.load(std::sync::atomic::Ordering::Relaxed));
- bytes.put_u64_le(stats.size_bytes_inconsistent());
- bytes.put_u64_le(stats.messages_count_inconsistent());
-}
-
-fn extend_consumer_group(
- root: &ConsumerGroupRoot,
- members: &SharedGetGuard<'_, Slab<Member>>,
- bytes: &mut BytesMut,
-) {
- bytes.put_u32_le(root.id() as u32);
- bytes.put_u32_le(root.partitions().len() as u32);
- bytes.put_u32_le(members.len() as u32);
- bytes.put_u8(root.key().len() as u8);
- bytes.put_slice(root.key().as_bytes());
-}
-
fn extend_client(client: &Client, bytes: &mut BytesMut) {
bytes.put_u32_le(client.session.client_id);
bytes.put_u32_le(client.user_id.unwrap_or(u32::MAX));
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index 13690a813..a665b63f9 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -27,6 +27,7 @@ use crate::{
system::{INDEX_EXTENSION, LOG_EXTENSION, SystemConfig},
},
io::fs_utils::{self, DirEntry},
+ metadata::{ConsumerGroupMeta, MetadataWriter, PartitionMeta, StreamMeta, TopicMeta, UserMeta},
server_error::ServerError,
shard::{
system::info::SystemInfo,
@@ -35,33 +36,18 @@ use crate::{
frame::ShardFrame,
},
},
- slab::{
- consumer_groups::ConsumerGroups, partitions::Partitions, streams::Streams, topics::Topics,
- traits_ext::IntoComponents, users::Users,
- },
- state::system::{StreamState, UserState},
+ state::system::{StreamState, TopicState, UserState},
streaming::{
- partitions::{
- consumer_offset::ConsumerOffset,
- helpers::create_message_deduplicator,
- journal::MemoryMessageJournal,
- log::SegmentedLog,
- partition,
- storage::{load_consumer_group_offsets, load_consumer_offsets},
- },
+ partitions::{journal::MemoryMessageJournal, log::SegmentedLog},
persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind},
- polling_consumer::ConsumerGroupId,
segments::{Segment, storage::Storage},
stats::{PartitionStats, StreamStats, TopicStats},
storage::SystemStorage,
- streams::stream,
- topics::{consumer_group, topic},
users::user::User,
utils::{crypto, file::overwrite},
},
versioning::SemanticVersion,
};
-use ahash::HashMap;
use compio::{fs::create_dir_all, runtime::Runtime};
use err_trail::ErrContext;
use iggy_common::{
@@ -71,157 +57,10 @@ use iggy_common::{
MIN_USERNAME_LENGTH,
},
};
+use slab::Slab;
use std::{env, path::Path, sync::Arc};
use tracing::{info, warn};
-pub async fn load_streams(
- state: impl IntoIterator<Item = StreamState>,
- config: &SystemConfig,
-) -> Result<Streams, IggyError> {
- let state: Vec<StreamState> = state.into_iter().collect();
- let mut stream_entries = Vec::with_capacity(state.len());
-
- for stream_state in state {
- let stream_id = stream_state.id as usize;
- info!(
- "Loading stream with ID: {}, name: {} from state...",
- stream_state.id, stream_state.name
- );
-
- let stream_stats = Arc::new(StreamStats::default());
- let mut topic_entries = Vec::new();
-
- for topic_state in stream_state.topics.into_values() {
- let topic_id = topic_state.id as usize;
- info!(
- "Loading topic with ID: {}, name: {} from state...",
- topic_state.id, topic_state.name
- );
-
- let topic_stats = Arc::new(TopicStats::new(stream_stats.clone()));
-
- // Build partitions
- let mut partition_entries = Vec::new();
- for partition_state in topic_state.partitions.into_values() {
- let partition_id = partition_state.id as usize;
- info!(
- "Loading partition with ID: {}, for topic with ID: {} from state...",
- partition_id, topic_id
- );
-
- let partition = load_partition(
- config,
- stream_id,
- topic_id,
- partition_state,
- topic_stats.clone(),
- )
- .await?;
- partition_entries.push((partition_id, partition));
-
- info!(
- "Loaded partition with ID: {}, for topic with ID: {} from state...",
- partition_id, topic_id
- );
- }
-
- // Build consumer groups
- let partition_ids: Vec<_> = partition_entries.iter().map(|(id, _)| *id).collect();
- let cg_entries: Vec<_> = topic_state
- .consumer_groups
- .into_values()
- .map(|cg_state| {
- info!(
- "Loading consumer group with ID: {}, name: {} for topic with ID: {} from state...",
- cg_state.id, cg_state.name, topic_id
- );
- let cg = consumer_group::ConsumerGroup::new(
- cg_state.name.clone(),
- Default::default(),
- partition_ids.clone(),
- );
- info!(
- "Loaded consumer group with ID: {}, name: {} for topic with ID: {} from state...",
- cg_state.id, cg_state.name, topic_id
- );
- (cg_state.id as usize, cg)
- })
- .collect();
-
- // Build topic with pre-built partitions and consumer groups
- let mut topic = topic::Topic::new(
- topic_state.name.clone(),
- topic_stats,
- topic_state.created_at,
- topic_state.replication_factor.unwrap_or(1),
- topic_state.message_expiry,
- topic_state.compression_algorithm,
- topic_state.max_topic_size,
- );
-
- // Decompose, set nested containers, recompose
- let (mut root, auxilary, stats) = topic.into_components();
- root.set_partitions(Partitions::from_entries(partition_entries));
- root.set_consumer_groups(ConsumerGroups::from_entries(cg_entries));
- topic = topic::Topic::new_with_components(root, auxilary, stats);
-
- topic_entries.push((topic_id, topic));
- info!(
- "Loaded topic with ID: {}, name: {} from state...",
- topic_state.id, topic_state.name
- );
- }
-
- // Build stream with pre-built topics
- let mut stream = stream::Stream::new(
- stream_state.name.clone(),
- stream_stats,
- stream_state.created_at,
- );
-
- // Decompose, set nested containers, recompose
- let (mut root, stats) = stream.into_components();
- root.set_topics(Topics::from_entries(topic_entries));
- stream = stream::Stream::new_with_components(root, stats);
-
- stream_entries.push((stream_id, stream));
- info!(
- "Loaded stream with ID: {}, name: {} from state...",
- stream_state.id, stream_state.name
- );
- }
-
- Ok(Streams::from_entries(stream_entries))
-}
-
-pub fn load_users(state: impl IntoIterator<Item = UserState>) -> Users {
- let users = Users::new();
- for user_state in state {
- let UserState {
- id,
- username,
- password_hash,
- status,
- created_at,
- permissions,
- personal_access_tokens,
- } = user_state;
- let mut user = User::with_password(id, &username, password_hash, status, permissions);
- user.created_at = created_at;
- user.personal_access_tokens = personal_access_tokens
- .into_values()
- .map(|token| {
- (
- Arc::from(token.token_hash.as_str()),
- PersonalAccessToken::raw(id, &token.name, &token.token_hash, token.expiry_at),
- )
- })
- .collect();
- users.insert(user);
- }
- users
-}
-
pub fn create_shard_connections(
shard_assignment: &[ShardInfo],
) -> (Vec<ShardConnector<ShardFrame>>, Vec<(u16, StopSender)>) {
@@ -620,86 +459,210 @@ pub async fn load_segments(
Ok(log)
}
-async fn load_partition(
- config: &SystemConfig,
- stream_id: usize,
- topic_id: usize,
- partition_state: crate::state::system::PartitionState,
- parent_stats: Arc<TopicStats>,
-) -> Result<partition::Partition, IggyError> {
- let stats = Arc::new(PartitionStats::new(parent_stats));
- let partition_id = partition_state.id;
-
- let partition_path = config.get_partition_path(stream_id, topic_id, partition_id as usize);
- let log_files = collect_log_files(&partition_path).await?;
- let should_increment_offset = !log_files.is_empty()
- && log_files
- .first()
- .map(|entry| {
- let log_file_name = entry
- .path
- .file_stem()
- .unwrap()
- .to_string_lossy()
- .to_string();
-
- let start_offset = log_file_name.parse::<u64>().unwrap();
-
- let messages_file_path = config.get_messages_file_path(
- stream_id,
- topic_id,
- partition_id as usize,
- start_offset,
- );
- let metadata = std::fs::metadata(&messages_file_path)
- .expect("failed to get metadata for first segment in log");
- metadata.len() > 0
- })
- .unwrap_or_else(|| false);
+/// Builds `InnerMetadata` from persisted user and stream state.
+/// Uses `from_entries` to construct the slabs directly.
+pub fn build_inner_metadata(
+ users_state: impl IntoIterator<Item = UserState>,
+ streams_state: impl IntoIterator<Item = StreamState>,
+) -> crate::metadata::InnerMetadata {
+ use crate::metadata::InnerMetadata;
+ use std::sync::atomic::AtomicUsize;
+
+ let mut user_entries = Vec::new();
+ let mut user_index = ahash::AHashMap::default();
+ let mut personal_access_tokens: ahash::AHashMap<
+ u32,
+ ahash::AHashMap<Arc<str>, PersonalAccessToken>,
+ > = ahash::AHashMap::default();
+ let mut users_count = 0;
+ let mut pats_count = 0;
+
+ for UserState {
+ id,
+ username,
+ password_hash,
+ status,
+ created_at,
+ permissions,
+ personal_access_tokens: user_pats,
+ } in users_state
+ {
+ let username_arc: Arc<str> = Arc::from(username.as_str());
+ let user_meta = UserMeta {
+ id,
+ username: username_arc.clone(),
+ password_hash: Arc::from(password_hash.as_str()),
+ status,
+ permissions: permissions.map(Arc::new),
+ created_at,
+ };
+ user_entries.push((id as usize, user_meta));
+ user_index.insert(username_arc, id);
+
+ if !user_pats.is_empty() {
+ let user_pat_map: ahash::AHashMap<Arc<str>, PersonalAccessToken> = user_pats
+ .into_values()
+ .map(|token| {
+ let pat = PersonalAccessToken::raw(
+ id,
+ &token.name,
+ &token.token_hash,
+ token.expiry_at,
+ );
+ (Arc::from(token.token_hash.as_str()), pat)
+ })
+ .collect();
+ pats_count += user_pat_map.len();
+ personal_access_tokens.insert(id, user_pat_map);
+ }
+ users_count += 1;
+ }
info!(
- "Loading partition with ID: {} for stream with ID: {} and topic with ID: {}, for path: {} from disk...",
- partition_id, stream_id, topic_id, partition_path
+ "Building metadata: {} users, {} personal access tokens",
+ users_count, pats_count
);
- // Load consumer offsets
- let message_deduplicator = create_message_deduplicator(config);
- let consumer_offset_path =
- config.get_consumer_offsets_path(stream_id, topic_id, partition_id as usize);
- let consumer_group_offsets_path =
- config.get_consumer_group_offsets_path(stream_id, topic_id, partition_id as usize);
-
- let consumer_offset = Arc::new(
- load_consumer_offsets(&consumer_offset_path)?
- .into_iter()
- .map(|offset| (offset.consumer_id as usize, offset))
- .collect::<HashMap<usize, ConsumerOffset>>()
- .into(),
- );
+ let mut stream_entries = Vec::new();
+ let mut stream_index = ahash::AHashMap::default();
+ let mut streams_count = 0;
+ let mut topics_count = 0;
+ let mut partitions_count = 0;
+ let mut consumer_groups_count = 0;
+
+ for StreamState {
+ name,
+ created_at,
+ id,
+ topics,
+ } in streams_state
+ {
+ info!(
+ "Building stream with ID: {}, name: {} metadata...",
+ id, name
+ );
+ let stream_id = id as usize;
+ let stream_name: Arc<str> = Arc::from(name.as_str());
- let consumer_group_offset = Arc::new(
- load_consumer_group_offsets(&consumer_group_offsets_path)?
- .into_iter()
- .collect::<HashMap<ConsumerGroupId, ConsumerOffset>>()
- .into(),
- );
+ let stream_stats = Arc::new(StreamStats::default());
- let log = Default::default();
- let partition = partition::Partition::new(
- partition_state.created_at,
- should_increment_offset,
- stats,
- message_deduplicator,
- Arc::new(Default::default()),
- consumer_offset,
- consumer_group_offset,
- log,
- );
+ let mut topic_entries = Vec::new();
+ let mut topic_index = ahash::AHashMap::default();
+
+ for TopicState {
+ id,
+ name,
+ created_at,
+ compression_algorithm,
+ message_expiry,
+ max_topic_size,
+ replication_factor,
+ consumer_groups,
+ partitions,
+ } in topics.into_values()
+ {
+ info!("Building topic with ID: {}, name: {} metadata...", id, name);
+ let topic_id = id as usize;
+ let topic_name: Arc<str> = Arc::from(name.as_str());
+
+ let topic_stats = Arc::new(TopicStats::new(stream_stats.clone()));
+
+ let mut partition_entries = Vec::new();
+ let mut partition_ids = Vec::new();
+
+ for partition_state in partitions.into_values() {
+ let partition_id = partition_state.id as usize;
+ partition_ids.push(partition_id);
+
+ let partition_stats = Arc::new(PartitionStats::new(topic_stats.clone()));
+ let partition_meta = PartitionMeta {
+ id: partition_id,
+ created_at: partition_state.created_at,
+ revision_id: 0,
+ stats: partition_stats,
+ consumer_offsets: None,
+ consumer_group_offsets: None,
+ };
+ partition_entries.push((partition_id, partition_meta));
+ partitions_count += 1;
+ }
+
+ partition_ids.sort_unstable();
+
+ let mut cg_entries = Vec::new();
+ let mut cg_index = ahash::AHashMap::default();
+
+ for cg_state in consumer_groups.into_values() {
+ info!(
+ "Building consumer group with ID: {}, name: {} for topic with ID: {} metadata...",
+ cg_state.id, cg_state.name, topic_id
+ );
+ let group_id = cg_state.id as usize;
+ let group_name: Arc<str> = Arc::from(cg_state.name.as_str());
+ let cg_meta = ConsumerGroupMeta {
+ id: group_id,
+ name: group_name.clone(),
+ partitions: partition_ids.clone(),
+ members: Slab::new(),
+ };
+ cg_entries.push((group_id, cg_meta));
+ cg_index.insert(group_name, group_id);
+ consumer_groups_count += 1;
+ }
+
+ let topic_meta = TopicMeta {
+ id: topic_id,
+ name: topic_name.clone(),
+ created_at,
+ message_expiry,
+ compression_algorithm,
+ max_topic_size,
+ replication_factor: replication_factor.unwrap_or(1),
+ stats: topic_stats,
+ partitions: partition_entries.into_iter().map(|(_, p)| p).collect(),
+ consumer_groups: cg_entries.into_iter().collect(),
+ consumer_group_index: cg_index,
+ round_robin_counter: Arc::new(AtomicUsize::new(0)),
+ };
+ topic_entries.push((topic_id, topic_meta));
+ topic_index.insert(topic_name, topic_id);
+ topics_count += 1;
+ }
+
+ let stream_meta = StreamMeta {
+ id: stream_id,
+ name: stream_name.clone(),
+ created_at,
+ stats: stream_stats,
+ topics: topic_entries.into_iter().collect(),
+ topic_index,
+ };
+ stream_entries.push((stream_id, stream_meta));
+ stream_index.insert(stream_name, stream_id);
+ streams_count += 1;
+ }
info!(
- "Loaded partition with ID: {} for stream with ID: {} and topic with ID: {}",
- partition_id, stream_id, topic_id
+ "Built metadata: {} streams, {} topics, {} partitions, {} consumer groups",
+ streams_count, topics_count, partitions_count, consumer_groups_count
);
- Ok(partition)
+ InnerMetadata {
+ streams: stream_entries.into_iter().collect(),
+ users: user_entries.into_iter().collect(),
+ stream_index,
+ user_index,
+ personal_access_tokens,
+ }
+}
+
+/// Loads all metadata from persisted state into metadata writer.
+/// Single atomic initialization via `Initialize` operation.
+pub fn load_metadata(
+ users_state: impl IntoIterator<Item = UserState>,
+ streams_state: impl IntoIterator<Item = StreamState>,
+ writer: &mut MetadataWriter,
+) {
+ let inner = build_inner_metadata(users_state, streams_state);
+ writer.initialize(inner);
}
diff --git a/core/server/src/configs/system.rs b/core/server/src/configs/system.rs
index c1b089a79..e3e39f317 100644
--- a/core/server/src/configs/system.rs
+++ b/core/server/src/configs/system.rs
@@ -16,13 +16,9 @@
* under the License.
*/
-use crate::configs::server::MemoryPoolConfig;
-use crate::slab::partitions;
-use crate::slab::streams;
-use crate::slab::topics;
-
use super::cache_indexes::CacheIndexesConfig;
use super::sharding::ShardingConfig;
+use crate::configs::server::MemoryPoolConfig;
use iggy_common::IggyByteSize;
use iggy_common::IggyError;
use iggy_common::IggyExpiry;
@@ -218,9 +214,9 @@ impl SystemConfig {
pub fn get_partition_path(
&self,
- stream_id: streams::ContainerId,
- topic_id: topics::ContainerId,
- partition_id: partitions::ContainerId,
+ stream_id: usize,
+ topic_id: usize,
+ partition_id: usize,
) -> String {
format!(
"{}/{}",
diff --git a/core/server/src/http/consumer_groups.rs b/core/server/src/http/consumer_groups.rs
index 6f66de799..bcd0b182c 100644
--- a/core/server/src/http/consumer_groups.rs
+++ b/core/server/src/http/consumer_groups.rs
@@ -21,7 +21,6 @@ use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::mapper;
use crate::http::shared::AppState;
-use crate::slab::traits_ext::{EntityComponentSystem, EntityMarker, IntoComponents};
use crate::state::command::EntryCommand;
use crate::state::models::CreateConsumerGroupWithId;
use crate::streaming::polling_consumer::ConsumerGroupId;
@@ -64,43 +63,38 @@ async fn get_consumer_group(
let identifier_group_id = Identifier::from_str_value(&group_id)?;
let session = Session::stateless(identity.user_id, identity.ip_address);
-
- let exists = state
- .shard
- .shard()
- .ensure_consumer_group_exists(
- &identifier_stream_id,
- &identifier_topic_id,
- &identifier_group_id,
- )
- .is_ok();
- if !exists {
- return Err(CustomError::ResourceNotFound);
- }
-
- let numeric_topic_id = state.shard.shard().streams.with_topic_by_id(
- &identifier_stream_id,
- &identifier_topic_id,
- crate::streaming::topics::helpers::get_topic_id(),
- );
- let numeric_stream_id = state.shard.shard().streams.with_stream_by_id(
- &identifier_stream_id,
- crate::streaming::streams::helpers::get_stream_id(),
- );
-
- state
- .shard
- .shard()
- .permissioner
- .borrow()
- .get_consumer_group(session.get_user_id(), numeric_stream_id, numeric_topic_id)?;
-
- let consumer_group = state.shard.shard().streams.with_consumer_group_by_id(
- &identifier_stream_id,
- &identifier_topic_id,
- &identifier_group_id,
- |(root, members)| mapper::map_consumer_group(root, members),
- );
+ let shard = state.shard.shard();
+
+ let numeric_stream_id = shard
+ .metadata
+ .get_stream_id(&identifier_stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let numeric_topic_id = shard
+ .metadata
+ .get_topic_id(numeric_stream_id, &identifier_topic_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let numeric_group_id = shard
+ .metadata
+ .get_consumer_group_id(numeric_stream_id, numeric_topic_id, &identifier_group_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ shard.permissioner.get_consumer_group(
+ session.get_user_id(),
+ numeric_stream_id,
+ numeric_topic_id,
+ )?;
+
+ let metadata = shard.metadata.load();
+ let cg_meta = metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(numeric_topic_id))
+ .and_then(|t| t.consumer_groups.get(numeric_group_id))
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let consumer_group = mapper::map_consumer_group_details_from_metadata(cg_meta);
Ok(Json(consumer_group))
}
@@ -114,39 +108,31 @@ async fn get_consumer_groups(
let identifier_topic_id = Identifier::from_str_value(&topic_id)?;
let session = Session::stateless(identity.user_id, identity.ip_address);
-
- state
- .shard
- .shard()
- .ensure_topic_exists(&identifier_stream_id, &identifier_topic_id)?;
-
- let numeric_topic_id = state.shard.shard().streams.with_topic_by_id(
- &identifier_stream_id,
- &identifier_topic_id,
- crate::streaming::topics::helpers::get_topic_id(),
- );
- let numeric_stream_id = state.shard.shard().streams.with_stream_by_id(
- &identifier_stream_id,
- crate::streaming::streams::helpers::get_stream_id(),
- );
-
- state
- .shard
- .shard()
- .permissioner
- .borrow()
- .get_consumer_groups(session.get_user_id(), numeric_stream_id, numeric_topic_id)?;
-
- let consumer_groups = state.shard.shard().streams.with_consumer_groups(
- &identifier_stream_id,
- &identifier_topic_id,
- |cgs| {
- cgs.with_components(|cgs| {
- let (roots, members) = cgs.into_components();
- mapper::map_consumer_groups(roots, members)
- })
- },
- );
+ let shard = state.shard.shard();
+
+ let numeric_stream_id = shard
+ .metadata
+ .get_stream_id(&identifier_stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let numeric_topic_id = shard
+ .metadata
+ .get_topic_id(numeric_stream_id, &identifier_topic_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ shard.permissioner.get_consumer_groups(
+ session.get_user_id(),
+ numeric_stream_id,
+ numeric_topic_id,
+ )?;
+
+ let metadata = shard.metadata.load();
+ let topic_meta = metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(numeric_topic_id))
+ .ok_or(CustomError::ResourceNotFound)?;
+ let consumer_groups = mapper::map_consumer_groups_from_metadata(topic_meta);
Ok(Json(consumer_groups))
}
@@ -165,8 +151,7 @@ async fn create_consumer_group(
let session = Session::stateless(identity.user_id, identity.ip_address);
- // Create consumer group using the new API
- let consumer_group = state.shard.shard().create_consumer_group(
+ let group_id = state.shard.create_consumer_group(
&session,
&command.stream_id,
&command.topic_id,
@@ -174,36 +159,20 @@ async fn create_consumer_group(
)
.error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to create consumer group, stream ID: {}, topic ID: {}, name: {}", stream_id, topic_id, command.name))?;
- let group_id = consumer_group.id();
-
- // Send event for consumer group creation
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::CreatedConsumerGroup {
- stream_id: command.stream_id.clone(),
- topic_id: command.topic_id.clone(),
- cg: consumer_group.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
- // Get the created consumer group details
- let group_id_identifier = Identifier::numeric(group_id as u32).unwrap();
- let consumer_group_details = state.shard.shard().streams.with_consumer_group_by_id(
- &command.stream_id,
- &command.topic_id,
- &group_id_identifier,
- |(root, members)| mapper::map_consumer_group(root, members),
- );
+ let shard = state.shard.shard();
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&command.stream_id, &command.topic_id)?;
+ let consumer_group_details = {
+ let metadata = shard.metadata.load();
+ let cg_meta = metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(numeric_topic_id))
+ .and_then(|t| t.consumer_groups.get(group_id))
+ .expect("Consumer group must exist after creation");
+ mapper::map_consumer_group_details_from_metadata(cg_meta)
+ };
- // Apply state change
let entry_command = EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId {
group_id: group_id as u32,
command,
@@ -235,8 +204,11 @@ async fn delete_consumer_group(
let result = SendWrapper::new(async move {
let session = Session::stateless(identity.user_id, identity.ip_address);
- // Delete using the new API
- let consumer_group = state.shard.shard().delete_consumer_group(
+ let shard = state.shard.shard();
+ let (numeric_stream_id, numeric_topic_id) =
+ shard.resolve_topic_id(&identifier_stream_id, &identifier_topic_id)?;
+
+ let cg_meta = shard.delete_consumer_group(
&session,
&identifier_stream_id,
&identifier_topic_id,
@@ -244,27 +216,13 @@ async fn delete_consumer_group(
)
.error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to delete consumer group with ID: {group_id} for topic with ID: {topic_id} in stream with ID: {stream_id}"))?;
- let cg_id = consumer_group.id();
+ let cg_id = cg_meta.id;
- // Remove all consumer group members from ClientManager
- let stream_id_usize = state.shard.shard().streams.with_stream_by_id(
- &identifier_stream_id,
- crate::streaming::streams::helpers::get_stream_id(),
- );
- let topic_id_usize = state.shard.shard().streams.with_topic_by_id(
- &identifier_stream_id,
- &identifier_topic_id,
- crate::streaming::topics::helpers::get_topic_id(),
- );
-
- // TODO: Tech debt, repeated code from `delete_consumer_group_handler.rs`
- // Get members from the deleted consumer group and make them leave
- let slab = consumer_group.members().inner().shared_get();
- for (_, member) in slab.iter() {
- if let Err(err) = state.shard.shard().client_manager.leave_consumer_group(
+ for (_, member) in cg_meta.members.iter() {
+ if let Err(err) = shard.client_manager.leave_consumer_group(
member.client_id,
- stream_id_usize,
- topic_id_usize,
+ numeric_stream_id,
+ numeric_topic_id,
cg_id,
) {
tracing::warn!(
@@ -276,13 +234,11 @@ async fn delete_consumer_group(
}
let cg_id_spez = ConsumerGroupId(cg_id);
- // Clean up consumer group offsets from all partitions using the specialized method
- let partition_ids = consumer_group.partitions();
state.shard.shard().delete_consumer_group_offsets(
cg_id_spez,
&identifier_stream_id,
&identifier_topic_id,
- partition_ids,
+ &cg_meta.partitions,
).await.error(|e: &IggyError| {
format!(
"{COMPONENT} (error: {e}) - failed to delete consumer group offsets for group ID: {} in stream: {}, topic: {}",
@@ -292,26 +248,6 @@ async fn delete_consumer_group(
)
})?;
- // Send event for consumer group deletion
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::DeletedConsumerGroup {
- id: cg_id,
- stream_id: identifier_stream_id.clone(),
- topic_id: identifier_topic_id.clone(),
- group_id: identifier_group_id.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
- // Apply state change
let entry_command = EntryCommand::DeleteConsumerGroup(DeleteConsumerGroup {
stream_id: identifier_stream_id,
topic_id: identifier_topic_id,
diff --git a/core/server/src/http/consumer_offsets.rs b/core/server/src/http/consumer_offsets.rs
index e7656a554..dff4150fd 100644
--- a/core/server/src/http/consumer_offsets.rs
+++ b/core/server/src/http/consumer_offsets.rs
@@ -118,6 +118,8 @@ async fn delete_consumer_offset(
Path((stream_id, topic_id, consumer_id)): Path<(String, String, String)>,
query: Query<DeleteConsumerOffset>,
) -> Result<StatusCode, CustomError> {
+ let stream_id_ident = Identifier::from_str_value(&stream_id)?;
+ let topic_id_ident = Identifier::from_str_value(&topic_id)?;
let consumer = Consumer::new(consumer_id.try_into()?);
let session = SendWrapper::new(Session::stateless(identity.user_id, identity.ip_address));
state
@@ -125,8 +127,8 @@ async fn delete_consumer_offset(
.delete_consumer_offset(
&session,
consumer,
- &query.stream_id,
- &query.topic_id,
+ &stream_id_ident,
+ &topic_id_ident,
query.partition_id,
)
.await
diff --git a/core/server/src/http/http_shard_wrapper.rs b/core/server/src/http/http_shard_wrapper.rs
index 7bcfdb65b..b48d8ac1a 100644
--- a/core/server/src/http/http_shard_wrapper.rs
+++ b/core/server/src/http/http_shard_wrapper.rs
@@ -149,11 +149,7 @@ impl HttpSafeShard {
future.await
}
- pub async fn create_stream(
- &self,
- session: &Session,
- name: String,
- ) -> Result<crate::streaming::streams::stream::Stream, IggyError> {
+ pub async fn create_stream(&self, session: &Session, name: String) -> Result<usize, IggyError> {
let future = SendWrapper::new(self.shard().create_stream(session, name));
future.await
}
@@ -302,31 +298,41 @@ impl HttpSafeShard {
) -> Result<(), IggyError> {
self.shard().ensure_topic_exists(&stream_id, &topic_id)?;
- let partition_id = self.shard().streams.with_topic_by_id(
- &stream_id,
- &topic_id,
- |(root, auxilary, ..)| match partitioning.kind {
- PartitioningKind::Balanced => {
- let upperbound = root.partitions().len();
- let pid = auxilary.get_next_partition_id(upperbound);
- Ok(pid)
- }
- PartitioningKind::PartitionId => Ok(u32::from_le_bytes(
- partitioning.value[..partitioning.length as usize]
- .try_into()
- .map_err(|_| IggyError::InvalidNumberEncoding)?,
- ) as usize),
- PartitioningKind::MessagesKey => {
- let upperbound = root.partitions().len();
- Ok(
- topics::helpers::calculate_partition_id_by_messages_key_hash(
- upperbound,
- &partitioning.value,
- ),
- )
- }
- },
- )?;
+ let numeric_stream_id = self
+ .shard()
+ .metadata
+ .get_stream_id(&stream_id)
+ .expect("Stream existence already verified");
+ let numeric_topic_id = self
+ .shard()
+ .metadata
+ .get_topic_id(numeric_stream_id, &topic_id)
+ .expect("Topic existence already verified");
+ let partition_id = match partitioning.kind {
+ PartitioningKind::Balanced => self
+ .shard()
+ .metadata
+ .get_next_partition_id(numeric_stream_id, numeric_topic_id)
+ .ok_or(IggyError::TopicIdNotFound(
+ stream_id.clone(),
+ topic_id.clone(),
+ ))?,
+ PartitioningKind::PartitionId => u32::from_le_bytes(
+ partitioning.value[..partitioning.length as usize]
+ .try_into()
+ .map_err(|_| IggyError::InvalidNumberEncoding)?,
+ ) as usize,
+ PartitioningKind::MessagesKey => {
+ let partitions_count = self
+ .shard()
+ .metadata
+ .partitions_count(numeric_stream_id, numeric_topic_id);
+ topics::helpers::calculate_partition_id_by_messages_key_hash(
+ partitions_count,
+ &partitioning.value,
+ )
+ }
+ };
let future = SendWrapper::new(self.shard().append_messages(
user_id,
@@ -337,4 +343,39 @@ impl HttpSafeShard {
));
future.await
}
+
+ pub fn create_consumer_group(
+ &self,
+ session: &Session,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ name: String,
+ ) -> Result<usize, IggyError> {
+ self.shard()
+ .create_consumer_group(session, stream_id, topic_id, name)
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ pub fn update_topic(
+ &self,
+ session: &Session,
+ stream_id: &Identifier,
+ topic_id: &Identifier,
+ name: String,
+ message_expiry: iggy_common::IggyExpiry,
+ compression_algorithm: iggy_common::CompressionAlgorithm,
+ max_topic_size: iggy_common::MaxTopicSize,
+ replication_factor: Option<u8>,
+ ) -> Result<(), IggyError> {
+ self.shard().update_topic(
+ session,
+ stream_id,
+ topic_id,
+ name,
+ message_expiry,
+ compression_algorithm,
+ max_topic_size,
+ replication_factor,
+ )
+ }
}
diff --git a/core/server/src/http/mapper.rs b/core/server/src/http/mapper.rs
index d7ec542f2..879d98338 100644
--- a/core/server/src/http/mapper.rs
+++ b/core/server/src/http/mapper.rs
@@ -17,95 +17,13 @@
*/
use crate::http::jwt::json_web_token::GeneratedToken;
-use crate::slab::Keyed;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
+use crate::metadata::{ConsumerGroupMeta, InnerMetadata, PartitionMeta, StreamMeta, TopicMeta};
use crate::streaming::clients::client_manager::Client;
-use crate::streaming::stats::TopicStats;
-use crate::streaming::topics::consumer_group::{ConsumerGroupMembers, ConsumerGroupRoot};
-use crate::streaming::topics::topic::TopicRoot;
use crate::streaming::users::user::User;
use iggy_common::PersonalAccessToken;
use iggy_common::{ConsumerGroupDetails, ConsumerGroupInfo, ConsumerGroupMember, IggyByteSize};
use iggy_common::{IdentityInfo, PersonalAccessTokenInfo, TokenInfo, TopicDetails};
use iggy_common::{UserInfo, UserInfoDetails};
-use slab::Slab;
-use std::sync::Arc;
-
-/// Map TopicRoot with partitions to TopicDetails for HTTP responses
-pub fn map_topic_details(root: &TopicRoot, stats: &TopicStats) -> TopicDetails {
- let mut partitions = Vec::new();
-
- // Get partition details similar to binary mapper
- root.partitions().with_components(|partition_components| {
- let (partition_roots, partition_stats, _, offsets, _, _, _) =
- partition_components.into_components();
- for (partition_root, partition_stat, offset) in partition_roots
- .iter()
- .map(|(_, val)| val)
- .zip(partition_stats.iter().map(|(_, val)| val))
- .zip(offsets.iter().map(|(_, val)| val))
- .map(|((root, stat), offset)| (root, stat, offset))
- {
- partitions.push(iggy_common::Partition {
- id: partition_root.id() as u32,
- created_at: partition_root.created_at(),
- segments_count: partition_stat.segments_count_inconsistent(),
- current_offset: offset.load(std::sync::atomic::Ordering::Relaxed),
- size: IggyByteSize::from(partition_stat.size_bytes_inconsistent()),
- messages_count: partition_stat.messages_count_inconsistent(),
- });
- }
- });
-
- // Sort partitions by ID
- partitions.sort_by(|a, b| a.id.cmp(&b.id));
-
- TopicDetails {
- id: root.id() as u32,
- created_at: root.created_at(),
- name: root.name().clone(),
- size: stats.size_bytes_inconsistent().into(),
- messages_count: stats.messages_count_inconsistent(),
- partitions_count: partitions.len() as u32,
- partitions,
- message_expiry: root.message_expiry(),
- compression_algorithm: root.compression_algorithm(),
- max_topic_size: root.max_topic_size(),
- replication_factor: root.replication_factor(),
- }
-}
-
-/// Map TopicRoot and TopicStats to Topic for HTTP responses
-pub fn map_topic(root: &TopicRoot, stats: &TopicStats) -> iggy_common::Topic {
- iggy_common::Topic {
- id: root.id() as u32,
- created_at: root.created_at(),
- name: root.name().clone(),
- size: stats.size_bytes_inconsistent().into(),
- partitions_count: root.partitions().len() as u32,
- messages_count: stats.messages_count_inconsistent(),
- message_expiry: root.message_expiry(),
- compression_algorithm: root.compression_algorithm(),
- max_topic_size: root.max_topic_size(),
- replication_factor: root.replication_factor(),
- }
-}
-
-/// Map multiple topics from slab components to Vec<Topic> for HTTP responses
-pub fn map_topics_from_components(
- roots: &Slab<TopicRoot>,
- stats: &Slab<Arc<TopicStats>>,
-) -> Vec<iggy_common::Topic> {
- let mut topics = roots
- .iter()
- .map(|(_, root)| root)
- .zip(stats.iter().map(|(_, stat)| stat))
- .map(|(root, stat)| map_topic(root, stat))
- .collect::<Vec<_>>();
-
- topics.sort_by(|a, b| a.id.cmp(&b.id));
- topics
-}
pub fn map_user(user: &User) -> UserInfoDetails {
UserInfoDetails {
@@ -183,52 +101,6 @@ pub fn map_clients(clients: &[Client]) -> Vec<iggy_common::ClientInfo> {
all_clients
}
-pub fn map_consumer_groups(
- roots: &slab::Slab<ConsumerGroupRoot>,
- members: &slab::Slab<ConsumerGroupMembers>,
-) -> Vec<iggy_common::ConsumerGroup> {
- let mut groups = Vec::new();
- for (root, member) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(members.iter().map(|(_, val)| val))
- {
- let members_guard = member.inner().shared_get();
- let consumer_group = iggy_common::ConsumerGroup {
- id: root.id() as u32,
- name: root.key().clone(),
- partitions_count: root.partitions().len() as u32,
- members_count: members_guard.len() as u32,
- };
- groups.push(consumer_group);
- }
- groups.sort_by(|a, b| a.id.cmp(&b.id));
- groups
-}
-
-pub fn map_consumer_group(
- root: &ConsumerGroupRoot,
- members: &ConsumerGroupMembers,
-) -> ConsumerGroupDetails {
- let members_guard = members.inner().shared_get();
- let mut consumer_group_details = ConsumerGroupDetails {
- id: root.id() as u32,
- name: root.key().clone(),
- partitions_count: root.partitions().len() as u32,
- members_count: members_guard.len() as u32,
- members: Vec::new(),
- };
-
- for (_, member) in members_guard.iter() {
- consumer_group_details.members.push(ConsumerGroupMember {
- id: member.id as u32,
- partitions_count: member.partitions.len() as u32,
- partitions: member.partitions.iter().map(|p| *p as u32).collect(),
- });
- }
- consumer_group_details
-}
-
pub fn map_generated_access_token_to_identity_info(token: GeneratedToken) -> IdentityInfo {
IdentityInfo {
user_id: token.user_id,
@@ -239,69 +111,220 @@ pub fn map_generated_access_token_to_identity_info(token: GeneratedToken) -> Ide
}
}
-/// Map StreamRoot and StreamStats to StreamDetails for HTTP responses
-pub fn map_stream_details(
- root: &crate::streaming::streams::stream::StreamRoot,
- stats: &crate::streaming::stats::StreamStats,
-) -> iggy_common::StreamDetails {
- // Get topics using the new slab-based API
- let topics = root.topics().with_components(|topic_ref| {
- let (topic_roots, _topic_auxiliaries, topic_stats) = topic_ref.into_components();
- let mut topics_vec = Vec::new();
-
- // Iterate over topics in the stream
- for (topic_root, topic_stat) in topic_roots
- .iter()
- .map(|(_, root)| root)
- .zip(topic_stats.iter().map(|(_, stat)| stat))
- {
- topics_vec.push(map_topic(topic_root, topic_stat));
+/// Map a stream from SharedMetadata to StreamDetails (with topics)
+pub fn map_stream_details_from_metadata(stream_meta: &StreamMeta) -> iggy_common::StreamDetails {
+ // Get topic IDs sorted
+ let mut topic_ids: Vec<_> = stream_meta.topics.iter().map(|(k, _)| k).collect();
+ topic_ids.sort_unstable();
+
+ // Map topics
+ let mut topics = Vec::with_capacity(topic_ids.len());
+ for topic_id in topic_ids {
+ if let Some(topic_meta) = stream_meta.topics.get(topic_id) {
+ topics.push(map_topic_from_metadata(topic_meta));
}
+ }
- // Sort topics by ID for consistent ordering
- topics_vec.sort_by(|a, b| a.id.cmp(&b.id));
- topics_vec
- });
+ // Aggregate stats
+ let (total_size, total_messages) = aggregate_stream_stats(stream_meta);
iggy_common::StreamDetails {
- id: root.id() as u32,
- created_at: root.created_at(),
- name: root.name().clone(),
- topics_count: root.topics_count() as u32,
- size: stats.size_bytes_inconsistent().into(),
- messages_count: stats.messages_count_inconsistent(),
+ id: stream_meta.id as u32,
+ created_at: stream_meta.created_at,
+ name: stream_meta.name.to_string(),
+ topics_count: topics.len() as u32,
+ size: IggyByteSize::from(total_size),
+ messages_count: total_messages,
topics,
}
}
-/// Map StreamRoot and StreamStats to Stream for HTTP responses
-pub fn map_stream(
- root: &crate::streaming::streams::stream::StreamRoot,
- stats: &crate::streaming::stats::StreamStats,
-) -> iggy_common::Stream {
+/// Map a stream from SharedMetadata to Stream (without topics)
+pub fn map_stream_from_metadata(stream_meta: &StreamMeta) -> iggy_common::Stream {
+ let (total_size, total_messages) = aggregate_stream_stats(stream_meta);
+
iggy_common::Stream {
- id: root.id() as u32,
- created_at: root.created_at(),
- name: root.name().clone(),
- topics_count: root.topics_count() as u32,
- size: stats.size_bytes_inconsistent().into(),
- messages_count: stats.messages_count_inconsistent(),
+ id: stream_meta.id as u32,
+ created_at: stream_meta.created_at,
+ name: stream_meta.name.to_string(),
+ topics_count: stream_meta.topics.len() as u32,
+ size: IggyByteSize::from(total_size),
+ messages_count: total_messages,
}
}
-/// Map multiple streams from slabs
-pub fn map_streams_from_slabs(
- roots: &slab::Slab<crate::streaming::streams::stream::StreamRoot>,
- stats: &slab::Slab<Arc<crate::streaming::stats::StreamStats>>,
-) -> Vec<iggy_common::Stream> {
- let mut streams = Vec::new();
- for (root, stat) in roots
- .iter()
- .map(|(_, val)| val)
- .zip(stats.iter().map(|(_, val)| val))
- {
- streams.push(map_stream(root, stat));
+/// Map all streams from SharedMetadata
+pub fn map_streams_from_metadata(metadata: &InnerMetadata) -> Vec<iggy_common::Stream> {
+ let mut stream_ids: Vec<_> = metadata.streams.iter().map(|(k, _)| k).collect();
+ stream_ids.sort_unstable();
+
+ let mut streams = Vec::with_capacity(stream_ids.len());
+ for stream_id in stream_ids {
+ if let Some(stream_meta) = metadata.streams.get(stream_id) {
+ streams.push(map_stream_from_metadata(stream_meta));
+ }
}
- streams.sort_by(|a, b| a.id.cmp(&b.id));
streams
}
+
+/// Map a topic from SharedMetadata to Topic (without partitions)
+pub fn map_topic_from_metadata(topic_meta: &TopicMeta) -> iggy_common::Topic {
+ let (total_size, total_messages) = aggregate_topic_stats(topic_meta);
+
+ iggy_common::Topic {
+ id: topic_meta.id as u32,
+ created_at: topic_meta.created_at,
+ name: topic_meta.name.to_string(),
+ size: IggyByteSize::from(total_size),
+ partitions_count: topic_meta.partitions.len() as u32,
+ messages_count: total_messages,
+ message_expiry: topic_meta.message_expiry,
+ compression_algorithm: topic_meta.compression_algorithm,
+ max_topic_size: topic_meta.max_topic_size,
+ replication_factor: topic_meta.replication_factor,
+ }
+}
+
+/// Map all topics for a stream from SharedMetadata
+pub fn map_topics_from_metadata(stream_meta: &StreamMeta) -> Vec<iggy_common::Topic> {
+ let mut topic_ids: Vec<_> = stream_meta.topics.iter().map(|(k, _)| k).collect();
+ topic_ids.sort_unstable();
+
+ let mut topics = Vec::with_capacity(topic_ids.len());
+ for topic_id in topic_ids {
+ if let Some(topic_meta) = stream_meta.topics.get(topic_id) {
+ topics.push(map_topic_from_metadata(topic_meta));
+ }
+ }
+ topics
+}
+
+/// Map a topic from SharedMetadata to TopicDetails (with partitions)
+pub fn map_topic_details_from_metadata(topic_meta: &TopicMeta) -> TopicDetails {
+ // Get partition IDs sorted
+ let mut partition_ids: Vec<_> = topic_meta
+ .partitions
+ .iter()
+ .enumerate()
+ .map(|(k, _)| k)
+ .collect();
+ partition_ids.sort_unstable();
+
+ // Map partitions
+ let mut partitions = Vec::with_capacity(partition_ids.len());
+ for partition_id in partition_ids {
+ if let Some(partition_meta) = topic_meta.partitions.get(partition_id) {
+ partitions.push(map_partition_from_metadata(partition_meta));
+ }
+ }
+
+ // Aggregate stats
+ let (total_size, total_messages) = aggregate_topic_stats(topic_meta);
+
+ TopicDetails {
+ id: topic_meta.id as u32,
+ created_at: topic_meta.created_at,
+ name: topic_meta.name.to_string(),
+ size: IggyByteSize::from(total_size),
+ messages_count: total_messages,
+ partitions_count: partitions.len() as u32,
+ partitions,
+ message_expiry: topic_meta.message_expiry,
+ compression_algorithm: topic_meta.compression_algorithm,
+ max_topic_size: topic_meta.max_topic_size,
+ replication_factor: topic_meta.replication_factor,
+ }
+}
+
+/// Map a partition from SharedMetadata
+pub fn map_partition_from_metadata(partition_meta: &PartitionMeta) -> iggy_common::Partition {
+ let stats = &partition_meta.stats;
+ let segments_count = stats.segments_count_inconsistent();
+ let size_bytes = stats.size_bytes_inconsistent();
+ let messages_count = stats.messages_count_inconsistent();
+ let current_offset = stats.current_offset();
+
+ iggy_common::Partition {
+ id: partition_meta.id as u32,
+ created_at: partition_meta.created_at,
+ segments_count,
+ current_offset,
+ size: IggyByteSize::from(size_bytes),
+ messages_count,
+ }
+}
+
+/// Map a consumer group from SharedMetadata
+pub fn map_consumer_group_from_metadata(cg_meta: &ConsumerGroupMeta) -> iggy_common::ConsumerGroup {
+ iggy_common::ConsumerGroup {
+ id: cg_meta.id as u32,
+ name: cg_meta.name.to_string(),
+ partitions_count: cg_meta.partitions.len() as u32,
+ members_count: cg_meta.members.len() as u32,
+ }
+}
+
+/// Map a consumer group to ConsumerGroupDetails from SharedMetadata
+pub fn map_consumer_group_details_from_metadata(
+ cg_meta: &ConsumerGroupMeta,
+) -> ConsumerGroupDetails {
+ let members: Vec<ConsumerGroupMember> = cg_meta
+ .members
+ .iter()
+ .map(|(_, member)| ConsumerGroupMember {
+ id: member.id as u32,
+ partitions_count: member.partitions.len() as u32,
+ partitions: member.partitions.iter().map(|&p| p as u32).collect(),
+ })
+ .collect();
+
+ ConsumerGroupDetails {
+ id: cg_meta.id as u32,
+ name: cg_meta.name.to_string(),
+ partitions_count: cg_meta.partitions.len() as u32,
+ members_count: members.len() as u32,
+ members,
+ }
+}
+
+/// Map all consumer groups for a topic from SharedMetadata
+pub fn map_consumer_groups_from_metadata(
+ topic_meta: &TopicMeta,
+) -> Vec<iggy_common::ConsumerGroup> {
+ let mut group_ids: Vec<_> = topic_meta.consumer_groups.iter().map(|(k, _)| k).collect();
+ group_ids.sort_unstable();
+
+ let mut groups = Vec::with_capacity(group_ids.len());
+ for group_id in group_ids {
+ if let Some(cg_meta) = topic_meta.consumer_groups.get(group_id) {
+ groups.push(map_consumer_group_from_metadata(cg_meta));
+ }
+ }
+ groups
+}
+
+fn aggregate_stream_stats(stream_meta: &StreamMeta) -> (u64, u64) {
+ let mut total_size = 0u64;
+ let mut total_messages = 0u64;
+
+ for (_, topic_meta) in stream_meta.topics.iter() {
+ for partition_meta in topic_meta.partitions.iter() {
+ total_size += partition_meta.stats.size_bytes_inconsistent();
+ total_messages += partition_meta.stats.messages_count_inconsistent();
+ }
+ }
+
+ (total_size, total_messages)
+}
+
+fn aggregate_topic_stats(topic_meta: &TopicMeta) -> (u64, u64) {
+ let mut total_size = 0u64;
+ let mut total_messages = 0u64;
+
+ for partition_meta in topic_meta.partitions.iter() {
+ total_size += partition_meta.stats.size_bytes_inconsistent();
+ total_messages += partition_meta.stats.messages_count_inconsistent();
+ }
+
+ (total_size, total_messages)
+}
diff --git a/core/server/src/http/partitions.rs b/core/server/src/http/partitions.rs
index 959e1fd2c..10f3f27a4 100644
--- a/core/server/src/http/partitions.rs
+++ b/core/server/src/http/partitions.rs
@@ -60,7 +60,7 @@ async fn create_partitions(
let _parititon_guard = state.shard.shard().fs_locks.partition_lock.lock().await;
let session = Session::stateless(identity.user_id, identity.ip_address);
- let partitions = SendWrapper::new(state.shard.shard().create_partitions(
+ let partition_infos = SendWrapper::new(state.shard.shard().create_partitions(
&session,
&command.stream_id,
&command.topic_id,
@@ -74,7 +74,7 @@ async fn create_partitions(
let event = ShardEvent::CreatedPartitions {
stream_id: command.stream_id.clone(),
topic_id: command.topic_id.clone(),
- partitions,
+ partitions: partition_infos,
};
let _responses = shard.broadcast_event_to_all_shards(event).await;
Ok::<(), CustomError>(())
diff --git a/core/server/src/http/streams.rs b/core/server/src/http/streams.rs
index d96fd4e1e..a4a6a9f80 100644
--- a/core/server/src/http/streams.rs
+++ b/core/server/src/http/streams.rs
@@ -20,7 +20,6 @@ use crate::http::COMPONENT;
use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::shared::AppState;
-use crate::slab::traits_ext::{EntityComponentSystem, IntoComponents};
use crate::streaming::session::Session;
use axum::extract::{Path, State};
use axum::http::StatusCode;
@@ -59,23 +58,21 @@ async fn get_stream(
Path(stream_id): Path<String>,
) -> Result<Json<StreamDetails>, CustomError> {
let stream_id = Identifier::from_str_value(&stream_id)?;
- let exists = state.shard.shard().ensure_stream_exists(&stream_id).is_ok();
- if !exists {
- return Err(CustomError::ResourceNotFound);
- }
let _session = Session::stateless(identity.user_id, identity.ip_address);
+ let shard = state.shard.shard();
+ let numeric_stream_id = shard
+ .metadata
+ .get_stream_id(&stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
- // Use direct slab access for thread-safe stream retrieval
- let stream_details = SendWrapper::new(|| {
- state
- .shard
- .shard()
- .streams
- .with_stream_by_id(&stream_id, |(root, stats)| {
- crate::http::mapper::map_stream_details(&root, &stats)
- })
- })();
+ let metadata = shard.metadata.load();
+ let stream_meta = metadata
+ .streams
+ .get(numeric_stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let stream_details = crate::http::mapper::map_stream_details_from_metadata(stream_meta);
Ok(Json(stream_details))
}
@@ -86,14 +83,9 @@ async fn get_streams(
Extension(identity): Extension<Identity>,
) -> Result<Json<Vec<Stream>>, CustomError> {
let _session = Session::stateless(identity.user_id, identity.ip_address);
-
- // Use direct slab access for thread-safe streams retrieval
- let streams = SendWrapper::new(|| {
- state.shard.shard().streams.with_components(|stream_ref| {
- let (roots, stats) = stream_ref.into_components();
- crate::http::mapper::map_streams_from_slabs(&roots, &stats)
- })
- })();
+ let shard = state.shard.shard();
+ let metadata = shard.metadata.load();
+ let streams = crate::http::mapper::map_streams_from_metadata(&metadata);
Ok(Json(streams))
}
@@ -112,7 +104,7 @@ async fn create_stream(
let _stream_guard = state.shard.shard().fs_locks.stream_lock.lock().await;
// Create stream using wrapper method
- let stream = state
+ let created_stream_id = state
.shard
.create_stream(&session, command.name.clone())
.await
@@ -123,23 +115,6 @@ async fn create_stream(
)
})?;
- let created_stream_id = stream.root().id();
-
- // Send event for stream creation - inlined from wrapper
- {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::CreatedStream {
- id: created_stream_id,
- stream,
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- }
-
- // Apply state change using wrapper method
let entry_command = EntryCommand::CreateStream(CreateStreamWithId {
stream_id: created_stream_id as u32,
command,
@@ -156,16 +131,13 @@ async fn create_stream(
)
})?;
- // Get the created stream details using direct slab access
- let response = SendWrapper::new(|| {
- state
- .shard
- .shard()
- .streams
- .with_components_by_id(created_stream_id, |(root, stats)| {
- crate::http::mapper::map_stream_details(&root, &stats)
- })
- })();
+ let shard = state.shard.shard();
+ let metadata = shard.metadata.load();
+ let stream_meta = metadata
+ .streams
+ .get(created_stream_id)
+ .expect("Stream must exist after creation");
+ let response = crate::http::mapper::map_stream_details_from_metadata(stream_meta);
Ok::<Json<StreamDetails>, CustomError>(Json(response))
});
@@ -197,24 +169,6 @@ async fn update_stream(
)
})?;
- // Send event for stream update
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::UpdatedStream {
- stream_id: command.stream_id.clone(),
- name: command.name.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
- // Apply state change using wrapper method
let entry_command = EntryCommand::UpdateStream(command);
state.shard.apply_state(identity.user_id, &entry_command).await.error(|e: &IggyError| {
format!(
@@ -241,8 +195,8 @@ async fn delete_stream(
let session = Session::stateless(identity.user_id, identity.ip_address);
let _stream_guard = state.shard.shard().fs_locks.stream_lock.lock().await;
- // Delete stream and get the stream entity
- let stream = {
+ // Delete stream
+ {
let future = SendWrapper::new(
state
.shard
@@ -255,26 +209,6 @@ async fn delete_stream(
format!("{COMPONENT} (error: {e}) - failed to delete stream with ID: {stream_id}",)
})?;
- let stream_id_numeric = stream.root().id();
-
- // Send event for stream deletion
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::DeletedStream {
- id: stream_id_numeric,
- stream_id: identifier_stream_id.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
- // Apply state change using wrapper method
let entry_command = EntryCommand::DeleteStream(DeleteStream {
stream_id: identifier_stream_id,
});
@@ -327,7 +261,6 @@ async fn purge_stream(
broadcast_future.await;
}
- // Apply state change using wrapper method
let entry_command = EntryCommand::PurgeStream(PurgeStream {
stream_id: identifier_stream_id,
});
diff --git a/core/server/src/http/topics.rs b/core/server/src/http/topics.rs
index cc686e69f..e7ab03827 100644
--- a/core/server/src/http/topics.rs
+++ b/core/server/src/http/topics.rs
@@ -20,11 +20,9 @@ use crate::http::COMPONENT;
use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::shared::AppState;
-use crate::slab::traits_ext::{EntityComponentSystem, EntityMarker, IntoComponents};
use crate::state::command::EntryCommand;
use crate::state::models::CreateTopicWithId;
use crate::streaming::session::Session;
-use crate::streaming::{streams, topics};
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::routing::{delete, get};
@@ -68,39 +66,20 @@ async fn get_topic(
let identity_topic_id = Identifier::from_str_value(&topic_id)?;
let session = Session::stateless(identity.user_id, identity.ip_address);
+ let shard = state.shard.shard();
- let stream_exists = state
- .shard
- .shard()
- .ensure_stream_exists(&identity_stream_id)
- .is_ok();
- if !stream_exists {
- return Err(CustomError::ResourceNotFound);
- }
+ let numeric_stream_id = shard
+ .metadata
+ .get_stream_id(&identity_stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
- let topic_exists = state
- .shard
- .shard()
- .ensure_topic_exists(&identity_stream_id, &identity_topic_id)
- .is_ok();
- if !topic_exists {
- return Err(CustomError::ResourceNotFound);
- }
+ let numeric_topic_id = shard
+ .metadata
+ .get_topic_id(numeric_stream_id, &identity_topic_id)
+ .ok_or(CustomError::ResourceNotFound)?;
- let numeric_stream_id = state
- .shard
- .shard()
- .streams
- .with_stream_by_id(&identity_stream_id, streams::helpers::get_stream_id());
- let numeric_topic_id = state.shard.shard().streams.with_topic_by_id(
- &identity_stream_id,
- &identity_topic_id,
- topics::helpers::get_topic_id(),
- );
-
- state.shard.shard()
+ shard
.permissioner
- .borrow()
.get_topic(session.get_user_id(), numeric_stream_id, numeric_topic_id)
.error(|e: &IggyError| {
format!(
@@ -109,12 +88,14 @@ async fn get_topic(
)
})?;
- // Get topic details using the new API
- let topic_details = state.shard.shard().streams.with_topic_by_id(
- &identity_stream_id,
- &identity_topic_id,
- |(root, _, stats)| crate::http::mapper::map_topic_details(&root, &stats),
- );
+ let metadata = shard.metadata.load();
+ let topic_meta = metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(numeric_topic_id))
+ .ok_or(CustomError::ResourceNotFound)?;
+
+ let topic_details = crate::http::mapper::map_topic_details_from_metadata(topic_meta);
Ok(Json(topic_details))
}
@@ -125,20 +106,17 @@ async fn get_topics(
Extension(identity): Extension<Identity>,
Path(stream_id): Path<String>,
) -> Result<Json<Vec<Topic>>, CustomError> {
- let stream_id = Identifier::from_str_value(&stream_id)?;
+ let stream_id_ident = Identifier::from_str_value(&stream_id)?;
let session = Session::stateless(identity.user_id, identity.ip_address);
+ let shard = state.shard.shard();
- state.shard.shard().ensure_stream_exists(&stream_id)?;
-
- let numeric_stream_id = state
- .shard
- .shard()
- .streams
- .with_stream_by_id(&stream_id, streams::helpers::get_stream_id());
+ let numeric_stream_id = shard
+ .metadata
+ .get_stream_id(&stream_id_ident)
+ .ok_or(CustomError::ResourceNotFound)?;
- state.shard.shard()
+ shard
.permissioner
- .borrow()
.get_topics(session.get_user_id(), numeric_stream_id)
.error(|e: &IggyError| {
format!(
@@ -147,17 +125,12 @@ async fn get_topics(
)
})?;
- // Get topics using the new API
- let topics = state
- .shard
- .shard()
+ let metadata = shard.metadata.load();
+ let stream_meta = metadata
.streams
- .with_topics(&stream_id, |topics| {
- topics.with_components(|topics| {
- let (roots, _, stats) = topics.into_components();
- crate::http::mapper::map_topics_from_components(&roots, &stats)
- })
- });
+ .get(numeric_stream_id)
+ .ok_or(CustomError::ResourceNotFound)?;
+ let topics = crate::http::mapper::map_topics_from_metadata(stream_meta);
Ok(Json(topics))
}
@@ -176,7 +149,7 @@ async fn create_topic(
let session = SendWrapper::new(Session::stateless(identity.user_id, identity.ip_address));
let _topic_guard = state.shard.shard().fs_locks.topic_lock.lock().await;
- let topic = {
+ let topic_id = {
let future = SendWrapper::new(state.shard.shard().create_topic(
&session,
&command.stream_id,
@@ -192,26 +165,30 @@ async fn create_topic(
format!("{COMPONENT} (error: {e}) - failed to create topic, stream ID: {stream_id}")
})?;
- // Update command with actual values from created topic
- command.message_expiry = topic.root().message_expiry();
- command.max_topic_size = topic.root().max_topic_size();
-
- let topic_id = topic.id();
+ {
+ let shard = state.shard.shard();
+ let numeric_stream_id = shard
+ .metadata
+ .get_stream_id(&command.stream_id)
+ .expect("Stream must exist");
+ let metadata = shard.metadata.load();
+ if let Some(topic_meta) = metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(topic_id))
+ {
+ command.message_expiry = topic_meta.message_expiry;
+ command.max_topic_size = topic_meta.max_topic_size;
+ }
+ }
- // Send events for topic creation
let broadcast_future = SendWrapper::new(async {
use crate::shard::transmission::event::ShardEvent;
let shard = state.shard.shard();
- let event = ShardEvent::CreatedTopic {
- stream_id: command.stream_id.clone(),
- topic,
- };
- let _responses = shard.broadcast_event_to_all_shards(event).await;
-
// Create partitions
- let partitions = shard
+ let partition_infos = shard
.create_partitions(
&session,
&command.stream_id,
@@ -223,7 +200,7 @@ async fn create_topic(
let event = ShardEvent::CreatedPartitions {
stream_id: command.stream_id.clone(),
topic_id: Identifier::numeric(topic_id as u32).unwrap(),
- partitions,
+ partitions: partition_infos,
};
let _responses = shard.broadcast_event_to_all_shards(event).await;
@@ -236,18 +213,22 @@ async fn create_topic(
)
})?;
- // Create response using the same approach as binary handler
let response = {
- let topic_identifier = Identifier::numeric(topic_id as u32).unwrap();
- let topic_response = state.shard.shard().streams.with_topic_by_id(
- &command.stream_id,
- &topic_identifier,
- |(root, _, stats)| crate::http::mapper::map_topic_details(&root, &stats),
- );
+ let shard = state.shard.shard();
+ let numeric_stream_id = shard
+ .metadata
+ .get_stream_id(&command.stream_id)
+ .expect("Stream must exist");
+ let metadata = shard.metadata.load();
+ let topic_meta = metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(topic_id))
+ .expect("Topic must exist after creation");
+ let topic_response = crate::http::mapper::map_topic_details_from_metadata(topic_meta);
Json(topic_response)
};
- // Apply state change like in binary handler
{
let entry_command = EntryCommand::CreateTopic(CreateTopicWithId {
topic_id: topic_id as u32,
@@ -284,7 +265,7 @@ async fn update_topic(
let session = Session::stateless(identity.user_id, identity.ip_address);
let name_changed = !command.name.is_empty();
- state.shard.shard().update_topic(
+ state.shard.update_topic(
&session,
&command.stream_id,
&command.topic_id,
@@ -299,41 +280,32 @@ async fn update_topic(
)
})?;
- // TODO: Tech debt.
- let topic_id = if name_changed {
+ let shard = state.shard.shard();
+ let numeric_stream_id = shard
+ .metadata
+ .get_stream_id(&command.stream_id)
+ .expect("Stream must exist");
+
+ let topic_id_for_lookup = if name_changed {
Identifier::named(&command.name.clone()).unwrap()
} else {
command.topic_id.clone()
};
- // Get the updated values from the topic
- let (message_expiry, max_topic_size) = state.shard.shard().streams.with_topic_by_id(
- &command.stream_id,
- &topic_id,
- |(root, _, _)| (root.message_expiry(), root.max_topic_size()),
- );
-
- // Send event for topic update
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::UpdatedTopic {
- stream_id: command.stream_id.clone(),
- topic_id: command.topic_id.clone(),
- name: command.name.clone(),
- message_expiry: command.message_expiry,
- compression_algorithm: command.compression_algorithm,
- max_topic_size: command.max_topic_size,
- replication_factor: command.replication_factor,
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
+ let numeric_topic_id = shard
+ .metadata
+ .get_topic_id(numeric_stream_id, &topic_id_for_lookup)
+ .expect("Topic must exist after update");
+
+ let (message_expiry, max_topic_size) = {
+ let metadata = shard.metadata.load();
+ let topic_meta = metadata
+ .streams
+ .get(numeric_stream_id)
+ .and_then(|s| s.topics.get(numeric_topic_id))
+ .expect("Topic metadata must exist");
+ (topic_meta.message_expiry, topic_meta.max_topic_size)
+ };
command.message_expiry = message_expiry;
command.max_topic_size = max_topic_size;
@@ -365,7 +337,7 @@ async fn delete_topic(
let session = Session::stateless(identity.user_id, identity.ip_address);
let _topic_guard = state.shard.shard().fs_locks.topic_lock.lock().await;
- let topic = {
+ {
let future = SendWrapper::new(state.shard.shard().delete_topic(
&session,
&identifier_stream_id,
@@ -378,26 +350,6 @@ async fn delete_topic(
)
})?;
- let topic_id_numeric = topic.root().id();
-
- // Send event for topic deletion
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::DeletedTopic {
- id: topic_id_numeric,
- stream_id: identifier_stream_id.clone(),
- topic_id: identifier_topic_id.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let entry_command = EntryCommand::DeleteTopic(DeleteTopic {
stream_id: identifier_stream_id,
@@ -443,7 +395,6 @@ async fn purge_topic(
)
})?;
- // Send event for topic purge
{
let broadcast_future = SendWrapper::new(async {
use crate::shard::transmission::event::ShardEvent;
diff --git a/core/server/src/http/users.rs b/core/server/src/http/users.rs
index 29ae19230..47110dfb9 100644
--- a/core/server/src/http/users.rs
+++ b/core/server/src/http/users.rs
@@ -137,26 +137,6 @@ async fn create_user(
let user_id = user.id;
let response = Json(mapper::map_user(&user));
- // Send event for user creation
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::CreatedUser {
- user_id,
- username: command.username.to_owned(),
- password: command.password.to_owned(),
- status: command.status,
- permissions: command.permissions.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let username = command.username.clone();
let entry_command = EntryCommand::CreateUser(CreateUserWithId {
@@ -212,24 +192,6 @@ async fn update_user(
format!("{COMPONENT} (error: {e}) - failed to update user, user ID: {user_id}")
})?;
- // Send event for user update
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::UpdatedUser {
- user_id: command.user_id.clone(),
- username: command.username.clone(),
- status: command.status,
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let username = command.username.clone();
let entry_command = EntryCommand::UpdateUser(command);
@@ -271,23 +233,6 @@ async fn update_permissions(
format!("{COMPONENT} (error: {e}) - failed to update permissions, user ID: {user_id}")
})?;
- // Send event for permissions update
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::UpdatedPermissions {
- user_id: command.user_id.clone(),
- permissions: command.permissions.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let entry_command = EntryCommand::UpdatePermissions(command);
let future = SendWrapper::new(
@@ -332,24 +277,6 @@ async fn change_password(
format!("{COMPONENT} (error: {e}) - failed to change password, user ID: {user_id}")
})?;
- // Send event for password change
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::ChangedPassword {
- user_id: command.user_id.clone(),
- current_password: command.current_password.clone(),
- new_password: command.new_password.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let entry_command = EntryCommand::ChangePassword(command);
let future = SendWrapper::new(
@@ -388,22 +315,6 @@ async fn delete_user(
format!("{COMPONENT} (error: {e}) - failed to delete user with ID: {user_id}")
})?;
- // Send event for user deletion
- {
- let broadcast_future = SendWrapper::new(async {
- use crate::shard::transmission::event::ShardEvent;
- let event = ShardEvent::DeletedUser {
- user_id: identifier_user_id.clone(),
- };
- let _responses = state
- .shard
- .shard()
- .broadcast_event_to_all_shards(event)
- .await;
- });
- broadcast_future.await;
- }
-
{
let entry_command = EntryCommand::DeleteUser(DeleteUser {
user_id: identifier_user_id,
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index 41669486f..b6a758006 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -41,7 +41,6 @@ pub mod metadata;
pub mod quic;
pub mod server_error;
pub mod shard;
-pub mod slab;
pub mod state;
pub mod streaming;
pub mod tcp;
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 3fe36e20d..756644ba8 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -28,19 +28,17 @@ use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError, MemoryPool};
use server::SEMANTIC_VERSION;
use server::args::Args;
use server::bootstrap::{
- create_directories, create_shard_connections, create_shard_executor, load_config, load_streams,
- load_users, resolve_persister, update_system_info,
+ create_directories, create_shard_connections, create_shard_executor, load_config,
+ load_metadata, resolve_persister, update_system_info,
};
use server::configs::sharding::ShardAllocator;
use server::diagnostics::{print_io_uring_permission_info, print_locked_memory_limit_info};
use server::io::fs_utils;
use server::log::logger::Logging;
+use server::metadata::{Metadata, create_metadata_handles};
use server::server_error::ServerError;
use server::shard::system::info::SystemInfo;
use server::shard::{IggyShard, calculate_shard_assignment};
-use server::slab::traits_ext::{
- EntityComponentSystem, EntityComponentSystemMutCell, IntoComponents,
-};
use server::state::file::FileState;
use server::state::system::SystemState;
use server::streaming::clients::client_manager::{Client, ClientManager};
@@ -278,8 +276,19 @@ fn main() -> Result<(), ServerError> {
);
let state = SystemState::load(state).await?;
let (streams_state, users_state) = state.decompose();
- let streams = load_streams(streams_state.into_values(), &config.system).await?;
- let users = load_users(users_state.into_values());
+
+ // Create left-right handles for metadata
+ let (mut metadata_writer, metadata_reader) = create_metadata_handles();
+
+ // Create shared metadata reader - each shard will get a clone
+ let shared_metadata = Metadata::new(metadata_reader);
+
+ // Load initial metadata using the writer
+ load_metadata(
+ users_state.into_values(),
+ streams_state.into_values(),
+ &mut metadata_writer,
+ );
// ELEVENTH DISCRETE LOADING STEP.
let shard_allocator = ShardAllocator::new(&config.system.sharding.cpu_allocation)?;
@@ -320,41 +329,34 @@ fn main() -> Result<(), ServerError> {
let client_manager: EternalPtr<DashMap<u32, Client>> = client_manager.into();
let client_manager = ClientManager::new(client_manager);
- streams.with_components(|components| {
- let (root, ..) = components.into_components();
- for (_, stream) in root.iter() {
- stream.topics().with_components(|components| {
- let (root, ..) = components.into_components();
- for (_, topic) in root.iter() {
- topic.partitions().with_components(|components| {
- let (root, ..) = components.into_components();
- for (_, partition) in root.iter() {
- let stream_id = stream.id();
- let topic_id = topic.id();
- let partition_id = partition.id();
- let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
- let shard_id = ShardId::new(calculate_shard_assignment(
- &ns,
- shard_assignment.len() as u32,
- ));
- // TODO(hubcio): LocalIdx is 0 until IggyPartitions is integratedds
- let location = PartitionLocation::new(shard_id, LocalIdx::new(0));
- shards_table.insert(ns, location);
- }
- });
+ // Populate shards_table from SharedMetadata partitions (hierarchical traversal)
+ {
+ let metadata = shared_metadata.load();
+ for (stream_id, stream_meta) in metadata.streams.iter() {
+ for (topic_id, topic_meta) in stream_meta.topics.iter() {
+ for (partition_id, _partition_meta) in topic_meta.partitions.iter().enumerate() {
+ let ns = IggyNamespace::new(stream_id, topic_id, partition_id);
+ let shard_id = ShardId::new(calculate_shard_assignment(
+ &ns,
+ shard_assignment.len() as u32,
+ ));
+ // TODO(hubcio): LocalIdx is 0 until IggyPartitions is integrated
+ let location = PartitionLocation::new(shard_id, LocalIdx::new(0));
+ shards_table.insert(ns, location);
}
- })
+ }
}
- });
+ }
+
+ // Wrap metadata_writer in Option so we can take it for shard 0
+ let mut metadata_writer_opt = Some(metadata_writer);
for (id, assignment) in shard_assignment
.into_iter()
.enumerate()
.map(|(idx, assignment)| (idx as u16, assignment))
{
- let streams = streams.clone();
let shards_table = shards_table.clone();
- let users = users.clone();
let connections = connections.clone();
let config = config.clone();
let encryptor = encryptor.clone();
@@ -372,29 +374,14 @@ fn main() -> Result<(), ServerError> {
state_term.clone(),
);
let client_manager = client_manager.clone();
+ let shard_metadata = shared_metadata.clone();
- // TODO: Explore decoupling the `Log` from `Partition` entity.
- // Ergh... I knew this will backfire to include `Log` as part of the `Partition` entity,
- // We have to initialize with a default log for every partition, once we `Clone` the Streams / Topics / Partitions,
- // because `Clone` impl for `Partition` does not clone the actual log, just creates an empty one.
- streams.with_components(|components| {
- let (root, ..) = components.into_components();
- for (_, stream) in root.iter() {
- stream.topics().with_components_mut(|components| {
- let (mut root, ..) = components.into_components();
- for (_, topic) in root.iter_mut() {
- let partitions_count = topic.partitions().len();
- for log_id in 0..partitions_count {
- let id = topic.partitions_mut().insert_default_log();
- assert_eq!(
- id, log_id,
- "main: partition_insert_default_log: id mismatch when creating default log"
- );
- }
- }
- })
- }
- });
+ // Take metadata_writer for shard 0 only
+ let shard_metadata_writer = if id == 0 {
+ metadata_writer_opt.take()
+ } else {
+ None
+ };
let shard_done_tx = shard_done_tx.clone();
let handle = std::thread::Builder::new()
@@ -411,12 +398,10 @@ fn main() -> Result<(), ServerError> {
let rt = create_shard_executor();
rt.block_on(async move {
- let builder = IggyShard::builder();
- let shard = builder
+ let mut builder = IggyShard::builder();
+ builder = builder
.id(id)
- .streams(streams)
.state(state)
- .users(users)
.shards_table(shards_table)
.connections(connections)
.clients_manager(client_manager)
@@ -425,7 +410,13 @@ fn main() -> Result<(), ServerError> {
.version(current_version)
.metrics(metrics)
.is_follower(is_follower)
- .build();
+ .shared_metadata(shard_metadata);
+
+ if let Some(writer) = shard_metadata_writer {
+ builder = builder.metadata_writer(writer);
+ }
+
+ let shard = builder.build();
let shard = Rc::new(shard);
diff --git a/core/server/src/metadata/mod.rs b/core/server/src/metadata/mod.rs
index 12804095b..14c5a8c37 100644
--- a/core/server/src/metadata/mod.rs
+++ b/core/server/src/metadata/mod.rs
@@ -17,8 +17,8 @@
//! Shared metadata module providing a single source of truth for all shards.
//!
-//! This module provides an `ArcSwap`-based approach where all shards read from
-//! a shared snapshot, and only shard 0 can write (swap in new snapshots).
+//! This module provides a `LeftRight`-based approach where all shards read from
+//! a shared snapshot, and only shard 0 can write.
//!
//! # Architecture
//!
diff --git a/core/server/src/metadata/ops.rs b/core/server/src/metadata/ops.rs
index 4d21c0d90..f489b7309 100644
--- a/core/server/src/metadata/ops.rs
+++ b/core/server/src/metadata/ops.rs
@@ -20,7 +20,8 @@ use crate::metadata::{
ConsumerGroupId, ConsumerGroupMeta, PartitionId, PartitionMeta, StreamId, StreamMeta, TopicId,
TopicMeta, UserId, UserMeta,
};
-use crate::streaming::partitions::partition::{ConsumerGroupOffsets, ConsumerOffsets};
+use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets;
+use crate::streaming::partitions::consumer_offsets::ConsumerOffsets;
use iggy_common::{CompressionAlgorithm, IggyExpiry, MaxTopicSize, PersonalAccessToken};
use std::sync::Arc;
use std::sync::atomic::AtomicUsize;
diff --git a/core/server/src/metadata/partition.rs b/core/server/src/metadata/partition.rs
index b2490b96a..13999222f 100644
--- a/core/server/src/metadata/partition.rs
+++ b/core/server/src/metadata/partition.rs
@@ -16,7 +16,8 @@
// under the License.
use crate::metadata::PartitionId;
-use crate::streaming::partitions::partition::{ConsumerGroupOffsets, ConsumerOffsets};
+use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets;
+use crate::streaming::partitions::consumer_offsets::ConsumerOffsets;
use crate::streaming::stats::PartitionStats;
use iggy_common::IggyTimestamp;
use std::sync::Arc;
@@ -25,12 +26,12 @@ use std::sync::Arc;
pub struct PartitionMeta {
pub id: PartitionId,
pub created_at: IggyTimestamp,
- /// Monotonically increasing version to detect stale partition_store entries.
+ /// Monotonically increasing version to detect stale local_partitions entries.
/// Set to the Metadata version when the partition was created.
pub revision_id: u64,
pub stats: Arc<PartitionStats>,
- /// Consumer offsets for this partition // TODO: move to partition store
+ /// Consumer offsets for this partition
pub consumer_offsets: Option<Arc<ConsumerOffsets>>,
- /// Consumer group offsets for this partition // TODO: move to partition store
+ /// Consumer group offsets for this partition
pub consumer_group_offsets: Option<Arc<ConsumerGroupOffsets>>,
}
diff --git a/core/server/src/metadata/reader.rs b/core/server/src/metadata/reader.rs
index ea85511a6..6b7aa537c 100644
--- a/core/server/src/metadata/reader.rs
+++ b/core/server/src/metadata/reader.rs
@@ -19,7 +19,8 @@ use crate::metadata::{
ConsumerGroupId, ConsumerGroupMeta, InnerMetadata, MetadataReadHandle, PartitionId,
PartitionMeta, StreamId, StreamMeta, TopicId, TopicMeta, UserId, UserMeta,
};
-use crate::streaming::partitions::partition::{ConsumerGroupOffsets, ConsumerOffsets};
+use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets;
+use crate::streaming::partitions::consumer_offsets::ConsumerOffsets;
use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats};
use iggy_common::sharding::IggyNamespace;
use iggy_common::{IdKind, Identifier, PersonalAccessToken};
diff --git a/core/server/src/metadata/writer.rs b/core/server/src/metadata/writer.rs
index daa1142bf..55d291c0b 100644
--- a/core/server/src/metadata/writer.rs
+++ b/core/server/src/metadata/writer.rs
@@ -22,7 +22,8 @@ use crate::metadata::{
ConsumerGroupId, ConsumerGroupMeta, PartitionId, PartitionMeta, StreamId, StreamMeta, TopicId,
TopicMeta, UserId, UserMeta,
};
-use crate::streaming::partitions::partition::{ConsumerGroupOffsets, ConsumerOffsets};
+use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets;
+use crate::streaming::partitions::consumer_offsets::ConsumerOffsets;
use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats};
use iggy_common::{
CompressionAlgorithm, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize,
diff --git a/core/server/src/quic/listener.rs b/core/server/src/quic/listener.rs
index 67a2d4937..69df4dadf 100644
--- a/core/server/src/quic/listener.rs
+++ b/core/server/src/quic/listener.rs
@@ -131,7 +131,7 @@ async fn handle_connection(
}
}
- shard.delete_client(client_id);
+ shard.delete_client(client_id).await;
shard.task_registry.remove_connection(&client_id);
info!("QUIC connection {} closed", client_id);
Ok(())
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 135a9b4bc..400cab0d0 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -20,28 +20,32 @@ use super::{
IggyShard, TaskRegistry, transmission::connector::ShardConnector,
transmission::frame::ShardFrame,
};
+use crate::metadata::{Metadata, MetadataWriter};
+use crate::streaming::partitions::local_partitions::LocalPartitions;
use crate::{
configs::server::ServerConfig,
- slab::{streams::Streams, users::Users},
state::file::FileState,
streaming::{
clients::client_manager::ClientManager, diagnostics::metrics::Metrics,
- utils::ptr::EternalPtr,
+ users::permissioner::Permissioner, utils::ptr::EternalPtr,
},
versioning::SemanticVersion,
};
+use ahash::AHashSet;
use dashmap::DashMap;
use iggy_common::EncryptorKind;
use iggy_common::sharding::{IggyNamespace, PartitionLocation};
-use std::{cell::Cell, rc::Rc, sync::atomic::AtomicBool};
+use std::{
+ cell::{Cell, RefCell},
+ rc::Rc,
+ sync::atomic::AtomicBool,
+};
#[derive(Default)]
pub struct IggyShardBuilder {
id: Option<u16>,
- streams: Option<Streams>,
shards_table: Option<EternalPtr<DashMap<IggyNamespace, PartitionLocation>>>,
state: Option<FileState>,
- users: Option<Users>,
client_manager: Option<ClientManager>,
connections: Option<Vec<ShardConnector<ShardFrame>>>,
config: Option<ServerConfig>,
@@ -49,6 +53,8 @@ pub struct IggyShardBuilder {
version: Option<SemanticVersion>,
metrics: Option<Metrics>,
is_follower: bool,
+ shared_metadata: Option<Metadata>,
+ metadata_writer: Option<MetadataWriter>,
}
impl IggyShardBuilder {
@@ -90,21 +96,11 @@ impl IggyShardBuilder {
self
}
- pub fn streams(mut self, streams: Streams) -> Self {
- self.streams = Some(streams);
- self
- }
-
pub fn state(mut self, state: FileState) -> Self {
self.state = Some(state);
self
}
- pub fn users(mut self, users: Users) -> Self {
- self.users = Some(users);
- self
- }
-
pub fn metrics(mut self, metrics: Metrics) -> Self {
self.metrics = Some(metrics);
self
@@ -115,18 +111,27 @@ impl IggyShardBuilder {
self
}
+ pub fn shared_metadata(mut self, shared_metadata: Metadata) -> Self {
+ self.shared_metadata = Some(shared_metadata);
+ self
+ }
+
+ pub fn metadata_writer(mut self, metadata_writer: MetadataWriter) -> Self {
+ self.metadata_writer = Some(metadata_writer);
+ self
+ }
+
// TODO: Too much happens in there, some of those bootstrapping logic should be moved outside.
pub fn build(self) -> IggyShard {
let id = self.id.unwrap();
- let streams = self.streams.unwrap();
let shards_table = self.shards_table.unwrap();
let state = self.state.unwrap();
- let users = self.users.unwrap();
let config = self.config.unwrap();
let connections = self.connections.unwrap();
let encryptor = self.encryptor;
let client_manager = self.client_manager.unwrap();
let version = self.version.unwrap();
+ let shared_metadata = self.shared_metadata.expect("shared_metadata is required");
let (stop_receiver, frame_receiver) = connections
.iter()
.filter(|c| c.id == id)
@@ -150,12 +155,18 @@ impl IggyShardBuilder {
// Trigger initial check in case servers bind before task starts
let _ = config_writer_notify.try_send(());
+ // Create per-shard stores (wrapped in RefCell for interior mutability)
+ let local_partitions = RefCell::new(LocalPartitions::new());
+ let permissioner = Permissioner::new(shared_metadata.clone());
+
IggyShard {
id,
shards,
shards_table,
- streams, // TODO: Fixme
- users,
+ metadata: shared_metadata,
+ metadata_writer: self.metadata_writer.map(RefCell::new),
+ local_partitions,
+ pending_partition_inits: RefCell::new(AHashSet::new()),
fs_locks: Default::default(),
encryptor,
config,
@@ -173,7 +184,7 @@ impl IggyShardBuilder {
config_writer_notify,
config_writer_receiver,
task_registry,
- permissioner: Default::default(),
+ permissioner,
client_manager,
}
}
diff --git a/core/server/src/shard/handlers.rs b/core/server/src/shard/handlers.rs
index 087b7a54d..2b988dd82 100644
--- a/core/server/src/shard/handlers.rs
+++ b/core/server/src/shard/handlers.rs
@@ -19,21 +19,21 @@ use super::*;
use crate::{
shard::{
IggyShard,
- namespace::IggyFullNamespace,
transmission::{
event::ShardEvent,
frame::ShardResponse,
message::{ShardMessage, ShardRequest, ShardRequestPayload},
},
},
- streaming::{session::Session, traits::MainOps},
+ streaming::session::Session,
tcp::{
connection_handler::{ConnectionAction, handle_connection, handle_error},
tcp_listener::cleanup_connection,
},
};
use compio_net::TcpStream;
-use iggy_common::{Identifier, IggyError, SenderKind, TransportProtocol};
+use iggy_common::sharding::IggyNamespace;
+use iggy_common::{Identifier, IggyError, IggyTimestamp, SenderKind, TransportProtocol};
use nix::sys::stat::SFlag;
use std::os::fd::{FromRawFd, IntoRawFd};
use tracing::info;
@@ -63,73 +63,220 @@ async fn handle_request(
let partition_id = request.partition_id;
match request.payload {
ShardRequestPayload::SendMessages { batch } => {
- let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
let batch = shard.maybe_encrypt_messages(batch)?;
let messages_count = batch.count();
- shard
+
+ let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?;
+ let namespace = IggyNamespace::new(stream, topic, partition_id);
+
+ let metadata = shard.metadata.load();
+ let partition_meta = metadata
.streams
- .append_messages(&shard.config.system, &shard.task_registry, &ns, batch)
+ .get(stream)
+ .and_then(|s| s.topics.get(topic))
+ .and_then(|t| t.partitions.get(partition_id));
+
+ let needs_init = {
+ let partitions = shard.local_partitions.borrow();
+ match (partitions.get(&namespace), partition_meta) {
+ (Some(data), Some(meta)) if data.revision_id == meta.revision_id => false,
+ (Some(_), _) => {
+ drop(partitions);
+ shard.local_partitions.borrow_mut().remove(&namespace);
+ true
+ }
+ (None, _) => true,
+ }
+ };
+
+ if needs_init {
+ let created_at = partition_meta
+ .map(|m| m.created_at)
+ .unwrap_or_else(IggyTimestamp::now);
+
+ shard
+ .init_partition_directly(stream, topic, partition_id, created_at)
+ .await?;
+ }
+
+ shard
+ .append_messages_to_local_partition(&namespace, batch, &shard.config.system)
.await?;
+
shard.metrics.increment_messages(messages_count as u64);
Ok(ShardResponse::SendMessages)
}
ShardRequestPayload::PollMessages { args, consumer } => {
let auto_commit = args.auto_commit;
- let ns = IggyFullNamespace::new(stream_id, topic_id, partition_id);
- let (metadata, batches) = shard.streams.poll_messages(&ns, consumer, args).await?;
+
+ let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?;
+ let namespace = IggyNamespace::new(stream, topic, partition_id);
+
+ let metadata = shard.metadata.load();
+ let partition_meta = metadata
+ .streams
+ .get(stream)
+ .and_then(|s| s.topics.get(topic))
+ .and_then(|t| t.partitions.get(partition_id));
+
+ let needs_init = {
+ let partitions = shard.local_partitions.borrow();
+ match (partitions.get(&namespace), partition_meta) {
+ (Some(data), Some(meta)) if data.revision_id == meta.revision_id => false,
+ (Some(_), _) => {
+ drop(partitions);
+ shard.local_partitions.borrow_mut().remove(&namespace);
+ true
+ }
+ (None, _) => true,
+ }
+ };
+
+ if needs_init {
+ let created_at = partition_meta
+ .map(|m| m.created_at)
+ .unwrap_or_else(IggyTimestamp::now);
+
+ shard
+ .init_partition_directly(stream, topic, partition_id, created_at)
+ .await?;
+ }
+
+ let (metadata, batches) = shard
+ .poll_messages_from_local_partitions(&namespace, consumer, args)
+ .await?;
if auto_commit && !batches.is_empty() {
let offset = batches
.last_offset()
.expect("Batch set should have at least one batch");
shard
- .streams
- .auto_commit_consumer_offset(
- &shard.config.system,
- ns.stream_id(),
- ns.topic_id(),
- partition_id,
- consumer,
- offset,
- )
+ .auto_commit_consumer_offset_from_local_partitions(&namespace, consumer, offset)
.await?;
}
Ok(ShardResponse::PollMessages((metadata, batches)))
}
ShardRequestPayload::FlushUnsavedBuffer { fsync } => {
+ let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?;
shard
- .flush_unsaved_buffer_base(&stream_id, &topic_id, partition_id, fsync)
+ .flush_unsaved_buffer_base(stream, topic, partition_id, fsync)
.await?;
Ok(ShardResponse::FlushUnsavedBuffer)
}
ShardRequestPayload::DeleteSegments { segments_count } => {
+ let (stream, topic) = shard.resolve_topic_id(&stream_id, &topic_id)?;
shard
- .delete_segments_base(&stream_id, &topic_id, partition_id, segments_count)
+ .delete_segments_base(stream, topic, partition_id, segments_count)
.await?;
Ok(ShardResponse::DeleteSegments)
}
- ShardRequestPayload::CreateStream { user_id, name } => {
- assert_eq!(shard.id, 0, "CreateStream should only be handled by shard0");
+ ShardRequestPayload::CreatePartitions {
+ user_id,
+ stream_id,
+ topic_id,
+ partitions_count,
+ } => {
+ assert_eq!(
+ shard.id, 0,
+ "CreatePartitions should only be handled by shard0"
+ );
let session = Session::stateless(
user_id,
std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0),
);
- // Acquire stream lock to serialize filesystem operations
- let _stream_guard = shard.fs_locks.stream_lock.lock().await;
+ let _partition_guard = shard.fs_locks.partition_lock.lock().await;
- let stream = shard.create_stream(&session, name.clone()).await?;
- let created_stream_id = stream.id();
+ let partition_infos = shard
... 13726 lines suppressed ...