You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2025/06/26 19:57:04 UTC
(iggy) branch io_uring_tpc updated (48107890 -> d2651b3d)
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a change to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
discard 48107890 fix iggy shard errors
discard f6b57add fixes
discard 4258422f fix issues with RefCell in core/server/src/streaming/topics/consumer_groups* files
discard 05bf966d begin fix cg
discard 9c6b9ffc temp
discard fc3f0fdb fix state initialization with gate
discard a0736342 refactors
discard 99aaccf2 todos
discard d4215002 begin shard init
discard 309d505a feat(io_uring): load server_config (#1798)
discard a435b97d feat(io_uring): bootstrap the main function (#1788)
discard d765fc44 initial setup and migration of shards
add d4288aaf feat(bench): add TCP TLS support to `iggy-bench` (#1899)
add 1c9f2715 refactor(foreign/go): replace the previous implementation of IggyMessage to option pattern (#1904)
add 80024ee3 style(repo): add various brand assets (#1910)
add ab9ca614 chore(repo): fix existing Iggy names in whole repo (#1908)
add 99a21a18 chore(csharp): update integration tests (#1913)
add 99b933a4 feat(bdd): add Python BDD tests and reorganize test structure (#1909)
add e748fd82 fix(server): refactor Rust code based on clippy lints (#1918)
new 3e599d25 initial setup and migration of shards
new d1ada2a1 feat(io_uring): bootstrap the main function (#1788)
new 4d172830 feat(io_uring): load server_config (#1798)
new 36ade47f begin shard init
new e53f18a0 todos
new 026e53be refactors
new 44743fb4 fix state initialization with gate
new 82735218 temp
new 5549b010 begin fix cg
new b42ba18a fix issues with RefCell in core/server/src/streaming/topics/consumer_groups* files
new a890252a fixes
new d2651b3d fix iggy shard errors
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (48107890)
\
N -- N -- N refs/heads/io_uring_tpc (d2651b3d)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/changed-files-config.json | 16 +-
.github/workflows/ci-check-pr.yml | 8 +
.github/workflows/ci-check-python-sdk.yml | 2 +-
.github/workflows/ci-check-rust.yml | 31 ---
.../{ci-check-shell.yml => ci-test-bdd.yml} | 39 +--
.gitignore | 1 +
Cargo.lock | 27 ++-
Cargo.toml | 6 +-
DEPENDENCIES.md | 4 +-
Dockerfile | 2 +-
Dockerfile.debug | 4 +-
README.md | 4 +-
assets/logo/0.5x/iggy-apache-b&wdark@0.5x.png | Bin 0 -> 36299 bytes
assets/logo/0.5x/iggy-apache-b&wlight@0.5x.png | Bin 0 -> 36828 bytes
assets/logo/0.5x/iggy-apache-color-darkbg@0.5x.png | Bin 0 -> 96024 bytes
.../logo/0.5x/iggy-apache-color-lightbg@0.5x.png | Bin 0 -> 97130 bytes
.../0.5x/iggy-apache-logo-wo-sygnet-dark@0.5x.png | Bin 0 -> 14714 bytes
.../0.5x/iggy-apache-logo-wo-sygnet-light@0.5x.png | Bin 0 -> 13915 bytes
.../logo/0.5x/iggy-apache-sygnet-b&wdark@0.5x.png | Bin 0 -> 23012 bytes
.../logo/0.5x/iggy-apache-sygnet-b&wlight@0.5x.png | Bin 0 -> 23210 bytes
.../0.5x/iggy-apache-sygnet-color-darkbg@0.5x.png | Bin 0 -> 68197 bytes
.../0.5x/iggy-apache-sygnet-color-lightbg@0.5x.png | Bin 0 -> 68525 bytes
assets/logo/4x/iggy-apache-b&wdark@4x.png | Bin 0 -> 440202 bytes
assets/logo/4x/iggy-apache-b&wlight@4x.png | Bin 0 -> 430715 bytes
assets/logo/4x/iggy-apache-color-darkbg@4x.png | Bin 0 -> 1312915 bytes
assets/logo/4x/iggy-apache-color-lightbg@4x.png | Bin 0 -> 1340584 bytes
.../logo/4x/iggy-apache-logo-wo-sygnet-dark@4x.png | Bin 0 -> 207686 bytes
.../4x/iggy-apache-logo-wo-sygnet-light@4x.png | Bin 0 -> 197503 bytes
assets/logo/4x/iggy-apache-sygnet-b&wdark@4x.png | Bin 0 -> 290806 bytes
assets/logo/4x/iggy-apache-sygnet-b&wlight@4x.png | Bin 0 -> 288572 bytes
.../logo/4x/iggy-apache-sygnet-color-darkbg@4x.png | Bin 0 -> 863513 bytes
.../4x/iggy-apache-sygnet-color-lightbg@4x.png | Bin 0 -> 869788 bytes
assets/logo/SVG/iggy-apache-b&wdark.svg | 63 +++++
assets/logo/SVG/iggy-apache-b&wlight.svg | 77 ++++++
.../logo/SVG/iggy-apache-color-darkbg.svg | 0
.../logo/SVG/iggy-apache-color-lightbg.svg | 0
.../logo/SVG/iggy-apache-logo-wo-sygnet-dark.svg | 11 +
.../logo/SVG/iggy-apache-logo-wo-sygnet-light.svg | 18 ++
assets/logo/SVG/iggy-apache-sygnet-b&wdark.svg | 49 ++++
assets/logo/SVG/iggy-apache-sygnet-b&wlight.svg | 61 +++++
.../logo/SVG/iggy-apache-sygnet-color-darkbg.svg | 88 ++++---
.../logo/SVG/iggy-apache-sygnet-color-lightbg.svg | 16 +-
Dockerfile.debug => bdd/Dockerfile | 41 +++-
bdd/README.md | 79 ++++++
bdd/docker-compose.yml | 68 ++++++
bdd/python/Dockerfile | 54 +++++
bdd/python/requirements.txt | 4 +
bdd/python/tests/conftest.py | 65 +++++
bdd/python/tests/test_basic_messaging.py | 237 ++++++++++++++++++
{core/bdd => bdd/rust}/Cargo.toml | 10 +-
.../post-install.sh => bdd/rust/Dockerfile | 22 +-
{core/bdd => bdd/rust}/tests/basic_messaging.rs | 2 +-
.../rust}/tests/common/global_context.rs | 3 +
{core/bdd => bdd/rust}/tests/common/mod.rs | 0
{core/bdd => bdd/rust}/tests/helpers/mod.rs | 0
{core/bdd => bdd/rust}/tests/helpers/test_data.rs | 0
{core/bdd => bdd/rust}/tests/steps/auth.rs | 0
{core/bdd => bdd/rust}/tests/steps/messages.rs | 15 +-
{core/bdd => bdd/rust}/tests/steps/mod.rs | 0
{core/bdd => bdd/rust}/tests/steps/server.rs | 25 +-
{core/bdd => bdd/rust}/tests/steps/streams.rs | 0
{core/bdd => bdd/rust}/tests/steps/topics.rs | 0
bdd/{ => scenarios}/basic_messaging.feature | 6 +-
core/bench/dashboard/frontend/src/api/mod.rs | 4 +-
.../frontend/src/components/chart/single_chart.rs | 2 +-
.../dashboard/frontend/src/components/footer.rs | 6 +-
.../components/tooltips/server_stats_tooltip.rs | 2 +-
core/bench/dashboard/frontend/src/config.rs | 2 +-
core/bench/dashboard/server/src/cache/mod.rs | 7 +-
core/bench/dashboard/server/src/github/client.rs | 8 +-
core/bench/dashboard/server/src/handlers.rs | 40 +---
core/bench/report/src/prints.rs | 25 +-
core/bench/report/src/types/numeric_parameter.rs | 4 +-
.../bench/runner/src/app/local_benchmark_runner.rs | 28 +--
.../src/actors/consumer/high_level_backend.rs | 1 +
.../src/actors/producer/benchmark_producer.rs | 1 +
.../benchmark_producing_consumer.rs | 8 +-
core/bench/src/analytics/report_builder.rs | 8 +-
core/bench/src/args/mod.rs | 2 +-
core/bench/src/args/transport.rs | 18 +-
core/bench/src/benchmarks/common.rs | 1 +
core/bench/src/runner.rs | 7 +-
core/bench/src/utils/client_factory.rs | 20 +-
core/bench/src/utils/mod.rs | 48 +---
core/bench/src/utils/server_starter.rs | 1 +
.../src/cli/binary_client/get_client.rs | 2 +-
.../src/cli/binary_client/get_clients.rs | 4 +-
.../create_consumer_group.rs | 2 +-
.../binary_consumer_groups/get_consumer_group.rs | 2 +-
.../src/cli/binary_context/get_contexts.rs | 2 +-
.../src/cli/binary_message/poll_messages.rs | 2 +-
.../src/cli/binary_message/send_messages.rs | 5 +-
.../create_personal_access_token.rs | 6 +-
.../src/cli/binary_streams/create_stream.rs | 2 +-
core/binary_protocol/src/cli/binary_system/me.rs | 2 +-
.../src/cli/binary_system/snapshot.rs | 2 +-
.../binary_protocol/src/cli/binary_system/stats.rs | 2 +-
.../src/cli/binary_topics/create_topic.rs | 4 +-
.../src/cli/binary_topics/get_topic.rs | 2 +-
.../src/cli/binary_topics/get_topics.rs | 4 +-
.../src/cli/binary_topics/update_topic.rs | 2 +-
.../src/cli/binary_users/get_user.rs | 6 +-
.../src/cli/binary_users/update_user.rs | 4 +-
core/cli/src/args/message.rs | 8 +-
core/cli/src/args/permissions/stream.rs | 4 +-
core/cli/src/args/permissions/topic.rs | 2 +-
core/cli/src/credentials.rs | 4 +-
core/common/Cargo.toml | 2 +
.../steps/server.rs => common/src/certificates.rs} | 22 +-
core/common/src/commands/system/get_snapshot.rs | 2 +-
core/common/src/lib.rs | 2 +
.../src/types/compression/compression_algorithm.rs | 2 +-
.../configuration/tcp_config/tcp_client_config.rs | 5 +
.../tcp_config/tcp_client_config_builder.rs | 6 +
core/common/src/types/message/iggy_message.rs | 6 +-
.../src/types/message/message_header_view.rs | 3 +-
core/common/src/types/message/user_headers.rs | 2 +-
.../src/types/permissions/permissions_global.rs | 10 +-
core/common/src/types/snapshot/mod.rs | 4 +-
core/common/src/utils/byte_size.rs | 2 +-
core/common/src/utils/duration.rs | 2 +-
core/common/src/utils/topic_size.rs | 4 +-
core/configs/server.toml | 17 +-
core/connectors/runtime/README.md | 4 +-
core/connectors/runtime/config.toml | 4 +-
core/connectors/runtime/src/api/config.rs | 2 +-
core/connectors/sdk/src/lib.rs | 4 +-
.../sdk/src/transforms/json/test_utils.rs | 8 +-
core/integration/src/tcp_client.rs | 26 +-
core/integration/src/test_server.rs | 5 +-
core/integration/tests/archiver/mod.rs | 2 +-
.../tests/cli/client/test_client_get_command.rs | 2 +-
core/integration/tests/cli/common/mod.rs | 4 +-
.../test_consumer_group_create_command.rs | 4 +-
.../test_consumer_group_delete_command.rs | 3 +-
.../test_consumer_group_get_command.rs | 3 +-
.../tests/cli/message/test_message_send_command.rs | 3 +-
.../test_pat_login_options.rs | 5 +-
.../tests/cli/stream/test_stream_create_command.rs | 4 +-
.../tests/cli/stream/test_stream_purge_command.rs | 5 +-
.../tests/cli/topic/test_topic_create_command.rs | 4 +-
.../tests/cli/topic/test_topic_delete_command.rs | 3 +-
.../tests/cli/topic/test_topic_get_command.rs | 6 +-
.../tests/cli/topic/test_topic_purge_command.rs | 5 +-
.../tests/cli/user/test_user_get_command.rs | 11 +-
core/integration/tests/config_provider/mod.rs | 6 +-
core/integration/tests/mod.rs | 2 +-
core/integration/tests/sdk/producer/mod.rs | 2 +-
...h_multiple_clients_polling_messages_scenario.rs | 4 +-
...with_single_client_polling_messages_scenario.rs | 4 +-
.../server/scenarios/create_message_payload.rs | 2 +-
.../server/scenarios/delete_segments_scenario.rs | 9 +-
.../server/scenarios/message_headers_scenario.rs | 2 +-
.../tests/server/scenarios/system_scenario.rs | 14 +-
core/integration/tests/server/specific.rs | 35 +++
core/integration/tests/state/mod.rs | 2 +-
core/integration/tests/streaming/get_by_offset.rs | 21 +-
.../tests/streaming/get_by_timestamp.rs | 13 +-
core/integration/tests/streaming/messages.rs | 155 +-----------
core/integration/tests/streaming/partition.rs | 6 +-
core/integration/tests/streaming/segment.rs | 11 +-
core/integration/tests/streaming/stream.rs | 10 +-
core/integration/tests/streaming/system.rs | 2 +-
core/integration/tests/streaming/topic.rs | 10 +-
core/integration/tests/streaming/topic_messages.rs | 2 +-
core/sdk/src/client_provider.rs | 1 +
core/sdk/src/clients/client_builder.rs | 8 +
core/sdk/src/clients/producer_error_callback.rs | 2 +-
core/sdk/src/http/binary_system.rs | 2 +-
core/sdk/src/http/binary_users.rs | 2 +-
.../stream_builder/config/config_iggy_consumer.rs | 2 +-
core/sdk/src/tcp/mod.rs | 1 +
core/sdk/src/tcp/tcp_client.rs | 58 +++--
.../tcp_tls_verifier.rs} | 45 ++--
core/server/Cargo.toml | 1 -
.../create_consumer_group_handler.rs | 3 +-
.../delete_consumer_group_handler.rs | 3 +-
.../partitions/create_partitions_handler.rs | 3 +-
.../partitions/delete_partitions_handler.rs | 3 +-
.../handlers/streams/create_stream_handler.rs | 6 +-
.../binary/handlers/topics/create_topic_handler.rs | 6 +-
.../binary/handlers/topics/update_topic_handler.rs | 3 +-
.../binary/handlers/users/update_user_handler.rs | 3 +-
core/server/src/configs/cache_indexes.rs | 5 +-
core/server/src/configs/config_provider.rs | 2 +-
core/server/src/configs/defaults.rs | 1 +
core/server/src/configs/tcp.rs | 1 +
core/server/src/configs/validators.rs | 3 +-
core/server/src/http/http_server.rs | 4 +-
core/server/src/http/jwt/jwt_manager.rs | 6 +-
core/server/src/http/messages.rs | 3 +-
core/server/src/http/partitions.rs | 12 +-
core/server/src/http/streams.rs | 14 +-
core/server/src/http/system.rs | 2 +-
core/server/src/http/topics.rs | 23 +-
core/server/src/http/users.rs | 22 +-
core/server/src/log/logger.rs | 5 +-
core/server/src/main.rs | 2 +-
core/server/src/quic/quic_server.rs | 13 +-
core/server/src/shard/system/consumer_groups.rs | 10 +-
core/server/src/shard/system/messages.rs | 4 +
core/server/src/shard/system/snapshot/mod.rs | 2 +-
core/server/src/shard/system/snapshot/procdump.rs | 12 +-
core/server/src/state/command.rs | 40 ++--
core/server/src/state/file.rs | 3 +-
.../src/streaming/partitions/consumer_offsets.rs | 7 +-
core/server/src/streaming/partitions/messages.rs | 11 +-
core/server/src/streaming/partitions/storage.rs | 5 +-
core/server/src/streaming/polling_consumer.rs | 6 +-
.../src/streaming/segments/indexes/indexes_mut.rs | 2 +-
.../streaming/segments/messages/persister_task.rs | 3 +-
core/server/src/streaming/segments/segment.rs | 4 +-
.../streaming/segments/types/messages_batch_mut.rs | 4 +-
.../src/streaming/segments/writing_messages.rs | 4 +-
core/server/src/streaming/utils/memory_pool.rs | 2 +-
core/server/src/tcp/tcp_tls_listener.rs | 111 +++++----
foreign/cpp/README.md | 4 +-
foreign/cpp/sdk/binary.h | 2 +-
foreign/cpp/sdk/model.h | 4 +-
foreign/cpp/tests/CMakeLists.txt | 4 +-
foreign/cpp/tests/e2e/e2e_testutils.cc | 2 +-
foreign/cpp/tests/e2e/server.toml | 20 +-
foreign/csharp/.editorconfig | 153 ++++++++++++
foreign/csharp/Benchmarks/Benchmarks.csproj | 3 +-
foreign/csharp/Benchmarks/Program.cs | 14 +-
foreign/csharp/DEPENDENCIES.md | 16 +-
foreign/csharp/Directory.Packages.props | 22 ++
.../Attributes/SkipHttpAttribute.cs | 36 +++
.../Attributes/SkipTcpAttribute.cs | 36 +++
.../ConsumerGroupTests.cs | 225 +++++++++++++++++
.../FetchMessagesTests.cs | 185 ++++++++++++++
.../Fixtures/ConsumerGroupFixture.cs | 38 +++
.../Fixtures/FetchMessagesFixture.cs | 148 ++++++++++++
.../Fixtures/FlushMessageFixture.cs | 54 +++++
.../Fixtures/IggyServerFixture.cs | 145 +++++++++++
.../Fixtures/OffsetFixtures.cs | 54 +++++
.../Fixtures/PartitionsFixture.cs | 38 +++
.../Fixtures/PollMessagesFixture.cs | 70 ++++++
.../Fixtures/SendMessageFixture.cs | 38 +++
.../Fixtures/SystemFixture.cs | 89 +++++++
.../FlushMessagesTests.cs | 69 ++++++
.../Helpers/StreamFactory.cs | 32 +++
.../Helpers/TopicFactory.cs | 34 +++
.../Iggy_SDK.Tests.Integration.csproj | 29 +++
.../Models/DummyMessage.cs | 45 ++++
.../Iggy_SDK.Tests.Integration/OffsetTests.cs | 104 ++++++++
.../Iggy_SDK.Tests.Integration/PartitionsTests.cs | 88 +++++++
.../PersonalAccessTokenTests.cs | 98 ++++++++
.../PollMessagesTests.cs | 60 +++++
.../SendMessagesTests.cs | 118 +++++++++
.../Iggy_SDK.Tests.Integration/StreamsTests.cs | 209 ++++++++++++++++
.../Iggy_SDK.Tests.Integration/SystemTests.cs | 195 +++++++++++++++
.../Iggy_SDK.Tests.Integration/TopicsTests.cs | 243 +++++++++++++++++++
.../Iggy_SDK.Tests.Integration/UsersTests.cs | 266 +++++++++++++++++++++
foreign/csharp/Iggy_SDK.sln | 15 ++
foreign/csharp/Iggy_SDK.sln.DotSettings | 23 +-
.../Iggy_SDK/Configuration/TlsConfiguration.cs | 4 +-
.../Iggy_SDK/Contracts/Http/Auth/AuthResponse.cs | 2 +-
.../csharp/Iggy_SDK/Contracts/Http/ClientInfo.cs | 2 +-
.../Iggy_SDK/Contracts/Http/ConsumerGroupInfo.cs | 2 +-
.../Contracts/Http/ConsumerGroupMembers.cs | 2 +-
.../Iggy_SDK/Contracts/Http/MessageResponse.cs | 2 +-
foreign/csharp/Iggy_SDK/Contracts/Http/Stats.cs | 2 +-
.../Iggy_SDK/Contracts/Http/StatsResponse.cs | 4 +-
.../Iggy_SDK/Contracts/Http/StreamResponse.cs | 2 +-
.../csharp/Iggy_SDK/Contracts/Http/TopicRequest.cs | 42 +++-
.../csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs | 12 -
.../Exceptions/InvalidResponseException.cs | 2 +-
foreign/csharp/Iggy_SDK/IggyClient/IIggyOffset.cs | 2 +-
.../Implementations/HttpMessageStream.cs | 4 +-
.../IggyClient/Implementations/TcpMessageStream.cs | 16 +-
foreign/csharp/Iggy_SDK/Iggy_SDK.csproj | 13 +-
.../JsonConfiguration/AuthResponseConverter.cs | 6 +-
.../JsonConfiguration/MessagesConverter.cs | 3 +-
.../JsonConfiguration/StatsResponseConverter.cs | 62 ++---
.../JsonConfiguration/StreamResponseConverter.cs | 27 ++-
.../JsonConfiguration/TopicResponseConverter.cs | 73 +++---
.../JsonConfiguration/UserResponseConverter.cs | 35 ---
foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 23 +-
.../Iggy_SDK_Tests/ContractTests/TcpContract.cs | 7 +-
.../Iggy_SDK_Tests/E2ETests/ConsumerGroupE2E.cs | 199 ---------------
.../Iggy_SDK_Tests/E2ETests/FetchMessagesE2E.cs | 158 ------------
.../Bootstraps/ConsumerGroupFixtureBootstrap.cs | 38 ---
.../Bootstraps/FetchMessagesFixtureBootstrap.cs | 66 -----
.../Bootstraps/FlushMessagesFixtureBootstrap.cs | 59 -----
.../Fixtures/Bootstraps/GeneralFixtureBootstrap.cs | 28 ---
.../Fixtures/Bootstraps/OffsetFixtureBootstrap.cs | 37 ---
.../Fixtures/Bootstraps/PATFixtureBootstrap.cs | 34 ---
.../Bootstraps/PartitionsFixtureBootstrap.cs | 37 ---
.../Bootstraps/PollMessagesFixtureBootstrap.cs | 70 ------
.../Bootstraps/SendMessagesFixtureBootstrap.cs | 47 ----
.../Fixtures/Bootstraps/StreamsFixtureBootstrap.cs | 32 ---
.../Fixtures/Bootstraps/SystemFixtureBootstrap.cs | 72 ------
.../Fixtures/Bootstraps/TopicsFixtureBootstrap.cs | 38 ---
.../Fixtures/Bootstraps/UsersFixtureBootstrap.cs | 34 ---
.../Fixtures/Configs/IggyFixtureClientConfigs.cs | 60 -----
.../E2ETests/Fixtures/IIggyBootstrap.cs | 25 --
.../E2ETests/Fixtures/IggyBaseFixture.cs | 122 ----------
.../E2ETests/Fixtures/IggyConsumerGroupFixture.cs | 29 ---
.../E2ETests/Fixtures/IggyFetchMessagesFixture.cs | 29 ---
.../E2ETests/Fixtures/IggyFlushMessagesFixture.cs | 29 ---
.../E2ETests/Fixtures/IggyGeneralFixture.cs | 30 ---
.../E2ETests/Fixtures/IggyOffsetFixture.cs | 30 ---
.../E2ETests/Fixtures/IggyPATFixture.cs | 30 ---
.../E2ETests/Fixtures/IggyPartitionFixture.cs | 31 ---
.../E2ETests/Fixtures/IggyPollMessagesFixture.cs | 30 ---
.../E2ETests/Fixtures/IggySendMessagesFixture.cs | 30 ---
.../E2ETests/Fixtures/IggyStreamFixture.cs | 30 ---
.../E2ETests/Fixtures/IggySystemFixture.cs | 29 ---
.../E2ETests/Fixtures/IggyTcpUsersFixture.cs | 30 ---
.../E2ETests/Fixtures/IggyTopicFixture.cs | 30 ---
.../E2ETests/Fixtures/Models/IggyClientModel.cs | 28 ---
.../Iggy_SDK_Tests/E2ETests/FlushMessagesE2E.cs | 106 --------
.../csharp/Iggy_SDK_Tests/E2ETests/OffsetE2E.cs | 76 ------
foreign/csharp/Iggy_SDK_Tests/E2ETests/PATE2E.cs | 122 ----------
.../Iggy_SDK_Tests/E2ETests/PartitionsE2E.cs | 103 --------
.../Iggy_SDK_Tests/E2ETests/PollMessagesE2E.cs | 64 -----
.../Iggy_SDK_Tests/E2ETests/SendMessagesE2E.cs | 114 ---------
.../csharp/Iggy_SDK_Tests/E2ETests/StreamsE2E.cs | 153 ------------
.../csharp/Iggy_SDK_Tests/E2ETests/SystemE2E.cs | 119 ---------
.../csharp/Iggy_SDK_Tests/E2ETests/TopicsE2E.cs | 209 ----------------
foreign/csharp/Iggy_SDK_Tests/E2ETests/UsersE2E.cs | 226 -----------------
.../csharp/Iggy_SDK_Tests/Iggy_SDK_Tests.csproj | 30 +--
.../Iggy_SDK_Tests/MapperTests/BinaryMapper.cs | 3 +-
.../csharp/Iggy_SDK_Tests/Utils/PriorityOrderer.cs | 13 +-
.../Iggy_SDK_Tests/Utils/Topics/TopicFactory.cs | 28 +--
.../Iggy_Sample_Consumer.csproj | 2 +-
foreign/csharp/Iggy_Sample_Consumer/Program.cs | 16 +-
.../Iggy_Sample_Producer.csproj | 6 +-
foreign/csharp/Iggy_Sample_Producer/Program.cs | 14 +-
foreign/csharp/README.md | 23 +-
.../go/benchmarks/send_messages_benchmark_test.go | 10 +-
.../binary_request_serializer.go | 2 +-
.../binary_response_deserializer.go | 4 +-
.../create_stream_serializer.go | 2 +-
.../create_stream_serializer_test.go | 2 +-
.../create_topic_serializer.go | 2 +-
.../fetch_messages_request_serializer.go | 2 +-
.../fetch_messages_request_serializer_test.go | 2 +-
.../binary_serialization/identifier_serializer.go | 2 +-
.../identifier_serializer_test.go | 2 +-
.../log_in_request_serializer.go | 2 +-
.../send_messages_request_serializer.go | 2 +-
.../send_messages_request_serializer_test.go | 10 +-
.../go/binary_serialization/stats_serializer.go | 2 +-
.../update_stream_serializer.go | 2 +-
.../update_stream_serializer_test.go | 2 +-
.../update_topic_serializer.go | 2 +-
.../update_topic_serializer_test.go | 2 +-
foreign/go/cli/cli.go | 4 +-
foreign/go/contracts/message_header.go | 26 +-
foreign/go/contracts/messages.go | 77 ++++--
.../go/e2e/tcp_test/consumers_feature_create.go | 2 +-
.../go/e2e/tcp_test/consumers_feature_delete.go | 4 +-
.../go/e2e/tcp_test/consumers_feature_get_all.go | 2 +-
.../go/e2e/tcp_test/consumers_feature_get_by_id.go | 4 +-
foreign/go/e2e/tcp_test/consumers_feature_join.go | 2 +-
foreign/go/e2e/tcp_test/consumers_feature_leave.go | 2 +-
foreign/go/e2e/tcp_test/consumers_steps.go | 4 +-
foreign/go/e2e/tcp_test/messages_feature_send.go | 2 +-
foreign/go/e2e/tcp_test/messages_steps.go | 13 +-
.../go/e2e/tcp_test/partitions_feature_create.go | 2 +-
.../go/e2e/tcp_test/partitions_feature_delete.go | 2 +-
foreign/go/e2e/tcp_test/partitions_steps.go | 4 +-
foreign/go/e2e/tcp_test/pat_feature_create.go | 2 +-
foreign/go/e2e/tcp_test/pat_feature_delete.go | 2 +-
foreign/go/e2e/tcp_test/pat_steps.go | 4 +-
foreign/go/e2e/tcp_test/session_feature_login.go | 2 +-
foreign/go/e2e/tcp_test/stream_feature_create.go | 2 +-
foreign/go/e2e/tcp_test/stream_feature_delete.go | 2 +-
.../go/e2e/tcp_test/stream_feature_get_by_id.go | 2 +-
foreign/go/e2e/tcp_test/stream_feature_update.go | 2 +-
foreign/go/e2e/tcp_test/stream_steps.go | 6 +-
foreign/go/e2e/tcp_test/test_helpers.go | 4 +-
foreign/go/e2e/tcp_test/test_shared_steps.go | 2 +-
foreign/go/e2e/tcp_test/topic_feature_create.go | 2 +-
foreign/go/e2e/tcp_test/topic_feature_delete.go | 4 +-
foreign/go/e2e/tcp_test/topic_feature_get_all.go | 2 +-
foreign/go/e2e/tcp_test/topic_feature_get_by_id.go | 4 +-
foreign/go/e2e/tcp_test/topic_feature_update.go | 2 +-
foreign/go/e2e/tcp_test/topic_steps.go | 6 +-
foreign/go/e2e/tcp_test/users_feature_create.go | 2 +-
foreign/go/e2e/tcp_test/users_feature_delete.go | 2 +-
foreign/go/e2e/tcp_test/users_feature_get_by_id.go | 2 +-
foreign/go/e2e/tcp_test/users_feature_password.go | 2 +-
.../go/e2e/tcp_test/users_feature_permissions.go | 2 +-
foreign/go/e2e/tcp_test/users_feature_update.go | 2 +-
foreign/go/e2e/tcp_test/users_steps.go | 4 +-
foreign/go/errors/constants.go | 20 +-
foreign/go/go.mod | 2 +-
foreign/go/message_stream.go | 2 +-
foreign/go/message_stream_factory.go | 4 +-
foreign/go/samples/README.md | 4 +-
foreign/go/samples/consumer/README.md | 2 +-
foreign/go/samples/consumer/consumer.go | 6 +-
foreign/go/samples/producer/README.md | 2 +-
foreign/go/samples/producer/message_generator.go | 2 +-
foreign/go/samples/producer/producer.go | 8 +-
foreign/go/tcp/tcp_access_token_managament.go | 4 +-
foreign/go/tcp/tcp_clients_managament.go | 4 +-
foreign/go/tcp/tcp_consumer_group_managament.go | 6 +-
foreign/go/tcp/tcp_core.go | 10 +-
foreign/go/tcp/tcp_messaging.go | 6 +-
foreign/go/tcp/tcp_offset_managament.go | 4 +-
foreign/go/tcp/tcp_partition_managament.go | 4 +-
foreign/go/tcp/tcp_session_managament.go | 4 +-
foreign/go/tcp/tcp_stream_managament.go | 6 +-
foreign/go/tcp/tcp_topic_managament.go | 6 +-
foreign/go/tcp/tcp_user_managament.go | 6 +-
foreign/go/tcp/tcp_utilities.go | 4 +-
foreign/node/README.md | 4 +-
foreign/node/docker-compose.yml | 2 +-
foreign/node/package.json | 3 +-
foreign/python/.devcontainer/docker-compose.yml | 2 +-
foreign/python/Cargo.toml | 2 +-
foreign/python/README.md | 2 +-
foreign/python/pyproject.toml | 10 +-
scripts/run-bdd-tests.sh | 66 +++++
web/README.md | 8 +-
419 files changed, 5086 insertions(+), 4268 deletions(-)
copy .github/workflows/{ci-check-shell.yml => ci-test-bdd.yml} (60%)
create mode 100644 assets/logo/0.5x/iggy-apache-b&wdark@0.5x.png
create mode 100644 assets/logo/0.5x/iggy-apache-b&wlight@0.5x.png
create mode 100644 assets/logo/0.5x/iggy-apache-color-darkbg@0.5x.png
create mode 100644 assets/logo/0.5x/iggy-apache-color-lightbg@0.5x.png
create mode 100644 assets/logo/0.5x/iggy-apache-logo-wo-sygnet-dark@0.5x.png
create mode 100644 assets/logo/0.5x/iggy-apache-logo-wo-sygnet-light@0.5x.png
create mode 100644 assets/logo/0.5x/iggy-apache-sygnet-b&wdark@0.5x.png
create mode 100644 assets/logo/0.5x/iggy-apache-sygnet-b&wlight@0.5x.png
create mode 100644 assets/logo/0.5x/iggy-apache-sygnet-color-darkbg@0.5x.png
create mode 100644 assets/logo/0.5x/iggy-apache-sygnet-color-lightbg@0.5x.png
create mode 100644 assets/logo/4x/iggy-apache-b&wdark@4x.png
create mode 100644 assets/logo/4x/iggy-apache-b&wlight@4x.png
create mode 100644 assets/logo/4x/iggy-apache-color-darkbg@4x.png
create mode 100644 assets/logo/4x/iggy-apache-color-lightbg@4x.png
create mode 100644 assets/logo/4x/iggy-apache-logo-wo-sygnet-dark@4x.png
create mode 100644 assets/logo/4x/iggy-apache-logo-wo-sygnet-light@4x.png
create mode 100644 assets/logo/4x/iggy-apache-sygnet-b&wdark@4x.png
create mode 100644 assets/logo/4x/iggy-apache-sygnet-b&wlight@4x.png
create mode 100644 assets/logo/4x/iggy-apache-sygnet-color-darkbg@4x.png
create mode 100644 assets/logo/4x/iggy-apache-sygnet-color-lightbg@4x.png
create mode 100644 assets/logo/SVG/iggy-apache-b&wdark.svg
create mode 100644 assets/logo/SVG/iggy-apache-b&wlight.svg
copy web/static/iggy-apache-darkbg.svg => assets/logo/SVG/iggy-apache-color-darkbg.svg (100%)
copy web/static/iggy-apache-lightbg.svg => assets/logo/SVG/iggy-apache-color-lightbg.svg (100%)
create mode 100644 assets/logo/SVG/iggy-apache-logo-wo-sygnet-dark.svg
create mode 100644 assets/logo/SVG/iggy-apache-logo-wo-sygnet-light.svg
create mode 100644 assets/logo/SVG/iggy-apache-sygnet-b&wdark.svg
create mode 100644 assets/logo/SVG/iggy-apache-sygnet-b&wlight.svg
copy web/src/lib/components/Logo/LogoMark.svelte => assets/logo/SVG/iggy-apache-sygnet-color-darkbg.svg (83%)
copy web/static/iggy-apache-lightbg.svg => assets/logo/SVG/iggy-apache-sygnet-color-lightbg.svg (83%)
copy Dockerfile.debug => bdd/Dockerfile (59%)
create mode 100644 bdd/README.md
create mode 100644 bdd/docker-compose.yml
create mode 100644 bdd/python/Dockerfile
create mode 100644 bdd/python/requirements.txt
create mode 100644 bdd/python/tests/conftest.py
create mode 100644 bdd/python/tests/test_basic_messaging.py
rename {core/bdd => bdd/rust}/Cargo.toml (79%)
copy foreign/cpp/.devcontainer/post-install.sh => bdd/rust/Dockerfile (70%)
mode change 100755 => 100644
rename {core/bdd => bdd/rust}/tests/basic_messaging.rs (92%)
rename {core/bdd => bdd/rust}/tests/common/global_context.rs (93%)
rename {core/bdd => bdd/rust}/tests/common/mod.rs (100%)
rename {core/bdd => bdd/rust}/tests/helpers/mod.rs (100%)
rename {core/bdd => bdd/rust}/tests/helpers/test_data.rs (100%)
rename {core/bdd => bdd/rust}/tests/steps/auth.rs (100%)
rename {core/bdd => bdd/rust}/tests/steps/messages.rs (93%)
rename {core/bdd => bdd/rust}/tests/steps/mod.rs (100%)
copy {core/bdd => bdd/rust}/tests/steps/server.rs (57%)
rename {core/bdd => bdd/rust}/tests/steps/streams.rs (100%)
rename {core/bdd => bdd/rust}/tests/steps/topics.rs (100%)
rename bdd/{ => scenarios}/basic_messaging.feature (99%)
rename core/{bdd/tests/steps/server.rs => common/src/certificates.rs} (58%)
copy core/sdk/src/{quic/skip_server_verification.rs => tcp/tcp_tls_verifier.rs} (51%)
create mode 100644 foreign/csharp/.editorconfig
create mode 100644 foreign/csharp/Directory.Packages.props
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Attributes/SkipHttpAttribute.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Attributes/SkipTcpAttribute.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/ConsumerGroupTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/ConsumerGroupFixture.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/FetchMessagesFixture.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/FlushMessageFixture.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/OffsetFixtures.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/PartitionsFixture.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/PollMessagesFixture.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/SendMessageFixture.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/SystemFixture.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/FlushMessagesTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Helpers/StreamFactory.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Helpers/TopicFactory.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Models/DummyMessage.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/OffsetTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/PartitionsTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/PersonalAccessTokenTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/PollMessagesTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/SystemTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs
delete mode 100644 foreign/csharp/Iggy_SDK/JsonConfiguration/UserResponseConverter.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/ConsumerGroupE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/FetchMessagesE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/ConsumerGroupFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/FetchMessagesFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/FlushMessagesFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/GeneralFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/OffsetFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/PATFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/PartitionsFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/PollMessagesFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/SendMessagesFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/StreamsFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/SystemFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/TopicsFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Bootstraps/UsersFixtureBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Configs/IggyFixtureClientConfigs.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IIggyBootstrap.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyBaseFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyConsumerGroupFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyFetchMessagesFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyFlushMessagesFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyGeneralFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyOffsetFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyPATFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyPartitionFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyPollMessagesFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggySendMessagesFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyStreamFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggySystemFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyTcpUsersFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/IggyTopicFixture.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/Fixtures/Models/IggyClientModel.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/FlushMessagesE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/OffsetE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/PATE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/PartitionsE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/PollMessagesE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/SendMessagesE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/StreamsE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/SystemE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/TopicsE2E.cs
delete mode 100644 foreign/csharp/Iggy_SDK_Tests/E2ETests/UsersE2E.cs
create mode 100755 scripts/run-bdd-tests.sh
(iggy) 09/12: begin fix cg
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 5549b010c48f798b1cd834a196256fa999624068
Author: numinex <g....@gmail.com>
AuthorDate: Tue Jun 24 18:24:30 2025 +0200
begin fix cg
---
Cargo.lock | 1 +
core/server/Cargo.toml | 1 +
core/server/src/binary/command.rs | 7 +-
core/server/src/binary/sender.rs | 28 ++++---
core/server/src/channels/commands/archive_state.rs | 4 +
.../commands/clean_personal_access_tokens.rs | 3 +
.../src/channels/commands/maintain_messages.rs | 5 ++
core/server/src/channels/commands/print_sysinfo.rs | 4 +
core/server/src/channels/commands/save_messages.rs | 4 +
.../src/channels/commands/verify_heartbeats.rs | 3 +
core/server/src/channels/handler.rs | 4 +
core/server/src/channels/server_command.rs | 4 +
core/server/src/configs/defaults.rs | 3 +-
core/server/src/http/http_server.rs | 3 +
core/server/src/quic/listener.rs | 9 +-
core/server/src/quic/quic_sender.rs | 15 +++-
core/server/src/quic/quic_server.rs | 4 +-
core/server/src/shard/builder.rs | 2 +-
core/server/src/shard/mod.rs | 95 +++++++++++++++++-----
core/server/src/shard/namespace.rs | 4 +-
core/server/src/shard/system/clients.rs | 10 +--
core/server/src/shard/system/snapshot/mod.rs | 1 -
core/server/src/shard/system/storage.rs | 1 +
core/server/src/shard/system/streams.rs | 1 -
core/server/src/shard/transmission/message.rs | 14 ++--
.../server/src/streaming/clients/client_manager.rs | 65 +++++++--------
core/server/src/streaming/partitions/messages.rs | 5 +-
core/server/src/streaming/partitions/partition.rs | 20 +++--
core/server/src/streaming/segments/segment.rs | 9 +-
core/server/src/streaming/session.rs | 21 +++--
core/server/src/streaming/storage.rs | 42 +++++-----
core/server/src/streaming/streams/storage.rs | 2 +-
core/server/src/streaming/streams/stream.rs | 10 +--
core/server/src/streaming/streams/topics.rs | 9 +-
core/server/src/streaming/topics/consumer_group.rs | 12 +--
.../server/src/streaming/topics/consumer_groups.rs | 16 ++--
core/server/src/streaming/topics/topic.rs | 18 ++--
core/server/src/streaming/utils/memory_pool.rs | 5 +-
core/server/src/tcp/connection_handler.rs | 32 +++++---
core/server/src/tcp/sender.rs | 53 +++++++-----
core/server/src/tcp/tcp_listener.rs | 45 ++++++----
core/server/src/tcp/tcp_sender.rs | 9 +-
core/server/src/tcp/tcp_server.rs | 4 +-
core/server/src/tcp/tcp_socket.rs | 24 ++++--
core/server/src/tcp/tcp_tls_listener.rs | 11 ++-
core/server/src/tcp/tcp_tls_sender.rs | 7 +-
46 files changed, 408 insertions(+), 241 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 944bbfc7..1b76ae4b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6952,6 +6952,7 @@ dependencies = [
"serde_with",
"serial_test",
"sharded_queue",
+ "socket2",
"static-toml",
"strum",
"sysinfo 0.35.2",
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index d2cfc24a..1246e049 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -61,6 +61,7 @@ futures = { workspace = true }
human-repr = { workspace = true }
iggy_common = { workspace = true }
jsonwebtoken = "9.3.1"
+socket2 = "0.5.10"
lending-iterator = "0.1.7"
hash32 = "1.0.0"
mimalloc = { workspace = true, optional = true }
diff --git a/core/server/src/binary/command.rs b/core/server/src/binary/command.rs
index 0b997941..aa935e09 100644
--- a/core/server/src/binary/command.rs
+++ b/core/server/src/binary/command.rs
@@ -16,8 +16,11 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::sender::SenderKind;
use crate::define_server_command_enum;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
use crate::streaming::systems::system::SharedSystem;
use bytes::{BufMut, Bytes, BytesMut};
@@ -130,8 +133,8 @@ pub trait ServerCommandHandler {
self,
sender: &mut SenderKind,
length: u32,
- session: &Session,
- system: &SharedSystem,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError>;
}
diff --git a/core/server/src/binary/sender.rs b/core/server/src/binary/sender.rs
index 2ccf349b..06e9c51f 100644
--- a/core/server/src/binary/sender.rs
+++ b/core/server/src/binary/sender.rs
@@ -22,10 +22,13 @@ use std::io::IoSlice;
use crate::tcp::tcp_sender::TcpSender;
use crate::tcp::tcp_tls_sender::TcpTlsSender;
use crate::{quic::quic_sender::QuicSender, server_error::ServerError};
+use bytes::BytesMut;
use iggy_common::IggyError;
-use quinn::{RecvStream, SendStream};
+use monoio::buf::IoBufMut;
use monoio::net::TcpStream;
-use tokio_native_tls::TlsStream;
+use monoio_native_tls::TlsStream;
+use nix::libc;
+use quinn::{RecvStream, SendStream};
macro_rules! forward_async_methods {
(
@@ -48,25 +51,24 @@ macro_rules! forward_async_methods {
}
pub trait Sender {
- fn read(&mut self, buffer: &mut [u8]) -> impl Future<Output = Result<usize, IggyError>>;
- fn send_empty_ok_response(&mut self) -> impl Future<Output = Result<(), IggyError>>;
- fn send_ok_response(
+ fn read(
&mut self,
- payload: &[u8],
- ) -> impl Future<Output = Result<(), IggyError>>;
+ buffer: BytesMut,
+ ) -> impl Future<Output = (Result<usize, IggyError>, BytesMut)>;
+ fn send_empty_ok_response(&mut self) -> impl Future<Output = Result<(), IggyError>>;
+ fn send_ok_response(&mut self, payload: &[u8]) -> impl Future<Output = Result<(), IggyError>>;
fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<IoSlice<'_>>,
+ slices: Vec<libc::iovec>,
) -> impl Future<Output = Result<(), IggyError>>;
fn send_error_response(
&mut self,
error: IggyError,
- ) -> impl Future<Output = Result<(), IggyError>>;
- fn shutdown(&mut self) -> impl Future<Output = Result<(), ServerError>>;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn shutdown(&mut self) -> impl Future<Output = Result<(), ServerError>>;
}
-#[allow(clippy::large_enum_variant)]
pub enum SenderKind {
Tcp(TcpSender),
TcpTls(TcpTlsSender),
@@ -90,10 +92,10 @@ impl SenderKind {
}
forward_async_methods! {
- async fn read(&mut self, buffer: &mut [u8]) -> Result<usize, IggyError>;
+ async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut);
async fn send_empty_ok_response(&mut self) -> Result<(), IggyError>;
async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError>;
- async fn send_ok_response_vectored(&mut self, length: &[u8], slices: Vec<IoSlice<'_>>) -> Result<(), IggyError>;
+ async fn send_ok_response_vectored(&mut self, length: &[u8], slices: Vec<libc::iovec>) -> Result<(), IggyError>;
async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError>;
async fn shutdown(&mut self) -> Result<(), ServerError>;
}
diff --git a/core/server/src/channels/commands/archive_state.rs b/core/server/src/channels/commands/archive_state.rs
index e80d17d3..f42393ae 100644
--- a/core/server/src/channels/commands/archive_state.rs
+++ b/core/server/src/channels/commands/archive_state.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//TODO: Fixme
+/*
use crate::channels::server_command::BackgroundServerCommand;
use crate::configs::server::StateMaintenanceConfig;
use crate::streaming::systems::system::SharedSystem;
@@ -137,3 +139,5 @@ impl BackgroundServerCommand<ArchiveStateCommand> for ArchiveStateExecutor {
});
}
}
+
+*/
diff --git a/core/server/src/channels/commands/clean_personal_access_tokens.rs b/core/server/src/channels/commands/clean_personal_access_tokens.rs
index c815a18f..fd2eaf11 100644
--- a/core/server/src/channels/commands/clean_personal_access_tokens.rs
+++ b/core/server/src/channels/commands/clean_personal_access_tokens.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+/*
use crate::channels::server_command::BackgroundServerCommand;
use crate::configs::server::PersonalAccessTokenCleanerConfig;
use crate::streaming::systems::system::SharedSystem;
@@ -135,3 +136,5 @@ impl BackgroundServerCommand<CleanPersonalAccessTokensCommand>
});
}
}
+
+*/
diff --git a/core/server/src/channels/commands/maintain_messages.rs b/core/server/src/channels/commands/maintain_messages.rs
index 16ef3647..2dc526e2 100644
--- a/core/server/src/channels/commands/maintain_messages.rs
+++ b/core/server/src/channels/commands/maintain_messages.rs
@@ -16,6 +16,9 @@
* under the License.
*/
+//TODO: Fixme
+/*
+
use crate::archiver::ArchiverKind;
use crate::channels::server_command::BackgroundServerCommand;
use crate::configs::server::MessagesMaintenanceConfig;
@@ -553,3 +556,5 @@ async fn delete_segments(
messages_count,
})
}
+
+*/
diff --git a/core/server/src/channels/commands/print_sysinfo.rs b/core/server/src/channels/commands/print_sysinfo.rs
index c130b58c..c0f2b1a2 100644
--- a/core/server/src/channels/commands/print_sysinfo.rs
+++ b/core/server/src/channels/commands/print_sysinfo.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+// TODO: Fixme
+/*
use crate::{
channels::server_command::BackgroundServerCommand,
configs::server::ServerConfig,
@@ -121,3 +123,5 @@ impl BackgroundServerCommand<SysInfoPrintCommand> for SysInfoPrintExecutor {
});
}
}
+
+*/
diff --git a/core/server/src/channels/commands/save_messages.rs b/core/server/src/channels/commands/save_messages.rs
index 586fb8b0..bf832762 100644
--- a/core/server/src/channels/commands/save_messages.rs
+++ b/core/server/src/channels/commands/save_messages.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//Todo: Fixme
+/*
use crate::channels::server_command::BackgroundServerCommand;
use crate::configs::server::MessageSaverConfig;
use crate::configs::server::ServerConfig;
@@ -116,3 +118,5 @@ impl BackgroundServerCommand<SaveMessagesCommand> for SaveMessagesExecutor {
});
}
}
+
+*/
diff --git a/core/server/src/channels/commands/verify_heartbeats.rs b/core/server/src/channels/commands/verify_heartbeats.rs
index ba48c69e..b2b01c49 100644
--- a/core/server/src/channels/commands/verify_heartbeats.rs
+++ b/core/server/src/channels/commands/verify_heartbeats.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//TODO: Fixme
+/*
use crate::channels::server_command::BackgroundServerCommand;
use crate::configs::server::HeartbeatConfig;
use crate::streaming::systems::system::SharedSystem;
@@ -148,3 +150,4 @@ impl BackgroundServerCommand<VerifyHeartbeatsCommand> for VerifyHeartbeatsExecut
});
}
}
+*/
diff --git a/core/server/src/channels/handler.rs b/core/server/src/channels/handler.rs
index 8bd82e0d..7612a259 100644
--- a/core/server/src/channels/handler.rs
+++ b/core/server/src/channels/handler.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//TODO: Fixme
+/*
use super::server_command::BackgroundServerCommand;
use crate::configs::server::ServerConfig;
use crate::streaming::systems::system::SharedSystem;
@@ -44,3 +46,5 @@ impl<'a> BackgroundServerCommandHandler<'a> {
}
}
}
+
+*/
diff --git a/core/server/src/channels/server_command.rs b/core/server/src/channels/server_command.rs
index 326b3d77..16b0dc2f 100644
--- a/core/server/src/channels/server_command.rs
+++ b/core/server/src/channels/server_command.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//TODO: Fixme
+/*
use crate::configs::server::ServerConfig;
use crate::streaming::systems::system::SharedSystem;
use flume::{Receiver, Sender};
@@ -38,3 +40,5 @@ pub trait BackgroundServerCommand<C> {
receiver: Receiver<C>,
);
}
+
+*/
diff --git a/core/server/src/configs/defaults.rs b/core/server/src/configs/defaults.rs
index 82b560cf..fc333ec7 100644
--- a/core/server/src/configs/defaults.rs
+++ b/core/server/src/configs/defaults.rs
@@ -36,6 +36,7 @@ use crate::configs::system::{
use crate::configs::tcp::{TcpConfig, TcpTlsConfig};
use iggy_common::IggyByteSize;
use iggy_common::IggyDuration;
+use std::rc::Rc;
use std::sync::Arc;
use std::time::Duration;
@@ -51,7 +52,7 @@ impl Default for ServerConfig {
heartbeat: HeartbeatConfig::default(),
message_saver: MessageSaverConfig::default(),
personal_access_token: PersonalAccessTokenConfig::default(),
- system: Arc::new(SystemConfig::default()),
+ system: Rc::new(SystemConfig::default()),
quic: QuicConfig::default(),
tcp: TcpConfig::default(),
http: HttpConfig::default(),
diff --git a/core/server/src/http/http_server.rs b/core/server/src/http/http_server.rs
index b91857f5..7dec1a7f 100644
--- a/core/server/src/http/http_server.rs
+++ b/core/server/src/http/http_server.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+//TODO: Fixme
+/*
use crate::configs::http::{HttpConfig, HttpCorsConfig};
use crate::http::diagnostics::request_diagnostics;
use crate::http::jwt::cleaner::start_expired_tokens_cleaner;
@@ -191,3 +193,4 @@ fn configure_cors(config: HttpCorsConfig) -> CorsLayer {
.allow_credentials(config.allow_credentials)
.allow_private_network(config.allow_private_network)
}
+*/
diff --git a/core/server/src/quic/listener.rs b/core/server/src/quic/listener.rs
index e9b329de..f5cfc429 100644
--- a/core/server/src/quic/listener.rs
+++ b/core/server/src/quic/listener.rs
@@ -16,12 +16,14 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{ServerCommand, ServerCommandHandler};
use crate::binary::sender::SenderKind;
use crate::server_error::ConnectionError;
+use crate::shard::IggyShard;
use crate::streaming::clients::client_manager::Transport;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::anyhow;
use iggy_common::IggyError;
use quinn::{Connection, Endpoint, RecvStream, SendStream};
@@ -29,6 +31,8 @@ use tracing::{error, info, trace};
const LISTENERS_COUNT: u32 = 10;
const INITIAL_BYTES_LENGTH: usize = 4;
+//TODO: Fixme
+/*
pub fn start(endpoint: Endpoint, system: SharedSystem) {
for _ in 0..LISTENERS_COUNT {
@@ -65,7 +69,7 @@ pub fn start(endpoint: Endpoint, system: SharedSystem) {
async fn handle_connection(
incoming_connection: quinn::Connecting,
- system: SharedSystem,
+ shard: Rc<IggyShard>,
) -> Result<(), ConnectionError> {
let connection = incoming_connection.await?;
let address = connection.remote_address();
@@ -179,3 +183,4 @@ async fn handle_stream(
}
}
}
+*/
diff --git a/core/server/src/quic/quic_sender.rs b/core/server/src/quic/quic_sender.rs
index f1631cb2..17b60818 100644
--- a/core/server/src/quic/quic_sender.rs
+++ b/core/server/src/quic/quic_sender.rs
@@ -18,8 +18,11 @@
use crate::quic::COMPONENT;
use crate::{binary::sender::Sender, server_error::ServerError};
+use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::IggyError;
+use monoio::buf::IoBufMut;
+use nix::libc;
use quinn::{RecvStream, SendStream};
use std::io::IoSlice;
use tracing::{debug, error};
@@ -33,15 +36,18 @@ pub struct QuicSender {
}
impl Sender for QuicSender {
- async fn read(&mut self, buffer: &mut [u8]) -> Result<usize, IggyError> {
+ async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut) {
// Not-so-nice code because quinn recv stream has different API for read_exact
+ /*
let read_bytes = buffer.len();
self.recv.read_exact(buffer).await.map_err(|error| {
error!("Failed to read from the stream: {:?}", error);
IggyError::QuicError
})?;
+ */
- Ok(read_bytes)
+ //Ok(read_bytes)
+ todo!();
}
async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> {
@@ -64,7 +70,7 @@ impl Sender for QuicSender {
async fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<IoSlice<'_>>,
+ slices: Vec<libc::iovec>,
) -> Result<(), IggyError> {
debug!("Sending vectored response with status: {:?}...", STATUS_OK);
@@ -79,6 +85,8 @@ impl Sender for QuicSender {
let mut total_bytes_written = 0;
+ //TODO: Fixme
+ /*
for slice in slices {
let slice_data = &*slice;
if !slice_data.is_empty() {
@@ -93,6 +101,7 @@ impl Sender for QuicSender {
total_bytes_written += slice_data.len();
}
}
+ */
debug!(
"Sent vectored response: {} bytes of payload",
diff --git a/core/server/src/quic/quic_server.rs b/core/server/src/quic/quic_server.rs
index 7b8edf21..0a8020b8 100644
--- a/core/server/src/quic/quic_server.rs
+++ b/core/server/src/quic/quic_server.rs
@@ -31,8 +31,9 @@ use crate::configs::quic::QuicConfig;
use crate::quic::COMPONENT;
use crate::quic::listener;
use crate::server_error::QuicError;
-use crate::streaming::systems::system::SharedSystem;
+//TODO: Fixme
+/*
/// Starts the QUIC server.
/// Returns the address the server is listening on.
pub fn start(config: QuicConfig, system: SharedSystem) -> SocketAddr {
@@ -136,3 +137,4 @@ fn load_certificates(
let key = keys.remove(0);
Ok((certs, key))
}
+ */
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 0428a483..a2a3fdfb 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -26,7 +26,7 @@ use crate::{
configs::server::ServerConfig,
map_toggle_str,
shard::Shard,
- state::{file::FileState, StateKind},
+ state::{StateKind, file::FileState},
streaming::{diagnostics::metrics::Metrics, storage::SystemStorage},
versioning::SemanticVersion,
};
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 3e7e10f1..12ba64bb 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -17,9 +17,9 @@
*/
pub mod builder;
-pub mod system;
pub mod gate;
pub mod namespace;
+pub mod system;
pub mod transmission;
use ahash::{AHashMap, AHashSet, HashMap};
@@ -29,18 +29,42 @@ use futures::future::try_join_all;
use iggy_common::{EncryptorKind, IggyError, UserId};
use namespace::IggyNamespace;
use std::{
- cell::{Cell, RefCell}, pin::Pin, rc::Rc, str::FromStr, sync::{atomic::{AtomicU32, Ordering}, Arc, RwLock}, time::Instant
+ cell::{Cell, RefCell},
+ pin::Pin,
+ rc::Rc,
+ str::FromStr,
+ sync::{
+ Arc, RwLock,
+ atomic::{AtomicU32, Ordering},
+ },
+ time::Instant,
};
use tracing::{error, info, instrument, trace, warn};
use transmission::connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use crate::{
configs::server::ServerConfig,
- shard::{system::info::SystemInfo, transmission::frame::ShardFrame},
+ shard::{
+ system::info::SystemInfo,
+ transmission::{
+ frame::ShardFrame,
+ message::{ShardEvent, ShardMessage},
+ },
+ },
state::{
- file::FileState, system::{StreamState, SystemState, UserState}, StateKind
+ StateKind,
+ file::FileState,
+ system::{StreamState, SystemState, UserState},
+ },
+ streaming::{
+ clients::client_manager::ClientManager,
+ diagnostics::metrics::Metrics,
+ personal_access_tokens::personal_access_token::PersonalAccessToken,
+ session::Session,
+ storage::SystemStorage,
+ streams::stream::Stream,
+ users::{permissioner::Permissioner, user::User},
},
- streaming::{clients::client_manager::ClientManager, diagnostics::metrics::Metrics, personal_access_tokens::personal_access_token::PersonalAccessToken, session::Session, storage::SystemStorage, streams::stream::Stream, users::{permissioner::Permissioner, user::User}},
versioning::SemanticVersion,
};
@@ -83,7 +107,7 @@ pub struct IggyShard {
pub(crate) config: ServerConfig,
//TODO: This could be shared.
pub(crate) client_manager: RefCell<ClientManager>,
- pub(crate) active_sessions: RefCell<Vec<Session>>,
+ pub(crate) active_sessions: RefCell<Vec<Rc<Session>>>,
pub(crate) permissioner: RefCell<Permissioner>,
pub(crate) users: RefCell<HashMap<UserId, User>>,
@@ -244,8 +268,8 @@ impl IggyShard {
info!("Loading streams from disk...");
let mut unloaded_streams = Vec::new();
// Does mononio has api for that ?
- let mut dir_entries = std::fs::read_dir(&self.config.system.get_streams_path())
- .map_err(|error| {
+ let mut dir_entries =
+ std::fs::read_dir(&self.config.system.get_streams_path()).map_err(|error| {
error!("Cannot read streams directory: {error}");
IggyError::CannotReadStreams
})?;
@@ -362,16 +386,21 @@ impl IggyShard {
self.metrics.increment_messages(stream.get_messages_count());
self.streams_ids
- .borrow_mut()
+ .borrow_mut()
.insert(stream.name.clone(), stream.stream_id);
self.streams.borrow_mut().insert(stream.stream_id, stream);
}
- info!("Loaded {} stream(s) from disk.", self.streams.borrow().len());
+ info!(
+ "Loaded {} stream(s) from disk.",
+ self.streams.borrow().len()
+ );
Ok(())
}
- pub fn assert_init(&self) -> Result<(), IggyError> { Ok(())}
+ pub fn assert_init(&self) -> Result<(), IggyError> {
+ Ok(())
+ }
#[instrument(skip_all, name = "trace_shutdown")]
pub async fn shutdown(&mut self) -> Result<(), IggyError> {
@@ -397,17 +426,41 @@ impl IggyShard {
self.shards.len() as u32
}
- pub fn ensure_authenticated(&self, client_id: u32) -> Result<u32, IggyError> {
+ pub fn broadcast_event_to_all_shards(&self, client_id: u32, event: Rc<ShardEvent>) {
+ self.shards
+ .iter()
+ .filter_map(|shard| {
+ if shard.id != self.id {
+ Some(shard.connection.clone())
+ } else {
+ None
+ }
+ })
+ .map(|conn| {
+ let message = ShardMessage::Event(event.clone());
+ conn.send(ShardFrame::new(client_id, message, None));
+ })
+ .collect::<Vec<_>>();
+ }
+
+ pub fn add_active_session(&self, session: Rc<Session>) {
+ self.active_sessions.borrow_mut().push(session);
+ }
+
+ pub fn ensure_authenticated(&self, session: &Session) -> Result<u32, IggyError> {
let active_sessions = self.active_sessions.borrow();
- let session = active_sessions
+ let user_id = active_sessions
.iter()
- .find(|s| s.client_id == client_id)
- .ok_or_else(|| IggyError::Unauthenticated)?;
- if session.is_authenticated() {
- Ok(session.get_user_id())
- } else {
- error!("{COMPONENT} - unauthenticated access attempt, session: {session}");
- Err(IggyError::Unauthenticated)
- }
+ .find(|s| s.get_user_id() == session.get_user_id())
+ .ok_or_else(|| IggyError::Unauthenticated)
+ .and_then(|session| {
+ if session.is_authenticated() {
+ Ok(session.get_user_id())
+ } else {
+ error!("{COMPONENT} - unauthenticated access attempt, session: {session}");
+ Err(IggyError::Unauthenticated)
+ }
+ })?;
+ Ok(user_id)
}
}
diff --git a/core/server/src/shard/namespace.rs b/core/server/src/shard/namespace.rs
index d9d68ec9..a7c899aa 100644
--- a/core/server/src/shard/namespace.rs
+++ b/core/server/src/shard/namespace.rs
@@ -16,9 +16,9 @@
* under the License.
*/
-use std::hash::Hasher as _;
-use iggy_common::Identifier;
use hash32::{Hasher, Murmur3Hasher};
+use iggy_common::Identifier;
+use std::hash::Hasher as _;
//TODO: Will probably want to move it to separate crate so we can share it with sdk.
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs
index 2cb0fbe8..fde182ce 100644
--- a/core/server/src/shard/system/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -16,22 +16,22 @@
* under the License.
*/
+use crate::shard::IggyShard;
use crate::streaming::clients::client_manager::{Client, Transport};
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
use iggy_common::locking::IggySharedMut;
use iggy_common::locking::IggySharedMutFn;
use std::net::SocketAddr;
+use std::rc::Rc;
use std::sync::Arc;
use tracing::{error, info};
impl IggyShard {
- pub async fn add_client(&self, address: &SocketAddr, transport: Transport) -> Arc<Session> {
- let mut client_manager = self.client_manager.write().await;
+ pub fn add_client(&self, address: &SocketAddr, transport: Transport) -> Rc<Session> {
+ let mut client_manager = self.client_manager.borrow_mut();
let session = client_manager.add_client(address, transport);
info!("Added {transport} client with session: {session} for IP address: {address}");
self.metrics.increment_clients(1);
@@ -42,7 +42,7 @@ impl IggyShard {
let consumer_groups: Vec<(u32, u32, u32)>;
{
- let mut client_manager = self.client_manager.write().await;
+ let mut client_manager = self.client_manager.borrow_mut();
let client = client_manager.delete_client(client_id).await;
if client.is_none() {
error!("Client with ID: {client_id} was not found in the client manager.",);
diff --git a/core/server/src/shard/system/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs
index 92b0c400..f6e07785 100644
--- a/core/server/src/shard/system/snapshot/mod.rs
+++ b/core/server/src/shard/system/snapshot/mod.rs
@@ -20,7 +20,6 @@ mod procdump;
use crate::configs::system::SystemConfig;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::System;
use async_zip::tokio::write::ZipFileWriter;
use async_zip::{Compression, ZipEntryBuilder};
use iggy_common::{IggyDuration, IggyError, Snapshot, SnapshotCompression, SystemSnapshotType};
diff --git a/core/server/src/shard/system/storage.rs b/core/server/src/shard/system/storage.rs
index 9a493751..4cb41120 100644
--- a/core/server/src/shard/system/storage.rs
+++ b/core/server/src/shard/system/storage.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use crate::shard::system::info::SystemInfo;
use crate::streaming::persistence::persister::PersisterKind;
use crate::streaming::storage::SystemInfoStorage;
use crate::streaming::utils::PooledBuffer;
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index efcfff2b..8a0772a9 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -26,7 +26,6 @@ use std::sync::atomic::{AtomicU32, Ordering};
use tokio::fs;
use tracing::{error, info, warn};
-
impl IggyShard {
pub fn get_streams(&self) -> Vec<&Stream> {
self.streams.values().collect()
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index 3572c0ec..8d1e66eb 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -1,3 +1,5 @@
+use std::rc::Rc;
+
/* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,16 +17,18 @@
* specific language governing permissions and limitations
* under the License.
*/
-use crate::binary::command::ServerCommand;
+use crate::{binary::command::ServerCommand, streaming::session::Session};
#[derive(Debug)]
pub enum ShardMessage {
Command(ServerCommand),
- Event(ShardEvent),
+ Event(Rc<ShardEvent>),
}
#[derive(Debug)]
-pub enum ShardEvent {}
+pub enum ShardEvent {
+ NewSession(Rc<Session>),
+}
impl From<ServerCommand> for ShardMessage {
fn from(command: ServerCommand) -> Self {
@@ -32,8 +36,8 @@ impl From<ServerCommand> for ShardMessage {
}
}
-impl From<ShardEvent> for ShardMessage {
- fn from(event: ShardEvent) -> Self {
+impl From<Rc<ShardEvent>> for ShardMessage {
+ fn from(event: Rc<ShardEvent>) -> Self {
ShardMessage::Event(event)
}
}
diff --git a/core/server/src/streaming/clients/client_manager.rs b/core/server/src/streaming/clients/client_manager.rs
index 4a84a58c..fc5c00cc 100644
--- a/core/server/src/streaming/clients/client_manager.rs
+++ b/core/server/src/streaming/clients/client_manager.rs
@@ -22,27 +22,26 @@ use ahash::AHashMap;
use iggy_common::IggyError;
use iggy_common::IggyTimestamp;
use iggy_common::UserId;
-use iggy_common::locking::IggySharedMut;
use iggy_common::locking::IggySharedMutFn;
use std::fmt::{Display, Formatter};
use std::net::SocketAddr;
-use std::sync::Arc;
+use std::rc::Rc;
#[derive(Debug, Default)]
pub struct ClientManager {
- clients: AHashMap<u32, IggySharedMut<Client>>,
+ clients: AHashMap<u32, Client>,
}
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct Client {
pub user_id: Option<u32>,
- pub session: Arc<Session>,
+ pub session: Rc<Session>,
pub transport: Transport,
pub consumer_groups: Vec<ConsumerGroup>,
pub last_heartbeat: IggyTimestamp,
}
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct ConsumerGroup {
pub stream_id: u32,
pub topic_id: u32,
@@ -65,9 +64,9 @@ impl Display for Transport {
}
impl ClientManager {
- pub fn add_client(&mut self, address: &SocketAddr, transport: Transport) -> Arc<Session> {
+ pub fn add_client(&mut self, address: &SocketAddr, transport: Transport) -> Rc<Session> {
let client_id = hash::calculate_32(address.to_string().as_bytes());
- let session = Arc::new(Session::from_client_id(client_id, *address));
+ let session = Rc::new(Session::from_client_id(client_id, *address));
let client = Client {
user_id: None,
session: session.clone(),
@@ -75,44 +74,43 @@ impl ClientManager {
consumer_groups: Vec::new(),
last_heartbeat: IggyTimestamp::now(),
};
- self.clients.insert(client_id, IggySharedMut::new(client));
+ self.clients.insert(client_id, client);
session
}
- pub async fn set_user_id(&mut self, client_id: u32, user_id: UserId) -> Result<(), IggyError> {
- let client = self.clients.get(&client_id);
+ pub fn set_user_id(&mut self, client_id: u32, user_id: UserId) -> Result<(), IggyError> {
+ let client = self.clients.get_mut(&client_id);
if client.is_none() {
return Err(IggyError::ClientNotFound(client_id));
}
- let mut client = client.unwrap().write().await;
+ let mut client = client.unwrap();
client.user_id = Some(user_id);
Ok(())
}
- pub async fn clear_user_id(&mut self, client_id: u32) -> Result<(), IggyError> {
- let client = self.clients.get(&client_id);
+ pub fn clear_user_id(&mut self, client_id: u32) -> Result<(), IggyError> {
+ let client = self.clients.get_mut(&client_id);
if client.is_none() {
return Err(IggyError::ClientNotFound(client_id));
}
- let mut client = client.unwrap().write().await;
+ let mut client = client.unwrap();
client.user_id = None;
Ok(())
}
- pub fn try_get_client(&self, client_id: u32) -> Option<IggySharedMut<Client>> {
+ pub fn try_get_client(&self, client_id: u32) -> Option<Client> {
self.clients.get(&client_id).cloned()
}
- pub fn get_clients(&self) -> Vec<IggySharedMut<Client>> {
+ pub fn get_clients(&self) -> Vec<Client> {
self.clients.values().cloned().collect()
}
- pub async fn delete_clients_for_user(&mut self, user_id: UserId) -> Result<(), IggyError> {
+ pub fn delete_clients_for_user(&mut self, user_id: UserId) -> Result<(), IggyError> {
let mut clients_to_remove = Vec::new();
for client in self.clients.values() {
- let client = client.read().await;
if let Some(client_user_id) = client.user_id {
if client_user_id == user_id {
clients_to_remove.push(client.session.client_id);
@@ -127,28 +125,27 @@ impl ClientManager {
Ok(())
}
- pub async fn delete_client(&mut self, client_id: u32) -> Option<IggySharedMut<Client>> {
+ pub fn delete_client(&mut self, client_id: u32) -> Option<Client> {
let client = self.clients.remove(&client_id);
if let Some(client) = client.as_ref() {
- let client = client.read().await;
client.session.clear_user_id();
}
client
}
- pub async fn join_consumer_group(
- &self,
+ pub fn join_consumer_group(
+ &mut self,
client_id: u32,
stream_id: u32,
topic_id: u32,
group_id: u32,
) -> Result<(), IggyError> {
- let client = self.clients.get(&client_id);
+ let client = self.clients.get_mut(&client_id);
if client.is_none() {
return Err(IggyError::ClientNotFound(client_id));
}
- let mut client = client.unwrap().write().await;
+ let client = client.unwrap();
if client.consumer_groups.iter().any(|consumer_group| {
consumer_group.group_id == group_id
&& consumer_group.topic_id == topic_id
@@ -165,18 +162,18 @@ impl ClientManager {
Ok(())
}
- pub async fn leave_consumer_group(
- &self,
+ pub fn leave_consumer_group(
+ &mut self,
client_id: u32,
stream_id: u32,
topic_id: u32,
consumer_group_id: u32,
) -> Result<(), IggyError> {
- let client = self.clients.get(&client_id);
+ let client = self.clients.get_mut(&client_id);
if client.is_none() {
return Err(IggyError::ClientNotFound(client_id));
}
- let mut client = client.unwrap().write().await;
+ let mut client = client.unwrap();
for (index, consumer_group) in client.consumer_groups.iter().enumerate() {
if consumer_group.stream_id == stream_id
&& consumer_group.topic_id == topic_id
@@ -189,9 +186,8 @@ impl ClientManager {
Ok(())
}
- pub async fn delete_consumer_groups_for_stream(&self, stream_id: u32) {
- for client in self.clients.values() {
- let mut client = client.write().await;
+ pub fn delete_consumer_groups_for_stream(&mut self, stream_id: u32) {
+ for client in self.clients.values_mut() {
let indexes_to_remove = client
.consumer_groups
.iter()
@@ -210,9 +206,8 @@ impl ClientManager {
}
}
- pub async fn delete_consumer_groups_for_topic(&self, stream_id: u32, topic_id: u32) {
- for client in self.clients.values() {
- let mut client = client.write().await;
+ pub fn delete_consumer_groups_for_topic(&mut self, stream_id: u32, topic_id: u32) {
+ for client in self.clients.values_mut() {
let indexes_to_remove = client
.consumer_groups
.iter()
diff --git a/core/server/src/streaming/partitions/messages.rs b/core/server/src/streaming/partitions/messages.rs
index a00b371f..a2d71c8e 100644
--- a/core/server/src/streaming/partitions/messages.rs
+++ b/core/server/src/streaming/partitions/messages.rs
@@ -368,6 +368,7 @@ mod tests {
use crate::streaming::utils::MemoryPool;
use bytes::Bytes;
use iggy_common::{IggyExpiry, IggyMessage};
+ use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64};
use tempfile::TempDir;
@@ -710,7 +711,7 @@ mod tests {
let partition_id = 3;
let with_segment = true;
let temp_dir = TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: temp_dir.path().to_path_buf().to_str().unwrap().to_string(),
message_deduplication: MessageDeduplicationConfig {
enabled: deduplication_enabled,
@@ -718,7 +719,7 @@ mod tests {
},
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
diff --git a/core/server/src/streaming/partitions/partition.rs b/core/server/src/streaming/partitions/partition.rs
index 73c190cc..4a00ca85 100644
--- a/core/server/src/streaming/partitions/partition.rs
+++ b/core/server/src/streaming/partitions/partition.rs
@@ -28,6 +28,7 @@ use iggy_common::IggyExpiry;
use iggy_common::IggyTimestamp;
use iggy_common::Sizeable;
use std::fmt;
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
@@ -58,8 +59,8 @@ pub struct Partition {
pub(crate) consumer_offsets: DashMap<u32, ConsumerOffset>,
pub(crate) consumer_group_offsets: DashMap<u32, ConsumerOffset>,
pub(crate) segments: Vec<Segment>,
- pub(crate) config: Arc<SystemConfig>,
- pub(crate) storage: Arc<SystemStorage>,
+ pub(crate) config: Rc<SystemConfig>,
+ pub(crate) storage: Rc<SystemStorage>,
}
#[derive(Debug, PartialEq, Clone)]
@@ -88,8 +89,8 @@ impl Partition {
topic_id: u32,
partition_id: u32,
with_segment: bool,
- config: Arc<SystemConfig>,
- storage: Arc<SystemStorage>,
+ config: Rc<SystemConfig>,
+ storage: Rc<SystemStorage>,
message_expiry: IggyExpiry,
messages_count_of_parent_stream: Arc<AtomicU64>,
messages_count_of_parent_topic: Arc<AtomicU64>,
@@ -209,17 +210,18 @@ mod tests {
use iggy_common::IggyDuration;
use iggy_common::IggyExpiry;
use iggy_common::IggyTimestamp;
+ use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64};
#[tokio::test]
async fn should_be_created_with_a_single_segment_given_valid_parameters() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
@@ -264,11 +266,11 @@ mod tests {
#[tokio::test]
async fn should_not_initialize_segments_given_false_with_segment_parameter() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
@@ -279,7 +281,7 @@ mod tests {
topic_id,
1,
false,
- Arc::new(SystemConfig::default()),
+ config,
storage,
IggyExpiry::NeverExpire,
Arc::new(AtomicU64::new(0)),
diff --git a/core/server/src/streaming/segments/segment.rs b/core/server/src/streaming/segments/segment.rs
index acfc98f4..252fe7f1 100644
--- a/core/server/src/streaming/segments/segment.rs
+++ b/core/server/src/streaming/segments/segment.rs
@@ -27,6 +27,7 @@ use iggy_common::IggyByteSize;
use iggy_common::IggyError;
use iggy_common::IggyExpiry;
use iggy_common::IggyTimestamp;
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::fs::remove_file;
@@ -60,7 +61,7 @@ pub struct Segment {
pub(super) index_reader: Option<IndexReader>,
pub(super) message_expiry: IggyExpiry,
pub(super) accumulator: MessagesAccumulator,
- pub(super) config: Arc<SystemConfig>,
+ pub(super) config: Rc<SystemConfig>,
pub(super) indexes: IggyIndexesMut,
pub(super) messages_size: Arc<AtomicU64>,
pub(super) indexes_size: Arc<AtomicU64>,
@@ -73,7 +74,7 @@ impl Segment {
topic_id: u32,
partition_id: u32,
start_offset: u64,
- config: Arc<SystemConfig>,
+ config: Rc<SystemConfig>,
message_expiry: IggyExpiry,
size_of_parent_stream: Arc<AtomicU64>,
size_of_parent_topic: Arc<AtomicU64>,
@@ -450,7 +451,7 @@ mod tests {
let topic_id = 2;
let partition_id = 3;
let start_offset = 0;
- let config = Arc::new(SystemConfig::default());
+ let config = Rc::new(SystemConfig::default());
let path = config.get_segment_path(stream_id, topic_id, partition_id, start_offset);
let messages_file_path = Segment::get_messages_file_path(&path);
let index_path = Segment::get_index_path(&path);
@@ -499,7 +500,7 @@ mod tests {
let topic_id = 2;
let partition_id = 3;
let start_offset = 0;
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
segment: SegmentConfig {
cache_indexes: CacheIndexesConfig::None,
..Default::default()
diff --git a/core/server/src/streaming/session.rs b/core/server/src/streaming/session.rs
index 1eae4eb0..3f412bce 100644
--- a/core/server/src/streaming/session.rs
+++ b/core/server/src/streaming/session.rs
@@ -19,13 +19,16 @@
use iggy_common::{AtomicUserId, UserId};
use std::fmt::Display;
use std::net::SocketAddr;
+use std::rc::Rc;
use std::sync::atomic::{AtomicBool, Ordering};
// This might be extended with more fields in the future e.g. custom name, permissions etc.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct Session {
- user_id: AtomicUserId,
- active: AtomicBool,
+ // TODO: Fixme, those don't have to be atomics anymore, simple integers will suffice
+ // Once the atomics are removed, we can impl Copy trait aswell.
+ user_id: Rc<AtomicUserId>,
+ active: Rc<AtomicBool>,
pub client_id: u32,
pub ip_address: SocketAddr,
}
@@ -34,8 +37,8 @@ impl Session {
pub fn new(client_id: u32, user_id: UserId, ip_address: SocketAddr) -> Self {
Self {
client_id,
- active: AtomicBool::new(true),
- user_id: AtomicUserId::new(user_id),
+ active: Rc::new(AtomicBool::new(true)),
+ user_id: Rc::new(AtomicUserId::new(user_id)),
ip_address,
}
}
@@ -49,15 +52,15 @@ impl Session {
}
pub fn get_user_id(&self) -> UserId {
- self.user_id.load(Ordering::Acquire)
+ self.user_id.load(Ordering::Relaxed)
}
pub fn set_user_id(&self, user_id: UserId) {
- self.user_id.store(user_id, Ordering::Release)
+ self.user_id.store(user_id, Ordering::Relaxed)
}
pub fn set_stale(&self) {
- self.active.store(false, Ordering::Release)
+ self.active.store(false, Ordering::Relaxed)
}
pub fn clear_user_id(&self) {
@@ -65,7 +68,7 @@ impl Session {
}
pub fn is_active(&self) -> bool {
- self.active.load(Ordering::Acquire)
+ self.active.load(Ordering::Relaxed)
}
pub fn is_authenticated(&self) -> bool {
diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs
index f423b842..51e2565e 100644
--- a/core/server/src/streaming/storage.rs
+++ b/core/server/src/streaming/storage.rs
@@ -18,13 +18,13 @@
use super::persistence::persister::PersisterKind;
use crate::configs::system::SystemConfig;
+use crate::shard::system::info::SystemInfo;
+use crate::shard::system::storage::FileSystemInfoStorage;
use crate::state::system::{PartitionState, StreamState, TopicState};
use crate::streaming::partitions::partition::{ConsumerOffset, Partition};
use crate::streaming::partitions::storage::FilePartitionStorage;
use crate::streaming::streams::storage::FileStreamStorage;
use crate::streaming::streams::stream::Stream;
-use crate::streaming::systems::info::SystemInfo;
-use crate::streaming::systems::storage::FileSystemInfoStorage;
use crate::streaming::topics::storage::FileTopicStorage;
use crate::streaming::topics::topic::Topic;
use iggy_common::ConsumerKind;
@@ -85,61 +85,61 @@ pub enum PartitionStorageKind {
}
#[cfg_attr(test, automock)]
-pub trait SystemInfoStorage: Send {
- fn load(&self) -> impl Future<Output = Result<SystemInfo, IggyError>> + Send;
- fn save(&self, system_info: &SystemInfo) -> impl Future<Output = Result<(), IggyError>> + Send;
+pub trait SystemInfoStorage {
+ fn load(&self) -> impl Future<Output = Result<SystemInfo, IggyError>>;
+ fn save(&self, system_info: &SystemInfo) -> impl Future<Output = Result<(), IggyError>>;
}
#[cfg_attr(test, automock)]
-pub trait StreamStorage: Send {
+pub trait StreamStorage {
fn load(
&self,
stream: &mut Stream,
state: StreamState,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn save(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn delete(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>>;
}
#[cfg_attr(test, automock)]
-pub trait TopicStorage: Send {
+pub trait TopicStorage {
fn load(
&self,
topic: &mut Topic,
state: TopicState,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn save(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn delete(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
}
#[cfg_attr(test, automock)]
-pub trait PartitionStorage: Send {
+pub trait PartitionStorage {
fn load(
&self,
partition: &mut Partition,
state: PartitionState,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
fn save(&self, partition: &mut Partition)
- -> impl Future<Output = Result<(), IggyError>> + Send;
- fn delete(&self, partition: &Partition) -> impl Future<Output = Result<(), IggyError>> + Send;
+ -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, partition: &Partition) -> impl Future<Output = Result<(), IggyError>>;
fn save_consumer_offset(
&self,
offset: u64,
path: &str,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
fn load_consumer_offsets(
&self,
kind: ConsumerKind,
path: &str,
- ) -> impl Future<Output = Result<Vec<ConsumerOffset>, IggyError>> + Send;
+ ) -> impl Future<Output = Result<Vec<ConsumerOffset>, IggyError>>;
fn delete_consumer_offsets(
&self,
path: &str,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
fn delete_consumer_offset(
&self,
path: &str,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
}
#[derive(Debug)]
diff --git a/core/server/src/streaming/streams/storage.rs b/core/server/src/streaming/streams/storage.rs
index 3e3e5e76..53639de9 100644
--- a/core/server/src/streaming/streams/storage.rs
+++ b/core/server/src/streaming/streams/storage.rs
@@ -156,7 +156,7 @@ impl StreamStorage for FileStreamStorage {
for mut topic in unloaded_topics {
let loaded_topics = loaded_topics.clone();
let topic_state = state.topics.remove(&topic.topic_id).unwrap();
- let load_topic = tokio::spawn(async move {
+ let load_topic = monoio::spawn(async move {
match topic.load(topic_state).await {
Ok(_) => loaded_topics.lock().await.push(topic),
Err(error) => error!(
diff --git a/core/server/src/streaming/streams/stream.rs b/core/server/src/streaming/streams/stream.rs
index 339fc08a..03681b75 100644
--- a/core/server/src/streaming/streams/stream.rs
+++ b/core/server/src/streaming/streams/stream.rs
@@ -40,8 +40,8 @@ pub struct Stream {
pub segments_count: Arc<AtomicU32>,
pub(crate) topics: AHashMap<u32, Topic>,
pub(crate) topics_ids: AHashMap<String, u32>,
- pub(crate) config: Arc<SystemConfig>,
- pub(crate) storage: Arc<SystemStorage>,
+ pub(crate) config: Rc<SystemConfig>,
+ pub(crate) storage: Rc<SystemStorage>,
}
impl Stream {
@@ -106,18 +106,18 @@ mod tests {
#[test]
fn should_be_created_given_valid_parameters() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
MemoryPool::init_pool(config.clone());
let id = 1;
let name = "test";
- let config = Arc::new(SystemConfig::default());
+ let config = Rc::new(SystemConfig::default());
let path = config.get_stream_path(id);
let topics_path = config.get_topics_path(id);
diff --git a/core/server/src/streaming/streams/topics.rs b/core/server/src/streaming/streams/topics.rs
index f8c742be..b975ee83 100644
--- a/core/server/src/streaming/streams/topics.rs
+++ b/core/server/src/streaming/streams/topics.rs
@@ -146,8 +146,7 @@ impl Stream {
topic.name = name.to_owned();
topic.message_expiry = message_expiry;
topic.compression_algorithm = compression_algorithm;
- for partition in topic.partitions.values_mut() {
- let mut partition = partition.write().await;
+ for partition in topic.partitions.borrow_mut().values_mut() {
partition.message_expiry = message_expiry;
for segment in partition.segments.iter_mut() {
segment.update_message_expiry(message_expiry);
@@ -282,16 +281,16 @@ mod tests {
},
};
use iggy_common::IggyByteSize;
- use std::sync::Arc;
+ use std::{rc::Rc, sync::Arc};
#[tokio::test]
async fn should_get_topic_by_id_and_name() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
diff --git a/core/server/src/streaming/topics/consumer_group.rs b/core/server/src/streaming/topics/consumer_group.rs
index dedb29c7..43d9611b 100644
--- a/core/server/src/streaming/topics/consumer_group.rs
+++ b/core/server/src/streaming/topics/consumer_group.rs
@@ -21,16 +21,16 @@ use iggy_common::IggyError;
use tokio::sync::RwLock;
use tracing::trace;
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct ConsumerGroup {
pub topic_id: u32,
pub group_id: u32,
pub name: String,
pub partitions_count: u32,
- members: AHashMap<u32, RwLock<ConsumerGroupMember>>,
+ members: AHashMap<u32, ConsumerGroupMember>,
}
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct ConsumerGroupMember {
pub id: u32,
partitions: AHashMap<u32, u32>,
@@ -49,8 +49,8 @@ impl ConsumerGroup {
}
}
- pub fn get_members(&self) -> Vec<&RwLock<ConsumerGroupMember>> {
- self.members.values().collect()
+ pub fn get_members(&self) -> Vec<ConsumerGroupMember> {
+ self.members.values().cloned().collect()
}
pub async fn reassign_partitions(&mut self, partitions_count: u32) {
@@ -61,7 +61,7 @@ impl ConsumerGroup {
pub async fn calculate_partition_id(&self, member_id: u32) -> Result<Option<u32>, IggyError> {
let member = self.members.get(&member_id);
if let Some(member) = member {
- return Ok(member.write().await.calculate_partition_id());
+ return Ok(member.await.calculate_partition_id());
}
Err(IggyError::ConsumerGroupMemberNotFound(
member_id,
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index dec2f163..26505355 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -91,7 +91,7 @@ impl Topic {
self.get_consumer_group_by_id(*group_id.unwrap())
}
- pub fn get_consumer_group_by_id(&self, id: u32) -> Result<&RwLock<ConsumerGroup>, IggyError> {
+ pub fn get_consumer_group_by_id(&self, id: u32) -> Result<ConsumerGroup, IggyError> {
let consumer_group = self.consumer_groups.get(&id);
if consumer_group.is_none() {
return Err(IggyError::ConsumerGroupIdNotFound(id, self.topic_id));
@@ -104,7 +104,7 @@ impl Topic {
&mut self,
group_id: Option<u32>,
name: &str,
- ) -> Result<&RwLock<ConsumerGroup>, IggyError> {
+ ) -> Result<ConsumerGroup, IggyError> {
if self.consumer_groups_ids.contains_key(name) {
return Err(IggyError::ConsumerGroupNameAlreadyExists(
name.to_owned(),
@@ -138,20 +138,20 @@ impl Topic {
}
let consumer_group =
- ConsumerGroup::new(self.topic_id, id, name, self.partitions.len() as u32);
- self.consumer_groups.insert(id, RwLock::new(consumer_group));
+ ConsumerGroup::new(self.topic_id, id, name, self.partitions.borrow().len() as u32);
+ self.consumer_groups.insert(id, consumer_group.clone());
self.consumer_groups_ids.insert(name.to_owned(), id);
info!(
"Created consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
id, self.topic_id, self.stream_id
);
- self.get_consumer_group_by_id(id)
+ consumer_group
}
pub async fn delete_consumer_group(
&mut self,
id: &Identifier,
- ) -> Result<RwLock<ConsumerGroup>, IggyError> {
+ ) -> Result<ConsumerGroup, IggyError> {
let group_id;
{
let consumer_group = self.get_consumer_group(id).with_error_context(|error| {
@@ -167,7 +167,6 @@ impl Topic {
}
let consumer_group = consumer_group.unwrap();
{
- let consumer_group = consumer_group.read().await;
let group_id = consumer_group.group_id;
self.consumer_groups_ids.remove(&consumer_group.name);
let current_group_id = self.current_consumer_group_id.load(Ordering::SeqCst);
@@ -176,8 +175,7 @@ impl Topic {
.store(group_id, Ordering::SeqCst);
}
- for (_, partition) in self.partitions.iter() {
- let partition = partition.read().await;
+ for (_, partition) in self.partitions.borrow().iter() {
if let Some((_, offset)) = partition.consumer_group_offsets.remove(&group_id) {
self.storage
.partition
diff --git a/core/server/src/streaming/topics/topic.rs b/core/server/src/streaming/topics/topic.rs
index b0913856..ad3e25ef 100644
--- a/core/server/src/streaming/topics/topic.rs
+++ b/core/server/src/streaming/topics/topic.rs
@@ -23,6 +23,8 @@ use crate::streaming::storage::SystemStorage;
use crate::streaming::topics::consumer_group::ConsumerGroup;
use ahash::AHashMap;
use core::fmt;
+use std::cell::RefCell;
+use std::rc::Rc;
use iggy_common::locking::IggySharedMut;
use iggy_common::{
CompressionAlgorithm, Consumer, ConsumerKind, IggyByteSize, IggyError, IggyExpiry,
@@ -48,10 +50,10 @@ pub struct Topic {
pub(crate) messages_count_of_parent_stream: Arc<AtomicU64>,
pub(crate) messages_count: Arc<AtomicU64>,
pub(crate) segments_count_of_parent_stream: Arc<AtomicU32>,
- pub(crate) config: Arc<SystemConfig>,
- pub(crate) partitions: AHashMap<u32, IggySharedMut<Partition>>,
- pub(crate) storage: Arc<SystemStorage>,
- pub(crate) consumer_groups: AHashMap<u32, RwLock<ConsumerGroup>>,
+ pub(crate) config: Rc<SystemConfig>,
+ pub(crate) partitions: RefCell<AHashMap<u32, Partition>>,
+ pub(crate) storage: Rc<SystemStorage>,
+ pub(crate) consumer_groups: AHashMap<u32, ConsumerGroup>,
pub(crate) consumer_groups_ids: AHashMap<String, u32>,
pub(crate) current_consumer_group_id: AtomicU32,
pub(crate) current_partition_id: AtomicU32,
@@ -71,8 +73,8 @@ impl Topic {
size_of_parent_stream: Arc<AtomicU64>,
messages_count_of_parent_stream: Arc<AtomicU64>,
segments_count_of_parent_stream: Arc<AtomicU32>,
- config: Arc<SystemConfig>,
- storage: Arc<SystemStorage>,
+ config: Rc<SystemConfig>,
+ storage: Rc<SystemStorage>,
) -> Topic {
Topic::create(
stream_id,
@@ -99,8 +101,8 @@ impl Topic {
topic_id: u32,
name: &str,
partitions_count: u32,
- config: Arc<SystemConfig>,
- storage: Arc<SystemStorage>,
+ config: Rc<SystemConfig>,
+ storage: Rc<SystemStorage>,
size_of_parent_stream: Arc<AtomicU64>,
messages_count_of_parent_stream: Arc<AtomicU64>,
segments_count_of_parent_stream: Arc<AtomicU32>,
diff --git a/core/server/src/streaming/utils/memory_pool.rs b/core/server/src/streaming/utils/memory_pool.rs
index 61962f0a..e05b7359 100644
--- a/core/server/src/streaming/utils/memory_pool.rs
+++ b/core/server/src/streaming/utils/memory_pool.rs
@@ -21,6 +21,7 @@ use bytes::BytesMut;
use crossbeam::queue::ArrayQueue;
use human_repr::HumanCount;
use once_cell::sync::OnceCell;
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use tracing::{info, trace, warn};
@@ -153,7 +154,7 @@ impl MemoryPool {
}
/// Initialize the global pool from the given config.
- pub fn init_pool(config: Arc<SystemConfig>) {
+ pub fn init_pool(config: Rc<SystemConfig>) {
let is_enabled = config.memory_pool.enabled;
let memory_limit = config.memory_pool.size.as_bytes_usize();
let bucket_capacity = config.memory_pool.bucket_capacity as usize;
@@ -467,7 +468,7 @@ mod tests {
fn initialize_pool_for_tests() {
TEST_INIT.call_once(|| {
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
memory_pool: MemoryPoolConfig {
enabled: true,
size: IggyByteSize::from_str("4GiB").unwrap(),
diff --git a/core/server/src/tcp/connection_handler.rs b/core/server/src/tcp/connection_handler.rs
index ec61b23b..9c92d27d 100644
--- a/core/server/src/tcp/connection_handler.rs
+++ b/core/server/src/tcp/connection_handler.rs
@@ -19,27 +19,28 @@
use crate::binary::command::ServerCommandHandler;
use crate::binary::{command, sender::SenderKind};
use crate::server_error::ConnectionError;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use crate::tcp::connection_handler::command::ServerCommand;
+use bytes::BytesMut;
use iggy_common::IggyError;
use std::io::ErrorKind;
-use std::sync::Arc;
+use std::rc::Rc;
use tracing::{debug, error, info};
const INITIAL_BYTES_LENGTH: usize = 4;
pub(crate) async fn handle_connection(
- session: Arc<Session>,
+ session: &Rc<Session>,
sender: &mut SenderKind,
- system: SharedSystem,
+ shard: &Rc<IggyShard>,
) -> Result<(), ConnectionError> {
- let mut length_buffer = [0u8; INITIAL_BYTES_LENGTH];
- let mut code_buffer = [0u8; INITIAL_BYTES_LENGTH];
+ let length_buffer = BytesMut::with_capacity(INITIAL_BYTES_LENGTH);
+ let code_buffer = BytesMut::with_capacity(INITIAL_BYTES_LENGTH);
loop {
- let read_length = match sender.read(&mut length_buffer).await {
- Ok(read_length) => read_length,
- Err(error) => {
+ let (read_length, initial_buffer) = match sender.read(length_buffer.clone()).await {
+ (Ok(read_length), initial_buffer) => (read_length, initial_buffer),
+ (Err(error), _) => {
if error.as_code() == IggyError::ConnectionClosed.as_code() {
return Err(ConnectionError::from(error));
} else {
@@ -56,13 +57,18 @@ pub(crate) async fn handle_connection(
continue;
}
- let length = u32::from_le_bytes(length_buffer);
- sender.read(&mut code_buffer).await?;
- let code = u32::from_le_bytes(code_buffer);
+ let initial_buffer = initial_buffer.freeze();
+ let length =
+ u32::from_le_bytes(initial_buffer[0..INITIAL_BYTES_LENGTH].try_into().unwrap());
+ let (res, code_buffer) = sender.read(code_buffer.clone()).await;
+ let _ = res?;
+ let code_buffer = code_buffer.freeze();
+ let code: u32 =
+ u32::from_le_bytes(code_buffer[0..INITIAL_BYTES_LENGTH].try_into().unwrap());
debug!("Received a TCP request, length: {length}, code: {code}");
let command = ServerCommand::from_code_and_reader(code, sender, length - 4).await?;
debug!("Received a TCP command: {command}, payload size: {length}");
- match command.handle(sender, length, &session, &system).await {
+ match command.handle(sender, length, session, shard).await {
Ok(_) => {
debug!(
"Command was handled successfully, session: {session}. TCP response was sent."
diff --git a/core/server/src/tcp/sender.rs b/core/server/src/tcp/sender.rs
index 4dd11725..6a10adb2 100644
--- a/core/server/src/tcp/sender.rs
+++ b/core/server/src/tcp/sender.rs
@@ -16,25 +16,33 @@
* under the License.
*/
+use bytes::BytesMut;
use iggy_common::IggyError;
-use monoio::io::{AsyncReadRent, AsyncReadRentExt, AsyncWriteRent, AsyncWriteRentExt};
+use monoio::{
+ buf::IoBufMut,
+ io::{AsyncReadRent, AsyncReadRentExt, AsyncWriteRent, AsyncWriteRentExt},
+};
+use nix::libc;
use std::io::IoSlice;
use tracing::debug;
const STATUS_OK: &[u8] = &[0; 4];
-pub(crate) async fn read<T>(stream: &mut T, buffer: &mut [u8]) -> Result<usize, IggyError>
+pub(crate) async fn read<T>(
+ stream: &mut T,
+ buffer: BytesMut,
+) -> (Result<usize, IggyError>, BytesMut)
where
T: AsyncReadRent + AsyncWriteRent + Unpin,
{
match stream.read_exact(buffer).await {
- Ok(0) => Err(IggyError::ConnectionClosed),
- Ok(read_bytes) => Ok(read_bytes),
- Err(error) => {
+ (Ok(0), buffer) => (Err(IggyError::ConnectionClosed), buffer),
+ (Ok(read_bytes), buffer) => (Ok(read_bytes), buffer),
+ (Err(error), buffer) => {
if error.kind() == std::io::ErrorKind::UnexpectedEof {
- Err(IggyError::ConnectionClosed)
+ (Err(IggyError::ConnectionClosed), buffer)
} else {
- Err(IggyError::TcpError)
+ (Err(IggyError::TcpError), buffer)
}
}
}
@@ -57,7 +65,7 @@ where
pub(crate) async fn send_ok_response_vectored<T>(
stream: &mut T,
length: &[u8],
- slices: Vec<IoSlice<'_>>,
+ slices: Vec<libc::iovec>,
) -> Result<(), IggyError>
where
T: AsyncReadRentExt + AsyncWriteRentExt + Unpin,
@@ -90,8 +98,9 @@ where
);
let length = (payload.len() as u32).to_le_bytes();
stream
- .write_all(&[status, &length, payload].as_slice().concat())
+ .write_all([status, &length, payload].concat())
.await
+ .0
.map_err(|_| IggyError::TcpError)?;
debug!("Sent response with status: {:?}", status);
Ok(())
@@ -101,7 +110,7 @@ pub(crate) async fn send_response_vectored<T>(
stream: &mut T,
status: &[u8],
length: &[u8],
- mut slices: Vec<IoSlice<'_>>,
+ mut slices: Vec<libc::iovec>,
) -> Result<(), IggyError>
where
T: AsyncReadRentExt + AsyncWriteRentExt + Unpin,
@@ -111,16 +120,22 @@ where
slices.len(),
status
);
- let prefix = [IoSlice::new(status), IoSlice::new(length)];
+ let prefix = [
+ libc::iovec {
+ iov_base: length.as_ptr() as _,
+ iov_len: length.len(),
+ },
+ libc::iovec {
+ iov_base: status.as_ptr() as _,
+ iov_len: status.len(),
+ },
+ ];
slices.splice(0..0, prefix);
- let mut slice_refs = slices.as_mut_slice();
- while !slice_refs.is_empty() {
- let bytes_written = stream
- .write_vectored(slice_refs)
- .await
- .map_err(|_| IggyError::TcpError)?;
- IoSlice::advance_slices(&mut slice_refs, bytes_written);
- }
+ stream
+ .write_vectored_all(slices)
+ .await
+ .0
+ .map_err(|_| IggyError::TcpError)?;
debug!("Sent response with status: {:?}", status);
Ok(())
}
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index 2e4f25be..ffa01eb1 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -18,39 +18,50 @@
use crate::binary::sender::SenderKind;
use crate::shard::IggyShard;
+use crate::shard::transmission::message::ShardEvent;
use crate::streaming::clients::client_manager::Transport;
use crate::tcp::connection_handler::{handle_connection, handle_error};
+use crate::tcp::tcp_socket;
use std::net::SocketAddr;
use std::rc::Rc;
-use monoio::net::TcpListener;
-use rustls::pki_types::Ipv4Addr;
-use tokio::net::TcpSocket;
-use tokio::sync::oneshot;
use tracing::{error, info};
-pub async fn start(server_name: &str, shard: Rc<IggyShard>) {
- let addr: SocketAddr = if shard.config.tcp.ipv6 {
- shard.config.tcp.address.parse().expect("Unable to parse IPv6 address")
- } else {
- shard.config.tcp.address.parse().expect("Unable to parse IPv4 address")
- };
+pub async fn start(server_name: &'static str, shard: Rc<IggyShard>) {
+ let ip_v6 = shard.config.tcp.ipv6;
+ let socket_config = &shard.config.tcp.socket;
+ let addr: SocketAddr = shard
+ .config
+ .tcp
+ .address
+ .parse()
+ .expect("Failed to parse TCP address");
+
+ let socket = tcp_socket::build(ip_v6, socket_config);
monoio::spawn(async move {
- let listener = TcpListener::bind(addr).expect(format!("Unable to start {server_name}.").as_ref());
+ socket
+ .bind(&addr.into())
+ .expect("Failed to bind eTCP listener");
+ socket.listen(1024);
+ let listener: std::net::TcpListener = socket.into();
+ let listener = monoio::net::TcpListener::from_std(listener).unwrap();
+ info!("{server_name} server has started on: {:?}", addr);
loop {
match listener.accept().await {
Ok((stream, address)) => {
let shard = shard.clone();
info!("Accepted new TCP connection: {address}");
- let session = shard
- .add_client(&address, Transport::Tcp);
+ let session = shard.add_client(&address, Transport::Tcp);
+ //TODO: Those can be shared with other shards.
+ shard.add_active_session(session.clone());
+ // Broadcast session to all shards.
+ let event = Rc::new(ShardEvent::NewSession(session.clone()));
+ shard.broadcast_event_to_all_shards(session.client_id, event);
- let client_id = session.client_id;
+ let _client_id = session.client_id;
info!("Created new session: {session}");
let mut sender = SenderKind::get_tcp_sender(stream);
monoio::spawn(async move {
- if let Err(error) =
- handle_connection(session, &mut sender, shard.clone()).await
- {
+ if let Err(error) = handle_connection(&session, &mut sender, &shard).await {
handle_error(error);
//TODO: Fixme
/*
diff --git a/core/server/src/tcp/tcp_sender.rs b/core/server/src/tcp/tcp_sender.rs
index a157dead..81089c85 100644
--- a/core/server/src/tcp/tcp_sender.rs
+++ b/core/server/src/tcp/tcp_sender.rs
@@ -19,10 +19,13 @@
use crate::binary::sender::Sender;
use crate::tcp::COMPONENT;
use crate::{server_error::ServerError, tcp::sender};
+use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::IggyError;
-use tokio::{io::AsyncWriteExt};
+use monoio::buf::IoBufMut;
+use monoio::io::AsyncWriteRent;
use monoio::net::TcpStream;
+use nix::libc;
#[derive(Debug)]
pub struct TcpSender {
@@ -30,7 +33,7 @@ pub struct TcpSender {
}
impl Sender for TcpSender {
- async fn read(&mut self, buffer: &mut [u8]) -> Result<usize, IggyError> {
+ async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut) {
sender::read(&mut self.stream, buffer).await
}
@@ -59,7 +62,7 @@ impl Sender for TcpSender {
async fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<std::io::IoSlice<'_>>,
+ slices: Vec<libc::iovec>,
) -> Result<(), IggyError> {
sender::send_ok_response_vectored(&mut self.stream, length, slices).await
}
diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs
index 82bbe0c1..9b8e1c04 100644
--- a/core/server/src/tcp/tcp_server.rs
+++ b/core/server/src/tcp/tcp_server.rs
@@ -17,9 +17,9 @@
*/
use crate::shard::IggyShard;
-use crate::tcp::{tcp_listener};
-use std::rc::Rc;
+use crate::tcp::tcp_listener;
use iggy_common::IggyError;
+use std::rc::Rc;
use tracing::info;
/// Starts the TCP server.
diff --git a/core/server/src/tcp/tcp_socket.rs b/core/server/src/tcp/tcp_socket.rs
index 69c1d408..747540fa 100644
--- a/core/server/src/tcp/tcp_socket.rs
+++ b/core/server/src/tcp/tcp_socket.rs
@@ -16,16 +16,18 @@
* under the License.
*/
+use socket2::{Domain, Protocol, Socket, Type};
use std::num::TryFromIntError;
-
use crate::configs::tcp::TcpSocketConfig;
-pub fn build(ipv6: bool, config: TcpSocketConfig) -> TcpSocket {
+pub fn build(ipv6: bool, config: &TcpSocketConfig) -> Socket {
let socket = if ipv6 {
- TcpSocket::new_v6().expect("Unable to create an ipv6 socket")
+ Socket::new(Domain::IPV6, Type::STREAM, Some(Protocol::TCP))
+ .expect("Unable to create an ipv6 socket")
} else {
- TcpSocket::new_v4().expect("Unable to create an ipv4 socket")
+ Socket::new(Domain::IPV4, Type::STREAM, Some(Protocol::TCP))
+ .expect("Unable to create an ipv4 socket")
};
if config.override_defaults {
@@ -34,7 +36,7 @@ pub fn build(ipv6: bool, config: TcpSocketConfig) -> TcpSocket {
.as_bytes_u64()
.try_into()
.map_err(|e: TryFromIntError| std::io::Error::other(e.to_string()))
- .and_then(|size: u32| socket.set_recv_buffer_size(size))
+ .and_then(|size| socket.set_recv_buffer_size(size))
.expect("Unable to set SO_RCVBUF on socket");
config
.send_buffer_size
@@ -52,6 +54,12 @@ pub fn build(ipv6: bool, config: TcpSocketConfig) -> TcpSocket {
socket
.set_linger(Some(config.linger.get_duration()))
.expect("Unable to set SO_LINGER on socket");
+ socket
+ .set_reuse_address(true)
+ .expect("Unable to set SO_REUSEADDR on socket");
+ socket
+ .set_reuse_port(true)
+ .expect("Unable to set SO_REUSEPORT on socket");
}
socket
@@ -77,9 +85,9 @@ mod tests {
nodelay: true,
linger: IggyDuration::new(linger_dur),
};
- let socket = build(false, config);
- assert!(socket.recv_buffer_size().unwrap() >= buffer_size as u32);
- assert!(socket.send_buffer_size().unwrap() >= buffer_size as u32);
+ let socket = build(false, &config);
+ assert!(socket.recv_buffer_size().unwrap() >= buffer_size as usize);
+ assert!(socket.send_buffer_size().unwrap() >= buffer_size as usize);
assert!(socket.keepalive().unwrap());
assert!(socket.nodelay().unwrap());
assert_eq!(socket.linger().unwrap(), Some(linger_dur));
diff --git a/core/server/src/tcp/tcp_tls_listener.rs b/core/server/src/tcp/tcp_tls_listener.rs
index d2bbf8da..53df4b0b 100644
--- a/core/server/src/tcp/tcp_tls_listener.rs
+++ b/core/server/src/tcp/tcp_tls_listener.rs
@@ -18,26 +18,30 @@
use crate::binary::sender::SenderKind;
use crate::configs::tcp::TcpTlsConfig;
+use crate::shard::IggyShard;
use crate::streaming::clients::client_manager::Transport;
-use crate::streaming::systems::system::SharedSystem;
use crate::tcp::connection_handler::{handle_connection, handle_error};
+use monoio_native_tls::TlsAcceptor;
use rustls::ServerConfig;
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
use rustls_pemfile::{certs, private_key};
use std::io::BufReader;
use std::net::SocketAddr;
+use std::rc::Rc;
use std::sync::Arc;
use tokio::net::TcpSocket;
use tokio::sync::oneshot;
-use tokio_rustls::{TlsAcceptor, rustls};
use tracing::{error, info};
pub(crate) async fn start(
address: &str,
config: TcpTlsConfig,
socket: TcpSocket,
- system: SharedSystem,
+ system: Rc<IggyShard>,
) -> SocketAddr {
+ //TODO: Fixme
+ todo!();
+ /*
let address = address.to_string();
let (tx, rx) = oneshot::channel();
tokio::spawn(async move {
@@ -131,6 +135,7 @@ pub(crate) async fn start(
Ok(addr) => addr,
Err(_) => panic!("Failed to get the local address for TCP TLS listener."),
}
+ */
}
fn generate_self_signed_cert()
diff --git a/core/server/src/tcp/tcp_tls_sender.rs b/core/server/src/tcp/tcp_tls_sender.rs
index 6ba62315..7213ad64 100644
--- a/core/server/src/tcp/tcp_tls_sender.rs
+++ b/core/server/src/tcp/tcp_tls_sender.rs
@@ -19,10 +19,13 @@
use crate::binary::sender::Sender;
use crate::tcp::COMPONENT;
use crate::{server_error::ServerError, tcp::sender};
+use bytes::BytesMut;
use error_set::ErrContext;
use iggy_common::IggyError;
+use monoio::io::AsyncWriteRent;
use monoio::net::TcpStream;
use monoio_native_tls::TlsStream;
+use nix::libc;
#[derive(Debug)]
pub struct TcpTlsSender {
@@ -30,7 +33,7 @@ pub struct TcpTlsSender {
}
impl Sender for TcpTlsSender {
- async fn read(&mut self, buffer: &mut [u8]) -> Result<usize, IggyError> {
+ async fn read(&mut self, buffer: BytesMut) -> (Result<usize, IggyError>, BytesMut) {
sender::read(&mut self.stream, buffer).await
}
@@ -59,7 +62,7 @@ impl Sender for TcpTlsSender {
async fn send_ok_response_vectored(
&mut self,
length: &[u8],
- slices: Vec<std::io::IoSlice<'_>>,
+ slices: Vec<libc::iovec>,
) -> Result<(), IggyError> {
sender::send_ok_response_vectored(&mut self.stream, length, slices).await
}
(iggy) 12/12: fix iggy shard errors
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit d2651b3d9301113880efd32f224e5dffcde62077
Author: numinex <g....@gmail.com>
AuthorDate: Thu Jun 26 16:34:52 2025 +0200
fix iggy shard errors
---
core/common/src/locking/mod.rs | 4 +-
core/common/src/locking/tokio_lock.rs | 3 +-
core/server/src/binary/command.rs | 1 -
.../create_consumer_group_handler.rs | 13 +-
core/server/src/binary/handlers/utils.rs | 15 +-
core/server/src/binary/mapper.rs | 14 +-
core/server/src/http/consumer_groups.rs | 6 +-
core/server/src/http/jwt/cleaner.rs | 8 +-
core/server/src/http/jwt/jwt_manager.rs | 6 +-
core/server/src/http/mapper.rs | 14 +-
core/server/src/http/messages.rs | 2 +-
core/server/src/http/mod.rs | 2 +
core/server/src/http/shared.rs | 5 +-
core/server/src/http/system.rs | 2 +-
core/server/src/shard/mod.rs | 2 +-
core/server/src/shard/system/clients.rs | 35 ++---
core/server/src/shard/system/consumer_groups.rs | 139 +++++++++++-------
core/server/src/shard/system/consumer_offsets.rs | 24 +++-
core/server/src/shard/system/messages.rs | 34 +++--
core/server/src/shard/system/partitions.rs | 48 +++++--
.../src/shard/system/personal_access_tokens.rs | 14 +-
core/server/src/shard/system/segments.rs | 40 +++++-
core/server/src/shard/system/snapshot/mod.rs | 62 ++++++---
core/server/src/shard/system/stats.rs | 8 +-
core/server/src/shard/system/storage.rs | 1 +
core/server/src/shard/system/streams.rs | 155 +++++++++++++--------
core/server/src/shard/system/topics.rs | 122 ++++++++--------
core/server/src/shard/system/users.rs | 154 ++++++++++++--------
.../src/streaming/partitions/consumer_offsets.rs | 5 +-
core/server/src/streaming/storage.rs | 29 ++--
core/server/src/streaming/topics/consumer_group.rs | 5 +-
.../server/src/streaming/topics/consumer_groups.rs | 22 +--
core/server/src/streaming/topics/topic.rs | 4 +-
core/server/src/streaming/users/user.rs | 2 +-
34 files changed, 595 insertions(+), 405 deletions(-)
diff --git a/core/common/src/locking/mod.rs b/core/common/src/locking/mod.rs
index c2e542c7..429b4f59 100644
--- a/core/common/src/locking/mod.rs
+++ b/core/common/src/locking/mod.rs
@@ -37,11 +37,11 @@ pub type IggyRwLock<T> = fast_async_lock::IggyFastAsyncRwLock<T>;
#[allow(async_fn_in_trait)]
pub trait IggySharedMutFn<T> {
- type ReadGuard<'a>: Deref<Target = T>
+ type ReadGuard<'a>: Deref<Target = T>
where
T: 'a,
Self: 'a;
- type WriteGuard<'a>: DerefMut<Target = T>
+ type WriteGuard<'a>: DerefMut<Target = T>
where
T: 'a,
Self: 'a;
diff --git a/core/common/src/locking/tokio_lock.rs b/core/common/src/locking/tokio_lock.rs
index ef5baaad..d6a96f06 100644
--- a/core/common/src/locking/tokio_lock.rs
+++ b/core/common/src/locking/tokio_lock.rs
@@ -24,8 +24,7 @@ use tokio::sync::{RwLock as TokioRwLock, RwLockReadGuard, RwLockWriteGuard};
#[derive(Debug)]
pub struct IggyTokioRwLock<T>(Arc<TokioRwLock<T>>);
-impl<T> IggySharedMutFn<T> for IggyTokioRwLock<T>
-{
+impl<T> IggySharedMutFn<T> for IggyTokioRwLock<T> {
type ReadGuard<'a>
= RwLockReadGuard<'a, T>
where
diff --git a/core/server/src/binary/command.rs b/core/server/src/binary/command.rs
index aa935e09..1ebac395 100644
--- a/core/server/src/binary/command.rs
+++ b/core/server/src/binary/command.rs
@@ -22,7 +22,6 @@ use crate::binary::sender::SenderKind;
use crate::define_server_command_enum;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use bytes::{BufMut, Bytes, BytesMut};
use enum_dispatch::enum_dispatch;
use iggy_common::change_password::ChangePassword;
diff --git a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
index 3d87fa17..5d695687 100644
--- a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
+++ b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs
@@ -16,14 +16,16 @@
* under the License.
*/
+use std::rc::Rc;
+
use crate::binary::command::{BinaryServerCommand, ServerCommand, ServerCommandHandler};
use crate::binary::handlers::utils::receive_and_validate;
use crate::binary::mapper;
use crate::binary::{handlers::consumer_groups::COMPONENT, sender::SenderKind};
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::state::models::CreateConsumerGroupWithId;
use crate::streaming::session::Session;
-use crate::streaming::systems::system::SharedSystem;
use anyhow::Result;
use error_set::ErrContext;
use iggy_common::IggyError;
@@ -39,14 +41,13 @@ impl ServerCommandHandler for CreateConsumerGroup {
async fn handle(
self,
sender: &mut SenderKind,
- _length: u32,
- session: &Session,
- system: &SharedSystem,
+ length: u32,
+ session: &Rc<Session>,
+ shard: &Rc<IggyShard>,
) -> Result<(), IggyError> {
debug!("session: {session}, command: {self}");
- let mut system = system.write().await;
- let consumer_group = system
+ let consumer_group = shard
.create_consumer_group(
session,
&self.stream_id,
diff --git a/core/server/src/binary/handlers/utils.rs b/core/server/src/binary/handlers/utils.rs
index 351f6297..80bcede4 100644
--- a/core/server/src/binary/handlers/utils.rs
+++ b/core/server/src/binary/handlers/utils.rs
@@ -26,12 +26,17 @@ pub async fn receive_and_validate(
length: u32,
) -> Result<ServerCommand, IggyError> {
let mut buffer = BytesMut::with_capacity(length as usize);
- if length > 0 {
- unsafe {
- buffer.set_len(length as usize);
- }
- sender.read(&mut buffer).await?;
+ unsafe {
+ buffer.set_len(length as usize);
}
+ let buffer = if length == 0 {
+ buffer
+ } else {
+ let (result, buffer) = sender.read(buffer).await;
+ result?;
+ buffer
+ };
+
let command = ServerCommand::from_code_and_payload(code, buffer.freeze())?;
command.validate()?;
Ok(command)
diff --git a/core/server/src/binary/mapper.rs b/core/server/src/binary/mapper.rs
index 61c65235..219ceed9 100644
--- a/core/server/src/binary/mapper.rs
+++ b/core/server/src/binary/mapper.rs
@@ -24,9 +24,8 @@ use crate::streaming::topics::consumer_group::ConsumerGroup;
use crate::streaming::topics::topic::Topic;
use crate::streaming::users::user::User;
use bytes::{BufMut, Bytes, BytesMut};
-use iggy_common::locking::{IggySharedMut, IggySharedMutFn};
+use iggy_common::locking::IggySharedMutFn;
use iggy_common::{BytesSerializable, ConsumerOffsetInfo, Sizeable, Stats, UserId};
-use tokio::sync::RwLock;
pub fn map_stats(stats: &Stats) -> Bytes {
let mut bytes = BytesMut::with_capacity(104);
@@ -95,11 +94,10 @@ pub fn map_client(client: &Client) -> Bytes {
bytes.freeze()
}
-pub async fn map_clients(clients: &[IggySharedMut<Client>]) -> Bytes {
+pub async fn map_clients(clients: &[Client]) -> Bytes {
let mut bytes = BytesMut::new();
for client in clients {
- let client = client.read().await;
- extend_client(&client, &mut bytes);
+ extend_client(client, &mut bytes);
}
bytes.freeze()
}
@@ -183,12 +181,11 @@ pub async fn map_topic(topic: &Topic) -> Bytes {
bytes.freeze()
}
-pub async fn map_consumer_group(consumer_group: &ConsumerGroup) -> Bytes {
+pub fn map_consumer_group(consumer_group: &ConsumerGroup) -> Bytes {
let mut bytes = BytesMut::new();
extend_consumer_group(consumer_group, &mut bytes);
let members = consumer_group.get_members();
for member in members {
- let member = member.read().await;
bytes.put_u32_le(member.id);
let partitions = member.get_partitions();
bytes.put_u32_le(partitions.len() as u32);
@@ -199,10 +196,9 @@ pub async fn map_consumer_group(consumer_group: &ConsumerGroup) -> Bytes {
bytes.freeze()
}
-pub async fn map_consumer_groups(consumer_groups: &[&RwLock<ConsumerGroup>]) -> Bytes {
+pub fn map_consumer_groups(consumer_groups: &[ConsumerGroup]) -> Bytes {
let mut bytes = BytesMut::new();
for consumer_group in consumer_groups {
- let consumer_group = consumer_group.read().await;
extend_consumer_group(&consumer_group, &mut bytes);
}
bytes.freeze()
diff --git a/core/server/src/http/consumer_groups.rs b/core/server/src/http/consumer_groups.rs
index dd48c8c2..d58faef3 100644
--- a/core/server/src/http/consumer_groups.rs
+++ b/core/server/src/http/consumer_groups.rs
@@ -72,7 +72,7 @@ async fn get_consumer_group(
};
let consumer_group = consumer_group.read().await;
- let consumer_group = mapper::map_consumer_group(&consumer_group).await;
+ let consumer_group = mapper::map_consumer_group(&consumer_group);
Ok(Json(consumer_group))
}
@@ -89,7 +89,7 @@ async fn get_consumer_groups(
&stream_id,
&topic_id,
)?;
- let consumer_groups = mapper::map_consumer_groups(&consumer_groups).await;
+ let consumer_groups = mapper::map_consumer_groups(&consumer_groups);
Ok(Json(consumer_groups))
}
@@ -116,7 +116,7 @@ async fn create_consumer_group(
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to create consumer group, stream ID: {}, topic ID: {}, group ID: {:?}", stream_id, topic_id, command.group_id))?;
let consumer_group = consumer_group.read().await;
let group_id = consumer_group.group_id;
- let consumer_group_details = mapper::map_consumer_group(&consumer_group).await;
+ let consumer_group_details = mapper::map_consumer_group(&consumer_group);
drop(consumer_group);
let system = system.downgrade();
diff --git a/core/server/src/http/jwt/cleaner.rs b/core/server/src/http/jwt/cleaner.rs
index 40003eca..6d4329e7 100644
--- a/core/server/src/http/jwt/cleaner.rs
+++ b/core/server/src/http/jwt/cleaner.rs
@@ -19,12 +19,12 @@
use crate::http::shared::AppState;
use iggy_common::IggyTimestamp;
use std::sync::Arc;
-use std::time::Duration;
+use monoio::time::Duration;
use tracing::{error, trace};
pub fn start_expired_tokens_cleaner(app_state: Arc<AppState>) {
- tokio::spawn(async move {
- let mut interval_timer = tokio::time::interval(Duration::from_secs(300));
+ monoio::spawn(async move {
+ let mut interval_timer = monoio::time::interval(Duration::from_secs(300));
loop {
interval_timer.tick().await;
trace!("Deleting expired tokens...");
@@ -34,7 +34,7 @@ pub fn start_expired_tokens_cleaner(app_state: Arc<AppState>) {
.delete_expired_revoked_tokens(now)
.await
.unwrap_or_else(|err| {
- error!("Failed to delete expired revoked access tokens. Error: {err}",);
+ error!("Failed to delete expired revoked access tokens. Error: {err}");
});
}
});
diff --git a/core/server/src/http/jwt/jwt_manager.rs b/core/server/src/http/jwt/jwt_manager.rs
index 56e2ce9b..dfe0f66b 100644
--- a/core/server/src/http/jwt/jwt_manager.rs
+++ b/core/server/src/http/jwt/jwt_manager.rs
@@ -28,7 +28,7 @@ use iggy_common::IggyError;
use iggy_common::IggyExpiry;
use iggy_common::IggyTimestamp;
use iggy_common::UserId;
-use iggy_common::locking::IggySharedMut;
+use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation, encode};
use std::sync::Arc;
@@ -54,7 +54,7 @@ pub struct JwtManager {
issuer: IssuerOptions,
validator: ValidatorOptions,
tokens_storage: TokenStorage,
- revoked_tokens: IggySharedMut<AHashMap<String, u64>>,
+ revoked_tokens: IggyRwLock<AHashMap<String, u64>>,
validations: AHashMap<Algorithm, Validation>,
}
@@ -77,7 +77,7 @@ impl JwtManager {
issuer,
validator,
tokens_storage: TokenStorage::new(persister, path),
- revoked_tokens: IggySharedMut::new(AHashMap::new()),
+ revoked_tokens: IggyRwLock::new(AHashMap::new()),
})
}
diff --git a/core/server/src/http/mapper.rs b/core/server/src/http/mapper.rs
index ed9ebba3..d839b9d9 100644
--- a/core/server/src/http/mapper.rs
+++ b/core/server/src/http/mapper.rs
@@ -28,12 +28,11 @@ use iggy_common::PersonalAccessTokenInfo;
use iggy_common::Sizeable;
use iggy_common::StreamDetails;
use iggy_common::TopicDetails;
-use iggy_common::locking::IggySharedMut;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{ConsumerGroupDetails, ConsumerGroupMember};
use iggy_common::{IdentityInfo, TokenInfo};
use iggy_common::{UserInfo, UserInfoDetails};
-use tokio::sync::RwLock;
+
pub fn map_stream(stream: &Stream) -> StreamDetails {
let topics = map_topics(&stream.get_topics());
@@ -177,10 +176,9 @@ pub fn map_client(client: &Client) -> iggy_common::ClientInfoDetails {
}
}
-pub async fn map_clients(clients: &[IggySharedMut<Client>]) -> Vec<iggy_common::ClientInfo> {
+pub fn map_clients(clients: &[Client]) -> Vec<iggy_common::ClientInfo> {
let mut all_clients = Vec::new();
for client in clients {
- let client = client.read().await;
let client = iggy_common::ClientInfo {
client_id: client.session.client_id,
user_id: client.user_id,
@@ -195,12 +193,11 @@ pub async fn map_clients(clients: &[IggySharedMut<Client>]) -> Vec<iggy_common::
all_clients
}
-pub async fn map_consumer_groups(
- consumer_groups: &[&RwLock<ConsumerGroup>],
+pub fn map_consumer_groups(
+ consumer_groups: &[ConsumerGroup],
) -> Vec<iggy_common::ConsumerGroup> {
let mut groups = Vec::new();
for consumer_group in consumer_groups {
- let consumer_group = consumer_group.read().await;
let consumer_group = iggy_common::ConsumerGroup {
id: consumer_group.group_id,
name: consumer_group.name.clone(),
@@ -213,7 +210,7 @@ pub async fn map_consumer_groups(
groups
}
-pub async fn map_consumer_group(consumer_group: &ConsumerGroup) -> ConsumerGroupDetails {
+pub fn map_consumer_group(consumer_group: &ConsumerGroup) -> ConsumerGroupDetails {
let mut consumer_group_details = ConsumerGroupDetails {
id: consumer_group.group_id,
name: consumer_group.name.clone(),
@@ -223,7 +220,6 @@ pub async fn map_consumer_group(consumer_group: &ConsumerGroup) -> ConsumerGroup
};
let members = consumer_group.get_members();
for member in members {
- let member = member.read().await;
let partitions = member.get_partitions();
consumer_group_details.members.push(ConsumerGroupMember {
id: member.id,
diff --git a/core/server/src/http/messages.rs b/core/server/src/http/messages.rs
index 251c503b..e289adb9 100644
--- a/core/server/src/http/messages.rs
+++ b/core/server/src/http/messages.rs
@@ -20,9 +20,9 @@ use crate::http::COMPONENT;
use crate::http::error::CustomError;
use crate::http::jwt::json_web_token::Identity;
use crate::http::shared::AppState;
+use crate::shard::system::messages::PollingArgs;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut};
use crate::streaming::session::Session;
-use crate::streaming::systems::messages::PollingArgs;
use crate::streaming::utils::PooledBuffer;
use axum::extract::{Path, Query, State};
use axum::http::StatusCode;
diff --git a/core/server/src/http/mod.rs b/core/server/src/http/mod.rs
index 194ed74a..7b24bfff 100644
--- a/core/server/src/http/mod.rs
+++ b/core/server/src/http/mod.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+/*
pub mod consumer_groups;
pub mod consumer_offsets;
pub mod diagnostics;
@@ -32,5 +33,6 @@ pub mod streams;
pub mod system;
pub mod topics;
pub mod users;
+*/
pub const COMPONENT: &str = "HTTP";
diff --git a/core/server/src/http/shared.rs b/core/server/src/http/shared.rs
index 50703357..a21ddb1c 100644
--- a/core/server/src/http/shared.rs
+++ b/core/server/src/http/shared.rs
@@ -16,14 +16,13 @@
* under the License.
*/
-use crate::http::jwt::jwt_manager::JwtManager;
-use crate::streaming::systems::system::SharedSystem;
+use crate::{http::jwt::jwt_manager::JwtManager, shard::IggyShard};
use std::net::SocketAddr;
use ulid::Ulid;
pub struct AppState {
pub jwt_manager: JwtManager,
- pub system: SharedSystem,
+ pub shard: IggyShard,
}
#[derive(Debug, Copy, Clone)]
diff --git a/core/server/src/http/system.rs b/core/server/src/http/system.rs
index 1c7166e3..4428d36c 100644
--- a/core/server/src/http/system.rs
+++ b/core/server/src/http/system.rs
@@ -114,7 +114,7 @@ async fn get_clients(
identity.user_id
)
})?;
- let clients = mapper::map_clients(&clients).await;
+ let clients = mapper::map_clients(&clients);
Ok(Json(clients))
}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 12ba64bb..b5e48525 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -461,6 +461,6 @@ impl IggyShard {
Err(IggyError::Unauthenticated)
}
})?;
- Ok(user_id)
+ Ok(user_id)
}
}
diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs
index a2debc42..ee9e1634 100644
--- a/core/server/src/shard/system/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -16,17 +16,16 @@
* under the License.
*/
+use super::COMPONENT;
use crate::shard::IggyShard;
use crate::streaming::clients::client_manager::{Client, Transport};
use crate::streaming::session::Session;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
-use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
use std::net::SocketAddr;
use std::rc::Rc;
-use std::sync::Arc;
use tracing::{error, info};
impl IggyShard {
@@ -43,7 +42,7 @@ impl IggyShard {
{
let mut client_manager = self.client_manager.borrow_mut();
- let client = client_manager.delete_client(client_id).await;
+ let client = client_manager.delete_client(client_id);
if client.is_none() {
error!("Client with ID: {client_id} was not found in the client manager.",);
return;
@@ -51,7 +50,6 @@ impl IggyShard {
self.metrics.decrement_clients(1);
let client = client.unwrap();
- let client = client.read().await;
consumer_groups = client
.consumer_groups
.iter()
@@ -65,14 +63,12 @@ impl IggyShard {
}
for (stream_id, topic_id, consumer_group_id) in consumer_groups.into_iter() {
- _ = self
- .leave_consumer_group_by_client(
- &Identifier::numeric(stream_id).unwrap(),
- &Identifier::numeric(topic_id).unwrap(),
- &Identifier::numeric(consumer_group_id).unwrap(),
- client_id,
- )
- .await
+ _ = self.leave_consumer_group_by_client(
+ &Identifier::numeric(stream_id).unwrap(),
+ &Identifier::numeric(topic_id).unwrap(),
+ &Identifier::numeric(consumer_group_id).unwrap(),
+ client_id,
+ )
}
}
@@ -80,9 +76,10 @@ impl IggyShard {
&self,
session: &Session,
client_id: u32,
- ) -> Result<Option<IggySharedMut<Client>>, IggyError> {
+ ) -> Result<Option<Client>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.get_client(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -91,16 +88,13 @@ impl IggyShard {
)
})?;
- let client_manager = self.client_manager.read().await;
- Ok(client_manager.try_get_client(client_id))
+ Ok(self.client_manager.borrow().try_get_client(client_id))
}
- pub async fn get_clients(
- &self,
- session: &Session,
- ) -> Result<Vec<IggySharedMut<Client>>, IggyError> {
+ pub async fn get_clients(&self, session: &Session) -> Result<Vec<Client>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.get_clients(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -109,7 +103,6 @@ impl IggyShard {
)
})?;
- let client_manager = self.client_manager.read().await;
- Ok(client_manager.get_clients())
+ Ok(self.client_manager.borrow().get_clients())
}
}
diff --git a/core/server/src/shard/system/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs
index d8d8818e..768d4e5c 100644
--- a/core/server/src/shard/system/consumer_groups.rs
+++ b/core/server/src/shard/system/consumer_groups.rs
@@ -16,29 +16,39 @@
* under the License.
*/
+use std::cell::Ref;
+
+use super::COMPONENT;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
+use crate::streaming::streams::stream::Stream;
use crate::streaming::topics::consumer_group::ConsumerGroup;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
use iggy_common::locking::IggySharedMutFn;
-use tokio::sync::RwLock;
impl IggyShard {
- pub fn get_consumer_group(
+ pub fn get_consumer_group<'cg, 'stream>(
&self,
session: &Session,
- stream_id: &Identifier,
+ stream: &'stream Stream,
topic_id: &Identifier,
group_id: &Identifier,
- ) -> Result<Option<&RwLock<ConsumerGroup>>, IggyError> {
+ ) -> Result<Option<Ref<'cg, ConsumerGroup>>, IggyError>
+ where
+ 'stream: 'cg,
+ {
self.ensure_authenticated(session)?;
- let Some(topic) = self.try_find_topic(session, stream_id, topic_id)? else {
- return Ok(None);
- };
+ let stream_id = stream.stream_id;
+ let topic = stream.get_topic(topic_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - topic with ID: {topic_id} was not found in stream with ID: {stream_id}",
+ )
+ })?;
self.permissioner
+ .borrow()
.get_consumer_group(session.get_user_id(), topic.stream_id, topic.topic_id)
.with_error_context(|error| {
format!(
@@ -55,12 +65,16 @@ impl IggyShard {
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
- ) -> Result<Vec<&RwLock<ConsumerGroup>>, IggyError> {
+ ) -> Result<Vec<ConsumerGroup>, IggyError> {
self.ensure_authenticated(session)?;
- let topic = self.find_topic(session, stream_id, topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream with ID: {stream_id} was not found")
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic with ID: {topic_id} was not found in stream with ID: {stream_id}"))?;
self.permissioner
+ .borrow()
.get_consumer_groups(session.get_user_id(), topic.stream_id, topic.topic_id)
.with_error_context(|error| {
format!(
@@ -72,40 +86,45 @@ impl IggyShard {
Ok(topic.get_consumer_groups())
}
- pub async fn create_consumer_group(
- &mut self,
+ pub fn create_consumer_group(
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
group_id: Option<u32>,
name: &str,
- ) -> Result<&RwLock<ConsumerGroup>, IggyError> {
+ ) -> Result<u32, IggyError> {
self.ensure_authenticated(session)?;
{
- let topic = self.find_topic(session, stream_id, topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}"
+ )
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.create_consumer_group(
+ self.permissioner.borrow().create_consumer_group(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
).with_error_context(|error| format!("{COMPONENT} (error: {error}) - permission denied to create consumer group for user {} on stream ID: {}, topic ID: {}", session.get_user_id(), topic.stream_id, topic.topic_id))?;
}
- let topic = self.get_stream_mut(stream_id)?
- .get_topic_mut(topic_id)
+ let mut stream = self.get_stream_mut(stream_id)
+ .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}"))?;
+ let topic = stream.get_topic_mut(topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
topic
.create_consumer_group(group_id, name)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to create consumer group with name: {name}")
- })
+ }).map(|cg| cg.group_id)
}
pub async fn delete_consumer_group(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -115,10 +134,15 @@ impl IggyShard {
let stream_id_value;
let topic_id_value;
{
- let topic = self.find_topic(session, stream_id, topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}"
+ )
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.delete_consumer_group(
+ self.permissioner.borrow().delete_consumer_group(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -130,7 +154,7 @@ impl IggyShard {
let consumer_group;
{
- let stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {stream_id}"
)
@@ -142,25 +166,20 @@ impl IggyShard {
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to delete consumer group with ID: {consumer_group_id}"))?
}
- let client_manager = self.client_manager.read().await;
- let consumer_group = consumer_group.read().await;
for member in consumer_group.get_members() {
- let member = member.read().await;
- client_manager
- .leave_consumer_group(
+ self.client_manager.borrow_mut().leave_consumer_group(
member.id,
stream_id_value,
topic_id_value,
consumer_group.group_id,
)
- .await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to make client leave consumer group for client ID: {}, group ID: {}", member.id, consumer_group.group_id))?;
}
Ok(())
}
- pub async fn join_consumer_group(
+ pub fn join_consumer_group(
&self,
session: &Session,
stream_id: &Identifier,
@@ -171,15 +190,20 @@ impl IggyShard {
let stream_id_value;
let topic_id_value;
{
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}"
+ )
+ })?;
let topic = self
- .find_topic(session, stream_id, topic_id)
+ .find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}",
)
})?;
- self.permissioner.join_consumer_group(
+ self.permissioner.borrow().join_consumer_group(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -191,7 +215,14 @@ impl IggyShard {
let group_id;
{
- let topic = self.find_topic(session, stream_id, topic_id)?;
+ let mut stream = self.get_stream_mut(stream_id)
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}")
+ })?;
+ let topic = stream.get_topic_mut(topic_id)
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get mutable reference to topic with ID: {topic_id}")
+ })?;
{
let consumer_group = topic
@@ -202,13 +233,11 @@ impl IggyShard {
)
})?;
- let consumer_group = consumer_group.read().await;
group_id = consumer_group.group_id;
}
topic
.join_consumer_group(consumer_group_id, session.client_id)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to join consumer group for group ID: {group_id}"
@@ -216,10 +245,7 @@ impl IggyShard {
})?;
}
- let client_manager = self.client_manager.read().await;
- client_manager
- .join_consumer_group(session.client_id, stream_id_value, topic_id_value, group_id)
- .await
+ self.client_manager.borrow_mut().join_consumer_group(session.client_id, stream_id_value, topic_id_value, group_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to make client join consumer group for client ID: {}",
@@ -239,15 +265,23 @@ impl IggyShard {
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
{
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
let topic = self
- .find_topic(session, stream_id, topic_id)
+ .find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!(
+<<<<<<< HEAD
"{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id:?}, topic_id: {topic_id:?}"
+=======
+ "{COMPONENT} (error: {error}) - topic not found for stream ID: {:?}, topic_id: {:?}",
+ stream.stream_id, topic_id
+>>>>>>> 48107890 (fix iggy shard errors)
)
})?;
- self.permissioner.leave_consumer_group(
+ self.permissioner.borrow().leave_consumer_group(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -260,7 +294,6 @@ impl IggyShard {
consumer_group_id,
session.client_id,
)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to leave consumer group for client ID: {}",
@@ -269,7 +302,7 @@ impl IggyShard {
})
}
- pub async fn leave_consumer_group_by_client(
+ pub fn leave_consumer_group_by_client(
&self,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -281,15 +314,17 @@ impl IggyShard {
let group_id;
{
- let stream = self.get_stream(stream_id).with_error_context(|error| {
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
})?;
- let topic = stream.get_topic(topic_id)
+ let stream_id = stream.stream_id;
+ let topic = stream.get_topic_mut(topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}",
)
})?;
+ let topic_id = topic.topic_id;
{
let consumer_group = topic
.get_consumer_group(consumer_group_id)
@@ -298,23 +333,23 @@ impl IggyShard {
"{COMPONENT} (error: {error}) - consumer group not found for group_id: {consumer_group_id}",
)
})?;
- let consumer_group = consumer_group.read().await;
group_id = consumer_group.group_id;
}
- stream_id_value = stream.stream_id;
- topic_id_value = topic.topic_id;
+ stream_id_value = stream_id;
+ topic_id_value = topic_id;
topic
.leave_consumer_group(consumer_group_id, client_id)
- .await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed leave consumer group, client ID {client_id}",)
})?;
}
- let client_manager = self.client_manager.read().await;
- client_manager
- .leave_consumer_group(client_id, stream_id_value, topic_id_value, group_id)
- .await
+ self.client_manager.borrow_mut().leave_consumer_group(
+ client_id,
+ stream_id_value,
+ topic_id_value,
+ group_id,
+ )
}
}
diff --git a/core/server/src/shard/system/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs
index 0c7438a6..e5504173 100644
--- a/core/server/src/shard/system/consumer_offsets.rs
+++ b/core/server/src/shard/system/consumer_offsets.rs
@@ -16,7 +16,8 @@
* under the License.
*/
-use crate::streaming::session::Session;
+use super::COMPONENT;
+use crate::{shard::IggyShard, streaming::session::Session};
use error_set::ErrContext;
use iggy_common::{Consumer, ConsumerOffsetInfo, Identifier, IggyError};
@@ -31,9 +32,12 @@ impl IggyShard {
offset: u64,
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
- let topic = self.find_topic(session, stream_id, topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream with ID: {stream_id} was not found")
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic with ID: {topic_id} was not found in stream with ID: {stream_id}"))?;
- self.permissioner.store_consumer_offset(
+ self.permissioner.borrow().store_consumer_offset(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -53,11 +57,14 @@ impl IggyShard {
partition_id: Option<u32>,
) -> Result<Option<ConsumerOffsetInfo>, IggyError> {
self.ensure_authenticated(session)?;
- let Some(topic) = self.try_find_topic(session, stream_id, topic_id)? else {
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream with ID: {stream_id} was not found")
+ })?;
+ let Some(topic) = self.try_find_topic(session, &stream, topic_id)? else {
return Ok(None);
};
- self.permissioner.get_consumer_offset(
+ self.permissioner.borrow().get_consumer_offset(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -82,9 +89,12 @@ impl IggyShard {
partition_id: Option<u32>,
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
- let topic = self.find_topic(session, stream_id, topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream with ID: {stream_id} was not found")
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id)
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic with ID: {topic_id} was not found in stream with ID: {stream_id}"))?;
- self.permissioner.delete_consumer_offset(
+ self.permissioner.borrow().delete_consumer_offset(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs
index 54046dba..06499bd3 100644
--- a/core/server/src/shard/system/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -16,11 +16,13 @@
* under the License.
*/
+use super::COMPONENT;
use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
use crate::shard::IggyShard;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet};
use crate::streaming::session::Session;
use crate::streaming::utils::PooledBuffer;
+use async_zip::tokio::read::stream;
use error_set::ErrContext;
use iggy_common::{
BytesSerializable, Confirmation, Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier,
@@ -43,8 +45,13 @@ impl IggyShard {
return Err(IggyError::InvalidMessagesCount);
}
- let topic = self.find_topic(session, stream_id, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
self.permissioner
+ .borrow()
.poll_messages(session.get_user_id(), topic.stream_id, topic.topic_id)
.with_error_context(|error| format!(
"{COMPONENT} (error: {error}) - permission denied to poll messages for user {} on stream ID: {}, topic ID: {}",
@@ -60,7 +67,6 @@ impl IggyShard {
// There might be no partition assigned, if it's the consumer group member without any partitions.
let Some((polling_consumer, partition_id)) = topic
.resolve_consumer_with_partition_id(consumer, session.client_id, partition_id, true)
- .await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer with partition id, consumer: {consumer}, client ID: {}, partition ID: {:?}", session.client_id, partition_id))? else {
return Ok((IggyPollMetadata::new(0, 0), IggyMessagesBatchSet::empty()));
};
@@ -80,11 +86,15 @@ impl IggyShard {
topic
.store_consumer_offset_internal(polling_consumer, offset, partition_id)
.await
+<<<<<<< HEAD
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to store consumer offset internal, polling consumer: {polling_consumer}, offset: {offset}, partition ID: {partition_id}")) ?;
+=======
+ .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to store consumer offset internal, polling consumer: {}, offset: {}, partition ID: {}", polling_consumer, offset, partition_id))?;
+>>>>>>> 48107890 (fix iggy shard errors)
}
let batch_set = if let Some(encryptor) = &self.encryptor {
- self.decrypt_messages(batch_set, encryptor.as_ref()).await?
+ self.decrypt_messages(batch_set, encryptor).await?
} else {
batch_set
};
@@ -102,8 +112,12 @@ impl IggyShard {
confirmation: Option<Confirmation>,
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
- let topic = self.find_topic(session, stream_id, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream_id: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.append_messages(
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream not found for stream_id: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream_id: {stream_id}, topic_id: {topic_id}"))?;
+ self.permissioner.borrow().append_messages(
session.get_user_id(),
topic.stream_id,
topic.topic_id
@@ -117,7 +131,7 @@ impl IggyShard {
// Encrypt messages if encryptor is configured
let messages = if let Some(encryptor) = &self.encryptor {
- self.encrypt_messages(messages, encryptor.as_ref())?
+ self.encrypt_messages(messages, encryptor)?
} else {
messages
};
@@ -139,8 +153,12 @@ impl IggyShard {
fsync: bool,
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
- let topic = self.find_topic(session, &stream_id, &topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.append_messages(
+ let stream = self.get_stream(&stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}")
+ })?;
+ let stream_id = stream.stream_id;
+ let topic = self.find_topic(session, &stream, &topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic ID: {topic_id}"))?;
+ self.permissioner.borrow().append_messages(
session.get_user_id(),
topic.stream_id,
topic.topic_id
diff --git a/core/server/src/shard/system/partitions.rs b/core/server/src/shard/system/partitions.rs
index 7712f498..01e02968 100644
--- a/core/server/src/shard/system/partitions.rs
+++ b/core/server/src/shard/system/partitions.rs
@@ -16,16 +16,16 @@
* under the License.
*/
+use super::COMPONENT;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
impl IggyShard {
pub async fn create_partitions(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -33,8 +33,13 @@ impl IggyShard {
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
{
- let topic = self.find_topic(session, stream_id, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.create_partitions(
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}"
+ )
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic ID: {topic_id}"))?;
+ self.permissioner.borrow().create_partitions(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -46,28 +51,33 @@ impl IggyShard {
))?;
}
- let topic = self
- .get_stream_mut(stream_id)?
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let topic = stream
.get_topic_mut(topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {stream_id}"
)
})?;
+
+ // TODO: Make add persisted partitions to topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
topic
.add_persisted_partitions(partitions_count)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to add persisted partitions, topic: {topic}")
})?;
- topic.reassign_consumer_groups().await;
+ topic.reassign_consumer_groups();
self.metrics.increment_partitions(partitions_count);
self.metrics.increment_segments(partitions_count);
Ok(())
}
pub async fn delete_partitions(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -75,8 +85,13 @@ impl IggyShard {
) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
{
- let topic = self.find_topic(session, stream_id, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.delete_partitions(
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream ID: {stream_id}"
+ )
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream ID: {stream_id}, topic_id: {topic_id}"))?;
+ self.permissioner.borrow().delete_partitions(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -88,21 +103,26 @@ impl IggyShard {
))?;
}
- let topic = self
- .get_stream_mut(stream_id)?
+ let mut stream = self.get_stream_mut(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let topic = stream
.get_topic_mut(topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {stream_id}"
)
})?;
+
+ // TODO: Make delete persisted partitions from topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
let partitions = topic
.delete_persisted_partitions(partitions_count)
.await
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to delete persisted partitions for topic: {topic}")
})?;
- topic.reassign_consumer_groups().await;
+ topic.reassign_consumer_groups();
if let Some(partitions) = partitions {
self.metrics.decrement_partitions(partitions_count);
self.metrics.decrement_segments(partitions.segments_count);
diff --git a/core/server/src/shard/system/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs
index b5452e9a..082fcd93 100644
--- a/core/server/src/shard/system/personal_access_tokens.rs
+++ b/core/server/src/shard/system/personal_access_tokens.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+use super::COMPONENT;
+use crate::shard::IggyShard;
use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
use crate::streaming::session::Session;
use crate::streaming::users::user::User;
@@ -64,7 +66,7 @@ impl IggyShard {
let user = self.get_user(&identifier).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get user with id: {user_id}")
})?;
- let max_token_per_user = self.personal_access_token.max_tokens_per_user;
+ let max_token_per_user = self.config.personal_access_token.max_tokens_per_user;
if user.personal_access_tokens.len() as u32 >= max_token_per_user {
error!(
"User with ID: {user_id} has reached the maximum number of personal access tokens: {max_token_per_user}.",
@@ -76,7 +78,7 @@ impl IggyShard {
}
}
- let user = self.get_user(&identifier).with_error_context(|error| {
+ let user = self.get_user_mut(&identifier).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get mutable reference to the user with id: {user_id}")
})?;
@@ -102,7 +104,7 @@ impl IggyShard {
}
pub async fn delete_personal_access_token(
- &mut self,
+ &self,
session: &Session,
name: &str,
) -> Result<(), IggyError> {
@@ -137,10 +139,11 @@ impl IggyShard {
&self,
token: &str,
session: Option<&Session>,
- ) -> Result<&User, IggyError> {
+ ) -> Result<User, IggyError> {
let token_hash = PersonalAccessToken::hash_token(token);
+ let users = self.users.borrow();
let mut personal_access_token = None;
- for user in self.users.values() {
+ for user in users.values() {
if let Some(pat) = user.personal_access_tokens.get(&token_hash) {
personal_access_token = Some(pat);
break;
@@ -173,6 +176,5 @@ impl IggyShard {
)
})?;
self.login_user_with_credentials(&user.username, None, session)
- .await
}
}
diff --git a/core/server/src/shard/system/segments.rs b/core/server/src/shard/system/segments.rs
index 27abb76b..6a8ad68e 100644
--- a/core/server/src/shard/system/segments.rs
+++ b/core/server/src/shard/system/segments.rs
@@ -16,6 +16,7 @@ use crate::shard::IggyShard;
* specific language governing permissions and limitations
* under the License.
*/
+use super::COMPONENT;
use crate::streaming::session::Session;
use error_set::ErrContext;
use iggy_common::Identifier;
@@ -24,7 +25,7 @@ use iggy_common::locking::IggySharedMutFn;
impl IggyShard {
pub async fn delete_segments(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -35,9 +36,14 @@ impl IggyShard {
self.ensure_authenticated(session)?;
{
- let topic = self.find_topic(session, stream_id, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream_id: {stream_id}, topic_id: {topic_id}"))?;
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - stream not found for stream_id: {stream_id}"
+ )
+ })?;
+ let topic = self.find_topic(session, &stream, topic_id).with_error_context(|error| format!("{COMPONENT} (error: {error}) - topic not found for stream_id: {stream_id}, topic_id: {topic_id}"))?;
- self.permissioner.delete_segments(
+ self.permissioner.borrow().delete_segments(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -49,9 +55,11 @@ impl IggyShard {
))?;
}
- let topic = self
- .get_stream_mut(stream_id)?
- .get_topic_mut(topic_id)
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
+ let topic = stream
+ .get_topic(topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}"
@@ -95,7 +103,25 @@ impl IggyShard {
(segments_count, messages_count)
};
- topic.reassign_consumer_groups().await;
+ drop(partition);
+ drop(topic);
+ drop(stream);
+
+ let mut stream = self
+ .get_stream_mut(stream_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get mutable reference to stream with ID: {stream_id}"
+ )
+ })?;
+ let topic = stream
+ .get_topic_mut(topic_id)
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to get mutable reference to topic with ID: {topic_id} in stream with ID: {stream_id}"
+ )
+ })?;
+ topic.reassign_consumer_groups();
self.metrics.decrement_segments(deleted_segments_count);
self.metrics.decrement_messages(deleted_messages_count);
diff --git a/core/server/src/shard/system/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs
index f6e07785..78c5d393 100644
--- a/core/server/src/shard/system/snapshot/mod.rs
+++ b/core/server/src/shard/system/snapshot/mod.rs
@@ -19,22 +19,23 @@
mod procdump;
use crate::configs::system::SystemConfig;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
use async_zip::tokio::write::ZipFileWriter;
use async_zip::{Compression, ZipEntryBuilder};
use iggy_common::{IggyDuration, IggyError, Snapshot, SnapshotCompression, SystemSnapshotType};
+use monoio::fs::{File, OpenOptions};
use std::io::Cursor;
use std::path::PathBuf;
-use std::sync::Arc;
+use std::rc::Rc;
use std::time::Instant;
use tempfile::NamedTempFile;
-use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::Command;
use tokio_util::compat::TokioAsyncWriteCompatExt;
use tracing::{error, info};
-impl System {
+impl IggyShard {
pub async fn get_snapshot(
&self,
session: &Session,
@@ -69,18 +70,19 @@ impl System {
let now = Instant::now();
for snapshot_type in snapshot_types {
- match get_command_result(snapshot_type, self.config.clone()).await {
+ match get_command_result(snapshot_type, self.config.system.clone()).await {
Ok(temp_file) => {
let filename = format!("{snapshot_type}.txt");
let entry = ZipEntryBuilder::new(filename.clone().into(), compression);
- let mut file = File::open(temp_file.path()).await.map_err(|e| {
+ let file = File::open(temp_file.path()).await.map_err(|e| {
error!("Failed to open temporary file: {}", e);
IggyError::SnapshotFileCompletionFailed
})?;
- let mut content = Vec::new();
- if let Err(e) = file.read_to_end(&mut content).await {
+ let content = Vec::new();
+ let (result, content) = file.read_exact_at(content, 0).await;
+ if let Err(e) = result {
error!("Failed to read temporary file: {}", e);
continue;
}
@@ -130,9 +132,10 @@ async fn write_command_output_to_temp_file(
) -> Result<NamedTempFile, std::io::Error> {
let output = command.output().await?;
let temp_file = NamedTempFile::new()?;
- let mut file = File::from_std(temp_file.as_file().try_clone()?);
- file.write_all(&output.stdout).await?;
- file.flush().await?;
+ let file = File::from_std(temp_file.as_file().try_clone()?).unwrap();
+ let (result, _) = file.write_all_at(output.stdout, 0).await;
+ result?;
+ file.sync_all().await?;
Ok(temp_file)
}
@@ -142,18 +145,35 @@ async fn get_filesystem_overview() -> Result<NamedTempFile, std::io::Error> {
async fn get_process_info() -> Result<NamedTempFile, std::io::Error> {
let temp_file = NamedTempFile::new()?;
- let mut file = File::from_std(temp_file.as_file().try_clone()?);
+ let file = File::from_std(temp_file.as_file().try_clone()?).unwrap();
+ let mut position = 0;
let ps_output = Command::new("ps").arg("aux").output().await?;
- file.write_all(b"=== Process List (ps aux) ===\n").await?;
- file.write_all(&ps_output.stdout).await?;
- file.write_all(b"\n\n").await?;
+ let (result, written) = file
+ .write_all_at(b"=== Process List (ps aux) ===\n", 0)
+ .await;
+ result?;
+ position += written.len() as u64;
+
+ let (result, written) = file.write_all_at(ps_output.stdout, position).await;
+ result?;
+ position += written.len() as u64;
+
+ let (result, written) = file.write_all_at(b"\n\n", position).await;
+ result?;
+ position += written.len() as u64;
+
+ let (result, written) = file
+ .write_all_at(b"=== Detailed Process Information ===\n", position)
+ .await;
+ result?;
+ position += written.len() as u64;
- file.write_all(b"=== Detailed Process Information ===\n")
- .await?;
let proc_info = procdump::get_proc_info().await?;
- file.write_all(proc_info.as_bytes()).await?;
- file.flush().await?;
+ let bytes = proc_info.as_bytes().to_owned();
+ let (result, _) = file.write_all_at(bytes, position).await;
+ result?;
+ file.sync_all().await?;
Ok(temp_file)
}
@@ -166,7 +186,7 @@ async fn get_test_snapshot() -> Result<NamedTempFile, std::io::Error> {
write_command_output_to_temp_file(Command::new("echo").arg("test")).await
}
-async fn get_server_logs(config: Arc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
+async fn get_server_logs(config: Rc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
let base_directory = PathBuf::from(config.get_system_path());
let logs_subdirectory = PathBuf::from(&config.logging.path);
let logs_path = base_directory.join(logs_subdirectory);
@@ -179,7 +199,7 @@ async fn get_server_logs(config: Arc<SystemConfig>) -> Result<NamedTempFile, std
write_command_output_to_temp_file(Command::new("sh").args(["-c", &list_and_cat])).await
}
-async fn get_server_config(config: Arc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
+async fn get_server_config(config: Rc<SystemConfig>) -> Result<NamedTempFile, std::io::Error> {
let base_directory = PathBuf::from(config.get_system_path());
let config_path = base_directory.join("runtime").join("current_config.toml");
@@ -188,7 +208,7 @@ async fn get_server_config(config: Arc<SystemConfig>) -> Result<NamedTempFile, s
async fn get_command_result(
snapshot_type: &SystemSnapshotType,
- config: Arc<SystemConfig>,
+ config: Rc<SystemConfig>,
) -> Result<NamedTempFile, std::io::Error> {
match snapshot_type {
SystemSnapshotType::FilesystemOverview => get_filesystem_overview().await,
diff --git a/core/server/src/shard/system/stats.rs b/core/server/src/shard/system/stats.rs
index 8410cac2..53e31322 100644
--- a/core/server/src/shard/system/stats.rs
+++ b/core/server/src/shard/system/stats.rs
@@ -16,8 +16,8 @@
* under the License.
*/
-use crate::shard::IggyShard;
use crate::VERSION;
+use crate::shard::IggyShard;
use crate::versioning::SemanticVersion;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{IggyDuration, IggyError, Stats};
@@ -45,7 +45,7 @@ impl IggyShard {
let total_cpu_usage = sys.global_cpu_usage();
let total_memory = sys.total_memory().into();
let available_memory = sys.available_memory().into();
- let clients_count = self.client_manager.read().await.get_clients().len() as u32;
+ let clients_count = self.client_manager.borrow().get_clients().len() as u32;
let hostname = sysinfo::System::host_name().unwrap_or("unknown_hostname".to_string());
let os_name = sysinfo::System::name().unwrap_or("unknown_os_name".to_string());
let os_version =
@@ -90,7 +90,7 @@ impl IggyShard {
drop(sys);
- for stream in self.streams.values() {
+ for stream in self.streams.borrow().values() {
stats.messages_count += stream.get_messages_count();
stats.segments_count += stream.get_segments_count();
stats.messages_size_bytes += stream.get_size();
@@ -104,7 +104,7 @@ impl IggyShard {
stats.consumer_groups_count += stream
.topics
.values()
- .map(|t| t.consumer_groups.len() as u32)
+ .map(|t| t.consumer_groups.borrow().len() as u32)
.sum::<u32>();
}
diff --git a/core/server/src/shard/system/storage.rs b/core/server/src/shard/system/storage.rs
index 4cb41120..cc413583 100644
--- a/core/server/src/shard/system/storage.rs
+++ b/core/server/src/shard/system/storage.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use super::COMPONENT;
use crate::shard::system::info::SystemInfo;
use crate::streaming::persistence::persister::PersisterKind;
use crate::streaming::storage::SystemInfoStorage;
diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs
index 8a0772a9..43b45246 100644
--- a/core/server/src/shard/system/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -16,24 +16,37 @@
* under the License.
*/
+use super::COMPONENT;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
use crate::streaming::streams::stream::Stream;
use error_set::ErrContext;
use futures::future::try_join_all;
-use std::cell::RefCell;
+use iggy_common::{IdKind, Identifier, IggyError};
+use std::cell::{Ref, RefCell, RefMut};
use std::sync::atomic::{AtomicU32, Ordering};
use tokio::fs;
use tracing::{error, info, warn};
+static CURRENT_STREAM_ID: AtomicU32 = AtomicU32::new(1);
+
impl IggyShard {
- pub fn get_streams(&self) -> Vec<&Stream> {
- self.streams.values().collect()
+ pub fn get_streams(&self) -> Vec<Ref<'_, Stream>> {
+ let len = self.streams.borrow().len();
+ let result = (0..len)
+ .map(|i| {
+ Ref::map(self.streams.borrow(), |streams| {
+ streams.values().nth(i).unwrap()
+ })
+ })
+ .collect();
+ result
}
- pub fn find_streams(&self, session: &Session) -> Result<Vec<&Stream>, IggyError> {
+ pub fn find_streams(&self, session: &Session) -> Result<Vec<Ref<'_, Stream>>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.get_streams(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -44,15 +57,16 @@ impl IggyShard {
Ok(self.get_streams())
}
- pub fn find_stream(
+ pub fn find_stream<'a>(
&self,
session: &Session,
identifier: &Identifier,
- ) -> Result<&Stream, IggyError> {
+ ) -> Result<Ref<'_, Stream>, IggyError> {
self.ensure_authenticated(session)?;
let stream = self.get_stream(identifier);
if let Ok(stream) = stream {
self.permissioner
+ .borrow()
.get_stream(session.get_user_id(), stream.stream_id)
.with_error_context(|error| {
format!(
@@ -70,13 +84,14 @@ impl IggyShard {
&self,
session: &Session,
identifier: &Identifier,
- ) -> Result<Option<&Stream>, IggyError> {
+ ) -> Result<Option<Ref<'_, Stream>>, IggyError> {
self.ensure_authenticated(session)?;
let Some(stream) = self.try_get_stream(identifier)? else {
return Ok(None);
};
self.permissioner
+ .borrow()
.get_stream(session.get_user_id(), stream.stream_id)
.with_error_context(|error| {
format!(
@@ -87,83 +102,92 @@ impl IggyShard {
Ok(Some(stream))
}
- pub fn try_get_stream(&self, identifier: &Identifier) -> Result<Option<&Stream>, IggyError> {
+ pub fn try_get_stream(
+ &self,
+ identifier: &Identifier,
+ ) -> Result<Option<Ref<'_, Stream>>, IggyError> {
match identifier.kind {
- IdKind::Numeric => Ok(self.streams.get(&identifier.get_u32_value()?)),
+ IdKind::Numeric => self.get_stream_by_id(identifier.get_u32_value()?).map(Some),
IdKind::String => Ok(self.try_get_stream_by_name(&identifier.get_cow_str_value()?)),
}
}
- fn try_get_stream_by_name(&self, name: &str) -> Option<&Stream> {
+ fn try_get_stream_by_name(&self, name: &str) -> Option<Ref<'_, Stream>> {
self.streams_ids
+ .borrow()
.get(name)
- .and_then(|id| self.streams.get(id))
+ .and_then(|id| Some(self.get_stream_ref(*id)))
}
- pub fn get_stream(&self, identifier: &Identifier) -> Result<&Stream, IggyError> {
+ pub fn get_stream(&self, identifier: &Identifier) -> Result<Ref<'_, Stream>, IggyError> {
match identifier.kind {
IdKind::Numeric => self.get_stream_by_id(identifier.get_u32_value()?),
IdKind::String => self.get_stream_by_name(&identifier.get_cow_str_value()?),
}
}
- pub fn get_stream_mut(&mut self, identifier: &Identifier) -> Result<&mut Stream, IggyError> {
+ pub fn get_stream_mut(&self, identifier: &Identifier) -> Result<RefMut<'_, Stream>, IggyError> {
match identifier.kind {
IdKind::Numeric => self.get_stream_by_id_mut(identifier.get_u32_value()?),
IdKind::String => self.get_stream_by_name_mut(&identifier.get_cow_str_value()?),
}
}
- fn get_stream_by_name(&self, name: &str) -> Result<&Stream, IggyError> {
- let stream_id = self.streams_ids.get(name);
- if stream_id.is_none() {
+ fn get_stream_by_name(&self, name: &str) -> Result<Ref<'_, Stream>, IggyError> {
+ let exists = self.streams_ids.borrow().iter().any(|s| s.0 == name);
+ if !exists {
return Err(IggyError::StreamNameNotFound(name.to_string()));
}
-
- self.get_stream_by_id(*stream_id.unwrap())
+ let stream_id = self.streams_ids.borrow().get(name).cloned().unwrap();
+ self.get_stream_by_id(stream_id)
}
- fn get_stream_by_id(&self, stream_id: u32) -> Result<&Stream, IggyError> {
- let stream = self.streams.get(&stream_id);
- if stream.is_none() {
+ fn get_stream_by_id(&self, stream_id: u32) -> Result<Ref<'_, Stream>, IggyError> {
+ let exists = self.streams.borrow().iter().any(|s| s.0 == &stream_id);
+ if !exists {
return Err(IggyError::StreamIdNotFound(stream_id));
}
-
- Ok(stream.unwrap())
+ Ok(self.get_stream_ref(stream_id))
}
- fn get_stream_by_name_mut(&mut self, name: &str) -> Result<&mut Stream, IggyError> {
- let stream_id;
- {
- let id = self.streams_ids.get_mut(name);
- if id.is_none() {
- return Err(IggyError::StreamNameNotFound(name.to_string()));
- }
+ fn get_stream_ref(&self, stream_id: u32) -> Ref<'_, Stream> {
+ Ref::map(self.streams.borrow(), |streams| {
+ streams.get(&stream_id).expect("Stream ID not found")
+ })
+ }
- stream_id = *id.unwrap();
+ fn get_stream_by_name_mut(&self, name: &str) -> Result<RefMut<'_, Stream>, IggyError> {
+ let exists = self.streams_ids.borrow().iter().any(|s| s.0 == name);
+ if !exists {
+ return Err(IggyError::StreamNameAlreadyExists(name.to_owned()));
}
-
- self.get_stream_by_id_mut(stream_id)
+ let streams_ids = self.streams_ids.borrow();
+ let id = streams_ids.get(name).cloned();
+ drop(streams_ids);
+ self.get_stream_by_id_mut(id.unwrap())
}
- fn get_stream_by_id_mut(&mut self, stream_id: u32) -> Result<&mut Stream, IggyError> {
- let stream = self.streams.get_mut(&stream_id);
- if stream.is_none() {
+ fn get_stream_by_id_mut(&self, stream_id: u32) -> Result<RefMut<'_, Stream>, IggyError> {
+ let exists = self.streams.borrow().iter().any(|s| s.0 == &stream_id);
+ if !exists {
return Err(IggyError::StreamIdNotFound(stream_id));
}
-
- Ok(stream.unwrap())
+ Ok(RefMut::map(self.streams.borrow_mut(), |s| {
+ s.get_mut(&stream_id).unwrap()
+ }))
}
pub async fn create_stream(
- &mut self,
+ &self,
session: &Session,
stream_id: Option<u32>,
name: &str,
- ) -> Result<&Stream, IggyError> {
+ ) -> Result<u32, IggyError> {
self.ensure_authenticated(session)?;
- self.permissioner.create_stream(session.get_user_id())?;
- if self.streams_ids.contains_key(name) {
+ self.permissioner
+ .borrow()
+ .create_stream(session.get_user_id())?;
+ if self.streams_ids.borrow().contains_key(name) {
return Err(IggyError::StreamNameAlreadyExists(name.to_owned()));
}
@@ -171,7 +195,7 @@ impl IggyShard {
if stream_id.is_none() {
id = CURRENT_STREAM_ID.fetch_add(1, Ordering::SeqCst);
loop {
- if self.streams.contains_key(&id) {
+ if self.streams.borrow().contains_key(&id) {
if id == u32::MAX {
return Err(IggyError::StreamIdAlreadyExists(id));
}
@@ -184,21 +208,23 @@ impl IggyShard {
id = stream_id.unwrap();
}
- if self.streams.contains_key(&id) {
+ if self.streams.borrow().contains_key(&id) {
return Err(IggyError::StreamIdAlreadyExists(id));
}
- let stream = Stream::create(id, name, self.config.clone(), self.storage.clone());
+ let stream = Stream::create(id, name, self.config.system.clone(), self.storage.clone());
stream.persist().await?;
info!("Created stream with ID: {id}, name: '{name}'.");
- self.streams_ids.insert(name.to_owned(), stream.stream_id);
- self.streams.insert(stream.stream_id, stream);
+ self.streams_ids
+ .borrow_mut()
+ .insert(name.to_owned(), stream.stream_id);
+ self.streams.borrow_mut().insert(stream.stream_id, stream);
self.metrics.increment_streams(1);
- self.get_stream_by_id(id)
+ Ok(id)
}
pub async fn update_stream(
- &mut self,
+ &self,
session: &Session,
id: &Identifier,
name: &str,
@@ -213,6 +239,7 @@ impl IggyShard {
}
self.permissioner
+ .borrow()
.update_stream(session.get_user_id(), stream_id)
.with_error_context(|error| {
format!(
@@ -223,7 +250,7 @@ impl IggyShard {
})?;
{
- if let Some(stream_id_by_name) = self.streams_ids.get(name) {
+ if let Some(stream_id_by_name) = self.streams_ids.borrow().get(name) {
if *stream_id_by_name != stream_id {
return Err(IggyError::StreamNameAlreadyExists(name.to_owned()));
}
@@ -232,7 +259,7 @@ impl IggyShard {
let old_name;
{
- let stream = self.get_stream_mut(id).with_error_context(|error| {
+ let mut stream = self.get_stream_mut(id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get mutable reference to stream with id: {id}")
})?;
old_name = stream.name.clone();
@@ -241,8 +268,10 @@ impl IggyShard {
}
{
- self.streams_ids.remove(&old_name);
- self.streams_ids.insert(name.to_owned(), stream_id);
+ self.streams_ids.borrow_mut().remove(&old_name);
+ self.streams_ids
+ .borrow_mut()
+ .insert(name.to_owned(), stream_id);
}
info!("Stream with ID '{id}' updated. Old name: '{old_name}' changed to: '{name}'.");
@@ -250,7 +279,7 @@ impl IggyShard {
}
pub async fn delete_stream(
- &mut self,
+ &self,
session: &Session,
id: &Identifier,
) -> Result<u32, IggyError> {
@@ -260,6 +289,7 @@ impl IggyShard {
})?;
let stream_id = stream.stream_id;
self.permissioner
+ .borrow()
.delete_stream(session.get_user_id(), stream_id)
.with_error_context(|error| {
format!(
@@ -279,17 +309,16 @@ impl IggyShard {
.decrement_partitions(stream.get_partitions_count());
self.metrics.decrement_messages(stream.get_messages_count());
self.metrics.decrement_segments(stream.get_segments_count());
- self.streams.remove(&stream_id);
- self.streams_ids.remove(&stream_name);
+ self.streams.borrow_mut().remove(&stream_id);
+ self.streams_ids.borrow_mut().remove(&stream_name);
let current_stream_id = CURRENT_STREAM_ID.load(Ordering::SeqCst);
if current_stream_id > stream_id {
CURRENT_STREAM_ID.store(stream_id, Ordering::SeqCst);
}
- let client_manager = self.client_manager.read().await;
- client_manager
- .delete_consumer_groups_for_stream(stream_id)
- .await;
+ self.client_manager
+ .borrow_mut()
+ .delete_consumer_groups_for_stream(stream_id);
Ok(stream_id)
}
@@ -302,6 +331,7 @@ impl IggyShard {
format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
})?;
self.permissioner
+ .borrow()
.purge_stream(session.get_user_id(), stream.stream_id)
.with_error_context(|error| {
format!(
@@ -329,10 +359,12 @@ mod tests {
sync::Arc,
};
+ //TODO: Fixme
+ /*
#[tokio::test]
async fn should_get_stream_by_id_and_name() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
@@ -378,4 +410,5 @@ mod tests {
assert_eq!(stream.stream_id, stream_id);
assert_eq!(stream.name, stream_name);
}
+ */
}
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index 145689af..d118090f 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -16,29 +16,31 @@
* under the License.
*/
+use super::COMPONENT;
use crate::shard::IggyShard;
use crate::streaming::session::Session;
+use crate::streaming::streams::stream::Stream;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize};
+use tokio_util::io::StreamReader;
impl IggyShard {
- pub fn find_topic(
+ pub fn find_topic<'topic, 'stream>(
&self,
session: &Session,
- stream_id: &Identifier,
+ stream: &'stream Stream,
topic_id: &Identifier,
- ) -> Result<&Topic, IggyError> {
+ ) -> Result<&'topic Topic, IggyError>
+ where
+ 'stream: 'topic,
+ {
self.ensure_authenticated(session)?;
- let stream = self
- .find_stream(session, stream_id)
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to find stream with ID: {stream_id}")
- })?;
- let topic = stream.get_topic(topic_id);
- if let Ok(topic) = topic {
- self.permissioner
+ let stream_id = stream.stream_id;
+ let topic = stream.get_topic(topic_id)?;
+ self.permissioner
+ .borrow()
.get_topic(session.get_user_id(), stream.stream_id, topic.topic_id)
.with_error_context(|error| {
format!(
@@ -46,22 +48,21 @@ impl IggyShard {
session.get_user_id(),
)
})?;
- return Ok(topic);
- }
-
- topic
+ Ok(topic)
}
- pub fn find_topics(
+ pub fn find_topics<'stream, 'topic>(
&self,
session: &Session,
- stream_id: &Identifier,
- ) -> Result<Vec<&Topic>, IggyError> {
+ stream: &'stream Stream,
+ ) -> Result<Vec<&'topic Topic>, IggyError>
+ where
+ 'stream: 'topic,
+ {
self.ensure_authenticated(session)?;
- let stream = self.get_stream(stream_id).with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
- })?;
+ let stream_id = stream.stream_id;
self.permissioner
+ .borrow()
.get_topics(session.get_user_id(), stream.stream_id)
.with_error_context(|error| {
format!(
@@ -72,28 +73,24 @@ impl IggyShard {
Ok(stream.get_topics())
}
- pub fn try_find_topic(
+ pub fn try_find_topic<'stream, 'topic>(
&self,
session: &Session,
- stream_id: &Identifier,
+ stream: &'stream Stream,
topic_id: &Identifier,
- ) -> Result<Option<&Topic>, IggyError> {
+ ) -> Result<Option<&'topic Topic>, IggyError>
+ where
+ 'stream: 'topic,
+ {
self.ensure_authenticated(session)?;
- let Some(stream) = self
- .try_find_stream(session, stream_id)
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to find stream with ID: {stream_id}")
- })?
- else {
- return Ok(None);
- };
-
+ let stream_id = stream.stream_id;
let Some(topic) = stream.try_get_topic(topic_id)? else {
return Ok(None);
};
self.permissioner
- .get_topic(session.get_user_id(), stream.stream_id, topic.topic_id)
+ .borrow()
+ .get_topic(session.get_user_id(), stream_id, topic.topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - permission denied to get topic with ID: {topic_id} in stream with ID: {stream_id} for user with ID: {}",
@@ -105,7 +102,7 @@ impl IggyShard {
#[allow(clippy::too_many_arguments)]
pub async fn create_topic(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: Option<u32>,
@@ -115,13 +112,14 @@ impl IggyShard {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
- ) -> Result<&Topic, IggyError> {
+ ) -> Result<u32, IggyError> {
self.ensure_authenticated(session)?;
{
let stream = self.get_stream(stream_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
})?;
self.permissioner
+ .borrow()
.create_topic(session.get_user_id(), stream.stream_id)
.with_error_context(|error| {
format!(
@@ -131,6 +129,8 @@ impl IggyShard {
})?;
}
+ // TODO: Make create topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
let created_topic_id = self
.get_stream_mut(stream_id)?
.create_topic(
@@ -151,21 +151,12 @@ impl IggyShard {
self.metrics.increment_partitions(partitions_count);
self.metrics.increment_segments(partitions_count);
- self.get_stream(stream_id)
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
- })?
- .get_topic(&created_topic_id.try_into()?)
- .with_error_context(|error| {
- format!(
- "{COMPONENT} (error: {error}) - failed to get created topic with ID: {created_topic_id} in stream with ID: {stream_id}",
- )
- })
+ Ok(created_topic_id)
}
#[allow(clippy::too_many_arguments)]
pub async fn update_topic(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -174,17 +165,20 @@ impl IggyShard {
compression_algorithm: CompressionAlgorithm,
max_topic_size: MaxTopicSize,
replication_factor: Option<u8>,
- ) -> Result<&Topic, IggyError> {
+ ) -> Result<u32, IggyError> {
self.ensure_authenticated(session)?;
{
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
let topic = self
- .find_topic(session, stream_id, topic_id)
+ .find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to find topic with ID: {topic_id}"
)
})?;
- self.permissioner.update_topic(
+ self.permissioner.borrow().update_topic(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -198,6 +192,8 @@ impl IggyShard {
})?;
}
+ // TODO: Make update topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
self.get_stream_mut(stream_id)?
.update_topic(
topic_id,
@@ -224,11 +220,11 @@ impl IggyShard {
.get_topic(topic_id)
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get topic with ID: {topic_id} in stream with ID: {stream_id}")
- })
+ }).map(|topic| topic.topic_id)
}
pub async fn delete_topic(
- &mut self,
+ &self,
session: &Session,
stream_id: &Identifier,
topic_id: &Identifier,
@@ -236,12 +232,15 @@ impl IggyShard {
self.ensure_authenticated(session)?;
let stream_id_value;
{
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
let topic = self
- .find_topic(session, stream_id, topic_id)
+ .find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to find topic with ID: {topic_id} in stream with ID: {stream_id}")
})?;
- self.permissioner.delete_topic(
+ self.permissioner.borrow().delete_topic(
session.get_user_id(),
topic.stream_id,
topic.topic_id,
@@ -254,6 +253,8 @@ impl IggyShard {
stream_id_value = topic.stream_id;
}
+ // TODO: Make delete topic sync, and extract the storage persister out of it
+ // perform disk i/o outside of the borrow_mut of the stream.
let topic = self
.get_stream_mut(stream_id)?
.delete_topic(topic_id)
@@ -266,10 +267,9 @@ impl IggyShard {
self.metrics.decrement_messages(topic.get_messages_count());
self.metrics
.decrement_segments(topic.get_segments_count().await);
- let client_manager = self.client_manager.read().await;
- client_manager
- .delete_consumer_groups_for_topic(stream_id_value, topic.topic_id)
- .await;
+ self.client_manager
+ .borrow_mut()
+ .delete_consumer_groups_for_topic(stream_id_value, topic.topic_id);
Ok(())
}
@@ -279,12 +279,16 @@ impl IggyShard {
stream_id: &Identifier,
topic_id: &Identifier,
) -> Result<(), IggyError> {
+ let stream = self.get_stream(stream_id).with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to get stream with ID: {stream_id}")
+ })?;
let topic = self
- .find_topic(session, stream_id, topic_id)
+ .find_topic(session, &stream, topic_id)
.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to find topic with ID: {topic_id} in stream with ID: {stream_id}")
})?;
self.permissioner
+ .borrow()
.purge_topic(session.get_user_id(), topic.stream_id, topic.topic_id)
.with_error_context(|error| {
format!(
diff --git a/core/server/src/shard/system/users.rs b/core/server/src/shard/system/users.rs
index f2405f1f..f3b271a7 100644
--- a/core/server/src/shard/system/users.rs
+++ b/core/server/src/shard/system/users.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use super::COMPONENT;
use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::state::models::CreateUserWithId;
@@ -33,6 +34,7 @@ use iggy_common::create_user::CreateUser;
use iggy_common::defaults::*;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{IdKind, Identifier};
+use std::cell::RefMut;
use std::env;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
@@ -46,7 +48,7 @@ impl IggyShard {
&self,
session: &Session,
user_id: &Identifier,
- ) -> Result<Option<&User>, IggyError> {
+ ) -> Result<Option<User>, IggyError> {
self.ensure_authenticated(session)?;
let Some(user) = self.try_get_user(user_id)? else {
return Ok(None);
@@ -54,7 +56,7 @@ impl IggyShard {
let session_user_id = session.get_user_id();
if user.id != session_user_id {
- self.permissioner.get_user(session_user_id).with_error_context(|error| {
+ self.permissioner.borrow().get_user(session_user_id).with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - permission denied to get user with ID: {user_id} for current user with ID: {session_user_id}"
)
@@ -64,45 +66,73 @@ impl IggyShard {
Ok(Some(user))
}
- pub fn get_user(&self, user_id: &Identifier) -> Result<&User, IggyError> {
+ pub fn get_user(&self, user_id: &Identifier) -> Result<User, IggyError> {
self.try_get_user(user_id)?
.ok_or(IggyError::ResourceNotFound(user_id.to_string()))
}
- pub fn try_get_user(&self, user_id: &Identifier) -> Result<Option<&User>, IggyError> {
+ pub fn try_get_user(&self, user_id: &Identifier) -> Result<Option<User>, IggyError> {
match user_id.kind {
- IdKind::Numeric => Ok(self.users.get(&user_id.get_u32_value()?)),
+ IdKind::Numeric => {
+ let user = self
+ .users
+ .borrow()
+ .get(&user_id.get_u32_value()?)
+ .map(|user| user.clone());
+ Ok(user)
+ }
IdKind::String => {
let username = user_id.get_cow_str_value()?;
- Ok(self
+ let user = self
.users
+ .borrow()
.iter()
.find(|(_, user)| user.username == username)
- .map(|(_, user)| user))
+ .map(|(_, user)| user.clone());
+ Ok(user)
}
}
}
- pub fn get_user_mut(&mut self, user_id: &Identifier) -> Result<&mut User, IggyError> {
+ pub fn get_user_mut(&self, user_id: &Identifier) -> Result<RefMut<'_, User>, IggyError> {
match user_id.kind {
- IdKind::Numeric => self
- .users
- .get_mut(&user_id.get_u32_value()?)
- .ok_or(IggyError::ResourceNotFound(user_id.to_string())),
+ IdKind::Numeric => {
+ let user_id = user_id.get_u32_value()?;
+ let users = self.users.borrow_mut();
+ let exists = users.contains_key(&user_id);
+ if !exists {
+ return Err(IggyError::ResourceNotFound(user_id.to_string()));
+ }
+ Ok(RefMut::map(users, |u| {
+ let user = u.get_mut(&user_id);
+ user.unwrap()
+ }))
+ }
IdKind::String => {
let username = user_id.get_cow_str_value()?;
- self.users
- .iter_mut()
+ let users = self.users.borrow_mut();
+ let exists = users
+ .iter()
.find(|(_, user)| user.username == username)
- .map(|(_, user)| user)
- .ok_or(IggyError::ResourceNotFound(user_id.to_string()))
+ .is_some();
+ if !exists {
+ return Err(IggyError::ResourceNotFound(user_id.to_string()));
+ }
+ Ok(RefMut::map(users, |u| {
+ let user = u
+ .iter_mut()
+ .find(|(_, user)| user.username == username)
+ .map(|(_, user)| user);
+ user.unwrap()
+ }))
}
}
}
- pub async fn get_users(&self, session: &Session) -> Result<Vec<&User>, IggyError> {
+ pub async fn get_users(&self, session: &Session) -> Result<Vec<User>, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.get_users(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -110,19 +140,20 @@ impl IggyShard {
session.get_user_id()
)
})?;
- Ok(self.users.values().collect())
+ Ok(self.users.borrow().values().cloned().collect())
}
pub async fn create_user(
- &mut self,
+ &self,
session: &Session,
username: &str,
password: &str,
status: UserStatus,
permissions: Option<Permissions>,
- ) -> Result<&User, IggyError> {
+ ) -> Result<User, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.create_user(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -131,12 +162,17 @@ impl IggyShard {
)
})?;
- if self.users.iter().any(|(_, user)| user.username == username) {
+ if self
+ .users
+ .borrow()
+ .iter()
+ .any(|(_, user)| user.username == username)
+ {
error!("User: {username} already exists.");
return Err(IggyError::UserAlreadyExists);
}
- if self.users.len() >= MAX_USERS {
+ if self.users.borrow().len() >= MAX_USERS {
error!("Available users limit reached.");
return Err(IggyError::UsersLimitReached);
}
@@ -145,8 +181,9 @@ impl IggyShard {
info!("Creating user: {username} with ID: {user_id}...");
let user = User::new(user_id, username, password, status, permissions.clone());
self.permissioner
+ .borrow_mut()
.init_permissions_for_user(user_id, permissions);
- self.users.insert(user.id, user);
+ self.users.borrow_mut().insert(user.id, user);
info!("Created user: {username} with ID: {user_id}.");
self.metrics.increment_users(1);
self.get_user(&user_id.try_into()?)
@@ -155,16 +192,13 @@ impl IggyShard {
})
}
- pub async fn delete_user(
- &mut self,
- session: &Session,
- user_id: &Identifier,
- ) -> Result<User, IggyError> {
+ pub fn delete_user(&self, session: &Session, user_id: &Identifier) -> Result<User, IggyError> {
self.ensure_authenticated(session)?;
let existing_user_id;
let existing_username;
{
self.permissioner
+ .borrow()
.delete_user(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -187,14 +221,14 @@ impl IggyShard {
info!("Deleting user: {existing_username} with ID: {user_id}...");
let user = self
.users
+ .borrow_mut()
.remove(&existing_user_id)
.ok_or(IggyError::ResourceNotFound(user_id.to_string()))?;
self.permissioner
+ .borrow_mut()
.delete_permissions_for_user(existing_user_id);
- let mut client_manager = self.client_manager.write().await;
- client_manager
+ self.client_manager.borrow_mut()
.delete_clients_for_user(existing_user_id)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to delete clients for user with ID: {existing_user_id}"
@@ -205,15 +239,16 @@ impl IggyShard {
Ok(user)
}
- pub async fn update_user(
- &mut self,
+ pub fn update_user(
+ &self,
session: &Session,
user_id: &Identifier,
username: Option<String>,
status: Option<UserStatus>,
- ) -> Result<&User, IggyError> {
+ ) -> Result<User, IggyError> {
self.ensure_authenticated(session)?;
self.permissioner
+ .borrow()
.update_user(session.get_user_id())
.with_error_context(|error| {
format!(
@@ -231,7 +266,7 @@ impl IggyShard {
}
}
- let user = self.get_user_mut(user_id).with_error_context(|error| {
+ let mut user = self.get_user_mut(user_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get mutable reference to the user with id: {user_id}")
})?;
if let Some(username) = username {
@@ -241,13 +276,18 @@ impl IggyShard {
if let Some(status) = status {
user.status = status;
}
+ let cloned_user = user.clone();
+ drop(user);
- info!("Updated user: {} with ID: {}.", user.username, user.id);
- Ok(user)
+ info!(
+ "Updated user: {} with ID: {}.",
+ cloned_user.username, cloned_user.id
+ );
+ Ok(cloned_user)
}
- pub async fn update_permissions(
- &mut self,
+ pub fn update_permissions(
+ &self,
session: &Session,
user_id: &Identifier,
permissions: Option<Permissions>,
@@ -256,13 +296,14 @@ impl IggyShard {
{
self.permissioner
+ .borrow()
.update_permissions(session.get_user_id())
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - permission denied to update permissions for user with id: {}", session.get_user_id()
)
})?;
- let user = self.get_user(user_id).with_error_context(|error| {
+ let user: User = self.get_user(user_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get user with id: {user_id}")
})?;
if user.is_root() {
@@ -271,11 +312,12 @@ impl IggyShard {
}
self.permissioner
+ .borrow_mut()
.update_permissions_for_user(user.id, permissions.clone());
}
{
- let user = self.get_user_mut(user_id).with_error_context(|error| {
+ let mut user = self.get_user_mut(user_id).with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get mutable reference to the user with id: {user_id}"
)
@@ -290,8 +332,8 @@ impl IggyShard {
Ok(())
}
- pub async fn change_password(
- &mut self,
+ pub fn change_password(
+ &self,
session: &Session,
user_id: &Identifier,
current_password: &str,
@@ -305,11 +347,13 @@ impl IggyShard {
})?;
let session_user_id = session.get_user_id();
if user.id != session_user_id {
- self.permissioner.change_password(session_user_id)?;
+ self.permissioner
+ .borrow()
+ .change_password(session_user_id)?;
}
}
- let user = self.get_user_mut(user_id).with_error_context(|error| {
+ let mut user = self.get_user_mut(user_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get mutable reference to the user with id: {user_id}")
})?;
if !crypto::verify_password(current_password, &user.password) {
@@ -328,22 +372,21 @@ impl IggyShard {
Ok(())
}
- pub async fn login_user(
+ pub fn login_user(
&self,
username: &str,
password: &str,
session: Option<&Session>,
- ) -> Result<&User, IggyError> {
+ ) -> Result<User, IggyError> {
self.login_user_with_credentials(username, Some(password), session)
- .await
}
- pub async fn login_user_with_credentials(
+ pub fn login_user_with_credentials(
&self,
username: &str,
password: Option<&str>,
session: Option<&Session>,
- ) -> Result<&User, IggyError> {
+ ) -> Result<User, IggyError> {
let user = match self.get_user(&username.try_into()?) {
Ok(user) => user,
Err(_) => {
@@ -380,14 +423,13 @@ impl IggyShard {
user.username,
session.get_user_id()
);
- self.logout_user(session).await?;
+ self.logout_user(session)?;
}
session.set_user_id(user.id);
- let mut client_manager = self.client_manager.write().await;
+ let mut client_manager = self.client_manager.borrow_mut();
client_manager
.set_user_id(session.client_id, user.id)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to set user_id to client, client ID: {}, user ID: {}",
@@ -397,7 +439,7 @@ impl IggyShard {
Ok(user)
}
- pub async fn logout_user(&self, session: &Session) -> Result<(), IggyError> {
+ pub fn logout_user(&self, session: &Session) -> Result<(), IggyError> {
self.ensure_authenticated(session)?;
let user = self
.get_user(&Identifier::numeric(session.get_user_id())?)
@@ -412,8 +454,8 @@ impl IggyShard {
user.username, user.id
);
if session.client_id > 0 {
- let mut client_manager = self.client_manager.write().await;
- client_manager.clear_user_id(session.client_id).await?;
+ let mut client_manager = self.client_manager.borrow_mut();
+ client_manager.clear_user_id(session.client_id)?;
info!(
"Cleared user ID: {} for client: {}.",
user.id, session.client_id
diff --git a/core/server/src/streaming/partitions/consumer_offsets.rs b/core/server/src/streaming/partitions/consumer_offsets.rs
index 96080c03..e9f5c57d 100644
--- a/core/server/src/streaming/partitions/consumer_offsets.rs
+++ b/core/server/src/streaming/partitions/consumer_offsets.rs
@@ -26,10 +26,7 @@ use iggy_common::IggyError;
use tracing::trace;
impl Partition {
- pub fn get_consumer_offset(
- &self,
- consumer: PollingConsumer,
- ) -> Result<Option<u64>, IggyError> {
+ pub fn get_consumer_offset(&self, consumer: PollingConsumer) -> Result<Option<u64>, IggyError> {
trace!(
"Getting consumer offset for {}, partition: {}, current: {}...",
consumer, self.partition_id, self.current_offset
diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs
index 978b97aa..eaf96dc3 100644
--- a/core/server/src/streaming/storage.rs
+++ b/core/server/src/streaming/storage.rs
@@ -87,7 +87,7 @@ pub enum PartitionStorageKind {
#[cfg_attr(test, automock)]
pub trait SystemInfoStorage {
fn load(&self) -> impl Future<Output = Result<SystemInfo, IggyError>>;
- fn save(&self, system_info: &SystemInfo) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, system_info: &SystemInfo) -> impl Future<Output = Result<(), IggyError>>;
}
#[cfg_attr(test, automock)]
@@ -96,8 +96,8 @@ pub trait StreamStorage {
&self,
stream: &mut Stream,
state: StreamState,
- ) -> impl Future<Output = Result<(), IggyError>>;
- fn save(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>>;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>>;
fn delete(&self, stream: &Stream) -> impl Future<Output = Result<(), IggyError>>;
}
@@ -107,9 +107,9 @@ pub trait TopicStorage {
&self,
topic: &mut Topic,
state: TopicState,
- ) -> impl Future<Output = Result<(), IggyError>>;
- fn save(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
- fn delete(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, topic: &Topic) -> impl Future<Output = Result<(), IggyError>>;
}
#[cfg_attr(test, automock)]
@@ -119,9 +119,8 @@ pub trait PartitionStorage {
partition: &mut Partition,
state: PartitionState,
) -> impl Future<Output = Result<(), IggyError>>;
- fn save(&self, partition: &mut Partition)
- -> impl Future<Output = Result<(), IggyError>>;
- fn delete(&self, partition: &Partition) -> impl Future<Output = Result<(), IggyError>>;
+ fn save(&self, partition: &mut Partition) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete(&self, partition: &Partition) -> impl Future<Output = Result<(), IggyError>>;
fn save_consumer_offset(
&self,
offset: u64,
@@ -131,15 +130,9 @@ pub trait PartitionStorage {
&self,
kind: ConsumerKind,
path: &str,
- ) -> impl Future<Output = Result<Vec<ConsumerOffset>, IggyError>>;
- fn delete_consumer_offsets(
- &self,
- path: &str,
- ) -> impl Future<Output = Result<(), IggyError>>;
- fn delete_consumer_offset(
- &self,
- path: &str,
- ) -> impl Future<Output = Result<(), IggyError>>;
+ ) -> impl Future<Output = Result<Vec<ConsumerOffset>, IggyError>>;
+ fn delete_consumer_offsets(&self, path: &str) -> impl Future<Output = Result<(), IggyError>>;
+ fn delete_consumer_offset(&self, path: &str) -> impl Future<Output = Result<(), IggyError>>;
}
#[derive(Debug)]
diff --git a/core/server/src/streaming/topics/consumer_group.rs b/core/server/src/streaming/topics/consumer_group.rs
index f000ce79..c810159c 100644
--- a/core/server/src/streaming/topics/consumer_group.rs
+++ b/core/server/src/streaming/topics/consumer_group.rs
@@ -57,10 +57,7 @@ impl ConsumerGroup {
self.assign_partitions();
}
- pub fn calculate_partition_id(
- &mut self,
- member_id: u32,
- ) -> Result<Option<u32>, IggyError> {
+ pub fn calculate_partition_id(&mut self, member_id: u32) -> Result<Option<u32>, IggyError> {
let member = self.members.get_mut(&member_id);
if let Some(member) = member {
return Ok(member.calculate_partition_id());
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index c1371096..92168738 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -17,12 +17,12 @@
*/
use crate::binary::handlers::topics::get_topic_handler;
-use crate::streaming::topics::{consumer_group, COMPONENT};
use crate::streaming::topics::consumer_group::ConsumerGroup;
use crate::streaming::topics::topic::Topic;
+use crate::streaming::topics::{COMPONENT, consumer_group};
use error_set::ErrContext;
-use iggy_common::locking::IggySharedMutFn;
use iggy_common::IggyError;
+use iggy_common::locking::IggySharedMutFn;
use iggy_common::{IdKind, Identifier};
use std::cell::{Ref, RefMut};
use std::sync::atomic::Ordering;
@@ -48,7 +48,10 @@ impl Topic {
self.consumer_groups.borrow().values().cloned().collect()
}
- pub fn get_consumer_group(&self, identifier: &Identifier) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
+ pub fn get_consumer_group(
+ &self,
+ identifier: &Identifier,
+ ) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
match identifier.kind {
IdKind::Numeric => self.get_consumer_group_by_id(identifier.get_u32_value().unwrap()),
IdKind::String => self.get_consumer_group_by_name(&identifier.get_cow_str_value()?),
@@ -93,7 +96,10 @@ impl Topic {
}))
}
- pub fn get_consumer_group_by_name(&self, name: &str) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
+ pub fn get_consumer_group_by_name(
+ &self,
+ name: &str,
+ ) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
let group_id = self.consumer_groups_ids.get(name);
if group_id.is_none() {
return Err(IggyError::ConsumerGroupNameNotFound(
@@ -199,12 +205,8 @@ impl Topic {
return Err(IggyError::ConsumerGroupIdAlreadyExists(id, self.topic_id));
}
- let consumer_group = ConsumerGroup::new(
- self.topic_id,
- id,
- name,
- self.partitions.len() as u32,
- );
+ let consumer_group =
+ ConsumerGroup::new(self.topic_id, id, name, self.partitions.len() as u32);
self.consumer_groups_ids.insert(name.to_owned(), id);
let cloned_group = consumer_group.clone();
self.consumer_groups.borrow_mut().insert(id, consumer_group);
diff --git a/core/server/src/streaming/topics/topic.rs b/core/server/src/streaming/topics/topic.rs
index 16aa324c..7e8ca4ac 100644
--- a/core/server/src/streaming/topics/topic.rs
+++ b/core/server/src/streaming/topics/topic.rs
@@ -23,13 +23,13 @@ use crate::streaming::storage::SystemStorage;
use crate::streaming::topics::consumer_group::ConsumerGroup;
use ahash::AHashMap;
use core::fmt;
-use std::cell::RefCell;
-use std::rc::Rc;
use iggy_common::locking::IggyRwLock;
use iggy_common::{
CompressionAlgorithm, Consumer, ConsumerKind, IggyByteSize, IggyError, IggyExpiry,
IggyTimestamp, MaxTopicSize, Sizeable,
};
+use std::cell::RefCell;
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
diff --git a/core/server/src/streaming/users/user.rs b/core/server/src/streaming/users/user.rs
index 01a9faa0..7e68d98f 100644
--- a/core/server/src/streaming/users/user.rs
+++ b/core/server/src/streaming/users/user.rs
@@ -24,7 +24,7 @@ use iggy_common::defaults::*;
use iggy_common::{Permissions, UserId};
use std::sync::Arc;
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct User {
pub id: UserId,
pub status: UserStatus,
(iggy) 02/12: feat(io_uring): bootstrap the main function (#1788)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit d1ada2a1ae5060467b70eb01668017363423151e
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Wed May 14 20:38:55 2025 +0200
feat(io_uring): bootstrap the main function (#1788)
---
Cargo.lock | 1 +
core/server/Cargo.toml | 1 +
core/server/src/bootstrap.rs | 20 ++++++++++
core/server/src/lib.rs | 4 +-
core/server/src/main.rs | 80 ++++++++++++++++++++++++++++++++++++--
core/server/src/shard/builder.rs | 73 ++++++++++++++++++++++++++++++++++
core/server/src/shard/connector.rs | 2 +
core/server/src/shard/frame.rs | 3 +-
core/server/src/shard/mod.rs | 79 +++++++++++++++++++++++++++----------
9 files changed, 234 insertions(+), 29 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 9d3922d5..2fa04bba 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6809,6 +6809,7 @@ dependencies = [
"futures",
"human-repr",
"iggy_common",
+ "io-uring",
"jsonwebtoken",
"lending-iterator",
"mimalloc",
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 8523c6bb..baa17e53 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -59,6 +59,7 @@ flume = { workspace = true }
futures = { workspace = true }
human-repr = { workspace = true }
iggy_common = { workspace = true }
+io-uring = "0.6"
jsonwebtoken = "9.3.1"
lending-iterator = "0.1.7"
mimalloc = { workspace = true, optional = true }
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index e69de29b..8eacf223 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -0,0 +1,20 @@
+use crate::shard::{connector::ShardConnector, frame::ShardFrame};
+use std::ops::Range;
+
+pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<ShardFrame>> {
+ let shards_count = shards_set.len();
+ let connectors = shards_set
+ .into_iter()
+ .map(|id| ShardConnector::new(id as u16, shards_count))
+ .collect();
+
+ connectors
+}
+
+pub async fn create_directories() {
+
+}
+
+pub async fn create_root_user() {
+
+}
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index 572301e0..e64837da 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -26,18 +26,18 @@ static GLOBAL: MiMalloc = MiMalloc;
#[cfg(windows)]
compile_error!("iggy-server doesn't support windows.");
-mod bootstrap;
pub mod archiver;
pub mod args;
pub mod binary;
+pub mod bootstrap;
pub mod channels;
pub(crate) mod compat;
pub mod configs;
-pub mod shard;
pub mod http;
pub mod log;
pub mod quic;
pub mod server_error;
+pub mod shard;
pub mod state;
pub mod streaming;
pub mod tcp;
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index d7351b5f..381ac578 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -16,11 +16,15 @@
* under the License.
*/
+use std::thread::available_parallelism;
+
use anyhow::Result;
use clap::Parser;
use dotenvy::dotenv;
use figlet_rs::FIGfont;
+use monoio::Buildable;
use server::args::Args;
+use server::bootstrap::{create_directories, create_root_user, create_shard_connections};
use server::channels::commands::archive_state::ArchiveStateExecutor;
use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
use server::channels::commands::maintain_messages::MaintainMessagesExecutor;
@@ -37,15 +41,15 @@ use server::log::logger::Logging;
use server::log::tokio_console::Logging;
use server::quic::quic_server;
use server::server_error::ServerError;
+use server::shard::IggyShard;
use server::streaming::systems::system::{SharedSystem, System};
use server::streaming::utils::MemoryPool;
use server::tcp::tcp_server;
use tokio::time::Instant;
use tracing::{info, instrument};
-#[tokio::main]
#[instrument(skip_all, name = "trace_start_server")]
-async fn main() -> Result<(), ServerError> {
+fn main() -> Result<(), ServerError> {
let startup_timestamp = Instant::now();
let standard_font = FIGfont::standard().unwrap();
let figure = standard_font.convert("Iggy Server");
@@ -68,6 +72,24 @@ async fn main() -> Result<(), ServerError> {
let args = Args::parse();
let config_provider = config_provider::resolve(&args.config_provider)?;
+ let config = ServerConfig::default();
+ //TODO: Load config.
+ /*
+ let xd = std::thread::scope(|scope| {
+ let config = scope.spawn(move || {
+ let mut rt = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new().build().unwrap();
+ let config: Result<ServerConfig, ServerError> = rt.block_on(async {
+ let config = ServerConfig::load(&config_provider).await?;
+ create_directories().await;
+ create_root_user().await;
+ Ok(config)
+ });
+ config
+ }).join().unwrap();
+ config
+ });
+ */
+ /*
let config = ServerConfig::load(&config_provider).await?;
if args.fresh {
let system_path = config.system.get_system_path();
@@ -80,9 +102,56 @@ async fn main() -> Result<(), ServerError> {
}
let mut logging = Logging::new(config.telemetry.clone());
logging.early_init();
+ */
+ // TODO: Make this configurable from config as a range
+ // for example this instance of Iggy will use cores from 0..4
+ let available_cpus = available_parallelism()
+ .expect("Failed to get num of cores")
+ .into();
+ let shards_count = available_cpus;
+ let shards_set = 0..shards_count;
+
+ let connections = create_shard_connections(shards_set.clone());
+
+ for shard_id in shards_set {
+ let id = shard_id as u16;
+ let connections = connections.clone();
+ let server_config = config.clone();
+ std::thread::Builder::new()
+ .name(format!("shard-{id}"))
+ .spawn(move || {
+ monoio::utils::bind_to_cpu_set(Some(shard_id))
+ .expect(format!("Failed to set CPU affinity for shard-{id}").as_str());
+ // TODO: Figure out what else we could tweak there
+ // We for sure want to disable the userspace interrupts on new cq entry (set_coop_taskrun)
+ // let urb = io_uring::IoUring::builder();
+ // TODO: Shall we make the size of ring be configureable ?
+ let mut rt = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new()
+ //.uring_builder(urb.setup_coop_taskrun())
+ .with_entries(1024) // Default size
+ .enable_timer()
+ .build()
+ .expect(format!("Failed to build monoio runtime for shard-{id}").as_str());
+ rt.block_on(async move {
+ let builder = IggyShard::builder();
+ let shard = builder
+ .id(id)
+ .connections(connections)
+ .server_config(server_config)
+ .build_and_init()
+ .await;
+
+ shard.assert_init();
+ })
+ })
+ .expect(format!("Failed to spawn thread for shard-{id}").as_str())
+ .join()
+ .expect(format!("Failed to join thread for shard-{id}").as_str());
+ }
// From this point on, we can use tracing macros to log messages.
+ /*
logging.late_init(config.system.get_system_path(), &config.system.logging)?;
#[cfg(feature = "disable-mimalloc")]
@@ -99,13 +168,15 @@ async fn main() -> Result<(), ServerError> {
config.data_maintenance.clone(),
config.personal_access_token.clone(),
));
+ */
// Workaround to ensure that the statistics are initialized before the server
// loads streams and starts accepting connections. This is necessary to
// have the correct statistics when the server starts.
- system.write().await.get_stats().await?;
- system.write().await.init().await?;
+ //system.write().await.get_stats().await?;
+ //system.write().await.init().await?;
+ /*
let _command_handler = BackgroundServerCommandHandler::new(system.clone(), &config)
.install_handler(SaveMessagesExecutor)
.install_handler(MaintainMessagesExecutor)
@@ -181,5 +252,6 @@ async fn main() -> Result<(), ServerError> {
"Iggy server has shutdown successfully. Shutdown took {} ms.",
elapsed_time.as_millis()
);
+ */
Ok(())
}
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
new file mode 100644
index 00000000..08a02e54
--- /dev/null
+++ b/core/server/src/shard/builder.rs
@@ -0,0 +1,73 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::{configs::server::ServerConfig, shard::Shard};
+
+use super::{connector::ShardConnector, frame::ShardFrame, IggyShard};
+
+#[derive(Default)]
+pub struct IggyShardBuilder {
+ id: Option<u16>,
+ connections: Option<Vec<ShardConnector<ShardFrame>>>,
+ config: Option<ServerConfig>,
+}
+
+impl IggyShardBuilder {
+ pub fn id(mut self, id: u16) -> Self {
+ self.id = Some(id);
+ self
+ }
+
+ pub fn connections(mut self, connections: Vec<ShardConnector<ShardFrame>>) -> Self {
+ self.connections = Some(connections);
+ self
+ }
+
+ pub fn server_config(mut self, config: ServerConfig) -> Self {
+ self.config = Some(config);
+ self
+ }
+
+ pub async fn build_and_init(self) -> IggyShard {
+ let id = self.id.unwrap();
+ let config = self.config.unwrap();
+ let connections = self.connections.unwrap();
+ let (stop_sender, stop_receiver, receiver) = connections
+ .iter()
+ .filter(|c| c.id == id)
+ .map(|c| {
+ (
+ c.stop_sender.clone(),
+ c.stop_receiver.clone(),
+ c.receiver.clone(),
+ )
+ })
+ .next()
+ .expect("Failed to find connection with the specified ID");
+ let shards = connections.into_iter().map(Shard::new).collect();
+
+ IggyShard::new(
+ id,
+ shards,
+ config,
+ stop_receiver,
+ stop_sender,
+ receiver,
+ )
+ }
+}
diff --git a/core/server/src/shard/connector.rs b/core/server/src/shard/connector.rs
index 349218ba..1d762d68 100644
--- a/core/server/src/shard/connector.rs
+++ b/core/server/src/shard/connector.rs
@@ -54,6 +54,8 @@ impl<T: Clone> ShardConnector<T> {
}
}
+// TODO: I think those Arcs can be replaced with 'static lifetimes...
+// Those shards will live for the entire duration of the application.
#[derive(Clone)]
pub struct Receiver<T> {
channel: Arc<ShardedChannel<T>>,
diff --git a/core/server/src/shard/frame.rs b/core/server/src/shard/frame.rs
index b1524ad6..a2c1220f 100644
--- a/core/server/src/shard/frame.rs
+++ b/core/server/src/shard/frame.rs
@@ -27,8 +27,7 @@ pub enum ShardMessage {
}
#[derive(Debug, Clone)]
-pub enum ShardEvent {
-}
+pub enum ShardEvent {}
#[derive(Debug)]
pub enum ShardResponse {
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 6ccbcab3..bffaca45 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -15,22 +15,33 @@
* specific language governing permissions and limitations
* under the License.
*/
-mod connector;
-mod namespace;
-mod frame;
+pub mod builder;
+pub mod connector;
+pub mod frame;
+pub mod namespace;
-
-
-use std::cell::RefCell;
use ahash::HashMap;
-use connector::ShardConnector;
+use builder::IggyShardBuilder;
+use connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use frame::ShardFrame;
use namespace::IggyNamespace;
-struct Shard {
+use std::cell::{Cell, RefCell};
+
+use crate::configs::server::ServerConfig;
+pub(crate) struct Shard {
id: u16,
connection: ShardConnector<ShardFrame>,
}
+impl Shard {
+ pub fn new(connection: ShardConnector<ShardFrame>) -> Self {
+ Self {
+ id: connection.id,
+ connection,
+ }
+ }
+}
+
struct ShardInfo {
id: u16,
}
@@ -38,22 +49,48 @@ struct ShardInfo {
pub struct IggyShard {
pub id: u16,
shards: Vec<Shard>,
- shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
+ //shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
- pub(crate) permissioner: RefCell<Permissioner>,
- pub(crate) storage: Rc<SystemStorage>,
- pub(crate) streams: RwLock<HashMap<u32, Stream>>,
- pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
- pub(crate) users: RefCell<HashMap<UserId, User>>,
+ //pub(crate) permissioner: RefCell<Permissioner>,
+ //pub(crate) streams: RwLock<HashMap<u32, Stream>>,
+ //pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
+ //pub(crate) users: RefCell<HashMap<UserId, User>>,
// TODO - get rid of this dynamic dispatch.
- pub(crate) state: Rc<FileState>,
- pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
- pub(crate) config: ServerConfig,
- pub(crate) client_manager: RefCell<ClientManager>,
- pub(crate) active_sessions: RefCell<Vec<Session>>,
- pub(crate) metrics: Metrics,
+ //pub(crate) state: Rc<FileState>,
+ //pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
+ config: ServerConfig,
+ //pub(crate) client_manager: RefCell<ClientManager>,
+ //pub(crate) active_sessions: RefCell<Vec<Session>>,
+ //pub(crate) metrics: Metrics,
pub message_receiver: Cell<Option<Receiver<ShardFrame>>>,
stop_receiver: StopReceiver,
stop_sender: StopSender,
-}
\ No newline at end of file
+}
+
+impl IggyShard {
+ pub fn builder() -> IggyShardBuilder {
+ Default::default()
+ }
+
+ pub(crate) fn new(
+ id: u16,
+ shards: Vec<Shard>,
+ config: ServerConfig,
+ stop_receiver: StopReceiver,
+ stop_sender: StopSender,
+ shard_messages_receiver: Receiver<ShardFrame>,
+ ) -> Self {
+ Self {
+ id,
+ shards,
+ config,
+ stop_receiver,
+ stop_sender,
+ message_receiver: Cell::new(Some(shard_messages_receiver)),
+ }
+ }
+
+ pub fn assert_init(&self) {
+ }
+}
(iggy) 03/12: feat(io_uring): load server_config (#1798)
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 4d17283057f5e22a6bb25dde2aae8203e4133fc7
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Fri May 16 21:37:23 2025 +0200
feat(io_uring): load server_config (#1798)
---
Cargo.lock | 31 +++++++++++---
core/server/src/bootstrap.rs | 13 +++++-
core/server/src/main.rs | 87 ++++++++++++++++++++++++++--------------
core/server/src/shard/builder.rs | 9 +----
core/server/src/shard/mod.rs | 3 +-
5 files changed, 96 insertions(+), 47 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 2fa04bba..3a3711a5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -142,7 +142,7 @@ dependencies = [
"actix-utils",
"futures-core",
"futures-util",
- "mio 1.0.3",
+ "mio 1.0.4",
"socket2",
"tokio",
"tracing",
@@ -639,7 +639,7 @@ checksum = "fd73835ad7deb4bd2b389e6f10333b143f025d607c55ca04c66a0bcc6bb2fc6d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.101",
+ "syn 2.0.104",
]
[[package]]
@@ -4626,6 +4626,15 @@ version = "2.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
+[[package]]
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+]
+
[[package]]
name = "mimalloc"
version = "0.1.47"
@@ -4675,6 +4684,18 @@ dependencies = [
"adler2",
]
+[[package]]
+name = "mio"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+dependencies = [
+ "libc",
+ "log",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "mio"
version = "1.0.4"
@@ -4763,7 +4784,7 @@ checksum = "176a5f5e69613d9e88337cf2a65e11135332b4efbcc628404a7c555e4452084c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.101",
+ "syn 2.0.104",
]
[[package]]
@@ -4858,7 +4879,7 @@ dependencies = [
"kqueue",
"libc",
"log",
- "mio 1.0.3",
+ "mio 1.0.4",
"notify-types",
"walkdir",
"windows-sys 0.59.0",
@@ -7404,7 +7425,7 @@ dependencies = [
"backtrace",
"bytes",
"libc",
- "mio",
+ "mio 1.0.4",
"parking_lot 0.12.4",
"pin-project-lite",
"signal-hook-registry",
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index 8eacf223..fd6a5aa0 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -1,3 +1,5 @@
+use monoio::{Buildable, Driver, Runtime};
+
use crate::shard::{connector::ShardConnector, frame::ShardFrame};
use std::ops::Range;
@@ -12,9 +14,18 @@ pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<
}
pub async fn create_directories() {
-
+ todo!();
}
pub async fn create_root_user() {
+ todo!();
+}
+pub fn create_default_executor<D>() -> Runtime<D>
+where
+ D: Driver + Buildable,
+{
+ let builder = monoio::RuntimeBuilder::<D>::new();
+ let rt = Buildable::build(builder).expect("Failed to create default runtime");
+ rt
}
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 381ac578..3b38700f 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -24,7 +24,9 @@ use dotenvy::dotenv;
use figlet_rs::FIGfont;
use monoio::Buildable;
use server::args::Args;
-use server::bootstrap::{create_directories, create_root_user, create_shard_connections};
+use server::bootstrap::{
+ create_default_executor, create_directories, create_root_user, create_shard_connections,
+};
use server::channels::commands::archive_state::ArchiveStateExecutor;
use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
use server::channels::commands::maintain_messages::MaintainMessagesExecutor;
@@ -32,7 +34,7 @@ use server::channels::commands::print_sysinfo::SysInfoPrintExecutor;
use server::channels::commands::save_messages::SaveMessagesExecutor;
use server::channels::commands::verify_heartbeats::VerifyHeartbeatsExecutor;
use server::channels::handler::BackgroundServerCommandHandler;
-use server::configs::config_provider;
+use server::configs::config_provider::{self, ConfigProviderKind};
use server::configs::server::ServerConfig;
use server::http::http_server;
#[cfg(not(feature = "tokio-console"))]
@@ -72,37 +74,60 @@ fn main() -> Result<(), ServerError> {
let args = Args::parse();
let config_provider = config_provider::resolve(&args.config_provider)?;
- let config = ServerConfig::default();
- //TODO: Load config.
- /*
- let xd = std::thread::scope(|scope| {
- let config = scope.spawn(move || {
- let mut rt = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new().build().unwrap();
- let config: Result<ServerConfig, ServerError> = rt.block_on(async {
- let config = ServerConfig::load(&config_provider).await?;
- create_directories().await;
- create_root_user().await;
- Ok(config)
- });
- config
- }).join().unwrap();
+ let config = std::thread::scope(|scope| {
+ let config = scope
+ .spawn(move || {
+ async fn load_config(
+ config_provider: &ConfigProviderKind,
+ ) -> Result<ServerConfig, ServerError> {
+ let config = ServerConfig::load(config_provider).await?;
+ Ok(config)
+ }
+
+ let mut rt = create_default_executor::<monoio::IoUringDriver>();
+ rt.block_on(load_config(&config_provider))
+ })
+ .join()
+ .expect("Failed to load config");
config
+ })?;
+
+ // Create directories and root user.
+ // Remove `local_data` directory if run with `--fresh` flag.
+ std::thread::scope(|scope| {
+ scope
+ .spawn(|| {
+ let mut rt = create_default_executor::<monoio::IoUringDriver>();
+ rt.block_on(async {
+ if args.fresh {
+ let system_path = config.system.get_system_path();
+ if monoio::fs::metadata(&system_path).await.is_ok() {
+ println!(
+ "Removing system path at: {} because `--fresh` flag was set",
+ system_path
+ );
+ //TODO: Impl dir walk and remove the files
+ /*
+ if let Err(e) = tokio::fs::remove_dir_all(&system_path).await {
+ eprintln!("Failed to remove system path at {}: {}", system_path, e);
+ }
+ */
+ }
+ }
+
+ // Create directories and root user
+ create_directories().await;
+ create_root_user().await;
+ });
+ })
+ .join()
+ .expect("Failed to create directories and root user");
});
- */
- /*
- let config = ServerConfig::load(&config_provider).await?;
- if args.fresh {
- let system_path = config.system.get_system_path();
- if tokio::fs::metadata(&system_path).await.is_ok() {
- println!("Removing system path at: {system_path} because `--fresh` flag was set");
- if let Err(e) = tokio::fs::remove_dir_all(&system_path).await {
- eprintln!("Failed to remove system path at {system_path}: {e}");
- }
- }
- }
+
+ // Initialize logging
let mut logging = Logging::new(config.telemetry.clone());
logging.early_init();
- */
+
// TODO: Make this configurable from config as a range
// for example this instance of Iggy will use cores from 0..4
let available_cpus = available_parallelism()
@@ -149,11 +174,11 @@ fn main() -> Result<(), ServerError> {
.join()
.expect(format!("Failed to join thread for shard-{id}").as_str());
}
- // From this point on, we can use tracing macros to log messages.
- /*
+ // From this point on, we can use tracing macros to log messages.
logging.late_init(config.system.get_system_path(), &config.system.logging)?;
+ /*
#[cfg(feature = "disable-mimalloc")]
tracing::warn!(
"Using default system allocator because code was build with `disable-mimalloc` feature"
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 08a02e54..750e2b71 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -61,13 +61,6 @@ impl IggyShardBuilder {
.expect("Failed to find connection with the specified ID");
let shards = connections.into_iter().map(Shard::new).collect();
- IggyShard::new(
- id,
- shards,
- config,
- stop_receiver,
- stop_sender,
- receiver,
- )
+ IggyShard::new(id, shards, config, stop_receiver, stop_sender, receiver)
}
}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index bffaca45..dcb089d9 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -91,6 +91,5 @@ impl IggyShard {
}
}
- pub fn assert_init(&self) {
- }
+ pub fn assert_init(&self) {}
}
(iggy) 10/12: fix issues with RefCell in core/server/src/streaming/topics/consumer_groups* files
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit b42ba18ab76952094d22dcf656a4db08002d2959
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Tue Jun 24 18:55:03 2025 +0200
fix issues with RefCell in core/server/src/streaming/topics/consumer_groups* files
---
core/server/src/streaming/topics/consumer_group.rs | 25 ++--
.../server/src/streaming/topics/consumer_groups.rs | 152 ++++++++++++---------
2 files changed, 94 insertions(+), 83 deletions(-)
diff --git a/core/server/src/streaming/topics/consumer_group.rs b/core/server/src/streaming/topics/consumer_group.rs
index 43d9611b..cc28ef6f 100644
--- a/core/server/src/streaming/topics/consumer_group.rs
+++ b/core/server/src/streaming/topics/consumer_group.rs
@@ -18,7 +18,6 @@
use ahash::AHashMap;
use iggy_common::IggyError;
-use tokio::sync::RwLock;
use tracing::trace;
#[derive(Debug, Clone)]
@@ -58,10 +57,13 @@ impl ConsumerGroup {
self.assign_partitions().await;
}
- pub async fn calculate_partition_id(&self, member_id: u32) -> Result<Option<u32>, IggyError> {
- let member = self.members.get(&member_id);
+ pub async fn calculate_partition_id(
+ &mut self,
+ member_id: u32,
+ ) -> Result<Option<u32>, IggyError> {
+ let member = self.members.get_mut(&member_id);
if let Some(member) = member {
- return Ok(member.await.calculate_partition_id());
+ return Ok(member.calculate_partition_id());
}
Err(IggyError::ConsumerGroupMemberNotFound(
member_id,
@@ -73,7 +75,7 @@ impl ConsumerGroup {
pub async fn get_current_partition_id(&self, member_id: u32) -> Result<Option<u32>, IggyError> {
let member = self.members.get(&member_id);
if let Some(member) = member {
- return Ok(member.read().await.current_partition_id);
+ return Ok(member.current_partition_id);
}
Err(IggyError::ConsumerGroupMemberNotFound(
member_id,
@@ -85,12 +87,12 @@ impl ConsumerGroup {
pub async fn add_member(&mut self, member_id: u32) {
self.members.insert(
member_id,
- RwLock::new(ConsumerGroupMember {
+ ConsumerGroupMember {
id: member_id,
partitions: AHashMap::new(),
current_partition_index: None,
current_partition_id: None,
- }),
+ },
);
trace!(
"Added member with ID: {} to consumer group: {} for topic with ID: {}",
@@ -117,7 +119,6 @@ impl ConsumerGroup {
let members_count = members.len() as u32;
for member in members.iter_mut() {
- let mut member = member.write().await;
member.current_partition_index = None;
member.current_partition_id = None;
member.partitions.clear();
@@ -126,8 +127,7 @@ impl ConsumerGroup {
for partition_index in 0..self.partitions_count {
let partition_id = partition_index + 1;
let member_index = partition_index % members_count;
- let member = members.get(member_index as usize).unwrap();
- let mut member = member.write().await;
+ let member = members.get_mut(member_index as usize).unwrap();
let member_partition_index = member.partitions.len() as u32;
member
.partitions
@@ -213,7 +213,6 @@ mod tests {
consumer_group.add_member(member_id).await;
let member = consumer_group.members.get(&member_id).unwrap();
- let member = member.read().await;
assert_eq!(
member.partitions.len() as u32,
consumer_group.partitions_count
@@ -240,8 +239,6 @@ mod tests {
consumer_group.add_member(member2_id).await;
let member1 = consumer_group.members.get(&member1_id).unwrap();
let member2 = consumer_group.members.get(&member2_id).unwrap();
- let member1 = member1.read().await;
- let member2 = member2.read().await;
assert_eq!(
member1.partitions.len() + member2.partitions.len(),
consumer_group.partitions_count as usize
@@ -277,8 +274,6 @@ mod tests {
consumer_group.add_member(member2_id).await;
let member1 = consumer_group.members.get(&member1_id).unwrap();
let member2 = consumer_group.members.get(&member2_id).unwrap();
- let member1 = member1.read().await;
- let member2 = member2.read().await;
if member1.partitions.len() == 1 {
assert_eq!(member2.partitions.len(), 0);
} else {
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index 26505355..1f50a1bb 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -21,37 +21,31 @@ use crate::streaming::topics::consumer_group::ConsumerGroup;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
use iggy_common::IggyError;
-use iggy_common::locking::IggySharedMutFn;
use iggy_common::{IdKind, Identifier};
use std::sync::atomic::Ordering;
-use tokio::sync::RwLock;
use tracing::info;
impl Topic {
- pub async fn reassign_consumer_groups(&mut self) {
+ pub fn reassign_consumer_groups(&mut self) {
if self.consumer_groups.is_empty() {
return;
}
- let partitions_count = self.partitions.len() as u32;
+ let partitions_count = self.partitions.borrow().len() as u32;
info!(
"Reassigning consumer groups for topic with ID: {} for stream with ID with {}, partitions count: {}",
self.topic_id, self.stream_id, partitions_count
);
for (_, consumer_group) in self.consumer_groups.iter_mut() {
- let mut consumer_group = consumer_group.write().await;
- consumer_group.reassign_partitions(partitions_count).await;
+ consumer_group.reassign_partitions(partitions_count);
}
}
- pub fn get_consumer_groups(&self) -> Vec<&RwLock<ConsumerGroup>> {
+ pub fn get_consumer_groups(&self) -> Vec<&ConsumerGroup> {
self.consumer_groups.values().collect()
}
- pub fn get_consumer_group(
- &self,
- identifier: &Identifier,
- ) -> Result<&RwLock<ConsumerGroup>, IggyError> {
+ pub fn get_consumer_group(&self, identifier: &Identifier) -> Result<&ConsumerGroup, IggyError> {
match identifier.kind {
IdKind::Numeric => self.get_consumer_group_by_id(identifier.get_u32_value().unwrap()),
IdKind::String => self.get_consumer_group_by_name(&identifier.get_cow_str_value()?),
@@ -61,7 +55,7 @@ impl Topic {
pub fn try_get_consumer_group(
&self,
identifier: &Identifier,
- ) -> Result<Option<&RwLock<ConsumerGroup>>, IggyError> {
+ ) -> Result<Option<&ConsumerGroup>, IggyError> {
match identifier.kind {
IdKind::Numeric => Ok(self.consumer_groups.get(&identifier.get_u32_value()?)),
IdKind::String => {
@@ -70,16 +64,13 @@ impl Topic {
}
}
- fn try_get_consumer_group_by_name(&self, name: &str) -> Option<&RwLock<ConsumerGroup>> {
+ fn try_get_consumer_group_by_name(&self, name: &str) -> Option<&ConsumerGroup> {
self.consumer_groups_ids
.get(name)
.and_then(|id| self.consumer_groups.get(id))
}
- pub fn get_consumer_group_by_name(
- &self,
- name: &str,
- ) -> Result<&RwLock<ConsumerGroup>, IggyError> {
+ pub fn get_consumer_group_by_name(&self, name: &str) -> Result<&ConsumerGroup, IggyError> {
let group_id = self.consumer_groups_ids.get(name);
if group_id.is_none() {
return Err(IggyError::ConsumerGroupNameNotFound(
@@ -91,7 +82,7 @@ impl Topic {
self.get_consumer_group_by_id(*group_id.unwrap())
}
- pub fn get_consumer_group_by_id(&self, id: u32) -> Result<ConsumerGroup, IggyError> {
+ pub fn get_consumer_group_by_id(&self, id: u32) -> Result<&ConsumerGroup, IggyError> {
let consumer_group = self.consumer_groups.get(&id);
if consumer_group.is_none() {
return Err(IggyError::ConsumerGroupIdNotFound(id, self.topic_id));
@@ -100,7 +91,46 @@ impl Topic {
Ok(consumer_group.unwrap())
}
- pub async fn create_consumer_group(
+ pub fn get_consumer_group_mut(
+ &mut self,
+ identifier: &Identifier,
+ ) -> Result<&mut ConsumerGroup, IggyError> {
+ match identifier.kind {
+ IdKind::Numeric => {
+ self.get_consumer_group_by_id_mut(identifier.get_u32_value().unwrap())
+ }
+ IdKind::String => self.get_consumer_group_by_name_mut(&identifier.get_cow_str_value()?),
+ }
+ }
+
+ pub fn get_consumer_group_by_id_mut(
+ &mut self,
+ id: u32,
+ ) -> Result<&mut ConsumerGroup, IggyError> {
+ let consumer_group = self.consumer_groups.get_mut(&id);
+ if consumer_group.is_none() {
+ return Err(IggyError::ConsumerGroupIdNotFound(id, self.topic_id));
+ }
+
+ Ok(consumer_group.unwrap())
+ }
+
+ pub fn get_consumer_group_by_name_mut(
+ &mut self,
+ name: &str,
+ ) -> Result<&mut ConsumerGroup, IggyError> {
+ let group_id = self.consumer_groups_ids.get(name).copied();
+ if group_id.is_none() {
+ return Err(IggyError::ConsumerGroupNameNotFound(
+ name.to_string(),
+ self.name.to_owned(),
+ ));
+ }
+
+ self.get_consumer_group_by_id_mut(group_id.unwrap())
+ }
+
+ pub fn create_consumer_group(
&mut self,
group_id: Option<u32>,
name: &str,
@@ -137,15 +167,20 @@ impl Topic {
return Err(IggyError::ConsumerGroupIdAlreadyExists(id, self.topic_id));
}
- let consumer_group =
- ConsumerGroup::new(self.topic_id, id, name, self.partitions.borrow().len() as u32);
- self.consumer_groups.insert(id, consumer_group.clone());
+ let consumer_group = ConsumerGroup::new(
+ self.topic_id,
+ id,
+ name,
+ self.partitions.borrow().len() as u32,
+ );
self.consumer_groups_ids.insert(name.to_owned(), id);
+ let cloned_group = consumer_group.clone();
+ self.consumer_groups.insert(id, consumer_group);
info!(
"Created consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
id, self.topic_id, self.stream_id
);
- consumer_group
+ Ok(cloned_group)
}
pub async fn delete_consumer_group(
@@ -157,7 +192,6 @@ impl Topic {
let consumer_group = self.get_consumer_group(id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get consumer group with id: {id}")
})?;
- let consumer_group = consumer_group.read().await;
group_id = consumer_group.group_id;
}
@@ -167,7 +201,6 @@ impl Topic {
}
let consumer_group = consumer_group.unwrap();
{
- let group_id = consumer_group.group_id;
self.consumer_groups_ids.remove(&consumer_group.name);
let current_group_id = self.current_consumer_group_id.load(Ordering::SeqCst);
if current_group_id > group_id {
@@ -193,16 +226,15 @@ impl Topic {
Ok(consumer_group)
}
- pub async fn join_consumer_group(
- &self,
+ pub fn join_consumer_group(
+ &mut self,
group_id: &Identifier,
member_id: u32,
) -> Result<(), IggyError> {
- let consumer_group = self.get_consumer_group(group_id).with_error_context(|error| {
+ let consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get consumer group with id: {group_id}")
})?;
- let mut consumer_group = consumer_group.write().await;
- consumer_group.add_member(member_id).await;
+ consumer_group.add_member(member_id);
info!(
"Member with ID: {} has joined consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
member_id, group_id, self.topic_id, self.stream_id
@@ -210,16 +242,15 @@ impl Topic {
Ok(())
}
- pub async fn leave_consumer_group(
- &self,
+ pub fn leave_consumer_group(
+ &mut self,
group_id: &Identifier,
member_id: u32,
) -> Result<(), IggyError> {
- let consumer_group = self.get_consumer_group(group_id).with_error_context(|error| {
+ let consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get consumer group with id: {group_id}")
})?;
- let mut consumer_group = consumer_group.write().await;
- consumer_group.delete_member(member_id).await;
+ consumer_group.delete_member(member_id);
info!(
"Member with ID: {} has left consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
member_id, group_id, self.topic_id, self.stream_id
@@ -236,6 +267,7 @@ mod tests {
use crate::streaming::storage::SystemStorage;
use crate::streaming::utils::MemoryPool;
use iggy_common::{CompressionAlgorithm, IggyExpiry, MaxTopicSize};
+ use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64};
@@ -245,10 +277,10 @@ mod tests {
let name = "test";
let mut topic = get_topic().await;
let topic_id = topic.topic_id;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
{
- let created_consumer_group = result.unwrap().read().await;
+ let created_consumer_group = result.unwrap();
assert_eq!(created_consumer_group.group_id, group_id);
assert_eq!(created_consumer_group.name, name);
assert_eq!(created_consumer_group.topic_id, topic_id);
@@ -258,13 +290,12 @@ mod tests {
let consumer_group = topic
.get_consumer_group(&Identifier::numeric(group_id).unwrap())
.unwrap();
- let consumer_group = consumer_group.read().await;
assert_eq!(consumer_group.group_id, group_id);
assert_eq!(consumer_group.name, name);
assert_eq!(consumer_group.topic_id, topic_id);
assert_eq!(
consumer_group.partitions_count,
- topic.partitions.len() as u32
+ topic.partitions.borrow().len() as u32
);
}
@@ -273,10 +304,10 @@ mod tests {
let group_id = 1;
let name = "test";
let mut topic = get_topic().await;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
assert_eq!(topic.consumer_groups.len(), 1);
- let result = topic.create_consumer_group(Some(group_id), "test2").await;
+ let result = topic.create_consumer_group(Some(group_id), "test2");
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, IggyError::ConsumerGroupIdAlreadyExists(_, _)));
@@ -288,11 +319,11 @@ mod tests {
let group_id = 1;
let name = "test";
let mut topic = get_topic().await;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
assert_eq!(topic.consumer_groups.len(), 1);
let group_id = group_id + 1;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(
@@ -307,7 +338,7 @@ mod tests {
let group_id = 1;
let name = "test";
let mut topic = get_topic().await;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
assert_eq!(topic.consumer_groups.len(), 1);
let result = topic
@@ -322,7 +353,7 @@ mod tests {
let group_id = 1;
let name = "test";
let mut topic = get_topic().await;
- let result = topic.create_consumer_group(Some(group_id), name).await;
+ let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
assert_eq!(topic.consumer_groups.len(), 1);
let group_id = group_id + 1;
@@ -339,19 +370,12 @@ mod tests {
let name = "test";
let member_id = 1;
let mut topic = get_topic().await;
- topic
- .create_consumer_group(Some(group_id), name)
- .await
- .unwrap();
- let result = topic
- .join_consumer_group(&Identifier::numeric(group_id).unwrap(), member_id)
- .await;
+ topic.create_consumer_group(Some(group_id), name).unwrap();
+ let result = topic.join_consumer_group(&Identifier::numeric(group_id).unwrap(), member_id);
assert!(result.is_ok());
let consumer_group = topic
.get_consumer_group(&Identifier::numeric(group_id).unwrap())
- .unwrap()
- .read()
- .await;
+ .unwrap();
let members = consumer_group.get_members();
assert_eq!(members.len(), 1);
}
@@ -362,34 +386,26 @@ mod tests {
let name = "test";
let member_id = 1;
let mut topic = get_topic().await;
- topic
- .create_consumer_group(Some(group_id), name)
- .await
- .unwrap();
+ topic.create_consumer_group(Some(group_id), name).unwrap();
topic
.join_consumer_group(&Identifier::numeric(group_id).unwrap(), member_id)
- .await
.unwrap();
- let result = topic
- .leave_consumer_group(&Identifier::numeric(group_id).unwrap(), member_id)
- .await;
+ let result = topic.leave_consumer_group(&Identifier::numeric(group_id).unwrap(), member_id);
assert!(result.is_ok());
let consumer_group = topic
.get_consumer_group(&Identifier::numeric(group_id).unwrap())
- .unwrap()
- .read()
- .await;
+ .unwrap();
let members = consumer_group.get_members();
assert!(members.is_empty())
}
async fn get_topic() -> Topic {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
(iggy) 04/12: begin shard init
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 36ade47f7871e63098a345ff4425259a2605a99a
Author: numinex <g....@gmail.com>
AuthorDate: Sat May 17 14:00:52 2025 +0200
begin shard init
---
core/server/Cargo.toml | 2 +-
core/server/src/bootstrap.rs | 131 +++++++++++++++++++++++++++++++++++--
core/server/src/main.rs | 33 +++++-----
core/server/src/shard/builder.rs | 54 +++++++++++++--
core/server/src/shard/mod.rs | 49 ++++++++------
core/server/src/shard/namespace.rs | 1 +
core/server/src/state/file.rs | 13 +---
7 files changed, 221 insertions(+), 62 deletions(-)
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index baa17e53..f8de2be5 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -64,7 +64,7 @@ jsonwebtoken = "9.3.1"
lending-iterator = "0.1.7"
mimalloc = { workspace = true, optional = true }
moka = { version = "0.12.10", features = ["future"] }
-monoio = "0.2.4"
+monoio = { version = "0.2.4", features = ["mkdirat", "unlinkat"] }
nix = { version = "0.30", features = ["fs"] }
once_cell = "1.21.3"
opentelemetry = { version = "0.30.0", features = ["trace", "logs"] }
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index fd6a5aa0..d51096a2 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -1,7 +1,24 @@
-use monoio::{Buildable, Driver, Runtime};
+use iggy_common::{
+ defaults::{
+ DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH,
+ MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH,
+ },
+ IggyError,
+};
+use monoio::{
+ fs::create_dir_all,
+ Buildable, Driver, Runtime,
+};
+use tracing::info;
-use crate::shard::{connector::ShardConnector, frame::ShardFrame};
-use std::ops::Range;
+use crate::{
+ configs::{config_provider::ConfigProviderKind, server::ServerConfig, system::SystemConfig},
+ server_error::ServerError,
+ shard::{connector::ShardConnector, frame::ShardFrame},
+ streaming::users::user::User,
+ IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
+};
+use std::{env, fs::remove_dir_all, ops::Range, path::Path};
pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<ShardFrame>> {
let shards_count = shards_set.len();
@@ -13,12 +30,112 @@ pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<
connectors
}
-pub async fn create_directories() {
- todo!();
+pub async fn load_config(
+ config_provider: &ConfigProviderKind,
+) -> Result<ServerConfig, ServerError> {
+ let config = ServerConfig::load(config_provider).await?;
+ Ok(config)
}
-pub async fn create_root_user() {
- todo!();
+pub async fn create_directories(config: &SystemConfig) -> Result<(), IggyError> {
+ let system_path = config.get_system_path();
+ if !Path::new(&system_path).exists() && create_dir_all(&system_path).await.is_err() {
+ return Err(IggyError::CannotCreateBaseDirectory(system_path));
+ }
+
+ let state_path = config.get_state_path();
+ if !Path::new(&state_path).exists() && create_dir_all(&state_path).await.is_err() {
+ return Err(IggyError::CannotCreateStateDirectory(state_path));
+ }
+
+ let streams_path = config.get_streams_path();
+ if !Path::new(&streams_path).exists() && create_dir_all(&streams_path).await.is_err() {
+ return Err(IggyError::CannotCreateStreamsDirectory(streams_path));
+ }
+
+ let runtime_path = config.get_runtime_path();
+ // TODO: Change remove_dir_all to async version, once we implement the dir walk using monoio `remove_dir` method.
+ if Path::new(&runtime_path).exists() && remove_dir_all(&runtime_path).is_err() {
+ return Err(IggyError::CannotRemoveRuntimeDirectory(runtime_path));
+ }
+
+ if create_dir_all(&runtime_path).await.is_err() {
+ return Err(IggyError::CannotCreateRuntimeDirectory(runtime_path));
+ }
+
+ info!(
+ "Initializing system, data will be stored at: {}",
+ config.get_system_path()
+ );
+ Ok(())
+
+ // TODO: Move this to individual shard level
+ /*
+ let state_entries = self.state.init().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to initialize state entries")
+ })?;
+ let system_state = SystemState::init(state_entries)
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to initialize system state")
+ })?;
+ let now = Instant::now();
+ self.load_version().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to load version")
+ })?;
+ self.load_users(system_state.users.into_values().collect())
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to load users")
+ })?;
+ self.load_streams(system_state.streams.into_values().collect())
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to load streams")
+ })?;
+ if let Some(archiver) = self.archiver.as_ref() {
+ archiver
+ .init()
+ .await
+ .expect("Failed to initialize archiver");
+ }
+ info!("Initialized system in {} ms.", now.elapsed().as_millis());
+ Ok(())
+ */
+}
+
+pub fn create_root_user() -> User {
+ info!("Creating root user...");
+ let username = env::var(IGGY_ROOT_USERNAME_ENV);
+ let password = env::var(IGGY_ROOT_PASSWORD_ENV);
+ if (username.is_ok() && password.is_err()) || (username.is_err() && password.is_ok()) {
+ panic!("When providing the custom root user credentials, both username and password must be set.");
+ }
+ if username.is_ok() && password.is_ok() {
+ info!("Using the custom root user credentials.");
+ } else {
+ info!("Using the default root user credentials.");
+ }
+
+ let username = username.unwrap_or(DEFAULT_ROOT_USERNAME.to_string());
+ let password = password.unwrap_or(DEFAULT_ROOT_PASSWORD.to_string());
+ if username.is_empty() || password.is_empty() {
+ panic!("Root user credentials are not set.");
+ }
+ if username.len() < MIN_USERNAME_LENGTH {
+ panic!("Root username is too short.");
+ }
+ if username.len() > MAX_USERNAME_LENGTH {
+ panic!("Root username is too long.");
+ }
+ if password.len() < MIN_PASSWORD_LENGTH {
+ panic!("Root password is too short.");
+ }
+ if password.len() > MAX_PASSWORD_LENGTH {
+ panic!("Root password is too long.");
+ }
+ let user = User::root(&username, &password);
+ user
}
pub fn create_default_executor<D>() -> Runtime<D>
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 3b38700f..bb4ea690 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -21,11 +21,13 @@ use std::thread::available_parallelism;
use anyhow::Result;
use clap::Parser;
use dotenvy::dotenv;
+use error_set::ErrContext;
use figlet_rs::FIGfont;
use monoio::Buildable;
use server::args::Args;
use server::bootstrap::{
create_default_executor, create_directories, create_root_user, create_shard_connections,
+ load_config,
};
use server::channels::commands::archive_state::ArchiveStateExecutor;
use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
@@ -73,17 +75,12 @@ fn main() -> Result<(), ServerError> {
}
let args = Args::parse();
+ // TODO: I think we could get rid of config provider, since we support only TOML
+ // as config provider.
let config_provider = config_provider::resolve(&args.config_provider)?;
let config = std::thread::scope(|scope| {
let config = scope
.spawn(move || {
- async fn load_config(
- config_provider: &ConfigProviderKind,
- ) -> Result<ServerConfig, ServerError> {
- let config = ServerConfig::load(config_provider).await?;
- Ok(config)
- }
-
let mut rt = create_default_executor::<monoio::IoUringDriver>();
rt.block_on(load_config(&config_provider))
})
@@ -93,7 +90,7 @@ fn main() -> Result<(), ServerError> {
})?;
// Create directories and root user.
- // Remove `local_data` directory if run with `--fresh` flag.
+ // Remove `local_data` directory if run with `--fresh` flag.
std::thread::scope(|scope| {
scope
.spawn(|| {
@@ -115,14 +112,15 @@ fn main() -> Result<(), ServerError> {
}
}
- // Create directories and root user
- create_directories().await;
- create_root_user().await;
- });
+ // Create directories.
+ create_directories(&config.system).await?;
+ Ok::<(), ServerError>(())
+ })
})
.join()
- .expect("Failed to create directories and root user");
- });
+ .expect("Failed to create directories and root user")
+ })
+ .with_error_context(|err| format!("Failed to create directories, err: {err}"))?;
// Initialize logging
let mut logging = Logging::new(config.telemetry.clone());
@@ -153,20 +151,21 @@ fn main() -> Result<(), ServerError> {
// let urb = io_uring::IoUring::builder();
// TODO: Shall we make the size of ring be configureable ?
let mut rt = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new()
- //.uring_builder(urb.setup_coop_taskrun())
+ //.uring_builder(urb.setup_coop_taskrun()) // broken shit.
.with_entries(1024) // Default size
.enable_timer()
.build()
.expect(format!("Failed to build monoio runtime for shard-{id}").as_str());
rt.block_on(async move {
let builder = IggyShard::builder();
- let shard = builder
+ let mut shard = builder
.id(id)
.connections(connections)
.server_config(server_config)
- .build_and_init()
+ .build()
.await;
+ shard.init().await;
shard.assert_init();
})
})
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 750e2b71..640ffb57 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -16,7 +16,12 @@
* under the License.
*/
-use crate::{configs::server::ServerConfig, shard::Shard};
+use std::{cell::Cell, rc::Rc, sync::Arc};
+
+use iggy_common::{Aes256GcmEncryptor, EncryptorKind};
+use tracing::info;
+
+use crate::{configs::server::ServerConfig, map_toggle_str, shard::Shard, state::{file::FileState, StateKind}, streaming::{persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind}, storage::SystemStorage}, versioning::SemanticVersion};
use super::{connector::ShardConnector, frame::ShardFrame, IggyShard};
@@ -43,11 +48,12 @@ impl IggyShardBuilder {
self
}
- pub async fn build_and_init(self) -> IggyShard {
+ // TODO: Too much happens in there, some of those bootstrapping logic should be moved outside.
+ pub async fn build(self) -> IggyShard {
let id = self.id.unwrap();
let config = self.config.unwrap();
let connections = self.connections.unwrap();
- let (stop_sender, stop_receiver, receiver) = connections
+ let (stop_sender, stop_receiver, frame_receiver) = connections
.iter()
.filter(|c| c.id == id)
.map(|c| {
@@ -60,7 +66,47 @@ impl IggyShardBuilder {
.next()
.expect("Failed to find connection with the specified ID");
let shards = connections.into_iter().map(Shard::new).collect();
+ let version = SemanticVersion::current().expect("Invalid version");
+
+ info!(
+ "Server-side encryption is {}.",
+ map_toggle_str(config.system.encryption.enabled)
+ );
+ let encryptor: Option<Arc<EncryptorKind>> = match config.system.encryption.enabled {
+ true => Some(Arc::new(EncryptorKind::Aes256Gcm(
+ Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key).unwrap(),
+ ))),
+ false => None,
+ };
+
+ let state_persister = Self::resolve_persister(config.system.state.enforce_fsync);
+ let state = Rc::new(StateKind::File(FileState::new(
+ &config.system.get_state_messages_file_path(),
+ &version,
+ state_persister,
+ encryptor.clone(),
+ )));
+
+ let partition_persister = Self::resolve_persister(config.system.partition.enforce_fsync);
+ let storage = SystemStorage::new(config.system, partition_persister);
+
+ IggyShard {
+ id: id,
+ shards: shards,
+ shards_table: Default::default(),
+ storage: storage,
+ state: state,
+ config: config,
+ stop_receiver: stop_receiver,
+ stop_sender: stop_sender,
+ frame_receiver: Cell::new(Some(frame_receiver)),
+ }
+ }
- IggyShard::new(id, shards, config, stop_receiver, stop_sender, receiver)
+ fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
+ match enforce_fsync {
+ true => Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister)),
+ false => Arc::new(PersisterKind::File(FilePersister)),
+ }
}
}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index dcb089d9..54b88fb1 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -15,6 +15,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
pub mod builder;
pub mod connector;
pub mod frame;
@@ -25,9 +26,9 @@ use builder::IggyShardBuilder;
use connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use frame::ShardFrame;
use namespace::IggyNamespace;
-use std::cell::{Cell, RefCell};
+use std::{cell::{Cell, RefCell}, rc::Rc, sync::Arc};
-use crate::configs::server::ServerConfig;
+use crate::{bootstrap::create_root_user, configs::server::ServerConfig, state::file::FileState, streaming::storage::SystemStorage};
pub(crate) struct Shard {
id: u16,
connection: ShardConnector<ShardFrame>,
@@ -49,21 +50,23 @@ struct ShardInfo {
pub struct IggyShard {
pub id: u16,
shards: Vec<Shard>,
- //shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
+ shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
//pub(crate) permissioner: RefCell<Permissioner>,
//pub(crate) streams: RwLock<HashMap<u32, Stream>>,
//pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
//pub(crate) users: RefCell<HashMap<UserId, User>>,
+ // TODO: Refactor.
+ pub(crate) storage: Arc<SystemStorage>,
// TODO - get rid of this dynamic dispatch.
- //pub(crate) state: Rc<FileState>,
+ pub(crate) state: Rc<FileState>,
//pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
config: ServerConfig,
//pub(crate) client_manager: RefCell<ClientManager>,
//pub(crate) active_sessions: RefCell<Vec<Session>>,
//pub(crate) metrics: Metrics,
- pub message_receiver: Cell<Option<Receiver<ShardFrame>>>,
+ pub frame_receiver: Cell<Option<Receiver<ShardFrame>>>,
stop_receiver: StopReceiver,
stop_sender: StopSender,
}
@@ -73,22 +76,26 @@ impl IggyShard {
Default::default()
}
- pub(crate) fn new(
- id: u16,
- shards: Vec<Shard>,
- config: ServerConfig,
- stop_receiver: StopReceiver,
- stop_sender: StopSender,
- shard_messages_receiver: Receiver<ShardFrame>,
- ) -> Self {
- Self {
- id,
- shards,
- config,
- stop_receiver,
- stop_sender,
- message_receiver: Cell::new(Some(shard_messages_receiver)),
- }
+ pub async fn init(&mut self) {
+ let user = create_root_user();
+ self.load_state().await;
+ self.load_users().await;
+ // Add default root user.
+ todo!();
+ self.load_streams().await;
+
+ }
+
+ async fn load_state(&self) {
+ todo!()
+ }
+
+ async fn load_users(&self) {
+ todo!()
+ }
+
+ async fn load_streams(&self) {
+ todo!()
}
pub fn assert_init(&self) {}
diff --git a/core/server/src/shard/namespace.rs b/core/server/src/shard/namespace.rs
index 4b972125..d9aedfb7 100644
--- a/core/server/src/shard/namespace.rs
+++ b/core/server/src/shard/namespace.rs
@@ -15,6 +15,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct IggyNamespace {
pub(crate) stream_id: u32,
diff --git a/core/server/src/state/file.rs b/core/server/src/state/file.rs
index e739066e..591b8072 100644
--- a/core/server/src/state/file.rs
+++ b/core/server/src/state/file.rs
@@ -84,18 +84,7 @@ impl FileState {
impl State for FileState {
async fn init(&self) -> Result<Vec<StateEntry>, IggyError> {
- if !Path::new(&self.path).exists() {
- info!("State file does not exist, creating a new one");
- self.persister
- .overwrite(&self.path, &[])
- .await
- .with_error_context(|error| {
- format!(
- "{COMPONENT} (error: {error}) - failed to overwrite state file, path: {}",
- self.path
- )
- })?;
- }
+ assert!(Path::new(&self.path).exists());
let entries = self.load_entries().await.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to load entries")
(iggy) 07/12: fix state initialization with gate
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 44743fb42bb3a2243f109778d579818ae28844f7
Author: numinex <g....@gmail.com>
AuthorDate: Sat Jun 21 13:51:27 2025 +0200
fix state initialization with gate
---
core/common/src/utils/crypto.rs | 3 +-
core/server/src/bootstrap.rs | 48 ++---
core/server/src/lib.rs | 8 +-
core/server/src/main.rs | 151 ++++++++++++---
core/server/src/shard/builder.rs | 65 +++----
core/server/src/shard/gate.rs | 42 +++++
core/server/src/shard/mod.rs | 256 ++++++++++++++++++++++++--
core/server/src/shard/transmission/frame.rs | 17 ++
core/server/src/shard/transmission/message.rs | 17 ++
core/server/src/shard/transmission/mod.rs | 18 ++
core/server/src/state/file.rs | 4 +-
core/server/src/streaming/systems/info.rs | 55 ------
core/server/src/streaming/systems/system.rs | 21 ++-
13 files changed, 524 insertions(+), 181 deletions(-)
diff --git a/core/common/src/utils/crypto.rs b/core/common/src/utils/crypto.rs
index 34cd72c7..11d51cb7 100644
--- a/core/common/src/utils/crypto.rs
+++ b/core/common/src/utils/crypto.rs
@@ -23,7 +23,7 @@ use aes_gcm::aead::{Aead, OsRng};
use aes_gcm::{AeadCore, Aes256Gcm, KeyInit};
use std::fmt::Debug;
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub enum EncryptorKind {
Aes256Gcm(Aes256GcmEncryptor),
}
@@ -46,6 +46,7 @@ pub trait Encryptor {
fn decrypt(&self, data: &[u8]) -> Result<Vec<u8>, IggyError>;
}
+#[derive(Clone)]
pub struct Aes256GcmEncryptor {
cipher: Aes256Gcm,
}
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index c6108568..4f356a9b 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -12,10 +12,13 @@ use crate::{
IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
configs::{config_provider::ConfigProviderKind, server::ServerConfig, system::SystemConfig},
server_error::ServerError,
- shard::{transmission::connector::ShardConnector, transmission::frame::ShardFrame},
- streaming::users::user::User,
+ shard::transmission::{connector::ShardConnector, frame::ShardFrame},
+ streaming::{
+ persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind},
+ users::user::User,
+ },
};
-use std::{env, fs::remove_dir_all, ops::Range, path::Path};
+use std::{env, fs::remove_dir_all, ops::Range, path::Path, sync::Arc};
pub fn create_shard_connections(shards_set: Range<usize>) -> Vec<ShardConnector<ShardFrame>> {
let shards_count = shards_set.len();
@@ -68,37 +71,7 @@ pub async fn create_directories(config: &SystemConfig) -> Result<(), IggyError>
// TODO: Move this to individual shard level
/*
- let state_entries = self.state.init().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to initialize state entries")
- })?;
- let system_state = SystemState::init(state_entries)
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to initialize system state")
- })?;
- let now = Instant::now();
- self.load_version().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load version")
- })?;
- self.load_users(system_state.users.into_values().collect())
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load users")
- })?;
- self.load_streams(system_state.streams.into_values().collect())
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load streams")
- })?;
- if let Some(archiver) = self.archiver.as_ref() {
- archiver
- .init()
- .await
- .expect("Failed to initialize archiver");
- }
- info!("Initialized system in {} ms.", now.elapsed().as_millis());
- Ok(())
- */
+ */
}
pub fn create_root_user() -> User {
@@ -158,3 +131,10 @@ pub fn create_shard_executor() -> Runtime<TimeDriver<monoio::IoUringDriver>> {
let rt = Buildable::build(builder).expect("Failed to create default runtime");
rt
}
+
+pub fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
+ match enforce_fsync {
+ true => Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister)),
+ false => Arc::new(PersisterKind::File(FilePersister)),
+ }
+}
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index e64837da..3dcf64b2 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -43,11 +43,11 @@ pub mod streaming;
pub mod tcp;
pub mod versioning;
-const VERSION: &str = env!("CARGO_PKG_VERSION");
-const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME";
-const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD";
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+pub const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME";
+pub const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD";
-pub(crate) fn map_toggle_str<'a>(enabled: bool) -> &'a str {
+pub fn map_toggle_str<'a>(enabled: bool) -> &'a str {
match enabled {
true => "enabled",
false => "disabled",
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 871ff2f1..3a6c1686 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -16,6 +16,7 @@
* under the License.
*/
+use std::sync::Arc;
use std::thread::available_parallelism;
use anyhow::Result;
@@ -23,33 +24,32 @@ use clap::Parser;
use dotenvy::dotenv;
use error_set::ErrContext;
use figlet_rs::FIGfont;
+use iggy_common::create_user::CreateUser;
+use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError};
use server::args::Args;
use server::bootstrap::{
create_default_executor, create_directories, create_root_user, create_shard_connections,
- create_shard_executor, load_config,
+ create_shard_executor, load_config, resolve_persister,
};
-use server::channels::commands::archive_state::ArchiveStateExecutor;
-use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
-use server::channels::commands::maintain_messages::MaintainMessagesExecutor;
-use server::channels::commands::print_sysinfo::SysInfoPrintExecutor;
-use server::channels::commands::save_messages::SaveMessagesExecutor;
-use server::channels::commands::verify_heartbeats::VerifyHeartbeatsExecutor;
-use server::channels::handler::BackgroundServerCommandHandler;
-use server::configs::config_provider::{self, ConfigProviderKind};
-use server::configs::server::ServerConfig;
-use server::http::http_server;
+use server::configs::config_provider::{self};
#[cfg(not(feature = "tokio-console"))]
use server::log::logger::Logging;
#[cfg(feature = "tokio-console")]
use server::log::tokio_console::Logging;
-use server::quic::quic_server;
use server::server_error::ServerError;
use server::shard::IggyShard;
-use server::streaming::systems::system::{SharedSystem, System};
-use server::streaming::utils::MemoryPool;
-use server::tcp::tcp_server;
+use server::shard::gate::Gate;
+use server::state::StateKind;
+use server::state::command::EntryCommand;
+use server::state::file::FileState;
+use server::state::models::CreateUserWithId;
+use server::state::system::SystemState;
+use server::versioning::SemanticVersion;
+use server::{IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV, map_toggle_str};
use tokio::time::Instant;
-use tracing::{info, instrument};
+use tracing::{error, info, instrument};
+
+const COMPONENT: &str = "MAIN";
#[instrument(skip_all, name = "trace_start_server")]
fn main() -> Result<(), ServerError> {
@@ -88,7 +88,11 @@ fn main() -> Result<(), ServerError> {
config
})?;
- // Create directories and root user.
+ // Initialize logging
+ let mut logging = Logging::new(config.telemetry.clone());
+ logging.early_init();
+
+ // Create directories.
// Remove `local_data` directory if run with `--fresh` flag.
std::thread::scope(|scope| {
scope
@@ -117,13 +121,9 @@ fn main() -> Result<(), ServerError> {
})
})
.join()
- .expect("Failed to create directories and root user")
+ .expect("Failed join thread")
})
- .with_error_context(|err| format!("Failed to create directories, err: {err}"))?;
-
- // Initialize logging
- let mut logging = Logging::new(config.telemetry.clone());
- logging.early_init();
+ .with_error_context(|err| format!("Failed to init server: {err}"))?;
// TODO: Make this configurable from config as a range
// for example this instance of Iggy will use cores from 0..4
@@ -132,9 +132,11 @@ fn main() -> Result<(), ServerError> {
let shards_set = 0..shards_count;
let connections = create_shard_connections(shards_set.clone());
for shard_id in shards_set {
+ let gate: Arc<Gate<()>> = Arc::new(Gate::new());
let id = shard_id as u16;
let connections = connections.clone();
- let server_config = config.clone();
+ let config = config.clone();
+ let state_persister = resolve_persister(config.system.state.enforce_fsync);
std::thread::Builder::new()
.name(format!("shard-{id}"))
.spawn(move || {
@@ -143,18 +145,109 @@ fn main() -> Result<(), ServerError> {
let mut rt = create_shard_executor();
rt.block_on(async move {
+ let version = SemanticVersion::current().expect("Invalid version");
+ info!(
+ "Server-side encryption is {}.",
+ map_toggle_str(config.system.encryption.enabled)
+ );
+ let encryptor: Option<EncryptorKind> = match config.system.encryption.enabled {
+ true => Some(EncryptorKind::Aes256Gcm(
+ Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key)
+ .unwrap(),
+ )),
+ false => None,
+ };
+
+ let state = StateKind::File(FileState::new(
+ &config.system.get_state_messages_file_path(),
+ &version,
+ state_persister,
+ encryptor.clone(),
+ ));
+
+ // We can't use std::sync::Once because it doesn't support async.
+ // Trait bound on the closure is FnOnce.
+ let gate = gate.clone();
+ // Peak into the state to check if the root user exists.
+ // If it does not exist, create it.
+ gate.with_async::<Result<(), IggyError>>(async |gate_state| {
+ // A thread already initialized state
+ // Thus, we can skip it.
+ if let Some(_) = gate_state.inner() {
+ return Ok(());
+ }
+
+ let state_entries = state.load_entries().await.with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to load state entries"
+ )
+ })?;
+ let root_exists = state_entries
+ .into_iter()
+ .find(|entry| {
+ entry
+ .command()
+ .and_then(|command| match command {
+ EntryCommand::CreateUser(payload)
+ if payload.command.username
+ == IGGY_ROOT_USERNAME_ENV && payload.command.password == IGGY_ROOT_PASSWORD_ENV =>
+ {
+ Ok(true)
+ }
+ _ => Ok(false),
+ })
+ .map_or_else(
+ |err| {
+ error!("Failed to check if root user exists: {err}");
+ false
+ },
+ |v| v,
+ )
+ })
+ .is_some();
+
+ if !root_exists {
+ info!("No users found, creating the root user...");
+ let root = create_root_user();
+ let command = CreateUser {
+ username: root.username.clone(),
+ password: root.password.clone(),
+ status: root.status,
+ permissions: root.permissions.clone(),
+ };
+ state
+ .apply(0, &EntryCommand::CreateUser(CreateUserWithId {
+ user_id: root.id,
+ command
+ }))
+ .await
+ .with_error_context(|error| {
+ format!(
+ "{COMPONENT} (error: {error}) - failed to apply create user command, username: {}",
+ root.username
+ )
+ })?;
+ }
+
+ gate_state.set_result(());
+ Ok(())
+ })
+ .await;
+
let builder = IggyShard::builder();
let mut shard = builder
.id(id)
.connections(connections)
- .server_config(server_config)
+ .config(config)
+ .encryptor(encryptor)
+ .version(version)
+ .state(state)
.build()
.await;
- if let Err(e) = shard.init().await {
- //TODO: If one of the shards fails to initialize, we should crash the whole program;
- panic!("Failed to initialize shard-{id}: {e}");
- }
+ //TODO: If one of the shards fails to initialize, we should crash the whole program;
+ shard.init().await.expect("Failed to initialize shard-{id}: {e}");
+ info!("Initiated shard with ID: {id}");
//TODO: If one of the shards fails to initialize, we should crash the whole program;
shard.assert_init();
})
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 36fe0e54..0fd5250f 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -22,14 +22,12 @@ use iggy_common::{Aes256GcmEncryptor, EncryptorKind};
use tracing::info;
use crate::{
+ bootstrap::resolve_persister,
configs::server::ServerConfig,
map_toggle_str,
shard::Shard,
state::{StateKind, file::FileState},
- streaming::{
- persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind},
- storage::SystemStorage,
- },
+ streaming::storage::SystemStorage,
versioning::SemanticVersion,
};
@@ -40,6 +38,9 @@ pub struct IggyShardBuilder {
id: Option<u16>,
connections: Option<Vec<ShardConnector<ShardFrame>>>,
config: Option<ServerConfig>,
+ encryptor: Option<EncryptorKind>,
+ version: Option<SemanticVersion>,
+ state: Option<StateKind>,
}
impl IggyShardBuilder {
@@ -53,16 +54,33 @@ impl IggyShardBuilder {
self
}
- pub fn server_config(mut self, config: ServerConfig) -> Self {
+ pub fn config(mut self, config: ServerConfig) -> Self {
self.config = Some(config);
self
}
+ pub fn encryptor(mut self, encryptor: Option<EncryptorKind>) -> Self {
+ self.encryptor = encryptor;
+ self
+ }
+
+ pub fn version(mut self, version: SemanticVersion) -> Self {
+ self.version = Some(version);
+ self
+ }
+
+ pub fn state(mut self, state: StateKind) -> Self {
+ self.state = Some(state);
+ self
+ }
+
// TODO: Too much happens in there, some of those bootstrapping logic should be moved outside.
pub async fn build(self) -> IggyShard {
let id = self.id.unwrap();
let config = self.config.unwrap();
let connections = self.connections.unwrap();
+ let state = self.state.unwrap();
+ let version = self.version.unwrap();
let (stop_sender, stop_receiver, frame_receiver) = connections
.iter()
.filter(|c| c.id == id)
@@ -76,33 +94,8 @@ impl IggyShardBuilder {
.next()
.expect("Failed to find connection with the specified ID");
let shards = connections.into_iter().map(Shard::new).collect();
-
- //TODO: This can be discrete step in the builder bootstrapped from main function.
- let version = SemanticVersion::current().expect("Invalid version");
-
- //TODO: This can be discrete step in the builder bootstrapped from main function.
- info!(
- "Server-side encryption is {}.",
- map_toggle_str(config.system.encryption.enabled)
- );
- let encryptor: Option<Arc<EncryptorKind>> = match config.system.encryption.enabled {
- true => Some(Arc::new(EncryptorKind::Aes256Gcm(
- Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key).unwrap(),
- ))),
- false => None,
- };
-
- //TODO: This can be discrete step in the builder bootstrapped from main function.
- let state_persister = Self::resolve_persister(config.system.state.enforce_fsync);
- let state = Rc::new(StateKind::File(FileState::new(
- &config.system.get_state_messages_file_path(),
- &version,
- state_persister,
- encryptor.clone(),
- )));
-
- //TODO: This can be discrete step in the builder bootstrapped from main function.
- let partition_persister = Self::resolve_persister(config.system.partition.enforce_fsync);
+ //TODO: Eghhhh.......
+ let partition_persister = resolve_persister(config.system.partition.enforce_fsync);
let storage = Rc::new(SystemStorage::new(
config.system.clone(),
partition_persister,
@@ -115,16 +108,10 @@ impl IggyShardBuilder {
storage: storage,
state: state,
config: config,
+ version: version,
stop_receiver: stop_receiver,
stop_sender: stop_sender,
frame_receiver: Cell::new(Some(frame_receiver)),
}
}
-
- fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
- match enforce_fsync {
- true => Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister)),
- false => Arc::new(PersisterKind::File(FilePersister)),
- }
- }
}
diff --git a/core/server/src/shard/gate.rs b/core/server/src/shard/gate.rs
new file mode 100644
index 00000000..a26950a4
--- /dev/null
+++ b/core/server/src/shard/gate.rs
@@ -0,0 +1,42 @@
+use std::sync::{Condvar, Mutex};
+
+#[derive(Default)]
+pub struct Gate<T> {
+ state: Mutex<GateState<T>>,
+}
+
+#[derive(Default)]
+pub struct GateState<T> {
+ result: Option<T>,
+}
+
+impl<T> GateState<T> {
+ pub fn set_result(&mut self, result: T) {
+ self.result = Some(result);
+ }
+
+ pub fn inner(&self) -> &Option<T> {
+ &self.result
+ }
+}
+
+impl<T> Gate<T>
+where
+ T: Default,
+{
+ pub fn new() -> Self {
+ Gate {
+ state: Default::default(),
+ }
+ }
+
+ pub async fn with_async<R>(&self, f: impl AsyncFnOnce(&mut GateState<T>) -> R) {
+ let mut guard = self.state.lock().unwrap();
+ f(&mut guard).await;
+ }
+
+ pub async fn with<R>(&self, f: impl FnOnce(&mut GateState<T>) -> R) {
+ let mut guard = self.state.lock().unwrap();
+ f(&mut guard);
+ }
+}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 6a6d35b8..23ef1e15 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -17,16 +17,19 @@
*/
pub mod builder;
+pub mod gate;
pub mod namespace;
pub mod transmission;
use ahash::HashMap;
use builder::IggyShardBuilder;
+use error_set::ErrContext;
use iggy_common::IggyError;
use namespace::IggyNamespace;
use std::{
cell::{Cell, RefCell},
rc::Rc,
+ str::FromStr,
sync::Arc,
time::Instant,
};
@@ -37,9 +40,17 @@ use crate::{
bootstrap::create_root_user,
configs::server::ServerConfig,
shard::transmission::frame::ShardFrame,
- state::{StateKind, file::FileState},
- streaming::storage::SystemStorage,
+ state::{
+ StateKind,
+ file::FileState,
+ system::{SystemState, UserState},
+ },
+ streaming::{storage::SystemStorage, systems::info::SystemInfo},
+ versioning::SemanticVersion,
};
+
+pub const COMPONENT: &str = "SHARD";
+
pub(crate) struct Shard {
id: u16,
connection: ShardConnector<ShardFrame>,
@@ -62,6 +73,7 @@ pub struct IggyShard {
pub id: u16,
shards: Vec<Shard>,
shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
+ version: SemanticVersion,
//pub(crate) permissioner: RefCell<Permissioner>,
//pub(crate) streams: RwLock<HashMap<u32, Stream>>,
@@ -70,7 +82,7 @@ pub struct IggyShard {
// TODO: Refactor.
pub(crate) storage: Rc<SystemStorage>,
- pub(crate) state: Rc<StateKind>,
+ pub(crate) state: StateKind,
//pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
config: ServerConfig,
//pub(crate) client_manager: RefCell<ClientManager>,
@@ -92,10 +104,12 @@ impl IggyShard {
//let state_entries = self.state.init().await?;
//let system_state = SystemState::init(state_entries).await?;
//let user = create_root_user();
- self.load_state().await;
- self.load_users().await;
+ self.load_version().await?;
+ let SystemState { users, streams } = self.load_state().await?;
+ self.load_users(users.into_values().collect()).await;
// Add default root user.
- self.load_streams().await;
+ self.load_streams(streams.into_values().collect()).await;
+
//TODO: Fix the archiver.
/*
if let Some(archiver) = self.archiver.as_ref() {
@@ -109,16 +123,234 @@ impl IggyShard {
Ok(())
}
- async fn load_state(&self) {
- todo!()
+ async fn load_version(&self) -> Result<(), IggyError> {
+ async fn update_system_info(
+ storage: &Rc<SystemStorage>,
+ system_info: &mut SystemInfo,
+ version: &SemanticVersion,
+ ) -> Result<(), IggyError> {
+ system_info.update_version(version);
+ storage.info.save(system_info).await?;
+ Ok(())
+ }
+
+ let current_version = &self.version;
+ let mut system_info;
+ let load_system_info = self.storage.info.load().await;
+ if load_system_info.is_err() {
+ let error = load_system_info.err().unwrap();
+ if let IggyError::ResourceNotFound(_) = error {
+ info!("System info not found, creating...");
+ system_info = SystemInfo::default();
+ update_system_info(&self.storage, &mut system_info, current_version).await?;
+ } else {
+ return Err(error);
+ }
+ } else {
+ system_info = load_system_info.unwrap();
+ }
+
+ info!("Loaded {system_info}.");
+ let loaded_version = SemanticVersion::from_str(&system_info.version.version)?;
+ if current_version.is_equal_to(&loaded_version) {
+ info!("System version {current_version} is up to date.");
+ } else if current_version.is_greater_than(&loaded_version) {
+ info!(
+ "System version {current_version} is greater than {loaded_version}, checking the available migrations..."
+ );
+ update_system_info(&self.storage, &mut system_info, current_version).await?;
+ } else {
+ info!(
+ "System version {current_version} is lower than {loaded_version}, possible downgrade."
+ );
+ update_system_info(&self.storage, &mut system_info, current_version).await?;
+ }
+
+ Ok(())
}
- async fn load_users(&self) {
- todo!()
+ async fn load_state(&self) -> Result<SystemState, IggyError> {
+ let state_entries = self.state.init().await.with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to initialize state entries")
+ })?;
+ let system_state = SystemState::init(state_entries)
+ .await
+ .with_error_context(|error| {
+ format!("{COMPONENT} (error: {error}) - failed to initialize system state")
+ })?;
+ Ok(system_state)
}
- async fn load_streams(&self) {
- todo!()
+ async fn load_users(&mut self, users: Vec<UserState>) -> Result<(), IggyError> {
+ info!("Loading users...");
+ /*
+
+ for user_state in users.into_iter() {
+ let mut user = User::with_password(
+ user_state.id,
+ &user_state.username,
+ user_state.password_hash,
+ user_state.status,
+ user_state.permissions,
+ );
+
+ user.created_at = user_state.created_at;
+ user.personal_access_tokens = user_state
+ .personal_access_tokens
+ .into_values()
+ .map(|token| {
+ (
+ Arc::new(token.token_hash.clone()),
+ PersonalAccessToken::raw(
+ user_state.id,
+ &token.name,
+ &token.token_hash,
+ token.expiry_at,
+ ),
+ )
+ })
+ .collect();
+ self.users.insert(user_state.id, user);
+ }
+
+ let users_count = self.users.len();
+ let current_user_id = self.users.keys().max().unwrap_or(&1);
+ USER_ID.store(current_user_id + 1, Ordering::SeqCst);
+ self.permissioner
+ .init(&self.users.values().collect::<Vec<&User>>());
+ self.metrics.increment_users(users_count as u32);
+ info!("Initialized {users_count} user(s).");
+ */
+ Ok(())
+ }
+
+ async fn load_streams(&mut self, streams: Vec<StreamState>) -> Result<(), IggyError> {
+ todo!();
+ info!("Loading streams from disk...");
+ let mut unloaded_streams = Vec::new();
+ let mut dir_entries = read_dir(&self.config.get_streams_path())
+ .await
+ .map_err(|error| {
+ error!("Cannot read streams directory: {error}");
+ IggyError::CannotReadStreams
+ })?;
+
+ while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) {
+ let name = dir_entry.file_name().into_string().unwrap();
+ let stream_id = name.parse::<u32>().map_err(|_| {
+ error!("Invalid stream ID file with name: '{name}'.");
+ IggyError::InvalidNumberValue
+ })?;
+ let stream_state = streams.iter().find(|s| s.id == stream_id);
+ if stream_state.is_none() {
+ error!(
+ "Stream with ID: '{stream_id}' was not found in state, but exists on disk and will be removed."
+ );
+ if let Err(error) = fs::remove_dir_all(&dir_entry.path()).await {
+ error!("Cannot remove stream directory: {error}");
+ } else {
+ warn!("Stream with ID: '{stream_id}' was removed.");
+ }
+ continue;
+ }
+
+ let stream_state = stream_state.unwrap();
+ let mut stream = Stream::empty(
+ stream_id,
+ &stream_state.name,
+ self.config.clone(),
+ self.storage.clone(),
+ );
+ stream.created_at = stream_state.created_at;
+ unloaded_streams.push(stream);
+ }
+
+ let state_stream_ids = streams
+ .iter()
+ .map(|stream| stream.id)
+ .collect::<AHashSet<u32>>();
+ let unloaded_stream_ids = unloaded_streams
+ .iter()
+ .map(|stream| stream.stream_id)
+ .collect::<AHashSet<u32>>();
+ let mut missing_ids = state_stream_ids
+ .difference(&unloaded_stream_ids)
+ .copied()
+ .collect::<AHashSet<u32>>();
+ if missing_ids.is_empty() {
+ info!("All streams found on disk were found in state.");
+ } else {
+ warn!("Streams with IDs: '{missing_ids:?}' were not found on disk.");
+ if self.config.recovery.recreate_missing_state {
+ info!(
+ "Recreating missing state in recovery config is enabled, missing streams will be created."
+ );
+ for stream_id in missing_ids.iter() {
+ let stream_id = *stream_id;
+ let stream_state = streams.iter().find(|s| s.id == stream_id).unwrap();
+ let stream = Stream::create(
+ stream_id,
+ &stream_state.name,
+ self.config.clone(),
+ self.storage.clone(),
+ );
+ stream.persist().await?;
+ unloaded_streams.push(stream);
+ info!(
+ "Missing stream with ID: '{stream_id}', name: {} was recreated.",
+ stream_state.name
+ );
+ }
+ missing_ids.clear();
+ } else {
+ warn!(
+ "Recreating missing state in recovery config is disabled, missing streams will not be created."
+ );
+ }
+ }
+
+ let mut streams_states = streams
+ .into_iter()
+ .filter(|s| !missing_ids.contains(&s.id))
+ .map(|s| (s.id, s))
+ .collect::<AHashMap<_, _>>();
+ let loaded_streams = RefCell::new(Vec::new());
+ let load_stream_tasks = unloaded_streams.into_iter().map(|mut stream| {
+ let state = streams_states.remove(&stream.stream_id).unwrap();
+
+ async {
+ stream.load(state).await?;
+ loaded_streams.borrow_mut().push(stream);
+ Result::<(), IggyError>::Ok(())
+ }
+ });
+ try_join_all(load_stream_tasks).await?;
+
+ for stream in loaded_streams.take() {
+ if self.streams.contains_key(&stream.stream_id) {
+ error!("Stream with ID: '{}' already exists.", &stream.stream_id);
+ continue;
+ }
+
+ if self.streams_ids.contains_key(&stream.name) {
+ error!("Stream with name: '{}' already exists.", &stream.name);
+ continue;
+ }
+
+ self.metrics.increment_streams(1);
+ self.metrics.increment_topics(stream.get_topics_count());
+ self.metrics
+ .increment_partitions(stream.get_partitions_count());
+ self.metrics.increment_segments(stream.get_segments_count());
+ self.metrics.increment_messages(stream.get_messages_count());
+
+ self.streams_ids
+ .insert(stream.name.clone(), stream.stream_id);
+ self.streams.insert(stream.stream_id, stream);
+ }
+
+ info!("Loaded {} stream(s) from disk.", self.streams.len());
+ Ok(())
}
pub fn assert_init(&self) {}
diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs
index ac60863d..ec63daf0 100644
--- a/core/server/src/shard/transmission/frame.rs
+++ b/core/server/src/shard/transmission/frame.rs
@@ -1,3 +1,20 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
use async_channel::Sender;
use bytes::Bytes;
use iggy_common::IggyError;
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
index e9f1dd4b..3572c0ec 100644
--- a/core/server/src/shard/transmission/message.rs
+++ b/core/server/src/shard/transmission/message.rs
@@ -1,3 +1,20 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
use crate::binary::command::ServerCommand;
#[derive(Debug)]
diff --git a/core/server/src/shard/transmission/mod.rs b/core/server/src/shard/transmission/mod.rs
index b871152a..1f9a79ed 100644
--- a/core/server/src/shard/transmission/mod.rs
+++ b/core/server/src/shard/transmission/mod.rs
@@ -1,3 +1,21 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
pub mod connector;
pub mod frame;
pub mod message;
diff --git a/core/server/src/state/file.rs b/core/server/src/state/file.rs
index 591b8072..41117c3a 100644
--- a/core/server/src/state/file.rs
+++ b/core/server/src/state/file.rs
@@ -47,7 +47,7 @@ pub struct FileState {
version: u32,
path: String,
persister: Arc<PersisterKind>,
- encryptor: Option<Arc<EncryptorKind>>,
+ encryptor: Option<EncryptorKind>,
}
impl FileState {
@@ -55,7 +55,7 @@ impl FileState {
path: &str,
version: &SemanticVersion,
persister: Arc<PersisterKind>,
- encryptor: Option<Arc<EncryptorKind>>,
+ encryptor: Option<EncryptorKind>,
) -> Self {
Self {
current_index: AtomicU64::new(0),
diff --git a/core/server/src/streaming/systems/info.rs b/core/server/src/streaming/systems/info.rs
index 61c1bcbb..3c20fd63 100644
--- a/core/server/src/streaming/systems/info.rs
+++ b/core/server/src/streaming/systems/info.rs
@@ -16,15 +16,11 @@
* under the License.
*/
-use crate::streaming::systems::system::System;
use crate::versioning::SemanticVersion;
-use iggy_common::IggyError;
use serde::{Deserialize, Serialize};
use std::collections::hash_map::DefaultHasher;
use std::fmt::Display;
use std::hash::{Hash, Hasher};
-use std::str::FromStr;
-use tracing::info;
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct SystemInfo {
@@ -46,57 +42,6 @@ pub struct Migration {
pub applied_at: u64,
}
-impl System {
- pub(crate) async fn load_version(&mut self) -> Result<(), IggyError> {
- let current_version = SemanticVersion::current()?;
- let mut system_info;
- let load_system_info = self.storage.info.load().await;
- if load_system_info.is_err() {
- let error = load_system_info.err().unwrap();
- if let IggyError::ResourceNotFound(_) = error {
- info!("System info not found, creating...");
- system_info = SystemInfo::default();
- self.update_system_info(&mut system_info, ¤t_version)
- .await?;
- } else {
- return Err(error);
- }
- } else {
- system_info = load_system_info.unwrap();
- }
-
- info!("Loaded {system_info}.");
- let loaded_version = SemanticVersion::from_str(&system_info.version.version)?;
- if current_version.is_equal_to(&loaded_version) {
- info!("System version {current_version} is up to date.");
- } else if current_version.is_greater_than(&loaded_version) {
- info!(
- "System version {current_version} is greater than {loaded_version}, checking the available migrations..."
- );
- self.update_system_info(&mut system_info, ¤t_version)
- .await?;
- } else {
- info!(
- "System version {current_version} is lower than {loaded_version}, possible downgrade."
- );
- self.update_system_info(&mut system_info, ¤t_version)
- .await?;
- }
-
- Ok(())
- }
-
- async fn update_system_info(
- &self,
- system_info: &mut SystemInfo,
- version: &SemanticVersion,
- ) -> Result<(), IggyError> {
- system_info.update_version(version);
- self.storage.info.save(system_info).await?;
- Ok(())
- }
-}
-
impl SystemInfo {
pub fn update_version(&mut self, version: &SemanticVersion) {
self.version.version = version.to_string();
diff --git a/core/server/src/streaming/systems/system.rs b/core/server/src/streaming/systems/system.rs
index 761f6d1b..5d830d71 100644
--- a/core/server/src/streaming/systems/system.rs
+++ b/core/server/src/streaming/systems/system.rs
@@ -102,10 +102,10 @@ impl System {
map_toggle_str(config.encryption.enabled)
);
- let encryptor: Option<Arc<EncryptorKind>> = match config.encryption.enabled {
- true => Some(Arc::new(EncryptorKind::Aes256Gcm(
+ let encryptor: Option<EncryptorKind> = match config.encryption.enabled {
+ true => Some(EncryptorKind::Aes256Gcm(
Aes256GcmEncryptor::from_base64_key(&config.encryption.key).unwrap(),
- ))),
+ )),
false => None,
};
@@ -116,13 +116,21 @@ impl System {
&config.get_state_messages_file_path(),
&version,
state_persister,
- encryptor.clone(),
+ encryptor,
)));
+
+ //TODO: Just shut the fuck up rust-analyzer.
+ let encryptor: Option<EncryptorKind> = match config.encryption.enabled {
+ true => Some(EncryptorKind::Aes256Gcm(
+ Aes256GcmEncryptor::from_base64_key(&config.encryption.key).unwrap(),
+ )),
+ false => None,
+ };
Self::create(
config.clone(),
SystemStorage::new(config, partition_persister),
state,
- encryptor,
+ encryptor.map(Arc::new),
data_maintenance_config,
pat_config,
)
@@ -224,9 +232,12 @@ impl System {
format!("{COMPONENT} (error: {error}) - failed to initialize system state")
})?;
let now = Instant::now();
+ //DONE
+ /*
self.load_version().await.with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to load version")
})?;
+ */
self.load_users(system_state.users.into_values().collect())
.await
.with_error_context(|error| {
(iggy) 06/12: refactors
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 026e53be79fa58ad68f3aa6a36d58f17616370eb
Author: numinex <g....@gmail.com>
AuthorDate: Fri Jun 20 10:52:31 2025 +0200
refactors
---
Cargo.lock | 13 +++++
Cargo.toml | 1 +
core/server/Cargo.toml | 1 +
core/server/src/bootstrap.rs | 7 +--
core/server/src/main.rs | 11 ++--
core/server/src/shard/builder.rs | 8 ++-
core/server/src/shard/frame.rs | 67 ----------------------
core/server/src/shard/mod.rs | 17 +++---
.../src/shard/{ => transmission}/connector.rs | 49 ++++++++++++----
core/server/src/shard/transmission/frame.rs | 42 ++++++++++++++
core/server/src/shard/transmission/message.rs | 22 +++++++
core/server/src/shard/transmission/mod.rs | 3 +
12 files changed, 142 insertions(+), 99 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 9c594a2c..826f6b36 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -467,6 +467,18 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "async-channel"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a"
+dependencies = [
+ "concurrent-queue",
+ "event-listener-strategy",
+ "futures-core",
+ "pin-project-lite",
+]
+
[[package]]
name = "async-compression"
version = "0.4.25"
@@ -6821,6 +6833,7 @@ version = "0.5.0"
dependencies = [
"ahash 0.8.12",
"anyhow",
+ "async-channel",
"async_zip",
"axum 0.8.4",
"axum-server",
diff --git a/Cargo.toml b/Cargo.toml
index 1fa553c3..d670c3b7 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -87,6 +87,7 @@ dlopen2 = "0.8.0"
enum_dispatch = "0.3.13"
figlet-rs = "0.1.5"
flume = "0.11.1"
+async-channel = "2.3.1"
futures = "0.3.31"
futures-util = "0.3.31"
human-repr = "1.1.0"
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index f8de2be5..02d39830 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -56,6 +56,7 @@ error_set = { version = "0.8.5", features = ["tracing"] }
figlet-rs = { workspace = true }
figment = { version = "0.10.19", features = ["toml", "env"] }
flume = { workspace = true }
+async-channel = { workspace = true }
futures = { workspace = true }
human-repr = { workspace = true }
iggy_common = { workspace = true }
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index d70d0317..c6108568 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -5,14 +5,14 @@ use iggy_common::{
MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH,
},
};
-use monoio::{fs::create_dir_all, time::TimeDriver, Buildable, Driver, Runtime};
+use monoio::{Buildable, Driver, Runtime, fs::create_dir_all, time::TimeDriver};
use tracing::info;
use crate::{
IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
configs::{config_provider::ConfigProviderKind, server::ServerConfig, system::SystemConfig},
server_error::ServerError,
- shard::{connector::ShardConnector, frame::ShardFrame},
+ shard::{transmission::connector::ShardConnector, transmission::frame::ShardFrame},
streaming::users::user::User,
};
use std::{env, fs::remove_dir_all, ops::Range, path::Path};
@@ -146,8 +146,7 @@ where
rt
}
-pub fn create_shard_executor() -> Runtime<TimeDriver<monoio::IoUringDriver>>
-{
+pub fn create_shard_executor() -> Runtime<TimeDriver<monoio::IoUringDriver>> {
// TODO: Figure out what else we could tweak there
// We for sure want to disable the userspace interrupts on new cq entry (set_coop_taskrun)
// let urb = io_uring::IoUring::builder();
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 06ffec35..871ff2f1 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -25,7 +25,8 @@ use error_set::ErrContext;
use figlet_rs::FIGfont;
use server::args::Args;
use server::bootstrap::{
- create_default_executor, create_directories, create_root_user, create_shard_connections, create_shard_executor, load_config
+ create_default_executor, create_directories, create_root_user, create_shard_connections,
+ create_shard_executor, load_config,
};
use server::channels::commands::archive_state::ArchiveStateExecutor;
use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
@@ -126,14 +127,10 @@ fn main() -> Result<(), ServerError> {
// TODO: Make this configurable from config as a range
// for example this instance of Iggy will use cores from 0..4
- let available_cpus = available_parallelism()
- .expect("Failed to get num of cores")
- .into();
- let shards_count = available_cpus;
+ let available_cpus = available_parallelism().expect("Failed to get num of cores");
+ let shards_count = available_cpus.into();
let shards_set = 0..shards_count;
-
let connections = create_shard_connections(shards_set.clone());
-
for shard_id in shards_set {
let id = shard_id as u16;
let connections = connections.clone();
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 06366ebf..36fe0e54 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -33,7 +33,7 @@ use crate::{
versioning::SemanticVersion,
};
-use super::{IggyShard, connector::ShardConnector, frame::ShardFrame};
+use super::{IggyShard, transmission::connector::ShardConnector, transmission::frame::ShardFrame};
#[derive(Default)]
pub struct IggyShardBuilder {
@@ -76,6 +76,7 @@ impl IggyShardBuilder {
.next()
.expect("Failed to find connection with the specified ID");
let shards = connections.into_iter().map(Shard::new).collect();
+
//TODO: This can be discrete step in the builder bootstrapped from main function.
let version = SemanticVersion::current().expect("Invalid version");
@@ -102,7 +103,10 @@ impl IggyShardBuilder {
//TODO: This can be discrete step in the builder bootstrapped from main function.
let partition_persister = Self::resolve_persister(config.system.partition.enforce_fsync);
- let storage = Rc::new(SystemStorage::new(config.system.clone(), partition_persister));
+ let storage = Rc::new(SystemStorage::new(
+ config.system.clone(),
+ partition_persister,
+ ));
IggyShard {
id: id,
diff --git a/core/server/src/shard/frame.rs b/core/server/src/shard/frame.rs
deleted file mode 100644
index a2c1220f..00000000
--- a/core/server/src/shard/frame.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-use bytes::Bytes;
-use flume::Sender;
-use iggy_common::IggyError;
-
-#[derive(Debug, Clone)]
-pub enum ShardMessage {
- //TODO: Fixme
- //Command(ServerCommand),
- Event(ShardEvent),
-}
-
-#[derive(Debug, Clone)]
-pub enum ShardEvent {}
-
-#[derive(Debug)]
-pub enum ShardResponse {
- BinaryResponse(Bytes),
- ErrorResponse(IggyError),
-}
-
-#[derive(Debug, Clone)]
-pub struct ShardFrame {
- pub client_id: u32,
- pub message: ShardMessage,
- pub response_sender: Option<Sender<ShardResponse>>,
-}
-
-impl ShardFrame {
- pub fn new(
- client_id: u32,
- message: ShardMessage,
- response_sender: Option<Sender<ShardResponse>>,
- ) -> Self {
- Self {
- client_id,
- message,
- response_sender,
- }
- }
-}
-
-#[macro_export]
-macro_rules! handle_response {
- ($sender:expr, $response:expr) => {
- match $response {
- ShardResponse::BinaryResponse(payload) => $sender.send_ok_response(&payload).await?,
- ShardResponse::ErrorResponse(err) => $sender.send_error_response(err).await?,
- }
- };
-}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index b98f6381..6a6d35b8 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -17,25 +17,27 @@
*/
pub mod builder;
-pub mod connector;
-pub mod frame;
pub mod namespace;
+pub mod transmission;
use ahash::HashMap;
use builder::IggyShardBuilder;
-use connector::{Receiver, ShardConnector, StopReceiver, StopSender};
-use frame::ShardFrame;
use iggy_common::IggyError;
use namespace::IggyNamespace;
-use tracing::info;
use std::{
cell::{Cell, RefCell},
rc::Rc,
- sync::Arc, time::Instant,
+ sync::Arc,
+ time::Instant,
};
+use tracing::info;
+use transmission::connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use crate::{
- bootstrap::create_root_user, configs::server::ServerConfig, state::{file::FileState, StateKind},
+ bootstrap::create_root_user,
+ configs::server::ServerConfig,
+ shard::transmission::frame::ShardFrame,
+ state::{StateKind, file::FileState},
streaming::storage::SystemStorage,
};
pub(crate) struct Shard {
@@ -68,7 +70,6 @@ pub struct IggyShard {
// TODO: Refactor.
pub(crate) storage: Rc<SystemStorage>,
- // TODO - get rid of this dynamic dispatch.
pub(crate) state: Rc<StateKind>,
//pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
config: ServerConfig,
diff --git a/core/server/src/shard/connector.rs b/core/server/src/shard/transmission/connector.rs
similarity index 74%
rename from core/server/src/shard/connector.rs
rename to core/server/src/shard/transmission/connector.rs
index 95aedc73..01ca4147 100644
--- a/core/server/src/shard/connector.rs
+++ b/core/server/src/shard/transmission/connector.rs
@@ -22,10 +22,9 @@ use std::{
task::Poll,
};
-pub type StopSender = flume::Sender<()>;
-pub type StopReceiver = flume::Receiver<()>;
+pub type StopSender = async_channel::Sender<()>;
+pub type StopReceiver = async_channel::Receiver<()>;
-#[derive(Clone)]
pub struct ShardConnector<T> {
pub id: u16,
pub sender: Sender<T>,
@@ -34,12 +33,25 @@ pub struct ShardConnector<T> {
pub stop_sender: StopSender,
}
-// TODO(numinex) - replace flume with async_channel
-impl<T: Clone> ShardConnector<T> {
+impl<T> Clone for ShardConnector<T> {
+ fn clone(&self) -> Self {
+ Self {
+ id: self.id,
+ sender: self.sender.clone(),
+ receiver: self.receiver.clone(),
+ stop_receiver: self.stop_receiver.clone(),
+ stop_sender: self.stop_sender.clone(),
+ }
+ }
+}
+
+// TODO(numinex) - replace async_channel with some other form of one shot async channel.
+// !!!!!IMPORTANT!!!! the one shot channel Sender/Receiver has to be Cloneable!!!!!
+impl<T> ShardConnector<T> {
pub fn new(id: u16, max_concurrent_thread_count: usize) -> Self {
let channel = Arc::new(ShardedChannel::new(max_concurrent_thread_count));
let (sender, receiver) = channel.unbounded();
- let (stop_sender, stop_receiver) = flume::bounded(1);
+ let (stop_sender, stop_receiver) = async_channel::bounded(1);
Self {
id,
receiver,
@@ -55,17 +67,32 @@ impl<T: Clone> ShardConnector<T> {
}
// TODO: I think those Arcs can be replaced with 'static lifetimes...
-// Those shards will live for the entire duration of the application.
-#[derive(Clone)]
+// Those connectors will live for the duration of the shard itself
+// and the shard lives for the duration of the entire application.
pub struct Receiver<T> {
channel: Arc<ShardedChannel<T>>,
}
-#[derive(Clone)]
+impl<T> Clone for Receiver<T> {
+ fn clone(&self) -> Self {
+ Self {
+ channel: self.channel.clone(),
+ }
+ }
+}
+
pub struct Sender<T> {
channel: Arc<ShardedChannel<T>>,
}
+impl<T> Clone for Sender<T> {
+ fn clone(&self) -> Self {
+ Self {
+ channel: self.channel.clone(),
+ }
+ }
+}
+
impl<T> Sender<T> {
pub fn send(&self, data: T) {
self.channel
@@ -94,13 +121,13 @@ impl<T> ShardedChannel<T> {
}
}
-pub trait ShardedChannelsSplit<T: Clone> {
+pub trait ShardedChannelsSplit<T> {
fn unbounded(&self) -> (Sender<T>, Receiver<T>);
fn sender(&self) -> Sender<T>;
}
-impl<T: Clone> ShardedChannelsSplit<T> for Arc<ShardedChannel<T>> {
+impl<T> ShardedChannelsSplit<T> for Arc<ShardedChannel<T>> {
fn unbounded(&self) -> (Sender<T>, Receiver<T>) {
let tx = self.sender();
let rx = Receiver {
diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs
new file mode 100644
index 00000000..ac60863d
--- /dev/null
+++ b/core/server/src/shard/transmission/frame.rs
@@ -0,0 +1,42 @@
+use async_channel::Sender;
+use bytes::Bytes;
+use iggy_common::IggyError;
+
+use crate::shard::transmission::message::ShardMessage;
+
+#[derive(Debug)]
+pub struct ShardFrame {
+ pub client_id: u32,
+ pub message: ShardMessage,
+ pub response_sender: Option<Sender<ShardResponse>>,
+}
+
+impl ShardFrame {
+ pub fn new(
+ client_id: u32,
+ message: ShardMessage,
+ response_sender: Option<Sender<ShardResponse>>,
+ ) -> Self {
+ Self {
+ client_id,
+ message,
+ response_sender,
+ }
+ }
+}
+
+#[derive(Debug)]
+pub enum ShardResponse {
+ BinaryResponse(Bytes),
+ ErrorResponse(IggyError),
+}
+
+#[macro_export]
+macro_rules! handle_response {
+ ($sender:expr, $response:expr) => {
+ match $response {
+ ShardResponse::BinaryResponse(payload) => $sender.send_ok_response(&payload).await?,
+ ShardResponse::ErrorResponse(err) => $sender.send_error_response(err).await?,
+ }
+ };
+}
diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs
new file mode 100644
index 00000000..e9f1dd4b
--- /dev/null
+++ b/core/server/src/shard/transmission/message.rs
@@ -0,0 +1,22 @@
+use crate::binary::command::ServerCommand;
+
+#[derive(Debug)]
+pub enum ShardMessage {
+ Command(ServerCommand),
+ Event(ShardEvent),
+}
+
+#[derive(Debug)]
+pub enum ShardEvent {}
+
+impl From<ServerCommand> for ShardMessage {
+ fn from(command: ServerCommand) -> Self {
+ ShardMessage::Command(command)
+ }
+}
+
+impl From<ShardEvent> for ShardMessage {
+ fn from(event: ShardEvent) -> Self {
+ ShardMessage::Event(event)
+ }
+}
diff --git a/core/server/src/shard/transmission/mod.rs b/core/server/src/shard/transmission/mod.rs
new file mode 100644
index 00000000..b871152a
--- /dev/null
+++ b/core/server/src/shard/transmission/mod.rs
@@ -0,0 +1,3 @@
+pub mod connector;
+pub mod frame;
+pub mod message;
(iggy) 08/12: temp
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 827352188f23737b54bafbcf1c25a8385ae4f424
Author: numinex <g....@gmail.com>
AuthorDate: Tue Jun 24 11:52:06 2025 +0200
temp
---
Cargo.lock | 83 ++++++++++-
core/server/Cargo.toml | 3 +-
core/server/src/binary/sender.rs | 16 +--
core/server/src/configs/server.rs | 3 +-
core/server/src/main.rs | 8 +-
core/server/src/quic/listener.rs | 3 +
core/server/src/shard/builder.rs | 16 ++-
core/server/src/shard/gate.rs | 2 +-
core/server/src/shard/mod.rs | 158 ++++++++++++++-------
core/server/src/shard/namespace.rs | 17 ++-
.../{streaming/systems => shard/system}/clients.rs | 2 +-
.../systems => shard/system}/consumer_groups.rs | 2 +-
.../systems => shard/system}/consumer_offsets.rs | 2 +-
.../{streaming/systems => shard/system}/info.rs | 0
.../systems => shard/system}/messages.rs | 5 +-
.../src/{streaming/systems => shard/system}/mod.rs | 2 +-
.../systems => shard/system}/partitions.rs | 2 +-
.../system}/personal_access_tokens.rs | 2 +-
.../systems => shard/system}/segments.rs | 2 +-
.../systems => shard/system}/snapshot/mod.rs | 0
.../systems => shard/system}/snapshot/procdump.rs | 0
.../{streaming/systems => shard/system}/stats.rs | 2 +-
.../{streaming/systems => shard/system}/storage.rs | 2 -
.../{streaming/systems => shard/system}/streams.rs | 142 +-----------------
.../{streaming/systems => shard/system}/system.rs | 0
.../{streaming/systems => shard/system}/topics.rs | 0
.../{streaming/systems => shard/system}/users.rs | 106 +-------------
core/server/src/streaming/mod.rs | 1 -
core/server/src/streaming/storage.rs | 5 +-
core/server/src/streaming/streams/stream.rs | 9 +-
core/server/src/tcp/sender.rs | 16 +--
core/server/src/tcp/tcp_listener.rs | 58 +++-----
core/server/src/tcp/tcp_sender.rs | 3 +-
core/server/src/tcp/tcp_server.rs | 21 ++-
core/server/src/tcp/tcp_socket.rs | 1 -
core/server/src/tcp/tcp_tls_sender.rs | 5 +-
36 files changed, 295 insertions(+), 404 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 826f6b36..944bbfc7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -653,6 +653,7 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
+ "syn 2.0.104",
]
[[package]]
@@ -1678,6 +1679,16 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "core-foundation"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
[[package]]
name = "core-foundation"
version = "0.10.1"
@@ -3311,6 +3322,12 @@ dependencies = [
"byteorder",
]
+[[package]]
+name = "hash32"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e4e11d13d42fc8d55326b323c63978c75721fbbb695a6e6686765bcb8b33917"
+
[[package]]
name = "hashbrown"
version = "0.12.3"
@@ -3366,7 +3383,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f"
dependencies = [
"atomic-polyfill",
- "hash32",
+ "hash32 0.2.1",
"rustc_version",
"serde",
"spin",
@@ -4801,6 +4818,15 @@ dependencies = [
"windows-sys 0.48.0",
]
+[[package]]
+name = "monoio-io-wrapper"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bcfaa76e5daf87cc4d31b4d1b6bc93c12db59c19df50b9200afdbde42077655"
+dependencies = [
+ "monoio",
+]
+
[[package]]
name = "monoio-macros"
version = "0.1.0"
@@ -4810,6 +4836,20 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
+ "syn 2.0.104",
+]
+
+[[package]]
+name = "monoio-native-tls"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9022f5aaa19f9688f97bfcfa0c4a4318d424851995badb356674ca742652cdb"
+dependencies = [
+ "bytes",
+ "monoio",
+ "monoio-io-wrapper",
+ "native-tls",
+ "thiserror 1.0.69",
]
[[package]]
@@ -4821,6 +4861,23 @@ dependencies = [
"getrandom 0.2.16",
]
+[[package]]
+name = "native-tls"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework 2.11.1",
+ "security-framework-sys",
+ "tempfile",
+]
+
[[package]]
name = "never-say-never"
version = "6.6.666"
@@ -6454,7 +6511,7 @@ dependencies = [
"openssl-probe",
"rustls-pki-types",
"schannel",
- "security-framework",
+ "security-framework 3.2.0",
]
[[package]]
@@ -6482,7 +6539,7 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1"
dependencies = [
- "core-foundation",
+ "core-foundation 0.10.1",
"core-foundation-sys",
"jni",
"log",
@@ -6491,7 +6548,7 @@ dependencies = [
"rustls-native-certs",
"rustls-platform-verifier-android",
"rustls-webpki",
- "security-framework",
+ "security-framework 3.2.0",
"security-framework-sys",
"webpki-root-certs 0.26.11",
"windows-sys 0.59.0",
@@ -6639,6 +6696,19 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "security-framework"
+version = "2.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
+dependencies = [
+ "bitflags 2.9.1",
+ "core-foundation 0.9.4",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
[[package]]
name = "security-framework"
version = "3.2.0"
@@ -6646,7 +6716,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
dependencies = [
"bitflags 2.9.1",
- "core-foundation",
+ "core-foundation 0.10.1",
"core-foundation-sys",
"libc",
"security-framework-sys",
@@ -6854,15 +6924,16 @@ dependencies = [
"figment",
"flume",
"futures",
+ "hash32 1.0.0",
"human-repr",
"iggy_common",
- "io-uring",
"jsonwebtoken",
"lending-iterator",
"mimalloc",
"mockall",
"moka",
"monoio",
+ "monoio-native-tls",
"nix 0.30.1",
"once_cell",
"opentelemetry",
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 02d39830..d2cfc24a 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -60,12 +60,13 @@ async-channel = { workspace = true }
futures = { workspace = true }
human-repr = { workspace = true }
iggy_common = { workspace = true }
-io-uring = "0.6"
jsonwebtoken = "9.3.1"
lending-iterator = "0.1.7"
+hash32 = "1.0.0"
mimalloc = { workspace = true, optional = true }
moka = { version = "0.12.10", features = ["future"] }
monoio = { version = "0.2.4", features = ["mkdirat", "unlinkat"] }
+monoio-native-tls = "0.4.0"
nix = { version = "0.30", features = ["fs"] }
once_cell = "1.21.3"
opentelemetry = { version = "0.30.0", features = ["trace", "logs"] }
diff --git a/core/server/src/binary/sender.rs b/core/server/src/binary/sender.rs
index 7273543c..2ccf349b 100644
--- a/core/server/src/binary/sender.rs
+++ b/core/server/src/binary/sender.rs
@@ -24,8 +24,8 @@ use crate::tcp::tcp_tls_sender::TcpTlsSender;
use crate::{quic::quic_sender::QuicSender, server_error::ServerError};
use iggy_common::IggyError;
use quinn::{RecvStream, SendStream};
-use tokio::net::TcpStream;
-use tokio_rustls::server::TlsStream;
+use monoio::net::TcpStream;
+use tokio_native_tls::TlsStream;
macro_rules! forward_async_methods {
(
@@ -48,22 +48,22 @@ macro_rules! forward_async_methods {
}
pub trait Sender {
- fn read(&mut self, buffer: &mut [u8]) -> impl Future<Output = Result<usize, IggyError>> + Send;
- fn send_empty_ok_response(&mut self) -> impl Future<Output = Result<(), IggyError>> + Send;
+ fn read(&mut self, buffer: &mut [u8]) -> impl Future<Output = Result<usize, IggyError>>;
+ fn send_empty_ok_response(&mut self) -> impl Future<Output = Result<(), IggyError>>;
fn send_ok_response(
&mut self,
payload: &[u8],
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
fn send_ok_response_vectored(
&mut self,
length: &[u8],
slices: Vec<IoSlice<'_>>,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
fn send_error_response(
&mut self,
error: IggyError,
- ) -> impl Future<Output = Result<(), IggyError>> + Send;
- fn shutdown(&mut self) -> impl Future<Output = Result<(), ServerError>> + Send;
+ ) -> impl Future<Output = Result<(), IggyError>>;
+ fn shutdown(&mut self) -> impl Future<Output = Result<(), ServerError>>;
}
#[allow(clippy::large_enum_variant)]
diff --git a/core/server/src/configs/server.rs b/core/server/src/configs/server.rs
index 252df9a7..059fd1c4 100644
--- a/core/server/src/configs/server.rs
+++ b/core/server/src/configs/server.rs
@@ -31,6 +31,7 @@ use iggy_common::Validatable;
use serde::{Deserialize, Serialize};
use serde_with::DisplayFromStr;
use serde_with::serde_as;
+use std::rc::Rc;
use std::str::FromStr;
use std::sync::Arc;
@@ -40,7 +41,7 @@ pub struct ServerConfig {
pub message_saver: MessageSaverConfig,
pub personal_access_token: PersonalAccessTokenConfig,
pub heartbeat: HeartbeatConfig,
- pub system: Arc<SystemConfig>,
+ pub system: Rc<SystemConfig>,
pub quic: QuicConfig,
pub tcp: TcpConfig,
pub http: HttpConfig,
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index 3a6c1686..78b3a1e9 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -235,7 +235,7 @@ fn main() -> Result<(), ServerError> {
.await;
let builder = IggyShard::builder();
- let mut shard = builder
+ let shard: Rc<IggyShard> = builder
.id(id)
.connections(connections)
.config(config)
@@ -243,13 +243,13 @@ fn main() -> Result<(), ServerError> {
.version(version)
.state(state)
.build()
- .await;
+ .into();
//TODO: If one of the shards fails to initialize, we should crash the whole program;
- shard.init().await.expect("Failed to initialize shard-{id}: {e}");
- info!("Initiated shard with ID: {id}");
+ shard.run().await.expect("Failed to run shard");
//TODO: If one of the shards fails to initialize, we should crash the whole program;
shard.assert_init();
+ let shard = Rc::new(shard);
})
})
.expect(format!("Failed to spawn thread for shard-{id}").as_str())
diff --git a/core/server/src/quic/listener.rs b/core/server/src/quic/listener.rs
index b5eb859d..e9b329de 100644
--- a/core/server/src/quic/listener.rs
+++ b/core/server/src/quic/listener.rs
@@ -34,6 +34,8 @@ pub fn start(endpoint: Endpoint, system: SharedSystem) {
for _ in 0..LISTENERS_COUNT {
let endpoint = endpoint.clone();
let system = system.clone();
+ //TODO: Fixme
+ /*
tokio::spawn(async move {
while let Some(incoming_connection) = endpoint.accept().await {
info!(
@@ -57,6 +59,7 @@ pub fn start(endpoint: Endpoint, system: SharedSystem) {
});
}
});
+ */
}
}
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 0fd5250f..0428a483 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -26,8 +26,8 @@ use crate::{
configs::server::ServerConfig,
map_toggle_str,
shard::Shard,
- state::{StateKind, file::FileState},
- streaming::storage::SystemStorage,
+ state::{file::FileState, StateKind},
+ streaming::{diagnostics::metrics::Metrics, storage::SystemStorage},
versioning::SemanticVersion,
};
@@ -75,11 +75,12 @@ impl IggyShardBuilder {
}
// TODO: Too much happens in there, some of those bootstrapping logic should be moved outside.
- pub async fn build(self) -> IggyShard {
+ pub fn build(self) -> IggyShard {
let id = self.id.unwrap();
let config = self.config.unwrap();
let connections = self.connections.unwrap();
let state = self.state.unwrap();
+ let encryptor = self.encryptor;
let version = self.version.unwrap();
let (stop_sender, stop_receiver, frame_receiver) = connections
.iter()
@@ -106,12 +107,21 @@ impl IggyShardBuilder {
shards: shards,
shards_table: Default::default(),
storage: storage,
+ encryptor: encryptor,
state: state,
config: config,
version: version,
stop_receiver: stop_receiver,
stop_sender: stop_sender,
frame_receiver: Cell::new(Some(frame_receiver)),
+ metrics: Metrics::init(),
+
+ users: Default::default(),
+ permissioner: Default::default(),
+ streams: Default::default(),
+ streams_ids: Default::default(),
+ client_manager: Default::default(),
+ active_sessions: Default::default(),
}
}
}
diff --git a/core/server/src/shard/gate.rs b/core/server/src/shard/gate.rs
index a26950a4..2ac20445 100644
--- a/core/server/src/shard/gate.rs
+++ b/core/server/src/shard/gate.rs
@@ -1,4 +1,4 @@
-use std::sync::{Condvar, Mutex};
+use std::sync::Mutex;
#[derive(Default)]
pub struct Gate<T> {
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 23ef1e15..3e7e10f1 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -17,39 +17,37 @@
*/
pub mod builder;
+pub mod system;
pub mod gate;
pub mod namespace;
pub mod transmission;
-use ahash::HashMap;
+use ahash::{AHashMap, AHashSet, HashMap};
use builder::IggyShardBuilder;
use error_set::ErrContext;
-use iggy_common::IggyError;
+use futures::future::try_join_all;
+use iggy_common::{EncryptorKind, IggyError, UserId};
use namespace::IggyNamespace;
use std::{
- cell::{Cell, RefCell},
- rc::Rc,
- str::FromStr,
- sync::Arc,
- time::Instant,
+ cell::{Cell, RefCell}, pin::Pin, rc::Rc, str::FromStr, sync::{atomic::{AtomicU32, Ordering}, Arc, RwLock}, time::Instant
};
-use tracing::info;
+use tracing::{error, info, instrument, trace, warn};
use transmission::connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use crate::{
- bootstrap::create_root_user,
configs::server::ServerConfig,
- shard::transmission::frame::ShardFrame,
+ shard::{system::info::SystemInfo, transmission::frame::ShardFrame},
state::{
- StateKind,
- file::FileState,
- system::{SystemState, UserState},
+ file::FileState, system::{StreamState, SystemState, UserState}, StateKind
},
- streaming::{storage::SystemStorage, systems::info::SystemInfo},
+ streaming::{clients::client_manager::ClientManager, diagnostics::metrics::Metrics, personal_access_tokens::personal_access_token::PersonalAccessToken, session::Session, storage::SystemStorage, streams::stream::Stream, users::{permissioner::Permissioner, user::User}},
versioning::SemanticVersion,
};
pub const COMPONENT: &str = "SHARD";
+static USER_ID: AtomicU32 = AtomicU32::new(1);
+
+type Task = Pin<Box<dyn Future<Output = Result<(), IggyError>>>>;
pub(crate) struct Shard {
id: u16,
@@ -75,19 +73,21 @@ pub struct IggyShard {
shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
version: SemanticVersion,
- //pub(crate) permissioner: RefCell<Permissioner>,
- //pub(crate) streams: RwLock<HashMap<u32, Stream>>,
- //pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
- //pub(crate) users: RefCell<HashMap<UserId, User>>,
+ pub(crate) streams: RefCell<HashMap<u32, Stream>>,
+ pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
// TODO: Refactor.
pub(crate) storage: Rc<SystemStorage>,
pub(crate) state: StateKind,
- //pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
- config: ServerConfig,
- //pub(crate) client_manager: RefCell<ClientManager>,
- //pub(crate) active_sessions: RefCell<Vec<Session>>,
- //pub(crate) metrics: Metrics,
+ pub(crate) encryptor: Option<EncryptorKind>,
+ pub(crate) config: ServerConfig,
+ //TODO: This could be shared.
+ pub(crate) client_manager: RefCell<ClientManager>,
+ pub(crate) active_sessions: RefCell<Vec<Session>>,
+ pub(crate) permissioner: RefCell<Permissioner>,
+ pub(crate) users: RefCell<HashMap<UserId, User>>,
+
+ pub(crate) metrics: Metrics,
pub frame_receiver: Cell<Option<Receiver<ShardFrame>>>,
stop_receiver: StopReceiver,
stop_sender: StopSender,
@@ -98,16 +98,12 @@ impl IggyShard {
Default::default()
}
- pub async fn init(&mut self) -> Result<(), IggyError> {
+ pub async fn init(&self) -> Result<(), IggyError> {
let now = Instant::now();
- //TODO: Fix this either by moving it to main function, or by using `run_once` barrier.
- //let state_entries = self.state.init().await?;
- //let system_state = SystemState::init(state_entries).await?;
- //let user = create_root_user();
+
self.load_version().await?;
let SystemState { users, streams } = self.load_state().await?;
self.load_users(users.into_values().collect()).await;
- // Add default root user.
self.load_streams(streams.into_values().collect()).await;
//TODO: Fix the archiver.
@@ -123,6 +119,27 @@ impl IggyShard {
Ok(())
}
+ pub async fn run(self: &Rc<Self>) -> Result<(), IggyError> {
+ // Workaround to ensure that the statistics are initialized before the server
+ // loads streams and starts accepting connections. This is necessary to
+ // have the correct statistics when the server starts.
+ //self.get_stats().await?;
+ self.init().await?;
+ self.assert_init();
+ info!("Initiated shard with ID: {}", self.id);
+ // Create all tasks (tcp listener, http listener, command processor, in the future also the background jobs).
+ /*
+ let mut tasks: Vec<Task> = vec![Box::pin(spawn_shard_message_task(shard.clone()))];
+ if self.config.tcp.enabled {
+ tasks.push(Box::pin(spawn_tcp_server(self.clone())));
+ }
+ let result = try_join_all(tasks).await;
+ result?;
+ */
+
+ Ok(())
+ }
+
async fn load_version(&self) -> Result<(), IggyError> {
async fn update_system_info(
storage: &Rc<SystemStorage>,
@@ -181,10 +198,8 @@ impl IggyShard {
Ok(system_state)
}
- async fn load_users(&mut self, users: Vec<UserState>) -> Result<(), IggyError> {
+ async fn load_users(&self, users: Vec<UserState>) -> Result<(), IggyError> {
info!("Loading users...");
- /*
-
for user_state in users.into_iter() {
let mut user = User::with_password(
user_state.id,
@@ -210,32 +225,34 @@ impl IggyShard {
)
})
.collect();
- self.users.insert(user_state.id, user);
+ self.users.borrow_mut().insert(user_state.id, user);
}
- let users_count = self.users.len();
- let current_user_id = self.users.keys().max().unwrap_or(&1);
+ let users = self.users.borrow();
+ let users_count = users.len();
+ let current_user_id = users.keys().max().unwrap_or(&1);
USER_ID.store(current_user_id + 1, Ordering::SeqCst);
self.permissioner
- .init(&self.users.values().collect::<Vec<&User>>());
+ .borrow_mut()
+ .init(&users.values().collect::<Vec<_>>());
self.metrics.increment_users(users_count as u32);
info!("Initialized {users_count} user(s).");
- */
Ok(())
}
- async fn load_streams(&mut self, streams: Vec<StreamState>) -> Result<(), IggyError> {
- todo!();
+ async fn load_streams(&self, streams: Vec<StreamState>) -> Result<(), IggyError> {
info!("Loading streams from disk...");
let mut unloaded_streams = Vec::new();
- let mut dir_entries = read_dir(&self.config.get_streams_path())
- .await
+ // Does mononio has api for that ?
+ let mut dir_entries = std::fs::read_dir(&self.config.system.get_streams_path())
.map_err(|error| {
error!("Cannot read streams directory: {error}");
IggyError::CannotReadStreams
})?;
- while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) {
+ //TODO: User the dir walk impl from main function, once implemented.
+ while let Some(dir_entry) = dir_entries.next() {
+ let dir_entry = dir_entry.unwrap();
let name = dir_entry.file_name().into_string().unwrap();
let stream_id = name.parse::<u32>().map_err(|_| {
error!("Invalid stream ID file with name: '{name}'.");
@@ -246,7 +263,7 @@ impl IggyShard {
error!(
"Stream with ID: '{stream_id}' was not found in state, but exists on disk and will be removed."
);
- if let Err(error) = fs::remove_dir_all(&dir_entry.path()).await {
+ if let Err(error) = std::fs::remove_dir_all(&dir_entry.path()) {
error!("Cannot remove stream directory: {error}");
} else {
warn!("Stream with ID: '{stream_id}' was removed.");
@@ -258,7 +275,7 @@ impl IggyShard {
let mut stream = Stream::empty(
stream_id,
&stream_state.name,
- self.config.clone(),
+ self.config.system.clone(),
self.storage.clone(),
);
stream.created_at = stream_state.created_at;
@@ -281,7 +298,7 @@ impl IggyShard {
info!("All streams found on disk were found in state.");
} else {
warn!("Streams with IDs: '{missing_ids:?}' were not found on disk.");
- if self.config.recovery.recreate_missing_state {
+ if self.config.system.recovery.recreate_missing_state {
info!(
"Recreating missing state in recovery config is enabled, missing streams will be created."
);
@@ -291,7 +308,7 @@ impl IggyShard {
let stream = Stream::create(
stream_id,
&stream_state.name,
- self.config.clone(),
+ self.config.system.clone(),
self.storage.clone(),
);
stream.persist().await?;
@@ -327,12 +344,12 @@ impl IggyShard {
try_join_all(load_stream_tasks).await?;
for stream in loaded_streams.take() {
- if self.streams.contains_key(&stream.stream_id) {
+ if self.streams.borrow().contains_key(&stream.stream_id) {
error!("Stream with ID: '{}' already exists.", &stream.stream_id);
continue;
}
- if self.streams_ids.contains_key(&stream.name) {
+ if self.streams_ids.borrow().contains_key(&stream.name) {
error!("Stream with name: '{}' already exists.", &stream.name);
continue;
}
@@ -345,13 +362,52 @@ impl IggyShard {
self.metrics.increment_messages(stream.get_messages_count());
self.streams_ids
+ .borrow_mut()
.insert(stream.name.clone(), stream.stream_id);
- self.streams.insert(stream.stream_id, stream);
+ self.streams.borrow_mut().insert(stream.stream_id, stream);
}
- info!("Loaded {} stream(s) from disk.", self.streams.len());
+ info!("Loaded {} stream(s) from disk.", self.streams.borrow().len());
Ok(())
}
- pub fn assert_init(&self) {}
+ pub fn assert_init(&self) -> Result<(), IggyError> { Ok(())}
+
+ #[instrument(skip_all, name = "trace_shutdown")]
+ pub async fn shutdown(&mut self) -> Result<(), IggyError> {
+ self.persist_messages().await?;
+ Ok(())
+ }
+
+ #[instrument(skip_all, name = "trace_persist_messages")]
+ pub async fn persist_messages(&self) -> Result<usize, IggyError> {
+ trace!("Saving buffered messages on disk...");
+ let mut saved_messages_number = 0;
+ //TODO: Fixme
+ /*
+ for stream in self.streams.values() {
+ saved_messages_number += stream.persist_messages().await?;
+ }
+ */
+
+ Ok(saved_messages_number)
+ }
+
+ pub fn get_available_shards_count(&self) -> u32 {
+ self.shards.len() as u32
+ }
+
+ pub fn ensure_authenticated(&self, client_id: u32) -> Result<u32, IggyError> {
+ let active_sessions = self.active_sessions.borrow();
+ let session = active_sessions
+ .iter()
+ .find(|s| s.client_id == client_id)
+ .ok_or_else(|| IggyError::Unauthenticated)?;
+ if session.is_authenticated() {
+ Ok(session.get_user_id())
+ } else {
+ error!("{COMPONENT} - unauthenticated access attempt, session: {session}");
+ Err(IggyError::Unauthenticated)
+ }
+ }
}
diff --git a/core/server/src/shard/namespace.rs b/core/server/src/shard/namespace.rs
index d9aedfb7..d9d68ec9 100644
--- a/core/server/src/shard/namespace.rs
+++ b/core/server/src/shard/namespace.rs
@@ -16,15 +16,20 @@
* under the License.
*/
+use std::hash::Hasher as _;
+use iggy_common::Identifier;
+use hash32::{Hasher, Murmur3Hasher};
+
+//TODO: Will probably want to move it to separate crate so we can share it with sdk.
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct IggyNamespace {
- pub(crate) stream_id: u32,
- pub(crate) topic_id: u32,
+ pub(crate) stream_id: Identifier,
+ pub(crate) topic_id: Identifier,
pub(crate) partition_id: u32,
}
impl IggyNamespace {
- pub fn new(stream_id: u32, topic_id: u32, partition_id: u32) -> Self {
+ pub fn new(stream_id: Identifier, topic_id: Identifier, partition_id: u32) -> Self {
Self {
stream_id,
topic_id,
@@ -33,6 +38,10 @@ impl IggyNamespace {
}
pub fn generate_hash(&self) -> u32 {
- todo!();
+ let mut hasher = Murmur3Hasher::default();
+ hasher.write(&self.stream_id.value);
+ hasher.write(&self.topic_id.value);
+ hasher.write_u32(self.partition_id);
+ hasher.finish32()
}
}
diff --git a/core/server/src/streaming/systems/clients.rs b/core/server/src/shard/system/clients.rs
similarity index 99%
rename from core/server/src/streaming/systems/clients.rs
rename to core/server/src/shard/system/clients.rs
index 050a0b58..2cb0fbe8 100644
--- a/core/server/src/streaming/systems/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -29,7 +29,7 @@ use std::net::SocketAddr;
use std::sync::Arc;
use tracing::{error, info};
-impl System {
+impl IggyShard {
pub async fn add_client(&self, address: &SocketAddr, transport: Transport) -> Arc<Session> {
let mut client_manager = self.client_manager.write().await;
let session = client_manager.add_client(address, transport);
diff --git a/core/server/src/streaming/systems/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs
similarity index 99%
rename from core/server/src/streaming/systems/consumer_groups.rs
rename to core/server/src/shard/system/consumer_groups.rs
index 19f29ccd..4975a92d 100644
--- a/core/server/src/streaming/systems/consumer_groups.rs
+++ b/core/server/src/shard/system/consumer_groups.rs
@@ -26,7 +26,7 @@ use iggy_common::IggyError;
use iggy_common::locking::IggySharedMutFn;
use tokio::sync::RwLock;
-impl System {
+impl IggyShard {
pub fn get_consumer_group(
&self,
session: &Session,
diff --git a/core/server/src/streaming/systems/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs
similarity index 99%
rename from core/server/src/streaming/systems/consumer_offsets.rs
rename to core/server/src/shard/system/consumer_offsets.rs
index fdac2b0c..e63815ff 100644
--- a/core/server/src/streaming/systems/consumer_offsets.rs
+++ b/core/server/src/shard/system/consumer_offsets.rs
@@ -22,7 +22,7 @@ use crate::streaming::systems::system::System;
use error_set::ErrContext;
use iggy_common::{Consumer, ConsumerOffsetInfo, Identifier, IggyError};
-impl System {
+impl IggyShard {
pub async fn store_consumer_offset(
&self,
session: &Session,
diff --git a/core/server/src/streaming/systems/info.rs b/core/server/src/shard/system/info.rs
similarity index 100%
rename from core/server/src/streaming/systems/info.rs
rename to core/server/src/shard/system/info.rs
diff --git a/core/server/src/streaming/systems/messages.rs b/core/server/src/shard/system/messages.rs
similarity index 99%
rename from core/server/src/streaming/systems/messages.rs
rename to core/server/src/shard/system/messages.rs
index 7bb772aa..54046dba 100644
--- a/core/server/src/streaming/systems/messages.rs
+++ b/core/server/src/shard/system/messages.rs
@@ -17,10 +17,9 @@
*/
use crate::binary::handlers::messages::poll_messages_handler::IggyPollMetadata;
+use crate::shard::IggyShard;
use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet};
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use crate::streaming::utils::PooledBuffer;
use error_set::ErrContext;
use iggy_common::{
@@ -29,7 +28,7 @@ use iggy_common::{
};
use tracing::{error, trace};
-impl System {
+impl IggyShard {
pub async fn poll_messages(
&self,
session: &Session,
diff --git a/core/server/src/streaming/systems/mod.rs b/core/server/src/shard/system/mod.rs
similarity index 95%
rename from core/server/src/streaming/systems/mod.rs
rename to core/server/src/shard/system/mod.rs
index 7039e919..1419abb9 100644
--- a/core/server/src/streaming/systems/mod.rs
+++ b/core/server/src/shard/system/mod.rs
@@ -32,4 +32,4 @@ pub mod system;
pub mod topics;
pub mod users;
-pub const COMPONENT: &str = "STREAMING_SYSTEMS";
+pub const COMPONENT: &str = "SYSTEM";
diff --git a/core/server/src/streaming/systems/partitions.rs b/core/server/src/shard/system/partitions.rs
similarity index 99%
rename from core/server/src/streaming/systems/partitions.rs
rename to core/server/src/shard/system/partitions.rs
index 5bb72875..7712f498 100644
--- a/core/server/src/streaming/systems/partitions.rs
+++ b/core/server/src/shard/system/partitions.rs
@@ -23,7 +23,7 @@ use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
-impl System {
+impl IggyShard {
pub async fn create_partitions(
&mut self,
session: &Session,
diff --git a/core/server/src/streaming/systems/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs
similarity index 99%
rename from core/server/src/streaming/systems/personal_access_tokens.rs
rename to core/server/src/shard/system/personal_access_tokens.rs
index 945c853b..18125bed 100644
--- a/core/server/src/streaming/systems/personal_access_tokens.rs
+++ b/core/server/src/shard/system/personal_access_tokens.rs
@@ -27,7 +27,7 @@ use iggy_common::IggyExpiry;
use iggy_common::IggyTimestamp;
use tracing::{error, info};
-impl System {
+impl IggyShard {
pub async fn get_personal_access_tokens(
&self,
session: &Session,
diff --git a/core/server/src/streaming/systems/segments.rs b/core/server/src/shard/system/segments.rs
similarity index 99%
rename from core/server/src/streaming/systems/segments.rs
rename to core/server/src/shard/system/segments.rs
index a15952c1..bd4043a2 100644
--- a/core/server/src/streaming/systems/segments.rs
+++ b/core/server/src/shard/system/segments.rs
@@ -23,7 +23,7 @@ use iggy_common::Identifier;
use iggy_common::IggyError;
use iggy_common::locking::IggySharedMutFn;
-impl System {
+impl IggyShard {
pub async fn delete_segments(
&mut self,
session: &Session,
diff --git a/core/server/src/streaming/systems/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs
similarity index 100%
rename from core/server/src/streaming/systems/snapshot/mod.rs
rename to core/server/src/shard/system/snapshot/mod.rs
diff --git a/core/server/src/streaming/systems/snapshot/procdump.rs b/core/server/src/shard/system/snapshot/procdump.rs
similarity index 100%
rename from core/server/src/streaming/systems/snapshot/procdump.rs
rename to core/server/src/shard/system/snapshot/procdump.rs
diff --git a/core/server/src/streaming/systems/stats.rs b/core/server/src/shard/system/stats.rs
similarity index 99%
rename from core/server/src/streaming/systems/stats.rs
rename to core/server/src/shard/system/stats.rs
index 80c588e6..2d2d2ea7 100644
--- a/core/server/src/streaming/systems/stats.rs
+++ b/core/server/src/shard/system/stats.rs
@@ -34,7 +34,7 @@ fn sysinfo() -> &'static Mutex<SysinfoSystem> {
})
}
-impl System {
+impl IggyShard {
pub async fn get_stats(&self) -> Result<Stats, IggyError> {
let mut sys = sysinfo().lock().await;
let process_id = std::process::id();
diff --git a/core/server/src/streaming/systems/storage.rs b/core/server/src/shard/system/storage.rs
similarity index 97%
rename from core/server/src/streaming/systems/storage.rs
rename to core/server/src/shard/system/storage.rs
index d4363631..9a493751 100644
--- a/core/server/src/streaming/systems/storage.rs
+++ b/core/server/src/shard/system/storage.rs
@@ -18,8 +18,6 @@
use crate::streaming::persistence::persister::PersisterKind;
use crate::streaming::storage::SystemInfoStorage;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::info::SystemInfo;
use crate::streaming::utils::PooledBuffer;
use crate::streaming::utils::file;
use anyhow::Context;
diff --git a/core/server/src/streaming/systems/streams.rs b/core/server/src/shard/system/streams.rs
similarity index 70%
rename from core/server/src/streaming/systems/streams.rs
rename to core/server/src/shard/system/streams.rs
index 18c6aa83..efcfff2b 100644
--- a/core/server/src/streaming/systems/streams.rs
+++ b/core/server/src/shard/system/streams.rs
@@ -16,156 +16,18 @@
* under the License.
*/
-use crate::state::system::StreamState;
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
use crate::streaming::streams::stream::Stream;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
-use ahash::{AHashMap, AHashSet};
use error_set::ErrContext;
use futures::future::try_join_all;
-use iggy_common::locking::IggySharedMutFn;
-use iggy_common::{IdKind, Identifier, IggyError};
use std::cell::RefCell;
use std::sync::atomic::{AtomicU32, Ordering};
use tokio::fs;
-use tokio::fs::read_dir;
use tracing::{error, info, warn};
-static CURRENT_STREAM_ID: AtomicU32 = AtomicU32::new(1);
-
-impl System {
- pub(crate) async fn load_streams(
- &mut self,
- streams: Vec<StreamState>,
- ) -> Result<(), IggyError> {
- info!("Loading streams from disk...");
- let mut unloaded_streams = Vec::new();
- let mut dir_entries = read_dir(&self.config.get_streams_path())
- .await
- .map_err(|error| {
- error!("Cannot read streams directory: {error}");
- IggyError::CannotReadStreams
- })?;
-
- while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) {
- let name = dir_entry.file_name().into_string().unwrap();
- let stream_id = name.parse::<u32>().map_err(|_| {
- error!("Invalid stream ID file with name: '{name}'.");
- IggyError::InvalidNumberValue
- })?;
- let stream_state = streams.iter().find(|s| s.id == stream_id);
- if stream_state.is_none() {
- error!(
- "Stream with ID: '{stream_id}' was not found in state, but exists on disk and will be removed."
- );
- if let Err(error) = fs::remove_dir_all(&dir_entry.path()).await {
- error!("Cannot remove stream directory: {error}");
- } else {
- warn!("Stream with ID: '{stream_id}' was removed.");
- }
- continue;
- }
-
- let stream_state = stream_state.unwrap();
- let mut stream = Stream::empty(
- stream_id,
- &stream_state.name,
- self.config.clone(),
- self.storage.clone(),
- );
- stream.created_at = stream_state.created_at;
- unloaded_streams.push(stream);
- }
-
- let state_stream_ids = streams
- .iter()
- .map(|stream| stream.id)
- .collect::<AHashSet<u32>>();
- let unloaded_stream_ids = unloaded_streams
- .iter()
- .map(|stream| stream.stream_id)
- .collect::<AHashSet<u32>>();
- let mut missing_ids = state_stream_ids
- .difference(&unloaded_stream_ids)
- .copied()
- .collect::<AHashSet<u32>>();
- if missing_ids.is_empty() {
- info!("All streams found on disk were found in state.");
- } else {
- warn!("Streams with IDs: '{missing_ids:?}' were not found on disk.");
- if self.config.recovery.recreate_missing_state {
- info!(
- "Recreating missing state in recovery config is enabled, missing streams will be created."
- );
- for stream_id in missing_ids.iter() {
- let stream_id = *stream_id;
- let stream_state = streams.iter().find(|s| s.id == stream_id).unwrap();
- let stream = Stream::create(
- stream_id,
- &stream_state.name,
- self.config.clone(),
- self.storage.clone(),
- );
- stream.persist().await?;
- unloaded_streams.push(stream);
- info!(
- "Missing stream with ID: '{stream_id}', name: {} was recreated.",
- stream_state.name
- );
- }
- missing_ids.clear();
- } else {
- warn!(
- "Recreating missing state in recovery config is disabled, missing streams will not be created."
- );
- }
- }
-
- let mut streams_states = streams
- .into_iter()
- .filter(|s| !missing_ids.contains(&s.id))
- .map(|s| (s.id, s))
- .collect::<AHashMap<_, _>>();
- let loaded_streams = RefCell::new(Vec::new());
- let load_stream_tasks = unloaded_streams.into_iter().map(|mut stream| {
- let state = streams_states.remove(&stream.stream_id).unwrap();
-
- async {
- stream.load(state).await?;
- loaded_streams.borrow_mut().push(stream);
- Result::<(), IggyError>::Ok(())
- }
- });
- try_join_all(load_stream_tasks).await?;
-
- for stream in loaded_streams.take() {
- if self.streams.contains_key(&stream.stream_id) {
- error!("Stream with ID: '{}' already exists.", &stream.stream_id);
- continue;
- }
-
- if self.streams_ids.contains_key(&stream.name) {
- error!("Stream with name: '{}' already exists.", &stream.name);
- continue;
- }
-
- self.metrics.increment_streams(1);
- self.metrics.increment_topics(stream.get_topics_count());
- self.metrics
- .increment_partitions(stream.get_partitions_count());
- self.metrics.increment_segments(stream.get_segments_count());
- self.metrics.increment_messages(stream.get_messages_count());
-
- self.streams_ids
- .insert(stream.name.clone(), stream.stream_id);
- self.streams.insert(stream.stream_id, stream);
- }
-
- info!("Loaded {} stream(s) from disk.", self.streams.len());
- Ok(())
- }
+impl IggyShard {
pub fn get_streams(&self) -> Vec<&Stream> {
self.streams.values().collect()
}
diff --git a/core/server/src/streaming/systems/system.rs b/core/server/src/shard/system/system.rs
similarity index 100%
rename from core/server/src/streaming/systems/system.rs
rename to core/server/src/shard/system/system.rs
diff --git a/core/server/src/streaming/systems/topics.rs b/core/server/src/shard/system/topics.rs
similarity index 100%
rename from core/server/src/streaming/systems/topics.rs
rename to core/server/src/shard/system/topics.rs
diff --git a/core/server/src/streaming/systems/users.rs b/core/server/src/shard/system/users.rs
similarity index 79%
rename from core/server/src/streaming/systems/users.rs
rename to core/server/src/shard/system/users.rs
index cc27fcae..f2405f1f 100644
--- a/core/server/src/streaming/systems/users.rs
+++ b/core/server/src/shard/system/users.rs
@@ -16,13 +16,12 @@
* under the License.
*/
+use crate::shard::IggyShard;
use crate::state::command::EntryCommand;
use crate::state::models::CreateUserWithId;
use crate::state::system::UserState;
use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use crate::streaming::users::user::User;
use crate::streaming::utils::crypto;
use crate::{IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV};
@@ -42,108 +41,7 @@ use tracing::{error, info, warn};
static USER_ID: AtomicU32 = AtomicU32::new(1);
const MAX_USERS: usize = u32::MAX as usize;
-impl System {
- pub(crate) async fn load_users(&mut self, users: Vec<UserState>) -> Result<(), IggyError> {
- info!("Loading users...");
- if users.is_empty() {
- info!("No users found, creating the root user...");
- let root = Self::create_root_user();
- let command = CreateUser {
- username: root.username.clone(),
- password: root.password.clone(),
- status: root.status,
- permissions: root.permissions.clone(),
- };
- self.state
- .apply(0, &EntryCommand::CreateUser(CreateUserWithId {
- user_id: root.id,
- command
- }))
- .await
- .with_error_context(|error| {
- format!(
- "{COMPONENT} (error: {error}) - failed to apply create user command, username: {}",
- root.username
- )
- })?;
-
- self.users.insert(root.id, root);
- info!("Created the root user.");
- }
-
- for user_state in users.into_iter() {
- let mut user = User::with_password(
- user_state.id,
- &user_state.username,
- user_state.password_hash,
- user_state.status,
- user_state.permissions,
- );
-
- user.created_at = user_state.created_at;
- user.personal_access_tokens = user_state
- .personal_access_tokens
- .into_values()
- .map(|token| {
- (
- Arc::new(token.token_hash.clone()),
- PersonalAccessToken::raw(
- user_state.id,
- &token.name,
- &token.token_hash,
- token.expiry_at,
- ),
- )
- })
- .collect();
- self.users.insert(user_state.id, user);
- }
-
- let users_count = self.users.len();
- let current_user_id = self.users.keys().max().unwrap_or(&1);
- USER_ID.store(current_user_id + 1, Ordering::SeqCst);
- self.permissioner
- .init(&self.users.values().collect::<Vec<&User>>());
- self.metrics.increment_users(users_count as u32);
- info!("Initialized {users_count} user(s).");
- Ok(())
- }
-
- fn create_root_user() -> User {
- let username = env::var(IGGY_ROOT_USERNAME_ENV);
- let password = env::var(IGGY_ROOT_PASSWORD_ENV);
- if (username.is_ok() && password.is_err()) || (username.is_err() && password.is_ok()) {
- panic!(
- "When providing the custom root user credentials, both username and password must be set."
- );
- }
- if username.is_ok() && password.is_ok() {
- info!("Using the custom root user credentials.");
- } else {
- info!("Using the default root user credentials.");
- }
-
- let username = username.unwrap_or(DEFAULT_ROOT_USERNAME.to_string());
- let password = password.unwrap_or(DEFAULT_ROOT_PASSWORD.to_string());
- if username.is_empty() || password.is_empty() {
- panic!("Root user credentials are not set.");
- }
- if username.len() < MIN_USERNAME_LENGTH {
- panic!("Root username is too short.");
- }
- if username.len() > MAX_USERNAME_LENGTH {
- panic!("Root username is too long.");
- }
- if password.len() < MIN_PASSWORD_LENGTH {
- panic!("Root password is too short.");
- }
- if password.len() > MAX_PASSWORD_LENGTH {
- panic!("Root password is too long.");
- }
-
- User::root(&username, &password)
- }
-
+impl IggyShard {
pub fn find_user(
&self,
session: &Session,
diff --git a/core/server/src/streaming/mod.rs b/core/server/src/streaming/mod.rs
index 5009cb9a..f9d40af8 100644
--- a/core/server/src/streaming/mod.rs
+++ b/core/server/src/streaming/mod.rs
@@ -28,7 +28,6 @@ pub mod segments;
pub mod session;
pub mod storage;
pub mod streams;
-pub mod systems;
pub mod topics;
pub mod users;
pub mod utils;
diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs
index fc0a752d..f423b842 100644
--- a/core/server/src/streaming/storage.rs
+++ b/core/server/src/streaming/storage.rs
@@ -33,6 +33,7 @@ use iggy_common::IggyError;
use mockall::automock;
use std::fmt::Debug;
use std::future::Future;
+use std::rc::Rc;
use std::sync::Arc;
macro_rules! forward_async_methods {
@@ -143,7 +144,7 @@ pub trait PartitionStorage: Send {
#[derive(Debug)]
pub struct SystemStorage {
- pub info: Arc<SystemInfoStorageKind>,
+ pub info: Rc<SystemInfoStorageKind>,
pub stream: Arc<StreamStorageKind>,
pub topic: Arc<TopicStorageKind>,
pub partition: Arc<PartitionStorageKind>,
@@ -151,7 +152,7 @@ pub struct SystemStorage {
}
impl SystemStorage {
- pub fn new(config: Arc<SystemConfig>, persister: Arc<PersisterKind>) -> Self {
+ pub fn new(config: Rc<SystemConfig>, persister: Arc<PersisterKind>) -> Self {
Self {
info: Arc::new(SystemInfoStorageKind::File(FileSystemInfoStorage::new(
config.get_state_info_path(),
diff --git a/core/server/src/streaming/streams/stream.rs b/core/server/src/streaming/streams/stream.rs
index ef528197..339fc08a 100644
--- a/core/server/src/streaming/streams/stream.rs
+++ b/core/server/src/streaming/streams/stream.rs
@@ -23,6 +23,7 @@ use ahash::AHashMap;
use iggy_common::IggyByteSize;
use iggy_common::IggyTimestamp;
use std::fmt::Display;
+use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
@@ -47,8 +48,8 @@ impl Stream {
pub fn empty(
id: u32,
name: &str,
- config: Arc<SystemConfig>,
- storage: Arc<SystemStorage>,
+ config: Rc<SystemConfig>,
+ storage: Rc<SystemStorage>,
) -> Self {
Stream::create(id, name, config, storage)
}
@@ -56,8 +57,8 @@ impl Stream {
pub fn create(
id: u32,
name: &str,
- config: Arc<SystemConfig>,
- storage: Arc<SystemStorage>,
+ config: Rc<SystemConfig>,
+ storage: Rc<SystemStorage>,
) -> Self {
let path = config.get_stream_path(id);
let topics_path = config.get_topics_path(id);
diff --git a/core/server/src/tcp/sender.rs b/core/server/src/tcp/sender.rs
index 2b6b6d17..4dd11725 100644
--- a/core/server/src/tcp/sender.rs
+++ b/core/server/src/tcp/sender.rs
@@ -17,15 +17,15 @@
*/
use iggy_common::IggyError;
+use monoio::io::{AsyncReadRent, AsyncReadRentExt, AsyncWriteRent, AsyncWriteRentExt};
use std::io::IoSlice;
-use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tracing::debug;
const STATUS_OK: &[u8] = &[0; 4];
pub(crate) async fn read<T>(stream: &mut T, buffer: &mut [u8]) -> Result<usize, IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRent + AsyncWriteRent + Unpin,
{
match stream.read_exact(buffer).await {
Ok(0) => Err(IggyError::ConnectionClosed),
@@ -42,14 +42,14 @@ where
pub(crate) async fn send_empty_ok_response<T>(stream: &mut T) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRent + AsyncWriteRent + Unpin,
{
send_ok_response(stream, &[]).await
}
pub(crate) async fn send_ok_response<T>(stream: &mut T, payload: &[u8]) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRent + AsyncWriteRent + Unpin,
{
send_response(stream, STATUS_OK, payload).await
}
@@ -60,7 +60,7 @@ pub(crate) async fn send_ok_response_vectored<T>(
slices: Vec<IoSlice<'_>>,
) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRentExt + AsyncWriteRentExt + Unpin,
{
send_response_vectored(stream, STATUS_OK, length, slices).await
}
@@ -70,7 +70,7 @@ pub(crate) async fn send_error_response<T>(
error: IggyError,
) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRent + AsyncWriteRent + Unpin,
{
send_response(stream, &error.as_code().to_le_bytes(), &[]).await
}
@@ -81,7 +81,7 @@ pub(crate) async fn send_response<T>(
payload: &[u8],
) -> Result<(), IggyError>
where
- T: AsyncRead + AsyncWrite + Unpin,
+ T: AsyncReadRent + AsyncWriteRent + Unpin,
{
debug!(
"Sending response of len: {} with status: {:?}...",
@@ -104,7 +104,7 @@ pub(crate) async fn send_response_vectored<T>(
mut slices: Vec<IoSlice<'_>>,
) -> Result<(), IggyError>
where
- T: AsyncReadExt + AsyncWriteExt + Unpin,
+ T: AsyncReadRentExt + AsyncWriteRentExt + Unpin,
{
debug!(
"Sending vectored response of len: {} with status: {:?}...",
diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs
index 635960d0..2e4f25be 100644
--- a/core/server/src/tcp/tcp_listener.rs
+++ b/core/server/src/tcp/tcp_listener.rs
@@ -17,57 +17,44 @@
*/
use crate::binary::sender::SenderKind;
+use crate::shard::IggyShard;
use crate::streaming::clients::client_manager::Transport;
-use crate::streaming::systems::system::SharedSystem;
use crate::tcp::connection_handler::{handle_connection, handle_error};
use std::net::SocketAddr;
+use std::rc::Rc;
+use monoio::net::TcpListener;
+use rustls::pki_types::Ipv4Addr;
use tokio::net::TcpSocket;
use tokio::sync::oneshot;
use tracing::{error, info};
-pub async fn start(address: &str, socket: TcpSocket, system: SharedSystem) -> SocketAddr {
- let address = address.to_string();
- let (tx, rx) = oneshot::channel();
- tokio::spawn(async move {
- let addr = address.parse();
- if addr.is_err() {
- panic!("Unable to parse address {address:?}");
- }
-
- socket
- .bind(addr.unwrap())
- .expect("Unable to bind socket to address");
-
- let listener = socket.listen(1024).expect("Unable to start TCP server.");
-
- let local_addr = listener
- .local_addr()
- .expect("Failed to get local address for TCP listener");
-
- tx.send(local_addr).unwrap_or_else(|_| {
- panic!("Failed to send the local address {local_addr:?} for TCP listener")
- });
-
+pub async fn start(server_name: &str, shard: Rc<IggyShard>) {
+ let addr: SocketAddr = if shard.config.tcp.ipv6 {
+ shard.config.tcp.address.parse().expect("Unable to parse IPv6 address")
+ } else {
+ shard.config.tcp.address.parse().expect("Unable to parse IPv4 address")
+ };
+ monoio::spawn(async move {
+ let listener = TcpListener::bind(addr).expect(format!("Unable to start {server_name}.").as_ref());
loop {
match listener.accept().await {
Ok((stream, address)) => {
+ let shard = shard.clone();
info!("Accepted new TCP connection: {address}");
- let session = system
- .read()
- .await
- .add_client(&address, Transport::Tcp)
- .await;
+ let session = shard
+ .add_client(&address, Transport::Tcp);
let client_id = session.client_id;
info!("Created new session: {session}");
- let system = system.clone();
let mut sender = SenderKind::get_tcp_sender(stream);
- tokio::spawn(async move {
+ monoio::spawn(async move {
if let Err(error) =
- handle_connection(session, &mut sender, system.clone()).await
+ handle_connection(session, &mut sender, shard.clone()).await
{
handle_error(error);
- system.read().await.delete_client(client_id).await;
+ //TODO: Fixme
+ /*
+ //system.read().await.delete_client(client_id).await;
if let Err(error) = sender.shutdown().await {
error!(
"Failed to shutdown TCP stream for client: {client_id}, address: {address}. {error}"
@@ -77,6 +64,7 @@ pub async fn start(address: &str, socket: TcpSocket, system: SharedSystem) -> So
"Successfully closed TCP stream for client: {client_id}, address: {address}."
);
}
+ */
}
});
}
@@ -84,8 +72,4 @@ pub async fn start(address: &str, socket: TcpSocket, system: SharedSystem) -> So
}
}
});
- match rx.await {
- Ok(addr) => addr,
- Err(_) => panic!("Failed to get the local address for TCP listener."),
- }
}
diff --git a/core/server/src/tcp/tcp_sender.rs b/core/server/src/tcp/tcp_sender.rs
index 837c8f06..a157dead 100644
--- a/core/server/src/tcp/tcp_sender.rs
+++ b/core/server/src/tcp/tcp_sender.rs
@@ -21,7 +21,8 @@ use crate::tcp::COMPONENT;
use crate::{server_error::ServerError, tcp::sender};
use error_set::ErrContext;
use iggy_common::IggyError;
-use tokio::{io::AsyncWriteExt, net::TcpStream};
+use tokio::{io::AsyncWriteExt};
+use monoio::net::TcpStream;
#[derive(Debug)]
pub struct TcpSender {
diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs
index a57b7c3f..82bbe0c1 100644
--- a/core/server/src/tcp/tcp_server.rs
+++ b/core/server/src/tcp/tcp_server.rs
@@ -16,26 +16,25 @@
* under the License.
*/
-use crate::configs::tcp::TcpConfig;
-use crate::streaming::systems::system::SharedSystem;
-use crate::tcp::{tcp_listener, tcp_socket, tcp_tls_listener};
-use std::net::SocketAddr;
+use crate::shard::IggyShard;
+use crate::tcp::{tcp_listener};
+use std::rc::Rc;
+use iggy_common::IggyError;
use tracing::info;
/// Starts the TCP server.
/// Returns the address the server is listening on.
-pub async fn start(config: TcpConfig, system: SharedSystem) -> SocketAddr {
- let server_name = if config.tls.enabled {
+pub async fn start(shard: Rc<IggyShard>) -> Result<(), IggyError> {
+ let server_name = if shard.config.tcp.tls.enabled {
"Iggy TCP TLS"
} else {
"Iggy TCP"
};
info!("Initializing {server_name} server...");
- let socket = tcp_socket::build(config.ipv6, config.socket);
- let addr = match config.tls.enabled {
- true => tcp_tls_listener::start(&config.address, config.tls, socket, system).await,
- false => tcp_listener::start(&config.address, socket, system).await,
+ let addr = match shard.config.tcp.tls.enabled {
+ true => unimplemented!("TLS support is not implemented yet"),
+ false => tcp_listener::start(server_name, shard).await,
};
info!("{server_name} server has started on: {:?}", addr);
- addr
+ Ok(())
}
diff --git a/core/server/src/tcp/tcp_socket.rs b/core/server/src/tcp/tcp_socket.rs
index 91ed105a..69c1d408 100644
--- a/core/server/src/tcp/tcp_socket.rs
+++ b/core/server/src/tcp/tcp_socket.rs
@@ -18,7 +18,6 @@
use std::num::TryFromIntError;
-use tokio::net::TcpSocket;
use crate::configs::tcp::TcpSocketConfig;
diff --git a/core/server/src/tcp/tcp_tls_sender.rs b/core/server/src/tcp/tcp_tls_sender.rs
index 6e3056a5..6ba62315 100644
--- a/core/server/src/tcp/tcp_tls_sender.rs
+++ b/core/server/src/tcp/tcp_tls_sender.rs
@@ -21,9 +21,8 @@ use crate::tcp::COMPONENT;
use crate::{server_error::ServerError, tcp::sender};
use error_set::ErrContext;
use iggy_common::IggyError;
-use tokio::io::AsyncWriteExt;
-use tokio::net::TcpStream;
-use tokio_rustls::server::TlsStream;
+use monoio::net::TcpStream;
+use monoio_native_tls::TlsStream;
#[derive(Debug)]
pub struct TcpTlsSender {
(iggy) 11/12: fixes
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit a890252aa72430082c871c86ceae96b625c2162e
Author: numinex <g....@gmail.com>
AuthorDate: Wed Jun 25 14:32:26 2025 +0200
fixes
---
core/common/src/locking/mod.rs | 10 +-
core/common/src/locking/tokio_lock.rs | 2 -
core/server/src/shard/system/clients.rs | 2 +-
core/server/src/shard/system/consumer_groups.rs | 3 +-
core/server/src/shard/system/consumer_offsets.rs | 2 -
core/server/src/shard/system/mod.rs | 3 +-
.../src/shard/system/personal_access_tokens.rs | 2 -
core/server/src/shard/system/segments.rs | 3 +-
core/server/src/shard/system/stats.rs | 2 +-
core/server/src/shard/system/system.rs | 291 ---------------------
core/server/src/shard/system/topics.rs | 5 +-
.../src/streaming/partitions/consumer_offsets.rs | 2 +-
core/server/src/streaming/storage.rs | 2 +-
core/server/src/streaming/streams/topics.rs | 3 +-
core/server/src/streaming/topics/consumer_group.rs | 31 ++-
.../server/src/streaming/topics/consumer_groups.rs | 136 ++++++----
.../src/streaming/topics/consumer_offsets.rs | 6 +-
core/server/src/streaming/topics/messages.rs | 5 +-
core/server/src/streaming/topics/partitions.rs | 4 +-
core/server/src/streaming/topics/storage.rs | 9 +-
core/server/src/streaming/topics/topic.rs | 25 +-
21 files changed, 142 insertions(+), 406 deletions(-)
diff --git a/core/common/src/locking/mod.rs b/core/common/src/locking/mod.rs
index 42d72ebb..c2e542c7 100644
--- a/core/common/src/locking/mod.rs
+++ b/core/common/src/locking/mod.rs
@@ -29,19 +29,19 @@ mod fast_async_lock;
#[cfg(feature = "tokio_lock")]
#[cfg(not(any(feature = "fast_async_lock")))]
-pub type IggySharedMut<T> = tokio_lock::IggyTokioRwLock<T>;
+pub type IggyRwLock<T> = tokio_lock::IggyTokioRwLock<T>;
//this can be used in the future to provide different locking mechanisms
#[cfg(feature = "fast_async_lock")]
-pub type IggySharedMut<T> = fast_async_lock::IggyFastAsyncRwLock<T>;
+pub type IggyRwLock<T> = fast_async_lock::IggyFastAsyncRwLock<T>;
#[allow(async_fn_in_trait)]
-pub trait IggySharedMutFn<T>: Send + Sync {
- type ReadGuard<'a>: Deref<Target = T> + Send
+pub trait IggySharedMutFn<T> {
+ type ReadGuard<'a>: Deref<Target = T>
where
T: 'a,
Self: 'a;
- type WriteGuard<'a>: DerefMut<Target = T> + Send
+ type WriteGuard<'a>: DerefMut<Target = T>
where
T: 'a,
Self: 'a;
diff --git a/core/common/src/locking/tokio_lock.rs b/core/common/src/locking/tokio_lock.rs
index 2e26b367..ef5baaad 100644
--- a/core/common/src/locking/tokio_lock.rs
+++ b/core/common/src/locking/tokio_lock.rs
@@ -25,8 +25,6 @@ use tokio::sync::{RwLock as TokioRwLock, RwLockReadGuard, RwLockWriteGuard};
pub struct IggyTokioRwLock<T>(Arc<TokioRwLock<T>>);
impl<T> IggySharedMutFn<T> for IggyTokioRwLock<T>
-where
- T: Send + Sync,
{
type ReadGuard<'a>
= RwLockReadGuard<'a, T>
diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs
index fde182ce..a2debc42 100644
--- a/core/server/src/shard/system/clients.rs
+++ b/core/server/src/shard/system/clients.rs
@@ -22,7 +22,7 @@ use crate::streaming::session::Session;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
-use iggy_common::locking::IggySharedMut;
+use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
use std::net::SocketAddr;
use std::rc::Rc;
diff --git a/core/server/src/shard/system/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs
index 4975a92d..d8d8818e 100644
--- a/core/server/src/shard/system/consumer_groups.rs
+++ b/core/server/src/shard/system/consumer_groups.rs
@@ -16,9 +16,8 @@
* under the License.
*/
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use crate::streaming::topics::consumer_group::ConsumerGroup;
use error_set::ErrContext;
use iggy_common::Identifier;
diff --git a/core/server/src/shard/system/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs
index e63815ff..0c7438a6 100644
--- a/core/server/src/shard/system/consumer_offsets.rs
+++ b/core/server/src/shard/system/consumer_offsets.rs
@@ -17,8 +17,6 @@
*/
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use error_set::ErrContext;
use iggy_common::{Consumer, ConsumerOffsetInfo, Identifier, IggyError};
diff --git a/core/server/src/shard/system/mod.rs b/core/server/src/shard/system/mod.rs
index 1419abb9..fae8ad2b 100644
--- a/core/server/src/shard/system/mod.rs
+++ b/core/server/src/shard/system/mod.rs
@@ -28,8 +28,7 @@ pub mod snapshot;
pub mod stats;
pub mod storage;
pub mod streams;
-pub mod system;
pub mod topics;
pub mod users;
-pub const COMPONENT: &str = "SYSTEM";
+pub const COMPONENT: &str = "SHARD_SYSTEM";
diff --git a/core/server/src/shard/system/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs
index 18125bed..b5452e9a 100644
--- a/core/server/src/shard/system/personal_access_tokens.rs
+++ b/core/server/src/shard/system/personal_access_tokens.rs
@@ -18,8 +18,6 @@
use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken;
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use crate::streaming::users::user::User;
use error_set::ErrContext;
use iggy_common::IggyError;
diff --git a/core/server/src/shard/system/segments.rs b/core/server/src/shard/system/segments.rs
index bd4043a2..27abb76b 100644
--- a/core/server/src/shard/system/segments.rs
+++ b/core/server/src/shard/system/segments.rs
@@ -1,3 +1,4 @@
+use crate::shard::IggyShard;
/* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,8 +17,6 @@
* under the License.
*/
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use error_set::ErrContext;
use iggy_common::Identifier;
use iggy_common::IggyError;
diff --git a/core/server/src/shard/system/stats.rs b/core/server/src/shard/system/stats.rs
index 2d2d2ea7..8410cac2 100644
--- a/core/server/src/shard/system/stats.rs
+++ b/core/server/src/shard/system/stats.rs
@@ -16,8 +16,8 @@
* under the License.
*/
+use crate::shard::IggyShard;
use crate::VERSION;
-use crate::streaming::systems::system::System;
use crate::versioning::SemanticVersion;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{IggyDuration, IggyError, Stats};
diff --git a/core/server/src/shard/system/system.rs b/core/server/src/shard/system/system.rs
deleted file mode 100644
index 5d830d71..00000000
--- a/core/server/src/shard/system/system.rs
+++ /dev/null
@@ -1,291 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-use crate::archiver::{ArchiverKind, ArchiverKindType};
-use crate::configs::server::{DataMaintenanceConfig, PersonalAccessTokenConfig};
-use crate::configs::system::SystemConfig;
-use crate::map_toggle_str;
-use crate::state::StateKind;
-use crate::state::file::FileState;
-use crate::state::system::SystemState;
-use crate::streaming::clients::client_manager::ClientManager;
-use crate::streaming::diagnostics::metrics::Metrics;
-use crate::streaming::persistence::persister::*;
-use crate::streaming::session::Session;
-use crate::streaming::storage::SystemStorage;
-use crate::streaming::streams::stream::Stream;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::users::permissioner::Permissioner;
-use crate::streaming::users::user::User;
-use crate::versioning::SemanticVersion;
-use ahash::AHashMap;
-use error_set::ErrContext;
-use iggy_common::locking::IggySharedMut;
-use iggy_common::locking::IggySharedMutFn;
-use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError, UserId};
-use std::path::Path;
-use std::sync::Arc;
-use tokio::fs::{create_dir_all, remove_dir_all};
-use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
-use tokio::time::Instant;
-use tracing::{error, info, instrument, trace};
-
-#[derive(Debug)]
-pub struct SharedSystem {
- system: Arc<RwLock<System>>,
-}
-
-impl SharedSystem {
- pub fn new(system: System) -> SharedSystem {
- SharedSystem {
- system: Arc::new(RwLock::new(system)),
- }
- }
-
- pub async fn read(&self) -> RwLockReadGuard<System> {
- self.system.read().await
- }
-
- pub async fn write(&self) -> RwLockWriteGuard<System> {
- self.system.write().await
- }
-}
-
-impl Clone for SharedSystem {
- fn clone(&self) -> Self {
- SharedSystem {
- system: self.system.clone(),
- }
- }
-}
-
-#[derive(Debug)]
-pub struct System {
- pub permissioner: Permissioner,
- pub(crate) storage: Arc<SystemStorage>,
- pub(crate) streams: AHashMap<u32, Stream>,
- pub(crate) streams_ids: AHashMap<String, u32>,
- pub(crate) users: AHashMap<UserId, User>,
- pub(crate) config: Arc<SystemConfig>,
- pub(crate) client_manager: IggySharedMut<ClientManager>,
- pub(crate) encryptor: Option<Arc<EncryptorKind>>,
- pub(crate) metrics: Metrics,
- pub(crate) state: Arc<StateKind>,
- pub(crate) archiver: Option<Arc<ArchiverKind>>,
- pub personal_access_token: PersonalAccessTokenConfig,
-}
-
-impl System {
- pub fn new(
- config: Arc<SystemConfig>,
- data_maintenance_config: DataMaintenanceConfig,
- pat_config: PersonalAccessTokenConfig,
- ) -> System {
- let version = SemanticVersion::current().expect("Invalid version");
- info!(
- "Server-side encryption is {}.",
- map_toggle_str(config.encryption.enabled)
- );
-
- let encryptor: Option<EncryptorKind> = match config.encryption.enabled {
- true => Some(EncryptorKind::Aes256Gcm(
- Aes256GcmEncryptor::from_base64_key(&config.encryption.key).unwrap(),
- )),
- false => None,
- };
-
- let state_persister = Self::resolve_persister(config.state.enforce_fsync);
- let partition_persister = Self::resolve_persister(config.partition.enforce_fsync);
-
- let state = Arc::new(StateKind::File(FileState::new(
- &config.get_state_messages_file_path(),
- &version,
- state_persister,
- encryptor,
- )));
-
- //TODO: Just shut the fuck up rust-analyzer.
- let encryptor: Option<EncryptorKind> = match config.encryption.enabled {
- true => Some(EncryptorKind::Aes256Gcm(
- Aes256GcmEncryptor::from_base64_key(&config.encryption.key).unwrap(),
- )),
- false => None,
- };
- Self::create(
- config.clone(),
- SystemStorage::new(config, partition_persister),
- state,
- encryptor.map(Arc::new),
- data_maintenance_config,
- pat_config,
- )
- }
-
- fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
- match enforce_fsync {
- true => Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister)),
- false => Arc::new(PersisterKind::File(FilePersister)),
- }
- }
-
- pub fn create(
- system_config: Arc<SystemConfig>,
- storage: SystemStorage,
- state: Arc<StateKind>,
- encryptor: Option<Arc<EncryptorKind>>,
- data_maintenance_config: DataMaintenanceConfig,
- pat_config: PersonalAccessTokenConfig,
- ) -> System {
- let archiver_config = data_maintenance_config.archiver;
- let archiver: Option<Arc<ArchiverKind>> = if archiver_config.enabled {
- info!("Archiving is enabled, kind: {}", archiver_config.kind);
- match archiver_config.kind {
- ArchiverKindType::Disk => Some(Arc::new(ArchiverKind::get_disk_archiver(
- archiver_config
- .disk
- .clone()
- .expect("Disk archiver config is missing"),
- ))),
- ArchiverKindType::S3 => Some(Arc::new(
- ArchiverKind::get_s3_archiver(
- archiver_config
- .s3
- .clone()
- .expect("S3 archiver config is missing"),
- )
- .expect("Failed to create S3 archiver"),
- )),
- }
- } else {
- info!("Archiving is disabled.");
- None
- };
-
- System {
- config: system_config,
- streams: AHashMap::new(),
- streams_ids: AHashMap::new(),
- storage: Arc::new(storage),
- encryptor,
- client_manager: IggySharedMut::new(ClientManager::default()),
- permissioner: Permissioner::default(),
- metrics: Metrics::init(),
- users: AHashMap::new(),
- state,
- personal_access_token: pat_config,
- archiver,
- }
- }
-
- #[instrument(skip_all, name = "trace_system_init")]
- pub async fn init(&mut self) -> Result<(), IggyError> {
- let system_path = self.config.get_system_path();
- if !Path::new(&system_path).exists() && create_dir_all(&system_path).await.is_err() {
- return Err(IggyError::CannotCreateBaseDirectory(system_path));
- }
-
- let state_path = self.config.get_state_path();
- if !Path::new(&state_path).exists() && create_dir_all(&state_path).await.is_err() {
- return Err(IggyError::CannotCreateStateDirectory(state_path));
- }
-
- let streams_path = self.config.get_streams_path();
- if !Path::new(&streams_path).exists() && create_dir_all(&streams_path).await.is_err() {
- return Err(IggyError::CannotCreateStreamsDirectory(streams_path));
- }
-
- let runtime_path = self.config.get_runtime_path();
- if Path::new(&runtime_path).exists() && remove_dir_all(&runtime_path).await.is_err() {
- return Err(IggyError::CannotRemoveRuntimeDirectory(runtime_path));
- }
-
- if create_dir_all(&runtime_path).await.is_err() {
- return Err(IggyError::CannotCreateRuntimeDirectory(runtime_path));
- }
-
- info!(
- "Initializing system, data will be stored at: {}",
- self.config.get_system_path()
- );
-
- let state_entries = self.state.init().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to initialize state entries")
- })?;
- let system_state = SystemState::init(state_entries)
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to initialize system state")
- })?;
- let now = Instant::now();
- //DONE
- /*
- self.load_version().await.with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load version")
- })?;
- */
- self.load_users(system_state.users.into_values().collect())
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load users")
- })?;
- self.load_streams(system_state.streams.into_values().collect())
- .await
- .with_error_context(|error| {
- format!("{COMPONENT} (error: {error}) - failed to load streams")
- })?;
- if let Some(archiver) = self.archiver.as_ref() {
- archiver
- .init()
- .await
- .expect("Failed to initialize archiver");
- }
- info!("Initialized system in {} ms.", now.elapsed().as_millis());
- Ok(())
- }
-
- #[instrument(skip_all, name = "trace_shutdown")]
- pub async fn shutdown(&mut self) -> Result<(), IggyError> {
- self.persist_messages().await?;
- Ok(())
- }
-
- #[instrument(skip_all, name = "trace_persist_messages")]
- pub async fn persist_messages(&self) -> Result<usize, IggyError> {
- trace!("Saving buffered messages on disk...");
- let mut saved_messages_number = 0;
- for stream in self.streams.values() {
- saved_messages_number += stream.persist_messages().await?;
- }
-
- Ok(saved_messages_number)
- }
-
- pub fn ensure_authenticated(&self, session: &Session) -> Result<(), IggyError> {
- if !session.is_active() {
- error!("{COMPONENT} - session is inactive, session: {session}");
- return Err(IggyError::StaleClient);
- }
-
- if session.is_authenticated() {
- Ok(())
- } else {
- error!("{COMPONENT} - unauthenticated access attempt, session: {session}");
- Err(IggyError::Unauthenticated)
- }
- }
-}
diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs
index a61f5464..145689af 100644
--- a/core/server/src/shard/system/topics.rs
+++ b/core/server/src/shard/system/topics.rs
@@ -16,15 +16,14 @@
* under the License.
*/
+use crate::shard::IggyShard;
use crate::streaming::session::Session;
-use crate::streaming::systems::COMPONENT;
-use crate::streaming::systems::system::System;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
use iggy_common::locking::IggySharedMutFn;
use iggy_common::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize};
-impl System {
+impl IggyShard {
pub fn find_topic(
&self,
session: &Session,
diff --git a/core/server/src/streaming/partitions/consumer_offsets.rs b/core/server/src/streaming/partitions/consumer_offsets.rs
index 177793c9..96080c03 100644
--- a/core/server/src/streaming/partitions/consumer_offsets.rs
+++ b/core/server/src/streaming/partitions/consumer_offsets.rs
@@ -26,7 +26,7 @@ use iggy_common::IggyError;
use tracing::trace;
impl Partition {
- pub async fn get_consumer_offset(
+ pub fn get_consumer_offset(
&self,
consumer: PollingConsumer,
) -> Result<Option<u64>, IggyError> {
diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs
index 51e2565e..978b97aa 100644
--- a/core/server/src/streaming/storage.rs
+++ b/core/server/src/streaming/storage.rs
@@ -154,7 +154,7 @@ pub struct SystemStorage {
impl SystemStorage {
pub fn new(config: Rc<SystemConfig>, persister: Arc<PersisterKind>) -> Self {
Self {
- info: Arc::new(SystemInfoStorageKind::File(FileSystemInfoStorage::new(
+ info: Rc::new(SystemInfoStorageKind::File(FileSystemInfoStorage::new(
config.get_state_info_path(),
persister.clone(),
))),
diff --git a/core/server/src/streaming/streams/topics.rs b/core/server/src/streaming/streams/topics.rs
index b975ee83..ed350bee 100644
--- a/core/server/src/streaming/streams/topics.rs
+++ b/core/server/src/streaming/streams/topics.rs
@@ -146,7 +146,8 @@ impl Stream {
topic.name = name.to_owned();
topic.message_expiry = message_expiry;
topic.compression_algorithm = compression_algorithm;
- for partition in topic.partitions.borrow_mut().values_mut() {
+ for partition in topic.partitions.values_mut() {
+ let mut partition = partition.write().await;
partition.message_expiry = message_expiry;
for segment in partition.segments.iter_mut() {
segment.update_message_expiry(message_expiry);
diff --git a/core/server/src/streaming/topics/consumer_group.rs b/core/server/src/streaming/topics/consumer_group.rs
index cc28ef6f..f000ce79 100644
--- a/core/server/src/streaming/topics/consumer_group.rs
+++ b/core/server/src/streaming/topics/consumer_group.rs
@@ -52,12 +52,12 @@ impl ConsumerGroup {
self.members.values().cloned().collect()
}
- pub async fn reassign_partitions(&mut self, partitions_count: u32) {
+ pub fn reassign_partitions(&mut self, partitions_count: u32) {
self.partitions_count = partitions_count;
- self.assign_partitions().await;
+ self.assign_partitions();
}
- pub async fn calculate_partition_id(
+ pub fn calculate_partition_id(
&mut self,
member_id: u32,
) -> Result<Option<u32>, IggyError> {
@@ -72,7 +72,7 @@ impl ConsumerGroup {
))
}
- pub async fn get_current_partition_id(&self, member_id: u32) -> Result<Option<u32>, IggyError> {
+ pub fn get_current_partition_id(&mut self, member_id: u32) -> Result<Option<u32>, IggyError> {
let member = self.members.get(&member_id);
if let Some(member) = member {
return Ok(member.current_partition_id);
@@ -84,7 +84,7 @@ impl ConsumerGroup {
))
}
- pub async fn add_member(&mut self, member_id: u32) {
+ pub fn add_member(&mut self, member_id: u32) {
self.members.insert(
member_id,
ConsumerGroupMember {
@@ -98,20 +98,20 @@ impl ConsumerGroup {
"Added member with ID: {} to consumer group: {} for topic with ID: {}",
member_id, self.group_id, self.topic_id
);
- self.assign_partitions().await;
+ self.assign_partitions();
}
- pub async fn delete_member(&mut self, member_id: u32) {
+ pub fn delete_member(&mut self, member_id: u32) {
if self.members.remove(&member_id).is_some() {
trace!(
"Deleted member with ID: {} in consumer group: {} for topic with ID: {}",
member_id, self.group_id, self.topic_id
);
- self.assign_partitions().await;
+ self.assign_partitions();
}
}
- async fn assign_partitions(&mut self) {
+ fn assign_partitions(&mut self) {
let mut members = self.members.values_mut().collect::<Vec<_>>();
if members.is_empty() {
return;
@@ -189,11 +189,10 @@ mod tests {
members: AHashMap::new(),
};
- consumer_group.add_member(member_id).await;
+ consumer_group.add_member(member_id);
for i in 0..1000 {
let partition_id = consumer_group
.calculate_partition_id(member_id)
- .await
.unwrap()
.expect("Partition ID not found");
assert_eq!(partition_id, (i % consumer_group.partitions_count) + 1);
@@ -211,7 +210,7 @@ mod tests {
members: AHashMap::new(),
};
- consumer_group.add_member(member_id).await;
+ consumer_group.add_member(member_id);
let member = consumer_group.members.get(&member_id).unwrap();
assert_eq!(
member.partitions.len() as u32,
@@ -235,8 +234,8 @@ mod tests {
members: AHashMap::new(),
};
- consumer_group.add_member(member1_id).await;
- consumer_group.add_member(member2_id).await;
+ consumer_group.add_member(member1_id);
+ consumer_group.add_member(member2_id);
let member1 = consumer_group.members.get(&member1_id).unwrap();
let member2 = consumer_group.members.get(&member2_id).unwrap();
assert_eq!(
@@ -270,8 +269,8 @@ mod tests {
members: AHashMap::new(),
};
- consumer_group.add_member(member1_id).await;
- consumer_group.add_member(member2_id).await;
+ consumer_group.add_member(member1_id);
+ consumer_group.add_member(member2_id);
let member1 = consumer_group.members.get(&member1_id).unwrap();
let member2 = consumer_group.members.get(&member2_id).unwrap();
if member1.partitions.len() == 1 {
diff --git a/core/server/src/streaming/topics/consumer_groups.rs b/core/server/src/streaming/topics/consumer_groups.rs
index 1f50a1bb..c1371096 100644
--- a/core/server/src/streaming/topics/consumer_groups.rs
+++ b/core/server/src/streaming/topics/consumer_groups.rs
@@ -16,36 +16,39 @@
* under the License.
*/
-use crate::streaming::topics::COMPONENT;
+use crate::binary::handlers::topics::get_topic_handler;
+use crate::streaming::topics::{consumer_group, COMPONENT};
use crate::streaming::topics::consumer_group::ConsumerGroup;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
+use iggy_common::locking::IggySharedMutFn;
use iggy_common::IggyError;
use iggy_common::{IdKind, Identifier};
+use std::cell::{Ref, RefMut};
use std::sync::atomic::Ordering;
use tracing::info;
impl Topic {
pub fn reassign_consumer_groups(&mut self) {
- if self.consumer_groups.is_empty() {
+ if self.consumer_groups.borrow().is_empty() {
return;
}
- let partitions_count = self.partitions.borrow().len() as u32;
+ let partitions_count = self.partitions.len() as u32;
info!(
"Reassigning consumer groups for topic with ID: {} for stream with ID with {}, partitions count: {}",
self.topic_id, self.stream_id, partitions_count
);
- for (_, consumer_group) in self.consumer_groups.iter_mut() {
+ for (_, consumer_group) in self.consumer_groups.borrow_mut().iter_mut() {
consumer_group.reassign_partitions(partitions_count);
}
}
- pub fn get_consumer_groups(&self) -> Vec<&ConsumerGroup> {
- self.consumer_groups.values().collect()
+ pub fn get_consumer_groups(&self) -> Vec<ConsumerGroup> {
+ self.consumer_groups.borrow().values().cloned().collect()
}
- pub fn get_consumer_group(&self, identifier: &Identifier) -> Result<&ConsumerGroup, IggyError> {
+ pub fn get_consumer_group(&self, identifier: &Identifier) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
match identifier.kind {
IdKind::Numeric => self.get_consumer_group_by_id(identifier.get_u32_value().unwrap()),
IdKind::String => self.get_consumer_group_by_name(&identifier.get_cow_str_value()?),
@@ -55,22 +58,42 @@ impl Topic {
pub fn try_get_consumer_group(
&self,
identifier: &Identifier,
- ) -> Result<Option<&ConsumerGroup>, IggyError> {
+ ) -> Result<Option<Ref<'_, ConsumerGroup>>, IggyError> {
match identifier.kind {
- IdKind::Numeric => Ok(self.consumer_groups.get(&identifier.get_u32_value()?)),
+ IdKind::Numeric => Ok(self.try_get_consumer_group_by_id(&identifier.get_u32_value()?)),
IdKind::String => {
Ok(self.try_get_consumer_group_by_name(&identifier.get_cow_str_value()?))
}
}
}
+ fn try_get_consumer_group_by_id(&self, id: &u32) -> Option<Ref<'_, ConsumerGroup>> {
+ let consumer_groups = self.consumer_groups.borrow();
+ let exists = consumer_groups.contains_key(id);
+ if !exists {
+ return None;
+ }
+
+ Some(Ref::map(consumer_groups, |cg| {
+ let consumer_group = cg.get(id);
+ consumer_group.unwrap()
+ }))
+ }
+
+ fn try_get_consumer_group_by_name(&self, name: &str) -> Option<Ref<'_, ConsumerGroup>> {
+ let consumer_groups = self.consumer_groups.borrow();
+ let exists = self.consumer_groups_ids.contains_key(name);
+ let id = self.consumer_groups_ids.get(name).unwrap();
+ if !exists {
+ return None;
+ }
- fn try_get_consumer_group_by_name(&self, name: &str) -> Option<&ConsumerGroup> {
- self.consumer_groups_ids
- .get(name)
- .and_then(|id| self.consumer_groups.get(id))
+ Some(Ref::map(consumer_groups, |cg| {
+ let consumer_group = cg.get(id);
+ consumer_group.unwrap()
+ }))
}
- pub fn get_consumer_group_by_name(&self, name: &str) -> Result<&ConsumerGroup, IggyError> {
+ pub fn get_consumer_group_by_name(&self, name: &str) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
let group_id = self.consumer_groups_ids.get(name);
if group_id.is_none() {
return Err(IggyError::ConsumerGroupNameNotFound(
@@ -82,19 +105,24 @@ impl Topic {
self.get_consumer_group_by_id(*group_id.unwrap())
}
- pub fn get_consumer_group_by_id(&self, id: u32) -> Result<&ConsumerGroup, IggyError> {
- let consumer_group = self.consumer_groups.get(&id);
- if consumer_group.is_none() {
+ pub fn get_consumer_group_by_id(&self, id: u32) -> Result<Ref<'_, ConsumerGroup>, IggyError> {
+ let consumer_groups = self.consumer_groups.borrow();
+ let exists = consumer_groups.contains_key(&id);
+ if !exists {
return Err(IggyError::ConsumerGroupIdNotFound(id, self.topic_id));
}
+ let consumer_group = Ref::map(consumer_groups, |cg| {
+ let consumer_group = cg.get(&id);
+ consumer_group.unwrap()
+ });
- Ok(consumer_group.unwrap())
+ Ok(consumer_group)
}
pub fn get_consumer_group_mut(
- &mut self,
+ &self,
identifier: &Identifier,
- ) -> Result<&mut ConsumerGroup, IggyError> {
+ ) -> Result<RefMut<'_, ConsumerGroup>, IggyError> {
match identifier.kind {
IdKind::Numeric => {
self.get_consumer_group_by_id_mut(identifier.get_u32_value().unwrap())
@@ -104,21 +132,25 @@ impl Topic {
}
pub fn get_consumer_group_by_id_mut(
- &mut self,
+ &self,
id: u32,
- ) -> Result<&mut ConsumerGroup, IggyError> {
- let consumer_group = self.consumer_groups.get_mut(&id);
- if consumer_group.is_none() {
+ ) -> Result<RefMut<'_, ConsumerGroup>, IggyError> {
+ let consumer_groups = self.consumer_groups.borrow_mut();
+ let exists = consumer_groups.contains_key(&id);
+ if !exists {
return Err(IggyError::ConsumerGroupIdNotFound(id, self.topic_id));
}
-
- Ok(consumer_group.unwrap())
+ let consumer_group = RefMut::map(consumer_groups, |cg| {
+ let consumer_group = cg.get_mut(&id);
+ consumer_group.unwrap()
+ });
+ Ok(consumer_group)
}
pub fn get_consumer_group_by_name_mut(
- &mut self,
+ &self,
name: &str,
- ) -> Result<&mut ConsumerGroup, IggyError> {
+ ) -> Result<RefMut<'_, ConsumerGroup>, IggyError> {
let group_id = self.consumer_groups_ids.get(name).copied();
if group_id.is_none() {
return Err(IggyError::ConsumerGroupNameNotFound(
@@ -148,7 +180,7 @@ impl Topic {
.current_consumer_group_id
.fetch_add(1, Ordering::SeqCst);
loop {
- if self.consumer_groups.contains_key(&id) {
+ if self.consumer_groups.borrow().contains_key(&id) {
if id == u32::MAX {
return Err(IggyError::ConsumerGroupIdAlreadyExists(id, self.topic_id));
}
@@ -163,7 +195,7 @@ impl Topic {
id = group_id.unwrap();
}
- if self.consumer_groups.contains_key(&id) {
+ if self.consumer_groups.borrow().contains_key(&id) {
return Err(IggyError::ConsumerGroupIdAlreadyExists(id, self.topic_id));
}
@@ -171,11 +203,11 @@ impl Topic {
self.topic_id,
id,
name,
- self.partitions.borrow().len() as u32,
+ self.partitions.len() as u32,
);
self.consumer_groups_ids.insert(name.to_owned(), id);
let cloned_group = consumer_group.clone();
- self.consumer_groups.insert(id, consumer_group);
+ self.consumer_groups.borrow_mut().insert(id, consumer_group);
info!(
"Created consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
id, self.topic_id, self.stream_id
@@ -195,11 +227,14 @@ impl Topic {
group_id = consumer_group.group_id;
}
- let consumer_group = self.consumer_groups.remove(&group_id);
+ let mut consumer_groups = self.consumer_groups.borrow_mut();
+ let consumer_group = consumer_groups.remove(&group_id);
if consumer_group.is_none() {
return Err(IggyError::ConsumerGroupIdNotFound(group_id, self.topic_id));
}
let consumer_group = consumer_group.unwrap();
+ let consumer_group = consumer_group.clone();
+ drop(consumer_groups);
{
self.consumer_groups_ids.remove(&consumer_group.name);
let current_group_id = self.current_consumer_group_id.load(Ordering::SeqCst);
@@ -208,7 +243,8 @@ impl Topic {
.store(group_id, Ordering::SeqCst);
}
- for (_, partition) in self.partitions.borrow().iter() {
+ for (_, partition) in self.partitions.iter() {
+ let partition = partition.read().await;
if let Some((_, offset)) = partition.consumer_group_offsets.remove(&group_id) {
self.storage
.partition
@@ -231,13 +267,15 @@ impl Topic {
group_id: &Identifier,
member_id: u32,
) -> Result<(), IggyError> {
- let consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
+ let topic_id = self.topic_id;
+ let stream_id = self.stream_id;
+ let mut consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get consumer group with id: {group_id}")
})?;
consumer_group.add_member(member_id);
info!(
"Member with ID: {} has joined consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
- member_id, group_id, self.topic_id, self.stream_id
+ member_id, group_id, topic_id, stream_id
);
Ok(())
}
@@ -247,13 +285,15 @@ impl Topic {
group_id: &Identifier,
member_id: u32,
) -> Result<(), IggyError> {
- let consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
+ let topic_id = self.topic_id;
+ let stream_id = self.stream_id;
+ let mut consumer_group = self.get_consumer_group_mut(group_id).with_error_context(|error| {
format!("{COMPONENT} (error: {error}) - failed to get consumer group with id: {group_id}")
})?;
consumer_group.delete_member(member_id);
info!(
"Member with ID: {} has left consumer group with ID: {} for topic with ID: {} and stream with ID: {}.",
- member_id, group_id, self.topic_id, self.stream_id
+ member_id, group_id, topic_id, stream_id
);
Ok(())
}
@@ -286,7 +326,7 @@ mod tests {
assert_eq!(created_consumer_group.topic_id, topic_id);
}
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
let consumer_group = topic
.get_consumer_group(&Identifier::numeric(group_id).unwrap())
.unwrap();
@@ -295,7 +335,7 @@ mod tests {
assert_eq!(consumer_group.topic_id, topic_id);
assert_eq!(
consumer_group.partitions_count,
- topic.partitions.borrow().len() as u32
+ topic.partitions.len() as u32
);
}
@@ -306,12 +346,12 @@ mod tests {
let mut topic = get_topic().await;
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
let result = topic.create_consumer_group(Some(group_id), "test2");
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, IggyError::ConsumerGroupIdAlreadyExists(_, _)));
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
}
#[tokio::test]
@@ -321,7 +361,7 @@ mod tests {
let mut topic = get_topic().await;
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
let group_id = group_id + 1;
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_err());
@@ -330,7 +370,7 @@ mod tests {
err,
IggyError::ConsumerGroupNameAlreadyExists(_, _)
));
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
}
#[tokio::test]
@@ -340,12 +380,12 @@ mod tests {
let mut topic = get_topic().await;
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
let result = topic
.delete_consumer_group(&Identifier::numeric(group_id).unwrap())
.await;
assert!(result.is_ok());
- assert!(topic.consumer_groups.is_empty());
+ assert!(topic.consumer_groups.borrow().is_empty());
}
#[tokio::test]
@@ -355,13 +395,13 @@ mod tests {
let mut topic = get_topic().await;
let result = topic.create_consumer_group(Some(group_id), name);
assert!(result.is_ok());
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
let group_id = group_id + 1;
let result = topic
.delete_consumer_group(&Identifier::numeric(group_id).unwrap())
.await;
assert!(result.is_err());
- assert_eq!(topic.consumer_groups.len(), 1);
+ assert_eq!(topic.consumer_groups.borrow().len(), 1);
}
#[tokio::test]
diff --git a/core/server/src/streaming/topics/consumer_offsets.rs b/core/server/src/streaming/topics/consumer_offsets.rs
index f59e9276..8748576b 100644
--- a/core/server/src/streaming/topics/consumer_offsets.rs
+++ b/core/server/src/streaming/topics/consumer_offsets.rs
@@ -34,7 +34,6 @@ impl Topic {
) -> Result<(), IggyError> {
let Some((polling_consumer, partition_id)) = self
.resolve_consumer_with_partition_id(&consumer, client_id, partition_id, false)
- .await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer with partition id, consumer ID: {}, client ID: {}, partition ID: {:?}", consumer.id, client_id, partition_id))? else {
return Err(IggyError::ConsumerOffsetNotFound(client_id));
};
@@ -78,8 +77,7 @@ impl Topic {
) -> Result<Option<ConsumerOffsetInfo>, IggyError> {
let Some((polling_consumer, partition_id)) = self
.resolve_consumer_with_partition_id(consumer, client_id, partition_id, false)
- .await
- .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer offset for consumer: {consumer}, client ID: {client_id}, partition ID: {partition_id:#?}"))? else {
+ .with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer offset for consumer: {consumer}, client ID: {client_id}, partition ID: {:#?}", partition_id))? else {
return Ok(None);
};
@@ -93,7 +91,6 @@ impl Topic {
let partition = partition.read().await;
let offset = partition
.get_consumer_offset(polling_consumer)
- .await
.with_error_context(|error| {
format!(
"{COMPONENT} (error: {error}) - failed to get consumer offset for consumer: {polling_consumer}"
@@ -118,7 +115,6 @@ impl Topic {
) -> Result<(), IggyError> {
let Some((polling_consumer, partition_id)) = self
.resolve_consumer_with_partition_id(&consumer, client_id, partition_id, false)
- .await
.with_error_context(|error| format!("{COMPONENT} (error: {error}) - failed to resolve consumer with partition id, consumer ID: {}, client ID: {}, partition ID: {:?}", consumer.id, client_id, partition_id))? else {
return Err(IggyError::ConsumerOffsetNotFound(client_id));
};
diff --git a/core/server/src/streaming/topics/messages.rs b/core/server/src/streaming/topics/messages.rs
index 56d7c1c7..e8c52385 100644
--- a/core/server/src/streaming/topics/messages.rs
+++ b/core/server/src/streaming/topics/messages.rs
@@ -209,6 +209,7 @@ mod tests {
use bytes::Bytes;
use iggy_common::CompressionAlgorithm;
use iggy_common::{IggyMessage, MaxTopicSize};
+ use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::AtomicU32;
use std::sync::atomic::AtomicU64;
@@ -321,11 +322,11 @@ mod tests {
async fn init_topic(partitions_count: u32) -> Topic {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
diff --git a/core/server/src/streaming/topics/partitions.rs b/core/server/src/streaming/topics/partitions.rs
index eb6b4280..dfedf0a9 100644
--- a/core/server/src/streaming/topics/partitions.rs
+++ b/core/server/src/streaming/topics/partitions.rs
@@ -20,7 +20,7 @@ use crate::streaming::partitions::partition::Partition;
use crate::streaming::topics::COMPONENT;
use crate::streaming::topics::topic::Topic;
use error_set::ErrContext;
-use iggy_common::locking::{IggySharedMut, IggySharedMutFn};
+use iggy_common::locking::{IggyRwLock, IggySharedMutFn};
use iggy_common::{IggyError, IggyTimestamp};
const MAX_PARTITIONS_COUNT: u32 = 100_000;
@@ -63,7 +63,7 @@ impl Topic {
)
.await;
self.partitions
- .insert(partition_id, IggySharedMut::new(partition));
+ .insert(partition_id, IggyRwLock::new(partition));
partition_ids.push(partition_id)
}
diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs
index 1cc33310..ae9435e5 100644
--- a/core/server/src/streaming/topics/storage.rs
+++ b/core/server/src/streaming/topics/storage.rs
@@ -27,7 +27,7 @@ use anyhow::Context;
use error_set::ErrContext;
use futures::future::join_all;
use iggy_common::IggyError;
-use iggy_common::locking::IggySharedMut;
+use iggy_common::locking::IggyRwLock;
use iggy_common::locking::IggySharedMutFn;
use serde::{Deserialize, Serialize};
use std::path::Path;
@@ -189,7 +189,7 @@ impl TopicStorage for FileTopicStorage {
for mut partition in unloaded_partitions {
let loaded_partitions = loaded_partitions.clone();
let partition_state = state.partitions.remove(&partition.partition_id).unwrap();
- let load_partition = tokio::spawn(async move {
+ let load_partition = monoio::spawn(async move {
match partition.load(partition_state).await {
Ok(_) => {
loaded_partitions.lock().await.push(partition);
@@ -209,7 +209,7 @@ impl TopicStorage for FileTopicStorage {
for partition in loaded_partitions.lock().await.drain(..) {
topic
.partitions
- .insert(partition.partition_id, IggySharedMut::new(partition));
+ .insert(partition.partition_id, IggyRwLock::new(partition));
}
for consumer_group in state.consumer_groups.into_values() {
@@ -224,7 +224,8 @@ impl TopicStorage for FileTopicStorage {
.insert(consumer_group.name.to_owned(), consumer_group.group_id);
topic
.consumer_groups
- .insert(consumer_group.group_id, RwLock::new(consumer_group));
+ .borrow_mut()
+ .insert(consumer_group.group_id, consumer_group);
}
info!("Loaded topic {topic}");
diff --git a/core/server/src/streaming/topics/topic.rs b/core/server/src/streaming/topics/topic.rs
index ad3e25ef..16aa324c 100644
--- a/core/server/src/streaming/topics/topic.rs
+++ b/core/server/src/streaming/topics/topic.rs
@@ -25,7 +25,7 @@ use ahash::AHashMap;
use core::fmt;
use std::cell::RefCell;
use std::rc::Rc;
-use iggy_common::locking::IggySharedMut;
+use iggy_common::locking::IggyRwLock;
use iggy_common::{
CompressionAlgorithm, Consumer, ConsumerKind, IggyByteSize, IggyError, IggyExpiry,
IggyTimestamp, MaxTopicSize, Sizeable,
@@ -33,7 +33,6 @@ use iggy_common::{
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
-use tokio::sync::RwLock;
use tracing::info;
const ALMOST_FULL_THRESHOLD: f64 = 0.9;
@@ -51,9 +50,9 @@ pub struct Topic {
pub(crate) messages_count: Arc<AtomicU64>,
pub(crate) segments_count_of_parent_stream: Arc<AtomicU32>,
pub(crate) config: Rc<SystemConfig>,
- pub(crate) partitions: RefCell<AHashMap<u32, Partition>>,
+ pub(crate) partitions: AHashMap<u32, IggyRwLock<Partition>>,
pub(crate) storage: Rc<SystemStorage>,
- pub(crate) consumer_groups: AHashMap<u32, ConsumerGroup>,
+ pub(crate) consumer_groups: RefCell<AHashMap<u32, ConsumerGroup>>,
pub(crate) consumer_groups_ids: AHashMap<String, u32>,
pub(crate) current_consumer_group_id: AtomicU32,
pub(crate) current_partition_id: AtomicU32,
@@ -126,7 +125,7 @@ impl Topic {
messages_count_of_parent_stream,
messages_count: Arc::new(AtomicU64::new(0)),
segments_count_of_parent_stream,
- consumer_groups: AHashMap::new(),
+ consumer_groups: RefCell::new(AHashMap::new()),
consumer_groups_ids: AHashMap::new(),
current_consumer_group_id: AtomicU32::new(1),
current_partition_id: AtomicU32::new(1),
@@ -172,11 +171,11 @@ impl Topic {
matches!(self.max_topic_size, MaxTopicSize::Unlimited)
}
- pub fn get_partitions(&self) -> Vec<IggySharedMut<Partition>> {
+ pub fn get_partitions(&self) -> Vec<IggyRwLock<Partition>> {
self.partitions.values().cloned().collect()
}
- pub fn get_partition(&self, partition_id: u32) -> Result<IggySharedMut<Partition>, IggyError> {
+ pub fn get_partition(&self, partition_id: u32) -> Result<IggyRwLock<Partition>, IggyError> {
match self.partitions.get(&partition_id) {
Some(partition_arc) => Ok(partition_arc.clone()),
None => Err(IggyError::PartitionNotFound(
@@ -187,7 +186,7 @@ impl Topic {
}
}
- pub async fn resolve_consumer_with_partition_id(
+ pub fn resolve_consumer_with_partition_id(
&self,
consumer: &Consumer,
client_id: u32,
@@ -203,7 +202,7 @@ impl Topic {
)))
}
ConsumerKind::ConsumerGroup => {
- let consumer_group = self.get_consumer_group(&consumer.id)?.read().await;
+ let mut consumer_group = self.get_consumer_group_mut(&consumer.id)?;
if let Some(partition_id) = partition_id {
return Ok(Some((
PollingConsumer::consumer_group(consumer_group.group_id, client_id),
@@ -212,9 +211,9 @@ impl Topic {
}
let partition_id = if calculate_partition_id {
- consumer_group.calculate_partition_id(client_id).await?
+ consumer_group.calculate_partition_id(client_id)?
} else {
- consumer_group.get_current_partition_id(client_id).await?
+ consumer_group.get_current_partition_id(client_id)?
};
let Some(partition_id) = partition_id else {
return Ok(None);
@@ -291,11 +290,11 @@ mod tests {
#[tokio::test]
async fn should_be_created_given_valid_parameters() {
let tempdir = tempfile::TempDir::new().unwrap();
- let config = Arc::new(SystemConfig {
+ let config = Rc::new(SystemConfig {
path: tempdir.path().to_str().unwrap().to_string(),
..Default::default()
});
- let storage = Arc::new(SystemStorage::new(
+ let storage = Rc::new(SystemStorage::new(
config.clone(),
Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister {})),
));
(iggy) 01/12: initial setup and migration of shards
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 3e599d25f6b6e7f8c58f0dd4546a6e6fb73e0b4a
Author: numinex <g....@gmail.com>
AuthorDate: Sun May 11 17:17:51 2025 +0200
initial setup and migration of shards
---
Cargo.lock | 91 +++++++++++++++-
core/server/Cargo.toml | 2 +
core/server/src/bootstrap.rs | 0
core/server/src/lib.rs | 2 +
core/server/src/shard/connector.rs | 140 +++++++++++++++++++++++++
core/server/src/shard/frame.rs | 68 ++++++++++++
core/server/src/shard/mod.rs | 59 +++++++++++
core/server/src/{lib.rs => shard/namespace.rs} | 48 +++------
8 files changed, 375 insertions(+), 35 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index ea6eb0e1..9d3922d5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -142,7 +142,7 @@ dependencies = [
"actix-utils",
"futures-core",
"futures-util",
- "mio",
+ "mio 1.0.3",
"socket2",
"tokio",
"tracing",
@@ -631,6 +631,17 @@ dependencies = [
"webpki-roots 0.26.11",
]
+[[package]]
+name = "auto-const-array"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd73835ad7deb4bd2b389e6f10333b143f025d607c55ca04c66a0bcc6bb2fc6d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
[[package]]
name = "autocfg"
version = "1.5.0"
@@ -2646,6 +2657,15 @@ dependencies = [
"slab",
]
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder",
+]
+
[[package]]
name = "generator"
version = "0.8.5"
@@ -4090,6 +4110,16 @@ dependencies = [
"rustversion",
]
+[[package]]
+name = "io-uring"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "595a0399f411a508feb2ec1e970a4a30c249351e30208960d58298de8660b0e5"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+]
+
[[package]]
name = "ipnet"
version = "2.11.0"
@@ -4705,6 +4735,37 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "monoio"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3bd0f8bcde87b1949f95338b547543fcab187bc7e7a5024247e359a5e828ba6a"
+dependencies = [
+ "auto-const-array",
+ "bytes",
+ "fxhash",
+ "io-uring",
+ "libc",
+ "memchr",
+ "mio 0.8.11",
+ "monoio-macros",
+ "nix 0.26.4",
+ "pin-project-lite",
+ "socket2",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "monoio-macros"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "176a5f5e69613d9e88337cf2a65e11135332b4efbcc628404a7c555e4452084c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
[[package]]
name = "nanorand"
version = "0.7.0"
@@ -4720,6 +4781,19 @@ version = "6.6.666"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf5a574dadd7941adeaa71823ecba5e28331b8313fb2e1c6a5c7e5981ea53ad6"
+[[package]]
+name = "nix"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "libc",
+ "memoffset",
+ "pin-utils",
+]
+
[[package]]
name = "nix"
version = "0.30.1"
@@ -4784,7 +4858,7 @@ dependencies = [
"kqueue",
"libc",
"log",
- "mio",
+ "mio 1.0.3",
"notify-types",
"walkdir",
"windows-sys 0.59.0",
@@ -6740,7 +6814,8 @@ dependencies = [
"mimalloc",
"mockall",
"moka",
- "nix",
+ "monoio",
+ "nix 0.30.1",
"once_cell",
"opentelemetry",
"opentelemetry-appender-tracing",
@@ -6757,6 +6832,7 @@ dependencies = [
"serde",
"serde_with",
"serial_test",
+ "sharded_queue",
"static-toml",
"strum",
"sysinfo 0.35.2",
@@ -6808,6 +6884,15 @@ dependencies = [
"lazy_static",
]
+[[package]]
+name = "sharded_queue"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3499be28bd82560e75ad10457698607ba3cc389175ab47ac93279834ce1fab4"
+dependencies = [
+ "crossbeam-utils",
+]
+
[[package]]
name = "shlex"
version = "1.3.0"
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index b0e46455..8523c6bb 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -63,6 +63,7 @@ jsonwebtoken = "9.3.1"
lending-iterator = "0.1.7"
mimalloc = { workspace = true, optional = true }
moka = { version = "0.12.10", features = ["future"] }
+monoio = "0.2.4"
nix = { version = "0.30", features = ["fs"] }
once_cell = "1.21.3"
opentelemetry = { version = "0.30.0", features = ["trace", "logs"] }
@@ -96,6 +97,7 @@ rustls-pemfile = "2.2.0"
serde = { workspace = true }
serde_with = { workspace = true }
static-toml = "1.3.0"
+sharded_queue = "2.0.1"
strum = { workspace = true }
sysinfo = { workspace = true }
tempfile = { workspace = true }
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
new file mode 100644
index 00000000..e69de29b
diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs
index 91d17586..572301e0 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/lib.rs
@@ -26,12 +26,14 @@ static GLOBAL: MiMalloc = MiMalloc;
#[cfg(windows)]
compile_error!("iggy-server doesn't support windows.");
+mod bootstrap;
pub mod archiver;
pub mod args;
pub mod binary;
pub mod channels;
pub(crate) mod compat;
pub mod configs;
+pub mod shard;
pub mod http;
pub mod log;
pub mod quic;
diff --git a/core/server/src/shard/connector.rs b/core/server/src/shard/connector.rs
new file mode 100644
index 00000000..349218ba
--- /dev/null
+++ b/core/server/src/shard/connector.rs
@@ -0,0 +1,140 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use futures::{task::AtomicWaker, Stream};
+use sharded_queue::ShardedQueue;
+use std::{
+ sync::{atomic::AtomicUsize, Arc},
+ task::Poll,
+};
+
+pub type StopSender = flume::Sender<()>;
+pub type StopReceiver = flume::Receiver<()>;
+
+#[derive(Clone)]
+pub struct ShardConnector<T> {
+ pub id: u16,
+ pub sender: Sender<T>,
+ pub receiver: Receiver<T>,
+ pub stop_receiver: StopReceiver,
+ pub stop_sender: StopSender,
+}
+
+// TODO(numinex) - replace flume with async_channel
+impl<T: Clone> ShardConnector<T> {
+ pub fn new(id: u16, max_concurrent_thread_count: usize) -> Self {
+ let channel = Arc::new(ShardedChannel::new(max_concurrent_thread_count));
+ let (sender, receiver) = channel.unbounded();
+ let (stop_sender, stop_receiver) = flume::bounded(1);
+ Self {
+ id,
+ receiver,
+ sender,
+ stop_receiver,
+ stop_sender,
+ }
+ }
+
+ pub fn send(&self, data: T) {
+ self.sender.send(data);
+ }
+}
+
+#[derive(Clone)]
+pub struct Receiver<T> {
+ channel: Arc<ShardedChannel<T>>,
+}
+
+#[derive(Clone)]
+pub struct Sender<T> {
+ channel: Arc<ShardedChannel<T>>,
+}
+
+impl<T> Sender<T> {
+ pub fn send(&self, data: T) {
+ self.channel
+ .task_queue
+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ self.channel.queue.push_back(data);
+ self.channel.waker.wake();
+ }
+}
+
+pub struct ShardedChannel<T> {
+ queue: ShardedQueue<T>,
+ task_queue: AtomicUsize,
+ waker: AtomicWaker,
+}
+
+impl<T> ShardedChannel<T> {
+ pub fn new(max_concurrent_thread_count: usize) -> Self {
+ let waker = AtomicWaker::new();
+
+ Self {
+ queue: ShardedQueue::new(max_concurrent_thread_count),
+ task_queue: AtomicUsize::new(0),
+ waker,
+ }
+ }
+}
+
+pub trait ShardedChannelsSplit<T: Clone> {
+ fn unbounded(&self) -> (Sender<T>, Receiver<T>);
+
+ fn sender(&self) -> Sender<T>;
+}
+
+impl<T: Clone> ShardedChannelsSplit<T> for Arc<ShardedChannel<T>> {
+ fn unbounded(&self) -> (Sender<T>, Receiver<T>) {
+ let tx = self.sender();
+ let rx = Receiver {
+ channel: Arc::clone(self),
+ };
+
+ (tx, rx)
+ }
+
+ fn sender(&self) -> Sender<T> {
+ Sender {
+ channel: Arc::clone(self),
+ }
+ }
+}
+
+impl<T> Stream for Receiver<T> {
+ type Item = T;
+ fn poll_next(
+ self: std::pin::Pin<&mut Self>,
+ cx: &mut std::task::Context<'_>,
+ ) -> std::task::Poll<Option<Self::Item>> {
+ let old = self
+ .channel
+ .task_queue
+ .load(std::sync::atomic::Ordering::Relaxed);
+ if old == 0 {
+ self.channel.waker.register(cx.waker());
+ return Poll::Pending;
+ }
+
+ assert!(old > 0);
+ self.channel
+ .task_queue
+ .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
+ let item = self.channel.queue.pop_front_or_spin_wait_item();
+ Poll::Ready(Some(item))
+ }
+}
diff --git a/core/server/src/shard/frame.rs b/core/server/src/shard/frame.rs
new file mode 100644
index 00000000..b1524ad6
--- /dev/null
+++ b/core/server/src/shard/frame.rs
@@ -0,0 +1,68 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use bytes::Bytes;
+use flume::Sender;
+use iggy_common::IggyError;
+
+#[derive(Debug, Clone)]
+pub enum ShardMessage {
+ //TODO: Fixme
+ //Command(ServerCommand),
+ Event(ShardEvent),
+}
+
+#[derive(Debug, Clone)]
+pub enum ShardEvent {
+}
+
+#[derive(Debug)]
+pub enum ShardResponse {
+ BinaryResponse(Bytes),
+ ErrorResponse(IggyError),
+}
+
+#[derive(Debug, Clone)]
+pub struct ShardFrame {
+ pub client_id: u32,
+ pub message: ShardMessage,
+ pub response_sender: Option<Sender<ShardResponse>>,
+}
+
+impl ShardFrame {
+ pub fn new(
+ client_id: u32,
+ message: ShardMessage,
+ response_sender: Option<Sender<ShardResponse>>,
+ ) -> Self {
+ Self {
+ client_id,
+ message,
+ response_sender,
+ }
+ }
+}
+
+#[macro_export]
+macro_rules! handle_response {
+ ($sender:expr, $response:expr) => {
+ match $response {
+ ShardResponse::BinaryResponse(payload) => $sender.send_ok_response(&payload).await?,
+ ShardResponse::ErrorResponse(err) => $sender.send_error_response(err).await?,
+ }
+ };
+}
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
new file mode 100644
index 00000000..6ccbcab3
--- /dev/null
+++ b/core/server/src/shard/mod.rs
@@ -0,0 +1,59 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+mod connector;
+mod namespace;
+mod frame;
+
+
+
+use std::cell::RefCell;
+use ahash::HashMap;
+use connector::ShardConnector;
+use frame::ShardFrame;
+use namespace::IggyNamespace;
+struct Shard {
+ id: u16,
+ connection: ShardConnector<ShardFrame>,
+}
+
+struct ShardInfo {
+ id: u16,
+}
+
+pub struct IggyShard {
+ pub id: u16,
+ shards: Vec<Shard>,
+ shards_table: RefCell<HashMap<IggyNamespace, ShardInfo>>,
+
+ pub(crate) permissioner: RefCell<Permissioner>,
+ pub(crate) storage: Rc<SystemStorage>,
+ pub(crate) streams: RwLock<HashMap<u32, Stream>>,
+ pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
+ pub(crate) users: RefCell<HashMap<UserId, User>>,
+
+ // TODO - get rid of this dynamic dispatch.
+ pub(crate) state: Rc<FileState>,
+ pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
+ pub(crate) config: ServerConfig,
+ pub(crate) client_manager: RefCell<ClientManager>,
+ pub(crate) active_sessions: RefCell<Vec<Session>>,
+ pub(crate) metrics: Metrics,
+ pub message_receiver: Cell<Option<Receiver<ShardFrame>>>,
+ stop_receiver: StopReceiver,
+ stop_sender: StopSender,
+}
\ No newline at end of file
diff --git a/core/server/src/lib.rs b/core/server/src/shard/namespace.rs
similarity index 51%
copy from core/server/src/lib.rs
copy to core/server/src/shard/namespace.rs
index 91d17586..4b972125 100644
--- a/core/server/src/lib.rs
+++ b/core/server/src/shard/namespace.rs
@@ -15,39 +15,23 @@
* specific language governing permissions and limitations
* under the License.
*/
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct IggyNamespace {
+ pub(crate) stream_id: u32,
+ pub(crate) topic_id: u32,
+ pub(crate) partition_id: u32,
+}
-#[cfg(not(feature = "disable-mimalloc"))]
-use mimalloc::MiMalloc;
-
-#[cfg(not(feature = "disable-mimalloc"))]
-#[global_allocator]
-static GLOBAL: MiMalloc = MiMalloc;
-
-#[cfg(windows)]
-compile_error!("iggy-server doesn't support windows.");
-
-pub mod archiver;
-pub mod args;
-pub mod binary;
-pub mod channels;
-pub(crate) mod compat;
-pub mod configs;
-pub mod http;
-pub mod log;
-pub mod quic;
-pub mod server_error;
-pub mod state;
-pub mod streaming;
-pub mod tcp;
-pub mod versioning;
-
-const VERSION: &str = env!("CARGO_PKG_VERSION");
-const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME";
-const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD";
+impl IggyNamespace {
+ pub fn new(stream_id: u32, topic_id: u32, partition_id: u32) -> Self {
+ Self {
+ stream_id,
+ topic_id,
+ partition_id,
+ }
+ }
-pub(crate) fn map_toggle_str<'a>(enabled: bool) -> &'a str {
- match enabled {
- true => "enabled",
- false => "disabled",
+ pub fn generate_hash(&self) -> u32 {
+ todo!();
}
}
(iggy) 05/12: todos
Posted by gk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch io_uring_tpc
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit e53f18a05ea8eb87e460e6c72ec8b36c80710985
Author: numinex <g....@gmail.com>
AuthorDate: Thu Jun 19 10:17:42 2025 +0200
todos
---
Cargo.lock | 14 +++++++++++++
core/sdk/src/tcp/tcp_client.rs | 1 +
core/server/src/binary/command.rs | 1 +
core/server/src/bootstrap.rs | 27 ++++++++++++++++++-------
core/server/src/main.rs | 23 ++++++++--------------
core/server/src/shard/builder.rs | 40 +++++++++++++++++++++++++-------------
core/server/src/shard/connector.rs | 4 ++--
core/server/src/shard/mod.rs | 38 ++++++++++++++++++++++++++++--------
8 files changed, 103 insertions(+), 45 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 3a3711a5..9c594a2c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -143,6 +143,7 @@ dependencies = [
"futures-core",
"futures-util",
"mio 1.0.4",
+ "mio 1.0.4",
"socket2",
"tokio",
"tracing",
@@ -4696,6 +4697,18 @@ dependencies = [
"windows-sys 0.48.0",
]
+[[package]]
+name = "mio"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+dependencies = [
+ "libc",
+ "log",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "mio"
version = "1.0.4"
@@ -7426,6 +7439,7 @@ dependencies = [
"bytes",
"libc",
"mio 1.0.4",
+ "mio 1.0.4",
"parking_lot 0.12.4",
"pin-project-lite",
"signal-hook-registry",
diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs
index 692d7689..2bab2efb 100644
--- a/core/sdk/src/tcp/tcp_client.rs
+++ b/core/sdk/src/tcp/tcp_client.rs
@@ -17,6 +17,7 @@
*/
use crate::prelude::Client;
+use crate::prelude::IggyConsumer;
use crate::prelude::TcpClientConfig;
use crate::tcp::tcp_connection_stream::TcpConnectionStream;
use crate::tcp::tcp_connection_stream_kind::ConnectionStreamKind;
diff --git a/core/server/src/binary/command.rs b/core/server/src/binary/command.rs
index c7802be9..0b997941 100644
--- a/core/server/src/binary/command.rs
+++ b/core/server/src/binary/command.rs
@@ -66,6 +66,7 @@ use iggy_common::update_stream::UpdateStream;
use iggy_common::update_topic::UpdateTopic;
use iggy_common::update_user::UpdateUser;
use iggy_common::*;
+use rustls::crypto::hash::Output;
use strum::EnumString;
use tracing::error;
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index d51096a2..d70d0317 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -1,22 +1,19 @@
use iggy_common::{
+ IggyError,
defaults::{
DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH,
MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH,
},
- IggyError,
-};
-use monoio::{
- fs::create_dir_all,
- Buildable, Driver, Runtime,
};
+use monoio::{fs::create_dir_all, time::TimeDriver, Buildable, Driver, Runtime};
use tracing::info;
use crate::{
+ IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
configs::{config_provider::ConfigProviderKind, server::ServerConfig, system::SystemConfig},
server_error::ServerError,
shard::{connector::ShardConnector, frame::ShardFrame},
streaming::users::user::User,
- IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV,
};
use std::{env, fs::remove_dir_all, ops::Range, path::Path};
@@ -109,7 +106,9 @@ pub fn create_root_user() -> User {
let username = env::var(IGGY_ROOT_USERNAME_ENV);
let password = env::var(IGGY_ROOT_PASSWORD_ENV);
if (username.is_ok() && password.is_err()) || (username.is_err() && password.is_ok()) {
- panic!("When providing the custom root user credentials, both username and password must be set.");
+ panic!(
+ "When providing the custom root user credentials, both username and password must be set."
+ );
}
if username.is_ok() && password.is_ok() {
info!("Using the custom root user credentials.");
@@ -146,3 +145,17 @@ where
let rt = Buildable::build(builder).expect("Failed to create default runtime");
rt
}
+
+pub fn create_shard_executor() -> Runtime<TimeDriver<monoio::IoUringDriver>>
+{
+ // TODO: Figure out what else we could tweak there
+ // We for sure want to disable the userspace interrupts on new cq entry (set_coop_taskrun)
+ // let urb = io_uring::IoUring::builder();
+ // TODO: Shall we make the size of ring be configureable ?
+ let builder = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new()
+ //.uring_builder(urb.setup_coop_taskrun()) // broken shit.
+ .with_entries(1024) // Default size
+ .enable_timer();
+ let rt = Buildable::build(builder).expect("Failed to create default runtime");
+ rt
+}
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index bb4ea690..06ffec35 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -23,11 +23,9 @@ use clap::Parser;
use dotenvy::dotenv;
use error_set::ErrContext;
use figlet_rs::FIGfont;
-use monoio::Buildable;
use server::args::Args;
use server::bootstrap::{
- create_default_executor, create_directories, create_root_user, create_shard_connections,
- load_config,
+ create_default_executor, create_directories, create_root_user, create_shard_connections, create_shard_executor, load_config
};
use server::channels::commands::archive_state::ArchiveStateExecutor;
use server::channels::commands::clean_personal_access_tokens::CleanPersonalAccessTokensExecutor;
@@ -112,7 +110,7 @@ fn main() -> Result<(), ServerError> {
}
}
- // Create directories.
+ // Create directories.
create_directories(&config.system).await?;
Ok::<(), ServerError>(())
})
@@ -146,16 +144,7 @@ fn main() -> Result<(), ServerError> {
monoio::utils::bind_to_cpu_set(Some(shard_id))
.expect(format!("Failed to set CPU affinity for shard-{id}").as_str());
- // TODO: Figure out what else we could tweak there
- // We for sure want to disable the userspace interrupts on new cq entry (set_coop_taskrun)
- // let urb = io_uring::IoUring::builder();
- // TODO: Shall we make the size of ring be configureable ?
- let mut rt = monoio::RuntimeBuilder::<monoio::IoUringDriver>::new()
- //.uring_builder(urb.setup_coop_taskrun()) // broken shit.
- .with_entries(1024) // Default size
- .enable_timer()
- .build()
- .expect(format!("Failed to build monoio runtime for shard-{id}").as_str());
+ let mut rt = create_shard_executor();
rt.block_on(async move {
let builder = IggyShard::builder();
let mut shard = builder
@@ -165,7 +154,11 @@ fn main() -> Result<(), ServerError> {
.build()
.await;
- shard.init().await;
+ if let Err(e) = shard.init().await {
+ //TODO: If one of the shards fails to initialize, we should crash the whole program;
+ panic!("Failed to initialize shard-{id}: {e}");
+ }
+ //TODO: If one of the shards fails to initialize, we should crash the whole program;
shard.assert_init();
})
})
diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs
index 640ffb57..06366ebf 100644
--- a/core/server/src/shard/builder.rs
+++ b/core/server/src/shard/builder.rs
@@ -21,9 +21,19 @@ use std::{cell::Cell, rc::Rc, sync::Arc};
use iggy_common::{Aes256GcmEncryptor, EncryptorKind};
use tracing::info;
-use crate::{configs::server::ServerConfig, map_toggle_str, shard::Shard, state::{file::FileState, StateKind}, streaming::{persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind}, storage::SystemStorage}, versioning::SemanticVersion};
+use crate::{
+ configs::server::ServerConfig,
+ map_toggle_str,
+ shard::Shard,
+ state::{StateKind, file::FileState},
+ streaming::{
+ persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind},
+ storage::SystemStorage,
+ },
+ versioning::SemanticVersion,
+};
-use super::{connector::ShardConnector, frame::ShardFrame, IggyShard};
+use super::{IggyShard, connector::ShardConnector, frame::ShardFrame};
#[derive(Default)]
pub struct IggyShardBuilder {
@@ -66,8 +76,10 @@ impl IggyShardBuilder {
.next()
.expect("Failed to find connection with the specified ID");
let shards = connections.into_iter().map(Shard::new).collect();
+ //TODO: This can be discrete step in the builder bootstrapped from main function.
let version = SemanticVersion::current().expect("Invalid version");
+ //TODO: This can be discrete step in the builder bootstrapped from main function.
info!(
"Server-side encryption is {}.",
map_toggle_str(config.system.encryption.enabled)
@@ -79,6 +91,7 @@ impl IggyShardBuilder {
false => None,
};
+ //TODO: This can be discrete step in the builder bootstrapped from main function.
let state_persister = Self::resolve_persister(config.system.state.enforce_fsync);
let state = Rc::new(StateKind::File(FileState::new(
&config.system.get_state_messages_file_path(),
@@ -87,20 +100,21 @@ impl IggyShardBuilder {
encryptor.clone(),
)));
+ //TODO: This can be discrete step in the builder bootstrapped from main function.
let partition_persister = Self::resolve_persister(config.system.partition.enforce_fsync);
- let storage = SystemStorage::new(config.system, partition_persister);
+ let storage = Rc::new(SystemStorage::new(config.system.clone(), partition_persister));
IggyShard {
- id: id,
- shards: shards,
- shards_table: Default::default(),
- storage: storage,
- state: state,
- config: config,
- stop_receiver: stop_receiver,
- stop_sender: stop_sender,
- frame_receiver: Cell::new(Some(frame_receiver)),
- }
+ id: id,
+ shards: shards,
+ shards_table: Default::default(),
+ storage: storage,
+ state: state,
+ config: config,
+ stop_receiver: stop_receiver,
+ stop_sender: stop_sender,
+ frame_receiver: Cell::new(Some(frame_receiver)),
+ }
}
fn resolve_persister(enforce_fsync: bool) -> Arc<PersisterKind> {
diff --git a/core/server/src/shard/connector.rs b/core/server/src/shard/connector.rs
index 1d762d68..95aedc73 100644
--- a/core/server/src/shard/connector.rs
+++ b/core/server/src/shard/connector.rs
@@ -15,10 +15,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-use futures::{task::AtomicWaker, Stream};
+use futures::{Stream, task::AtomicWaker};
use sharded_queue::ShardedQueue;
use std::{
- sync::{atomic::AtomicUsize, Arc},
+ sync::{Arc, atomic::AtomicUsize},
task::Poll,
};
diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs
index 54b88fb1..b98f6381 100644
--- a/core/server/src/shard/mod.rs
+++ b/core/server/src/shard/mod.rs
@@ -25,10 +25,19 @@ use ahash::HashMap;
use builder::IggyShardBuilder;
use connector::{Receiver, ShardConnector, StopReceiver, StopSender};
use frame::ShardFrame;
+use iggy_common::IggyError;
use namespace::IggyNamespace;
-use std::{cell::{Cell, RefCell}, rc::Rc, sync::Arc};
+use tracing::info;
+use std::{
+ cell::{Cell, RefCell},
+ rc::Rc,
+ sync::Arc, time::Instant,
+};
-use crate::{bootstrap::create_root_user, configs::server::ServerConfig, state::file::FileState, streaming::storage::SystemStorage};
+use crate::{
+ bootstrap::create_root_user, configs::server::ServerConfig, state::{file::FileState, StateKind},
+ streaming::storage::SystemStorage,
+};
pub(crate) struct Shard {
id: u16,
connection: ShardConnector<ShardFrame>,
@@ -57,10 +66,10 @@ pub struct IggyShard {
//pub(crate) streams_ids: RefCell<HashMap<String, u32>>,
//pub(crate) users: RefCell<HashMap<UserId, User>>,
// TODO: Refactor.
- pub(crate) storage: Arc<SystemStorage>,
+ pub(crate) storage: Rc<SystemStorage>,
// TODO - get rid of this dynamic dispatch.
- pub(crate) state: Rc<FileState>,
+ pub(crate) state: Rc<StateKind>,
//pub(crate) encryptor: Option<Rc<dyn Encryptor>>,
config: ServerConfig,
//pub(crate) client_manager: RefCell<ClientManager>,
@@ -76,14 +85,27 @@ impl IggyShard {
Default::default()
}
- pub async fn init(&mut self) {
- let user = create_root_user();
+ pub async fn init(&mut self) -> Result<(), IggyError> {
+ let now = Instant::now();
+ //TODO: Fix this either by moving it to main function, or by using `run_once` barrier.
+ //let state_entries = self.state.init().await?;
+ //let system_state = SystemState::init(state_entries).await?;
+ //let user = create_root_user();
self.load_state().await;
self.load_users().await;
// Add default root user.
- todo!();
self.load_streams().await;
-
+ //TODO: Fix the archiver.
+ /*
+ if let Some(archiver) = self.archiver.as_ref() {
+ archiver
+ .init()
+ .await
+ .expect("Failed to initialize archiver");
+ }
+ */
+ info!("Initialized system in {} ms.", now.elapsed().as_millis());
+ Ok(())
}
async fn load_state(&self) {