You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by pi...@apache.org on 2026/10/22 21:52:32 UTC
(iggy) branch resource_options updated (9a619dafc -> dd2e433cc)
This is an automated email from the ASF dual-hosted git repository.
spetz pushed a change to branch resource_options
in repository https://gitbox.apache.org/repos/asf/iggy.git
from 9a619dafc Merge branch 'master' into resource_options
add 786382afe feat(cluster)!: spread and rebalance partition primaries (#4619)
new dd2e433cc merge and fix
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.dockerignore | 2 +
.github/config/components.yml | 1 +
Cargo.lock | 3 +
bdd/README.md | 5 +-
.../step_definitions/session_contract_steps.cpp | 93 +
bdd/cpp/features/step_definitions/world.hpp | 1 +
bdd/cpp/scripts/entrypoint.sh | 1 +
bdd/docker-compose.coverage.yml | 4 +-
bdd/docker-compose.no-quorum.yml | 134 +
bdd/docker-compose.yml | 35 +-
bdd/go/tests/env/env.go | 5 +
bdd/go/tests/session_contract.go | 139 +
bdd/go/tests/suite_test.go | 11 +
bdd/java/build.gradle.kts | 3 +
.../org/apache/iggy/bdd/BasicMessagingSteps.java | 45 +
.../test/java/org/apache/iggy/bdd/TestContext.java | 1 +
.../java/org/apache/iggy/bdd/TestEnvironment.java | 4 +
bdd/node/Dockerfile | 1 +
...tureTest.php => SessionContractFeatureTest.php} | 67 +-
bdd/php/tests/SharedFeatureParser.php | 4 +-
bdd/php/tests/TestEnvironment.php | 5 +
bdd/python/tests/conftest.py | 8 +
...est_raw_command.py => test_session_contract.py} | 79 +-
bdd/rust/Cargo.toml | 5 +
bdd/rust/tests/common/global_context.rs | 1 +
bdd/rust/tests/helpers/env.rs | 4 +
.../{stream_options.rs => session_contract.rs} | 2 +-
bdd/rust/tests/steps/mod.rs | 1 +
bdd/rust/tests/steps/session.rs | 65 +
...aw_command.feature => session_contract.feature} | 31 +-
core/ai/mcp/src/service/mod.rs | 11 +-
.../actors/producer/typed_benchmark_producer.rs | 2 +-
.../typed_benchmark_producing_consumer.rs | 2 +-
core/binary_protocol/src/codes.rs | 3 +-
core/binary_protocol/src/consensus/command.rs | 10 +-
core/binary_protocol/src/consensus/header.rs | 39 +
core/binary_protocol/src/consensus/mod.rs | 4 +
.../src/consensus/primary_rebalance.rs | 375 +
.../src/consensus/segment_delete.rs | 193 +
core/binary_protocol/src/lib.rs | 9 +-
core/binary_protocol/src/primitives/consumer.rs | 32 +-
core/binary_protocol/src/primitives/identifier.rs | 87 +-
.../src/primitives/partition_assignment.rs | 24 +-
.../src/requests/messages/poll_messages.rs | 114 +
.../create_partitions_with_assignments.rs | 114 +-
.../topics/create_topic_with_assignments.rs | 95 +-
core/common/src/consumer_group_client_state.rs | 198 +-
core/common/src/error/iggy_error.rs | 69 +-
core/common/src/lib.rs | 5 +-
core/common/src/traits/binary_impls/messages.rs | 251 +-
core/common/src/traits/binary_impls/mod.rs | 41 +-
core/common/src/traits/binary_impls/partitions.rs | 2 -
.../traits/binary_impls/personal_access_tokens.rs | 30 +-
core/common/src/traits/binary_impls/streams.rs | 1 -
core/common/src/traits/binary_impls/topics.rs | 6 +-
core/common/src/traits/binary_impls/users.rs | 53 +-
core/common/src/traits/binary_transport.rs | 291 +-
core/common/src/traits/mod.rs | 2 +-
core/common/src/types/identifier/mod.rs | 84 +-
core/common/src/wire_conversions.rs | 29 +-
core/configs/src/server_config/cluster.rs | 65 +-
core/configs/src/server_config/defaults.rs | 2 +
core/configs/src/server_config/displays.rs | 7 +-
core/configs/src/server_config/message_bus.rs | 69 +
core/configs/src/server_config/metadata.rs | 5 +-
core/connectors/runtime/src/main.rs | 2 -
core/connectors/runtime/src/manager/source.rs | 4 +-
core/connectors/runtime/src/source.rs | 151 +-
core/consensus/src/client_table.rs | 89 +-
core/consensus/src/dvc_merge.rs | 65 +-
core/consensus/src/impls.rs | 2289 ++++-
core/consensus/src/lib.rs | 2 +-
core/consensus/src/metadata_helpers.rs | 49 +-
core/consensus/src/observability.rs | 14 +-
core/consensus/src/plane_helpers.rs | 508 +-
core/consensus/src/view_change_quorum.rs | 3 +-
core/consensus/src/vsr_state.rs | 638 +-
core/integration/Cargo.toml | 3 +
core/integration/src/harness/handle/server.rs | 42 +-
core/integration/src/harness/helpers.rs | 75 +-
core/integration/src/harness/mod.rs | 4 +-
.../src/harness/orchestrator/harness.rs | 63 +-
core/integration/tests/cli/common/mod.rs | 7 +
.../tests/cli/message/test_message_send_command.rs | 24 +-
.../tests/cli/topic/test_topic_get_command.rs | 42 +-
.../tests/cluster/client_table_restart.rs | 362 +-
.../tests/cluster/crash_recovery_corruption.rs | 178 +
.../tests/cluster/parked_frame_redispatch.rs | 21 +-
core/integration/tests/cluster/partition_dedup.rs | 158 +
.../tests/cluster/partition_primary_routing.rs | 1949 +++-
.../tests/cluster/staggered_bootstrap.rs | 13 +-
.../verify_consumer_group_partition_assignment.rs | 77 +-
core/integration/tests/sdk/options.rs | 16 +-
core/integration/tests/server/general.rs | 6 +
core/integration/tests/server/http_vsr.rs | 89 +-
.../server/scenarios/log_rotation_scenario.rs | 5 +-
.../scenarios/segment_rotation_race_scenario.rs | 8 +-
.../tests/server/shared_jwt_secrets.toml | 8 +-
core/journal/src/prepare_journal.rs | 57 +
core/journal/src/superblock.rs | 49 +-
core/message_bus/src/config.rs | 5 +
core/message_bus/src/error.rs | 34 +-
core/message_bus/src/installer/mod.rs | 55 +-
core/message_bus/src/installer/quic.rs | 8 +-
core/message_bus/src/installer/tcp.rs | 14 +-
core/message_bus/src/installer/ws.rs | 11 +-
core/message_bus/src/lib.rs | 382 +-
.../src/lifecycle/connection_registry.rs | 368 +-
core/message_bus/src/transports/mod.rs | 2 +-
core/message_bus/src/transports/quic.rs | 229 +-
core/message_bus/tests/duplicate_client_id.rs | 51 +
core/message_bus/tests/quic_client_roundtrip.rs | 18 +-
core/message_bus/tests/tcp_client_roundtrip.rs | 258 +-
core/message_bus/tests/tcp_tls_client_listener.rs | 20 +-
core/message_bus/tests/tcp_tls_client_roundtrip.rs | 15 +-
core/message_bus/tests/ws_client_roundtrip.rs | 25 +-
core/message_bus/tests/wss_client_listener.rs | 15 +-
core/message_bus/tests/wss_client_roundtrip.rs | 15 +-
core/metadata/src/applied_frontier.rs | 132 +-
core/metadata/src/impls/metadata.rs | 3505 +++++++-
core/metadata/src/impls/mod.rs | 1 +
core/metadata/src/impls/placement.rs | 949 ++
core/metadata/src/impls/recovery.rs | 348 +-
core/metadata/src/stm/authz.rs | 2 +-
core/metadata/src/stm/consumer_group.rs | 4 +-
core/metadata/src/stm/lifecycle.rs | 2 +-
core/metadata/src/stm/snapshot.rs | 17 +-
core/metadata/src/stm/stream.rs | 305 +-
core/metadata/src/stm/user.rs | 5 +-
core/partitions/src/iggy_partition.rs | 1902 +++-
core/partitions/src/iggy_partitions.rs | 88 +-
core/partitions/src/io.rs | 27 +-
core/partitions/src/lib.rs | 15 +-
core/partitions/src/persistence.rs | 47 +-
core/partitions/src/poll_plan.rs | 17 +-
core/partitions/src/segment_recovery.rs | 99 +-
core/partitions/src/state_transfer.rs | 12 +-
core/partitions/src/types.rs | 11 +
core/partitions/src/wal_storage.rs | 442 +
core/partitions/tests/consumer_offset_history.rs | 4 +-
core/sdk/README.md | 46 +-
core/sdk/src/binary/mod.rs | 77 +-
core/sdk/src/client_wrappers/client_wrapper.rs | 38 +-
core/sdk/src/clients/client.rs | 89 +-
core/sdk/src/clients/consumer.rs | 355 +-
core/sdk/src/clients/producer.rs | 753 +-
core/sdk/src/clients/producer_dispatcher.rs | 15 +-
core/sdk/src/clients/producer_sharding.rs | 26 +-
core/sdk/src/consumer_ext/consumer_message_ext.rs | 49 +-
core/sdk/src/driver.rs | 1178 ++-
core/sdk/src/driver/route_cache.rs | 298 +-
core/sdk/src/http/http_client.rs | 161 +-
core/sdk/src/http/messages.rs | 12 +-
core/sdk/src/http/scripted_server.rs | 24 +-
core/sdk/src/links/fake_cluster.rs | 139 +-
core/sdk/src/links/mux.rs | 148 +-
core/sdk/src/links/replicas.rs | 227 +-
core/sdk/src/session_owner.rs | 237 +-
core/sdk/src/session_owner/lanes.rs | 54 +-
core/sdk/src/tcp/tcp_client.rs | 737 +-
core/sdk/src/websocket/websocket_client.rs | 15 +
core/server/Cargo.toml | 2 +
core/server/README.md | 203 +-
core/server/config.toml | 42 +-
core/server/src/boot/mod.rs | 34 +-
core/server/src/boot/recovery.rs | 26 +-
core/server/src/boot/threads.rs | 2 +-
core/server/src/consumer_group.rs | 2 +-
core/server/src/consumer_group/liveness.rs | 156 +-
core/server/src/dispatch/authz.rs | 52 +-
core/server/src/dispatch/failure.rs | 287 +-
core/server/src/dispatch/host.rs | 194 +-
core/server/src/dispatch/mod.rs | 1531 +++-
core/server/src/dispatch/partition.rs | 1434 ++-
core/server/src/dispatch/reads.rs | 134 +-
core/server/src/dispatch/segment_delete.rs | 201 +
core/server/src/dispatch/session_ops.rs | 97 +-
core/server/src/dispatch/submit.rs | 14 +-
core/server/src/dispatch/test_support.rs | 188 +-
core/server/src/http.rs | 44 +-
core/server/src/http/admission.rs | 139 +-
core/server/src/http/error.rs | 135 +-
core/server/src/http/extractor.rs | 239 +-
core/server/src/http/forward.rs | 1940 +++-
core/server/src/http/handlers.rs | 460 +-
core/server/src/http/metrics.rs | 37 +
core/server/src/http/reads.rs | 361 +-
core/server/src/http/relay_proof.rs | 356 +
core/server/src/http/reply.rs | 9 +
core/server/src/http/state.rs | 126 +-
core/server/src/http/submit.rs | 145 +-
core/server/src/lib.rs | 5 +-
core/server/src/namespace.rs | 102 +-
core/server/src/partition_helpers.rs | 49 +-
core/server/src/partition_reconciler.rs | 319 +-
core/server/src/rewrite.rs | 4 +-
core/server/src/session_manager.rs | 175 +-
core/server_common/src/consensus_message.rs | 151 +-
core/shard/src/builder.rs | 39 +-
core/shard/src/lib.rs | 2876 +++++-
core/shard/src/metrics.rs | 147 +-
core/shard/src/partition_io.rs | 2875 +++++-
core/shard/src/poll.rs | 20 +-
core/shard/src/poll/completion_tests.rs | 71 +-
core/shard/src/poll/timeout_tests.rs | 4 +-
core/shard/src/primary_rebalance.rs | 845 ++
core/shard/src/primary_rebalance/tests.rs | 1191 +++
core/shard/src/router.rs | 203 +-
core/simulator/Cargo.toml | 2 +-
core/simulator/README.md | 23 +-
core/simulator/src/bin/workload-fuzz.rs | 197 +-
core/simulator/src/bus.rs | 140 +-
core/simulator/src/client.rs | 67 +-
core/simulator/src/deps.rs | 15 +
core/simulator/src/executor/time.rs | 17 +-
core/simulator/src/lib.rs | 9368 ++++++++++++++------
core/simulator/src/packet.rs | 33 +-
core/simulator/src/partition_wal.rs | 119 +
core/simulator/src/replica.rs | 107 +-
core/simulator/src/workload/actions.rs | 1 +
core/simulator/src/workload/invariants.rs | 57 +-
core/simulator/src/workload/mod.rs | 526 +-
core/simulator/src/workload/ops/mod.rs | 2 +
.../{store_consumer_offset.rs => poll_messages.rs} | 72 +-
.../src/workload/ops/store_stale_owner_offset.rs | 4 +-
core/simulator/src/workload/options.rs | 17 +-
core/simulator/src/workload/oracle.rs | 497 +-
core/simulator/src/workload/readback.rs | 420 +
core/simulator/src/workload/state_checker.rs | 288 +-
core/simulator/tests/lifecycle_seeds.rs | 127 +
core/system_stats/src/lib.rs | 9 +-
foreign/cpp/BUILD.bazel | 1 +
foreign/cpp/README.md | 15 +
foreign/cpp/include/iggy.hpp | 24 +-
foreign/cpp/src/client.rs | 101 +-
foreign/cpp/tests/unit/unit_tests.cpp | 21 +
.../Iggy_SDK.Tests.BDD/Context/TestContext.cs | 2 +
.../Iggy_SDK.Tests.BDD/Context/TestEnvironment.cs | 1 +
.../csharp/Iggy_SDK.Tests.BDD/Context/TestHooks.cs | 1 +
.../Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj | 3 +
.../StepDefinitions/SessionContractSteps.cs | 89 +
.../Fixtures/IggyServerFixture.cs | 16 +
.../IggyConsumerTests.cs | 10 +-
.../IggyTypedConsumerTests.cs | 2 +
.../Iggy_SDK.Tests.Integration/StreamsTests.cs | 4 +-
.../Iggy_SDK.Tests.Integration/TopicsTests.cs | 5 +-
.../csharp/Iggy_SDK/Consumers/AutoCommitMode.cs | 4 +-
.../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 37 +-
foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 313 +-
.../csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs | 23 +-
.../Exceptions/IggyInvalidStatusCodeException.cs | 3 +-
.../csharp/Iggy_SDK/Factory/IggyClientFactory.cs | 11 +-
.../TcpMessageStream.PollRouting.cs | 241 +-
.../TcpMessageStream.ReplicaLinks.cs | 109 +-
.../Implementations/TcpMessageStream.Vsr.cs | 327 +-
.../IggyClient/Implementations/TcpMessageStream.cs | 209 +-
foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs | 2 +-
foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs | 126 +-
foreign/csharp/Iggy_SDK/Vsr/MutationLanes.cs | 437 +-
foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs | 37 +-
foreign/csharp/Iggy_SDK/Vsr/VsrError.cs | 6 +-
foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs | 11 +-
.../ClientTests/IggyClientFactoryTests.cs | 23 +
.../PartitionIncarnationTests.Disposal.cs | 287 +
.../ConsumerTests/PartitionIncarnationTests.cs | 2 +-
.../ConsumerTests/PollFailureTests.cs | 457 +
.../VsrTests/ConsensusSessionTests.cs | 17 +-
.../VsrTests/EndpointFailoverTests.cs | 69 +-
foreign/csharp/Iggy_SDK_Tests/VsrTests/MockNode.cs | 12 +-
.../Iggy_SDK_Tests/VsrTests/MutationLaneTests.cs | 302 +-
.../VsrTests/NativeRecoveryTests.SignIn.cs | 710 ++
.../Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.cs | 678 +-
.../Iggy_SDK_Tests/VsrTests/PrimaryPollingTests.cs | 134 +-
.../Iggy_SDK_Tests/VsrTests/ReplicaLinkTests.cs | 177 +-
.../VsrTests/VsrMultiplexingTests.cs | 6 +-
foreign/csharp/README.md | 55 +-
foreign/go/README.md | 54 +-
foreign/go/client/iggy_client.go | 11 +-
foreign/go/client/iggy_client_test.go | 48 +
foreign/go/client/tcp/cluster.go | 55 +-
foreign/go/client/tcp/tcp_connect_test.go | 536 +-
foreign/go/client/tcp/tcp_core.go | 98 +-
foreign/go/client/tcp/tcp_core_review_test.go | 21 +-
foreign/go/client/tcp/tcp_core_test.go | 13 +-
foreign/go/client/tcp/tcp_driver.go | 158 +-
foreign/go/client/tcp/tcp_driver_test.go | 175 +-
foreign/go/client/tcp/tcp_failover_test.go | 117 +-
foreign/go/client/tcp/tcp_group_polling.go | 79 +-
foreign/go/client/tcp/tcp_group_polling_test.go | 48 +-
foreign/go/client/tcp/tcp_link.go | 36 +-
foreign/go/client/tcp/tcp_link_test.go | 275 +
foreign/go/client/tcp/tcp_messaging.go | 12 +-
foreign/go/client/tcp/tcp_offset_management.go | 9 +-
foreign/go/client/tcp/tcp_pending.go | 335 +-
foreign/go/client/tcp/tcp_poll_routing.go | 273 +-
foreign/go/client/tcp/tcp_poll_routing_test.go | 895 +-
foreign/go/client/tcp/tcp_replicas.go | 42 +-
foreign/go/client/tcp/tcp_replicas_test.go | 721 +-
.../go/client/tcp/tcp_session_credentials_test.go | 160 +-
foreign/go/client/tcp/tcp_session_management.go | 456 +-
foreign/go/client/tcp/tcp_testing_test.go | 19 +-
foreign/go/client/tcp/tcp_topic_cache_test.go | 14 +-
foreign/go/client/tcp/tcp_user_management.go | 17 +-
foreign/go/contracts/client.go | 2 +-
foreign/go/errors/errors.yaml | 8 +-
foreign/go/errors/errors_gen.go | 27 +-
foreign/go/errors/errors_test.go | 6 +-
foreign/go/internal/command/code.go | 2 +-
foreign/go/internal/vsr/reply.go | 46 +-
foreign/go/internal/vsr/reply_test.go | 117 +-
foreign/go/internal/vsr/session.go | 18 +-
foreign/go/internal/vsr/session_test.go | 36 +-
foreign/go/tests/e2e_helpers_test.go | 43 +
foreign/go/tests/e2e_test.go | 31 +-
foreign/java/README.md | 29 +-
.../iggy/client/async/tcp/AsyncIggyTcpClient.java | 766 +-
.../async/tcp/AsyncIggyTcpClientBuilder.java | 20 +-
.../iggy/client/async/tcp/AsyncTcpConnection.java | 1001 +--
.../iggy/client/async/tcp/ConnectionHeartbeat.java | 132 +
.../client/async/tcp/ConsumerGroupsTcpClient.java | 11 +-
.../client/async/tcp/ConsumerOffsetsTcpClient.java | 18 +-
.../async/tcp/CredentialsListener.java} | 9 +-
.../iggy/client/async/tcp/IggyAuthenticator.java | 78 +
.../iggy/client/async/tcp/LeaderAwareness.java | 23 -
.../iggy/client/async/tcp/LoginRoutingHook.java | 8 +-
.../iggy/client/async/tcp/MessagesTcpClient.java | 135 +-
.../apache/iggy/client/async/tcp/PollRouter.java | 448 +-
.../iggy/client/async/tcp/PoolChannelHandler.java | 93 +
.../client/async/tcp/RawPartitionRequests.java | 2 +-
.../apache/iggy/client/async/tcp/SessionLogin.java | 155 +
.../tcp/TransientFailoverHandler.java} | 27 +-
.../iggy/client/async/tcp/UsersTcpClient.java | 20 +-
.../async/tcp/VsrExchange.java} | 39 +-
.../client/async/tcp/vsr/ConsensusSession.java | 356 +-
.../client/async/tcp/vsr/VsrResponseHandler.java | 60 +-
.../java/org/apache/iggy/serde/CommandCode.java | 2 +-
.../java/org/apache/iggy/ProtocolParityTest.java | 217 +
...nnectionExceptionTest.java => RustSources.java} | 32 +-
.../client/async/AsyncClientIntegrationTest.java | 157 +
.../client/async/AsyncConnectionPoolAuthTest.java | 22 +-
.../async/tcp/AsyncIggyTcpClientBuilderTest.java | 9 +
.../AsyncIggyTcpClientCredentialChangeTest.java | 293 +
.../AsyncIggyTcpClientEndpointFailoverTest.java | 695 +-
.../tcp/AsyncIggyTcpClientFailoverSignInTest.java | 406 +
.../async/tcp/AsyncIggyTcpClientLockOrderTest.java | 183 +
.../tcp/AsyncIggyTcpClientLoginRoutingTest.java | 93 +-
.../tcp/AsyncIggyTcpClientLostLinkReplayTest.java | 295 +
.../tcp/AsyncIggyTcpClientNamespaceMissTest.java | 508 ++
.../tcp/AsyncIggyTcpClientPartitionWriteTest.java | 118 +
.../async/tcp/AsyncIggyTcpClientRecoveryTest.java | 397 +
.../tcp/AsyncIggyTcpClientSessionRulesTest.java | 1225 +++
.../AsyncIggyTcpClientTransientFailoverTest.java | 85 +-
.../tcp/AsyncTcpConnectionConcurrencyTest.java | 90 +
.../tcp/AsyncTcpConnectionLifecycleBusyTest.java | 112 +-
.../tcp/AsyncTcpConnectionRequestTimeoutTest.java | 61 +
.../iggy/client/async/tcp/LeaderAwarenessTest.java | 30 -
.../iggy/client/async/tcp/PollRouterTest.java | 862 +-
.../client/async/tcp/vsr/ConsensusSessionTest.java | 68 +
.../client/async/tcp/vsr/VsrLoginCodecTest.java | 11 +-
.../async/tcp/vsr/VsrResponseHandlerTest.java | 11 +
foreign/node/README.md | 88 +-
foreign/node/src/bdd/README.md | 1 +
foreign/node/src/bdd/env.ts | 10 +-
foreign/node/src/bdd/session.ts | 63 +
foreign/node/src/bdd/world.ts | 3 +-
foreign/node/src/client/client.connection.test.ts | 37 +-
foreign/node/src/client/client.connection.ts | 28 +-
foreign/node/src/client/client.operation.test.ts | 25 +-
foreign/node/src/client/client.operation.ts | 19 +-
foreign/node/src/client/client.socket.test.ts | 1635 +++-
foreign/node/src/client/client.socket.ts | 496 +-
foreign/node/src/client/client.type.ts | 15 +-
foreign/node/src/client/index.ts | 1 +
foreign/node/src/stream/consumer-stream.ts | 69 +-
foreign/node/src/wire/command.code.ts | 2 +-
foreign/node/src/wire/error.code.ts | 2 +-
.../src/wire/message/poll-messages.command.test.ts | 73 +-
.../node/src/wire/message/poll-messages.command.ts | 42 +-
.../node/src/wire/message/send-messages.command.ts | 15 +-
.../src/wire/session/login-with-token.command.ts | 10 +-
foreign/node/src/wire/session/login.command.ts | 10 +-
foreign/node/src/wire/vsr/session.test.ts | 8 +-
foreign/node/src/wire/vsr/session.ts | 5 -
foreign/php/README.md | 8 +
foreign/php/src/client.rs | 2 -
foreign/python/README.md | 40 +-
foreign/python/apache_iggy.pyi | 33 +-
foreign/python/src/bin/stub_gen.rs | 2 +
foreign/python/src/client.rs | 97 +-
foreign/python/src/identifier.rs | 11 +-
foreign/python/src/lib.rs | 3 +-
foreign/python/src/producer.rs | 80 +-
foreign/python/tests/test_quic_config.py | 7 +
foreign/python/tests/test_raw_command.py | 27 +-
foreign/swift/README.md | 70 +-
.../Sources/Iggy/Client/ClientConfiguration.swift | 8 +-
foreign/swift/Sources/Iggy/Client/ClientCore.swift | 663 +-
.../Sources/Iggy/Client/IggyClient+Messages.swift | 106 +-
.../Sources/Iggy/Client/IggyClient+Resources.swift | 132 +-
foreign/swift/Sources/Iggy/Client/IggyClient.swift | 101 +-
foreign/swift/Sources/Iggy/Client/PollRouter.swift | 620 +-
foreign/swift/Sources/Iggy/Errors/IggyError.swift | 18 +-
.../swift/Sources/Iggy/Protocol/CommandCode.swift | 2 +-
foreign/swift/Sources/Iggy/Protocol/Requests.swift | 26 +
.../swift/Sources/Iggy/Protocol/Responses.swift | 8 +-
foreign/swift/Sources/Iggy/Protocol/VSRFrame.swift | 6 +-
.../swift/Tests/IggyE2ETests/ClientE2ETests.swift | 17 +-
.../Tests/IggyTests/ClientTransportTests.swift | 1402 ++-
foreign/swift/Tests/IggyTests/MessagingTests.swift | 1288 ++-
foreign/swift/Tests/IggyTests/ProtocolTests.swift | 6 +-
.../Tests/IggyTests/Support/FakeVSRServer.swift | 121 +-
gateways/kafka/README.md | 6 +-
gateways/kafka/src/bridge/error.rs | 19 +-
gateways/kafka/src/bridge/iggy_bridge/produce.rs | 19 +-
.../kafka/tests/offset_groups_real_bridge_tests.rs | 3 +-
scripts/ci/apt-install.sh | 72 +-
scripts/run-bdd-tests.sh | 11 +-
417 files changed, 69821 insertions(+), 11480 deletions(-)
create mode 100644 bdd/cpp/features/step_definitions/session_contract_steps.cpp
create mode 100644 bdd/docker-compose.no-quorum.yml
create mode 100644 bdd/go/tests/session_contract.go
copy bdd/php/tests/{RawCommandFeatureTest.php => SessionContractFeatureTest.php} (56%)
copy bdd/python/tests/{test_raw_command.py => test_session_contract.py} (50%)
copy bdd/rust/tests/{stream_options.rs => session_contract.rs} (93%)
create mode 100644 bdd/rust/tests/steps/session.rs
copy bdd/scenarios/{raw_command.feature => session_contract.feature} (52%)
create mode 100644 core/binary_protocol/src/consensus/primary_rebalance.rs
create mode 100644 core/binary_protocol/src/consensus/segment_delete.rs
copy .claude/skills/connector-review/agents/openai.yaml => core/integration/tests/server/shared_jwt_secrets.toml (72%)
create mode 100644 core/metadata/src/impls/placement.rs
create mode 100644 core/partitions/src/wal_storage.rs
create mode 100644 core/server/src/dispatch/segment_delete.rs
create mode 100644 core/server/src/http/relay_proof.rs
create mode 100644 core/shard/src/primary_rebalance.rs
create mode 100644 core/shard/src/primary_rebalance/tests.rs
create mode 100644 core/simulator/src/partition_wal.rs
copy core/simulator/src/workload/ops/{store_consumer_offset.rs => poll_messages.rs} (53%)
create mode 100644 core/simulator/src/workload/readback.rs
create mode 100644 core/simulator/tests/lifecycle_seeds.rs
create mode 100644 foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/SessionContractSteps.cs
create mode 100644 foreign/csharp/Iggy_SDK_Tests/ConsumerTests/PartitionIncarnationTests.Disposal.cs
create mode 100644 foreign/csharp/Iggy_SDK_Tests/ConsumerTests/PollFailureTests.cs
create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.SignIn.cs
create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConnectionHeartbeat.java
copy foreign/java/java-sdk/src/main/java/org/apache/iggy/{personalaccesstoken/PersonalAccessTokenInfo.java => client/async/tcp/CredentialsListener.java} (78%)
create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PoolChannelHandler.java
create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/SessionLogin.java
copy foreign/java/java-sdk/src/main/java/org/apache/iggy/client/{ConnectionInfo.java => async/tcp/TransientFailoverHandler.java} (60%)
copy foreign/java/java-sdk/src/main/java/org/apache/iggy/{message/MessageId.java => client/async/tcp/VsrExchange.java} (57%)
create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/ProtocolParityTest.java
copy foreign/java/java-sdk/src/test/java/org/apache/iggy/{exception/IggyConnectionExceptionTest.java => RustSources.java} (52%)
create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientCredentialChangeTest.java
create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientFailoverSignInTest.java
create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLockOrderTest.java
create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLostLinkReplayTest.java
create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientNamespaceMissTest.java
create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientPartitionWriteTest.java
create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientSessionRulesTest.java
create mode 100644 foreign/node/src/bdd/session.ts
(iggy) 01/01: merge and fix
Posted by pi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spetz pushed a commit to branch resource_options
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit dd2e433cc4d372b8a743e8462f2ba3c91027575f
Merge: 9a619dafc 786382afe
Author: spetz <pi...@gmail.com>
AuthorDate: Thu Oct 22 23:52:23 2026 +0200
merge and fix
.dockerignore | 2 +
.github/config/components.yml | 1 +
Cargo.lock | 3 +
bdd/README.md | 5 +-
.../step_definitions/session_contract_steps.cpp | 93 +
bdd/cpp/features/step_definitions/world.hpp | 1 +
bdd/cpp/scripts/entrypoint.sh | 1 +
bdd/docker-compose.coverage.yml | 4 +-
bdd/docker-compose.no-quorum.yml | 134 +
bdd/docker-compose.yml | 35 +-
bdd/go/tests/env/env.go | 5 +
bdd/go/tests/session_contract.go | 139 +
bdd/go/tests/suite_test.go | 11 +
bdd/java/build.gradle.kts | 3 +
.../org/apache/iggy/bdd/BasicMessagingSteps.java | 45 +
.../test/java/org/apache/iggy/bdd/TestContext.java | 1 +
.../java/org/apache/iggy/bdd/TestEnvironment.java | 4 +
bdd/node/Dockerfile | 1 +
bdd/php/tests/SessionContractFeatureTest.php | 118 +
bdd/php/tests/SharedFeatureParser.php | 4 +-
bdd/php/tests/TestEnvironment.php | 5 +
bdd/python/tests/conftest.py | 8 +
bdd/python/tests/test_session_contract.py | 101 +
bdd/rust/Cargo.toml | 5 +
bdd/rust/tests/common/global_context.rs | 1 +
bdd/rust/tests/helpers/env.rs | 4 +
.../index.ts => bdd/rust/tests/session_contract.rs | 20 +-
bdd/rust/tests/steps/mod.rs | 1 +
bdd/rust/tests/steps/session.rs | 65 +
.../session_contract.feature} | 26 +-
core/ai/mcp/src/service/mod.rs | 11 +-
.../actors/producer/typed_benchmark_producer.rs | 2 +-
.../typed_benchmark_producing_consumer.rs | 2 +-
core/binary_protocol/src/codes.rs | 3 +-
core/binary_protocol/src/consensus/command.rs | 10 +-
core/binary_protocol/src/consensus/header.rs | 39 +
core/binary_protocol/src/consensus/mod.rs | 4 +
.../src/consensus/primary_rebalance.rs | 375 +
.../src/consensus/segment_delete.rs | 193 +
core/binary_protocol/src/lib.rs | 9 +-
core/binary_protocol/src/primitives/consumer.rs | 32 +-
core/binary_protocol/src/primitives/identifier.rs | 87 +-
.../src/primitives/partition_assignment.rs | 24 +-
.../src/requests/messages/poll_messages.rs | 114 +
.../create_partitions_with_assignments.rs | 114 +-
.../topics/create_topic_with_assignments.rs | 95 +-
core/common/src/consumer_group_client_state.rs | 198 +-
core/common/src/error/iggy_error.rs | 69 +-
core/common/src/lib.rs | 5 +-
core/common/src/traits/binary_impls/messages.rs | 251 +-
core/common/src/traits/binary_impls/mod.rs | 41 +-
core/common/src/traits/binary_impls/partitions.rs | 2 -
.../traits/binary_impls/personal_access_tokens.rs | 30 +-
core/common/src/traits/binary_impls/streams.rs | 1 -
core/common/src/traits/binary_impls/topics.rs | 6 +-
core/common/src/traits/binary_impls/users.rs | 53 +-
core/common/src/traits/binary_transport.rs | 291 +-
core/common/src/traits/mod.rs | 2 +-
core/common/src/types/identifier/mod.rs | 84 +-
core/common/src/wire_conversions.rs | 29 +-
core/configs/src/server_config/cluster.rs | 65 +-
core/configs/src/server_config/defaults.rs | 2 +
core/configs/src/server_config/displays.rs | 7 +-
core/configs/src/server_config/message_bus.rs | 69 +
core/configs/src/server_config/metadata.rs | 5 +-
core/connectors/runtime/src/main.rs | 2 -
core/connectors/runtime/src/manager/source.rs | 4 +-
core/connectors/runtime/src/source.rs | 151 +-
core/consensus/src/client_table.rs | 89 +-
core/consensus/src/dvc_merge.rs | 65 +-
core/consensus/src/impls.rs | 2289 ++++-
core/consensus/src/lib.rs | 2 +-
core/consensus/src/metadata_helpers.rs | 49 +-
core/consensus/src/observability.rs | 14 +-
core/consensus/src/plane_helpers.rs | 508 +-
core/consensus/src/view_change_quorum.rs | 3 +-
core/consensus/src/vsr_state.rs | 638 +-
core/integration/Cargo.toml | 3 +
core/integration/src/harness/handle/server.rs | 42 +-
core/integration/src/harness/helpers.rs | 75 +-
core/integration/src/harness/mod.rs | 4 +-
.../src/harness/orchestrator/harness.rs | 63 +-
core/integration/tests/cli/common/mod.rs | 7 +
.../tests/cli/message/test_message_send_command.rs | 24 +-
.../tests/cli/topic/test_topic_get_command.rs | 42 +-
.../tests/cluster/client_table_restart.rs | 362 +-
.../tests/cluster/crash_recovery_corruption.rs | 178 +
.../tests/cluster/parked_frame_redispatch.rs | 21 +-
core/integration/tests/cluster/partition_dedup.rs | 158 +
.../tests/cluster/partition_primary_routing.rs | 1949 +++-
.../tests/cluster/staggered_bootstrap.rs | 13 +-
.../verify_consumer_group_partition_assignment.rs | 77 +-
core/integration/tests/sdk/options.rs | 16 +-
core/integration/tests/server/general.rs | 6 +
core/integration/tests/server/http_vsr.rs | 89 +-
.../server/scenarios/log_rotation_scenario.rs | 5 +-
.../scenarios/segment_rotation_race_scenario.rs | 8 +-
.../tests/server/shared_jwt_secrets.toml | 19 +-
core/journal/src/prepare_journal.rs | 57 +
core/journal/src/superblock.rs | 49 +-
core/message_bus/src/config.rs | 5 +
core/message_bus/src/error.rs | 34 +-
core/message_bus/src/installer/mod.rs | 55 +-
core/message_bus/src/installer/quic.rs | 8 +-
core/message_bus/src/installer/tcp.rs | 14 +-
core/message_bus/src/installer/ws.rs | 11 +-
core/message_bus/src/lib.rs | 382 +-
.../src/lifecycle/connection_registry.rs | 368 +-
core/message_bus/src/transports/mod.rs | 2 +-
core/message_bus/src/transports/quic.rs | 229 +-
core/message_bus/tests/duplicate_client_id.rs | 51 +
core/message_bus/tests/quic_client_roundtrip.rs | 18 +-
core/message_bus/tests/tcp_client_roundtrip.rs | 258 +-
core/message_bus/tests/tcp_tls_client_listener.rs | 20 +-
core/message_bus/tests/tcp_tls_client_roundtrip.rs | 15 +-
core/message_bus/tests/ws_client_roundtrip.rs | 25 +-
core/message_bus/tests/wss_client_listener.rs | 15 +-
core/message_bus/tests/wss_client_roundtrip.rs | 15 +-
core/metadata/src/applied_frontier.rs | 132 +-
core/metadata/src/impls/metadata.rs | 3505 +++++++-
core/metadata/src/impls/mod.rs | 1 +
core/metadata/src/impls/placement.rs | 949 ++
core/metadata/src/impls/recovery.rs | 348 +-
core/metadata/src/stm/authz.rs | 2 +-
core/metadata/src/stm/consumer_group.rs | 4 +-
core/metadata/src/stm/lifecycle.rs | 2 +-
core/metadata/src/stm/snapshot.rs | 17 +-
core/metadata/src/stm/stream.rs | 305 +-
core/metadata/src/stm/user.rs | 5 +-
core/partitions/src/iggy_partition.rs | 1902 +++-
core/partitions/src/iggy_partitions.rs | 88 +-
core/partitions/src/io.rs | 27 +-
core/partitions/src/lib.rs | 15 +-
core/partitions/src/persistence.rs | 47 +-
core/partitions/src/poll_plan.rs | 17 +-
core/partitions/src/segment_recovery.rs | 99 +-
core/partitions/src/state_transfer.rs | 12 +-
core/partitions/src/types.rs | 11 +
core/partitions/src/wal_storage.rs | 442 +
core/partitions/tests/consumer_offset_history.rs | 4 +-
core/sdk/README.md | 46 +-
core/sdk/src/binary/mod.rs | 77 +-
core/sdk/src/client_wrappers/client_wrapper.rs | 38 +-
core/sdk/src/clients/client.rs | 89 +-
core/sdk/src/clients/consumer.rs | 355 +-
core/sdk/src/clients/producer.rs | 753 +-
core/sdk/src/clients/producer_dispatcher.rs | 15 +-
core/sdk/src/clients/producer_sharding.rs | 26 +-
core/sdk/src/consumer_ext/consumer_message_ext.rs | 49 +-
core/sdk/src/driver.rs | 1178 ++-
core/sdk/src/driver/route_cache.rs | 298 +-
core/sdk/src/http/http_client.rs | 161 +-
core/sdk/src/http/messages.rs | 12 +-
core/sdk/src/http/scripted_server.rs | 24 +-
core/sdk/src/links/fake_cluster.rs | 139 +-
core/sdk/src/links/mux.rs | 148 +-
core/sdk/src/links/replicas.rs | 227 +-
core/sdk/src/session_owner.rs | 237 +-
core/sdk/src/session_owner/lanes.rs | 54 +-
core/sdk/src/tcp/tcp_client.rs | 737 +-
core/sdk/src/websocket/websocket_client.rs | 15 +
core/server/Cargo.toml | 2 +
core/server/README.md | 203 +-
core/server/config.toml | 42 +-
core/server/src/boot/mod.rs | 34 +-
core/server/src/boot/recovery.rs | 26 +-
core/server/src/boot/threads.rs | 2 +-
core/server/src/consumer_group.rs | 2 +-
core/server/src/consumer_group/liveness.rs | 156 +-
core/server/src/dispatch/authz.rs | 52 +-
core/server/src/dispatch/failure.rs | 287 +-
core/server/src/dispatch/host.rs | 194 +-
core/server/src/dispatch/mod.rs | 1531 +++-
core/server/src/dispatch/partition.rs | 1434 ++-
core/server/src/dispatch/reads.rs | 134 +-
core/server/src/dispatch/segment_delete.rs | 201 +
core/server/src/dispatch/session_ops.rs | 97 +-
core/server/src/dispatch/submit.rs | 14 +-
core/server/src/dispatch/test_support.rs | 188 +-
core/server/src/http.rs | 44 +-
core/server/src/http/admission.rs | 139 +-
core/server/src/http/error.rs | 135 +-
core/server/src/http/extractor.rs | 239 +-
core/server/src/http/forward.rs | 1940 +++-
core/server/src/http/handlers.rs | 460 +-
core/server/src/http/metrics.rs | 37 +
core/server/src/http/reads.rs | 361 +-
core/server/src/http/relay_proof.rs | 356 +
core/server/src/http/reply.rs | 9 +
core/server/src/http/state.rs | 126 +-
core/server/src/http/submit.rs | 145 +-
core/server/src/lib.rs | 5 +-
core/server/src/namespace.rs | 102 +-
core/server/src/partition_helpers.rs | 49 +-
core/server/src/partition_reconciler.rs | 319 +-
core/server/src/rewrite.rs | 4 +-
core/server/src/session_manager.rs | 175 +-
core/server_common/src/consensus_message.rs | 151 +-
core/shard/src/builder.rs | 39 +-
core/shard/src/lib.rs | 2876 +++++-
core/shard/src/metrics.rs | 147 +-
core/shard/src/partition_io.rs | 2875 +++++-
core/shard/src/poll.rs | 20 +-
core/shard/src/poll/completion_tests.rs | 71 +-
core/shard/src/poll/timeout_tests.rs | 4 +-
core/shard/src/primary_rebalance.rs | 845 ++
core/shard/src/primary_rebalance/tests.rs | 1191 +++
core/shard/src/router.rs | 203 +-
core/simulator/Cargo.toml | 2 +-
core/simulator/README.md | 23 +-
core/simulator/src/bin/workload-fuzz.rs | 197 +-
core/simulator/src/bus.rs | 140 +-
core/simulator/src/client.rs | 67 +-
core/simulator/src/deps.rs | 15 +
core/simulator/src/executor/time.rs | 17 +-
core/simulator/src/lib.rs | 9368 ++++++++++++++------
core/simulator/src/packet.rs | 33 +-
core/simulator/src/partition_wal.rs | 119 +
core/simulator/src/replica.rs | 107 +-
core/simulator/src/workload/actions.rs | 1 +
core/simulator/src/workload/invariants.rs | 57 +-
core/simulator/src/workload/mod.rs | 526 +-
core/simulator/src/workload/ops/mod.rs | 2 +
core/simulator/src/workload/ops/poll_messages.rs | 117 +
.../src/workload/ops/store_stale_owner_offset.rs | 4 +-
core/simulator/src/workload/options.rs | 17 +-
core/simulator/src/workload/oracle.rs | 497 +-
core/simulator/src/workload/readback.rs | 420 +
core/simulator/src/workload/state_checker.rs | 288 +-
core/simulator/tests/lifecycle_seeds.rs | 127 +
core/system_stats/src/lib.rs | 9 +-
foreign/cpp/BUILD.bazel | 1 +
foreign/cpp/README.md | 15 +
foreign/cpp/include/iggy.hpp | 24 +-
foreign/cpp/src/client.rs | 101 +-
foreign/cpp/tests/unit/unit_tests.cpp | 21 +
.../Iggy_SDK.Tests.BDD/Context/TestContext.cs | 2 +
.../Iggy_SDK.Tests.BDD/Context/TestEnvironment.cs | 1 +
.../csharp/Iggy_SDK.Tests.BDD/Context/TestHooks.cs | 1 +
.../Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj | 3 +
.../StepDefinitions/SessionContractSteps.cs | 89 +
.../Fixtures/IggyServerFixture.cs | 16 +
.../IggyConsumerTests.cs | 10 +-
.../IggyTypedConsumerTests.cs | 2 +
.../Iggy_SDK.Tests.Integration/StreamsTests.cs | 4 +-
.../Iggy_SDK.Tests.Integration/TopicsTests.cs | 5 +-
.../csharp/Iggy_SDK/Consumers/AutoCommitMode.cs | 4 +-
.../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 37 +-
foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 313 +-
.../csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs | 23 +-
.../Exceptions/IggyInvalidStatusCodeException.cs | 3 +-
.../csharp/Iggy_SDK/Factory/IggyClientFactory.cs | 11 +-
.../TcpMessageStream.PollRouting.cs | 241 +-
.../TcpMessageStream.ReplicaLinks.cs | 109 +-
.../Implementations/TcpMessageStream.Vsr.cs | 327 +-
.../IggyClient/Implementations/TcpMessageStream.cs | 209 +-
foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs | 2 +-
foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs | 126 +-
foreign/csharp/Iggy_SDK/Vsr/MutationLanes.cs | 437 +-
foreign/csharp/Iggy_SDK/Vsr/VsrConnection.cs | 37 +-
foreign/csharp/Iggy_SDK/Vsr/VsrError.cs | 6 +-
foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs | 11 +-
.../ClientTests/IggyClientFactoryTests.cs | 23 +
.../PartitionIncarnationTests.Disposal.cs | 287 +
.../ConsumerTests/PartitionIncarnationTests.cs | 2 +-
.../ConsumerTests/PollFailureTests.cs | 457 +
.../VsrTests/ConsensusSessionTests.cs | 17 +-
.../VsrTests/EndpointFailoverTests.cs | 69 +-
foreign/csharp/Iggy_SDK_Tests/VsrTests/MockNode.cs | 12 +-
.../Iggy_SDK_Tests/VsrTests/MutationLaneTests.cs | 302 +-
.../VsrTests/NativeRecoveryTests.SignIn.cs | 710 ++
.../Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.cs | 678 +-
.../Iggy_SDK_Tests/VsrTests/PrimaryPollingTests.cs | 134 +-
.../Iggy_SDK_Tests/VsrTests/ReplicaLinkTests.cs | 177 +-
.../VsrTests/VsrMultiplexingTests.cs | 6 +-
foreign/csharp/README.md | 55 +-
foreign/go/README.md | 54 +-
foreign/go/client/iggy_client.go | 11 +-
foreign/go/client/iggy_client_test.go | 48 +
foreign/go/client/tcp/cluster.go | 55 +-
foreign/go/client/tcp/tcp_connect_test.go | 536 +-
foreign/go/client/tcp/tcp_core.go | 98 +-
foreign/go/client/tcp/tcp_core_review_test.go | 21 +-
foreign/go/client/tcp/tcp_core_test.go | 13 +-
foreign/go/client/tcp/tcp_driver.go | 158 +-
foreign/go/client/tcp/tcp_driver_test.go | 175 +-
foreign/go/client/tcp/tcp_failover_test.go | 117 +-
foreign/go/client/tcp/tcp_group_polling.go | 79 +-
foreign/go/client/tcp/tcp_group_polling_test.go | 48 +-
foreign/go/client/tcp/tcp_link.go | 36 +-
foreign/go/client/tcp/tcp_link_test.go | 275 +
foreign/go/client/tcp/tcp_messaging.go | 12 +-
foreign/go/client/tcp/tcp_offset_management.go | 9 +-
foreign/go/client/tcp/tcp_pending.go | 335 +-
foreign/go/client/tcp/tcp_poll_routing.go | 273 +-
foreign/go/client/tcp/tcp_poll_routing_test.go | 895 +-
foreign/go/client/tcp/tcp_replicas.go | 42 +-
foreign/go/client/tcp/tcp_replicas_test.go | 721 +-
.../go/client/tcp/tcp_session_credentials_test.go | 160 +-
foreign/go/client/tcp/tcp_session_management.go | 456 +-
foreign/go/client/tcp/tcp_testing_test.go | 19 +-
foreign/go/client/tcp/tcp_topic_cache_test.go | 14 +-
foreign/go/client/tcp/tcp_user_management.go | 17 +-
foreign/go/contracts/client.go | 2 +-
foreign/go/errors/errors.yaml | 8 +-
foreign/go/errors/errors_gen.go | 27 +-
foreign/go/errors/errors_test.go | 6 +-
foreign/go/internal/command/code.go | 2 +-
foreign/go/internal/vsr/reply.go | 46 +-
foreign/go/internal/vsr/reply_test.go | 117 +-
foreign/go/internal/vsr/session.go | 18 +-
foreign/go/internal/vsr/session_test.go | 36 +-
foreign/go/tests/e2e_helpers_test.go | 43 +
foreign/go/tests/e2e_test.go | 31 +-
foreign/java/README.md | 29 +-
.../iggy/client/async/tcp/AsyncIggyTcpClient.java | 766 +-
.../async/tcp/AsyncIggyTcpClientBuilder.java | 20 +-
.../iggy/client/async/tcp/AsyncTcpConnection.java | 1001 +--
.../iggy/client/async/tcp/ConnectionHeartbeat.java | 132 +
.../client/async/tcp/ConsumerGroupsTcpClient.java | 11 +-
.../client/async/tcp/ConsumerOffsetsTcpClient.java | 18 +-
.../iggy/client/async/tcp/CredentialsListener.java | 22 +-
.../iggy/client/async/tcp/IggyAuthenticator.java | 78 +
.../iggy/client/async/tcp/LeaderAwareness.java | 23 -
.../iggy/client/async/tcp/LoginRoutingHook.java | 8 +-
.../iggy/client/async/tcp/MessagesTcpClient.java | 135 +-
.../apache/iggy/client/async/tcp/PollRouter.java | 448 +-
.../iggy/client/async/tcp/PoolChannelHandler.java | 93 +
.../client/async/tcp/RawPartitionRequests.java | 2 +-
.../apache/iggy/client/async/tcp/SessionLogin.java | 155 +
.../client/async/tcp/TransientFailoverHandler.java | 29 +-
.../iggy/client/async/tcp/UsersTcpClient.java | 20 +-
.../apache/iggy/client/async/tcp/VsrExchange.java | 34 +-
.../client/async/tcp/vsr/ConsensusSession.java | 356 +-
.../client/async/tcp/vsr/VsrResponseHandler.java | 60 +-
.../java/org/apache/iggy/serde/CommandCode.java | 2 +-
.../java/org/apache/iggy/ProtocolParityTest.java | 217 +
.../src/test/java/org/apache/iggy/RustSources.java | 43 +
.../client/async/AsyncClientIntegrationTest.java | 157 +
.../client/async/AsyncConnectionPoolAuthTest.java | 22 +-
.../async/tcp/AsyncIggyTcpClientBuilderTest.java | 9 +
.../AsyncIggyTcpClientCredentialChangeTest.java | 293 +
.../AsyncIggyTcpClientEndpointFailoverTest.java | 695 +-
.../tcp/AsyncIggyTcpClientFailoverSignInTest.java | 406 +
.../async/tcp/AsyncIggyTcpClientLockOrderTest.java | 183 +
.../tcp/AsyncIggyTcpClientLoginRoutingTest.java | 93 +-
.../tcp/AsyncIggyTcpClientLostLinkReplayTest.java | 295 +
.../tcp/AsyncIggyTcpClientNamespaceMissTest.java | 508 ++
.../tcp/AsyncIggyTcpClientPartitionWriteTest.java | 118 +
.../async/tcp/AsyncIggyTcpClientRecoveryTest.java | 397 +
.../tcp/AsyncIggyTcpClientSessionRulesTest.java | 1225 +++
.../AsyncIggyTcpClientTransientFailoverTest.java | 85 +-
.../tcp/AsyncTcpConnectionConcurrencyTest.java | 90 +
.../tcp/AsyncTcpConnectionLifecycleBusyTest.java | 112 +-
.../tcp/AsyncTcpConnectionRequestTimeoutTest.java | 61 +
.../iggy/client/async/tcp/LeaderAwarenessTest.java | 30 -
.../iggy/client/async/tcp/PollRouterTest.java | 862 +-
.../client/async/tcp/vsr/ConsensusSessionTest.java | 68 +
.../client/async/tcp/vsr/VsrLoginCodecTest.java | 11 +-
.../async/tcp/vsr/VsrResponseHandlerTest.java | 11 +
foreign/node/README.md | 88 +-
foreign/node/src/bdd/README.md | 1 +
foreign/node/src/bdd/env.ts | 10 +-
foreign/node/src/bdd/session.ts | 63 +
foreign/node/src/bdd/world.ts | 3 +-
foreign/node/src/client/client.connection.test.ts | 37 +-
foreign/node/src/client/client.connection.ts | 28 +-
foreign/node/src/client/client.operation.test.ts | 25 +-
foreign/node/src/client/client.operation.ts | 19 +-
foreign/node/src/client/client.socket.test.ts | 1635 +++-
foreign/node/src/client/client.socket.ts | 496 +-
foreign/node/src/client/client.type.ts | 15 +-
foreign/node/src/client/index.ts | 1 +
foreign/node/src/stream/consumer-stream.ts | 69 +-
foreign/node/src/wire/command.code.ts | 2 +-
foreign/node/src/wire/error.code.ts | 2 +-
.../src/wire/message/poll-messages.command.test.ts | 73 +-
.../node/src/wire/message/poll-messages.command.ts | 42 +-
.../node/src/wire/message/send-messages.command.ts | 15 +-
.../src/wire/session/login-with-token.command.ts | 10 +-
foreign/node/src/wire/session/login.command.ts | 10 +-
foreign/node/src/wire/vsr/session.test.ts | 8 +-
foreign/node/src/wire/vsr/session.ts | 5 -
foreign/php/README.md | 8 +
foreign/php/src/client.rs | 2 -
foreign/python/README.md | 40 +-
foreign/python/apache_iggy.pyi | 33 +-
foreign/python/src/bin/stub_gen.rs | 2 +
foreign/python/src/client.rs | 97 +-
foreign/python/src/identifier.rs | 11 +-
foreign/python/src/lib.rs | 3 +-
foreign/python/src/producer.rs | 80 +-
foreign/python/tests/test_quic_config.py | 7 +
foreign/python/tests/test_raw_command.py | 27 +-
foreign/swift/README.md | 70 +-
.../Sources/Iggy/Client/ClientConfiguration.swift | 8 +-
foreign/swift/Sources/Iggy/Client/ClientCore.swift | 663 +-
.../Sources/Iggy/Client/IggyClient+Messages.swift | 106 +-
.../Sources/Iggy/Client/IggyClient+Resources.swift | 132 +-
foreign/swift/Sources/Iggy/Client/IggyClient.swift | 101 +-
foreign/swift/Sources/Iggy/Client/PollRouter.swift | 620 +-
foreign/swift/Sources/Iggy/Errors/IggyError.swift | 18 +-
.../swift/Sources/Iggy/Protocol/CommandCode.swift | 2 +-
foreign/swift/Sources/Iggy/Protocol/Requests.swift | 26 +
.../swift/Sources/Iggy/Protocol/Responses.swift | 8 +-
foreign/swift/Sources/Iggy/Protocol/VSRFrame.swift | 6 +-
.../swift/Tests/IggyE2ETests/ClientE2ETests.swift | 17 +-
.../Tests/IggyTests/ClientTransportTests.swift | 1402 ++-
foreign/swift/Tests/IggyTests/MessagingTests.swift | 1288 ++-
foreign/swift/Tests/IggyTests/ProtocolTests.swift | 6 +-
.../Tests/IggyTests/Support/FakeVSRServer.swift | 121 +-
gateways/kafka/README.md | 6 +-
gateways/kafka/src/bridge/error.rs | 19 +-
gateways/kafka/src/bridge/iggy_bridge/produce.rs | 19 +-
.../kafka/tests/offset_groups_real_bridge_tests.rs | 3 +-
scripts/ci/apt-install.sh | 72 +-
scripts/run-bdd-tests.sh | 11 +-
417 files changed, 70074 insertions(+), 11390 deletions(-)
diff --cc .dockerignore
index e411c8b85,b4405acc2..30d0a2bb6
--- a/.dockerignore
+++ b/.dockerignore
@@@ -10,19 -10,15 +10,21 @@@
/helm
/licenses
/local_data
+/local_state
/performance_results
/test_logs
+**/.venv
+**/node_modules
+**/.gradle
/foreign/swift/.build
+/foreign/java/**/build
/bdd/java/build
+ /examples/java/**/build
-/foreign/java/**/build
+ /web/build
-**/.gradle
-**/node_modules
+/foreign/csharp/**/bin
+/foreign/csharp/**/obj
+/foreign/cpp/bazel-*
+/foreign/php/vendor
# Selectively exclude scripts/ subdirs not needed in container builds.
# scripts/ci/third-party-licenses.sh + render-node-licenses.mjs are
# required by the convenience-binary Dockerfiles to render LICENSE-binary
diff --cc bdd/cpp/features/step_definitions/world.hpp
index 424e32c8f,886be4fe4..a10214778
--- a/bdd/cpp/features/step_definitions/world.hpp
+++ b/bdd/cpp/features/step_definitions/world.hpp
@@@ -51,14 -49,7 +51,15 @@@ struct GlobalContext
PolledData polled;
std::vector<std::uint8_t> raw_response;
std::string raw_error;
+ // Public client for the option steps, connected on first use.
+ std::optional<iggy::IggyBlockingClient> sdk;
+ std::optional<std::uint32_t> last_stream_id;
+ std::optional<std::uint32_t> last_topic_id;
+ std::optional<std::uint32_t> last_user_id;
+ std::optional<std::vector<std::uint8_t>> last_stream_key;
+ bool last_update_rejected = false;
+ std::vector<iggy::OptionSpec> last_option_specs;
+ std::string call_error;
GlobalContext() = default;
GlobalContext(const GlobalContext &) = delete;
diff --cc bdd/cpp/scripts/entrypoint.sh
index e201b2788,94c4eed4b..f04fcfd4e
--- a/bdd/cpp/scripts/entrypoint.sh
+++ b/bdd/cpp/scripts/entrypoint.sh
@@@ -58,8 -58,7 +58,9 @@@ tags=(
case "${BDD_FEATURE:-all}" in
basic_messaging) tags=(--tags @basic-messaging) ;;
raw_command) tags=(--tags @raw-command) ;;
+ stream_options) tags=(--tags @stream-options) ;;
+ resource_options_patch) tags=(--tags @resource-options-patch) ;;
+ session_contract) tags=(--tags @session-contract) ;;
esac
# --strict fails the run on undefined steps; without it a feature step with no matching C++
diff --cc bdd/docker-compose.coverage.yml
index 3bb49068a,7caa2153b..1bdf7614c
--- a/bdd/docker-compose.coverage.yml
+++ b/bdd/docker-compose.coverage.yml
@@@ -39,7 -39,7 +39,7 @@@ services
trap 'cargo llvm-cov report --lcov --output-path /reports/rust-bdd-coverage.lcov' EXIT
TEST_ARGS=()
case "$$BDD_FEATURE" in
- basic_messaging|leader_redirection|raw_command|stream_crud|stream_options|resource_options_patch) TEST_ARGS=(--test "$$BDD_FEATURE") ;;
- basic_messaging|leader_redirection|raw_command|session_contract|stream_crud) TEST_ARGS=(--test "$$BDD_FEATURE") ;;
++ basic_messaging|leader_redirection|raw_command|session_contract|stream_crud|stream_options|resource_options_patch) TEST_ARGS=(--test "$$BDD_FEATURE") ;;
esac
cargo test --locked -p bdd --features "$$BDD_RUST_FEATURES" "$${TEST_ARGS[@]}"
@@@ -103,8 -103,7 +103,9 @@@
case "$$BDD_FEATURE" in
basic_messaging) FILTER='--filter-trait Category=basic-messaging' ;;
leader_redirection) FILTER='--filter-trait Category=requires-leader-awareness' ;;
+ stream_options) FILTER='--filter-trait Category=stream-options' ;;
+ resource_options_patch) FILTER='--filter-trait Category=resource-options-patch' ;;
+ session_contract) FILTER='--filter-trait Category=session-contract' ;;
esac
dotnet test $$FILTER --coverage --coverage-output-format cobertura --coverage-output csharp-bdd-coverage.cobertura.xml --results-directory /reports
@@@ -119,8 -118,7 +120,9 @@@
basic_messaging) export CUCUMBER_FILTER_TAGS='@basic-messaging' ;;
leader_redirection) export CUCUMBER_FILTER_TAGS='@requires-leader-awareness' ;;
raw_command) export CUCUMBER_FILTER_TAGS='@raw-command' ;;
+ session_contract) export CUCUMBER_FILTER_TAGS='@session-contract' ;;
stream_crud) export CUCUMBER_FILTER_TAGS='@stream-crud' ;;
+ stream_options) export CUCUMBER_FILTER_TAGS='@stream-options' ;;
+ resource_options_patch) export CUCUMBER_FILTER_TAGS='@resource-options-patch' ;;
esac
gradle --no-daemon test jacocoTestReport && cp build/reports/jacoco/test/jacocoTestReport.xml /reports/java-bdd-coverage.xml
diff --cc bdd/docker-compose.yml
index a28bb575e,3dd3e1049..681f689c4
--- a/bdd/docker-compose.yml
+++ b/bdd/docker-compose.yml
@@@ -34,9 -34,8 +34,10 @@@ services
- ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature
- ./scenarios/leader_redirection.feature:/app/features/leader_redirection.feature
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
+ - ./scenarios/session_contract.feature:/app/features/session_contract.feature
- ./scenarios/stream_crud.feature:/app/features/stream_crud.feature
+ - ./scenarios/stream_options.feature:/app/features/stream_options.feature
+ - ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature
command:
- sh
- -c
@@@ -45,9 -44,8 +46,10 @@@
basic_messaging) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test basic_messaging ;;
leader_redirection) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test leader_redirection ;;
raw_command) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test raw_command ;;
+ session_contract) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test session_contract ;;
stream_crud) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test stream_crud ;;
+ stream_options) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test stream_options ;;
+ resource_options_patch) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test resource_options_patch ;;
*) cargo test -p bdd --features "$$BDD_RUST_FEATURES" ;;
esac
networks:
@@@ -65,19 -63,17 +67,21 @@@
volumes:
- ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
+ - ./scenarios/stream_options.feature:/app/features/stream_options.feature
+ - ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature
+ - ./scenarios/session_contract.feature:/app/features/session_contract.feature
working_dir: /app
command:
- sh
- -c
- |
case "$$BDD_FEATURE" in
- basic_messaging) TEST_PATH='tests/test_basic_messaging.py' ;;
- raw_command) TEST_PATH='tests/test_raw_command.py' ;;
- stream_options) TEST_PATH='tests/test_stream_options.py' ;;
- basic_messaging) TEST_PATH='tests/test_basic_messaging.py' ;;
- raw_command) TEST_PATH='tests/test_raw_command.py' ;;
- session_contract) TEST_PATH='tests/test_session_contract.py' ;;
- *) TEST_PATH='tests/' ;;
++ basic_messaging) TEST_PATH='tests/test_basic_messaging.py' ;;
++ raw_command) TEST_PATH='tests/test_raw_command.py' ;;
++ session_contract) TEST_PATH='tests/test_session_contract.py' ;;
++ stream_options) TEST_PATH='tests/test_stream_options.py' ;;
+ resource_options_patch) TEST_PATH='tests/test_resource_options_patch.py' ;;
- *) TEST_PATH='tests/' ;;
++ *) TEST_PATH='tests/' ;;
esac
uv run --project /workspace/bdd/python pytest $$TEST_PATH -v
networks:
@@@ -94,16 -90,16 +98,18 @@@
volumes:
- ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
+ - ./scenarios/stream_options.feature:/app/features/stream_options.feature
+ - ./scenarios/session_contract.feature:/app/features/session_contract.feature
command:
- sh
- -c
- |
case "$$BDD_FEATURE" in
- basic_messaging) GROUP='--group basic-messaging' ;;
- raw_command) GROUP='--group raw-command' ;;
- stream_options) GROUP='--group stream-options' ;;
- *) GROUP='' ;;
+ basic_messaging) GROUP='--group basic-messaging' ;;
+ raw_command) GROUP='--group raw-command' ;;
+ session_contract) GROUP='--group session-contract' ;;
++ stream_options) GROUP='--group stream-options' ;;
+ *) GROUP='' ;;
esac
./scripts/test.sh --configuration /workspace/bdd/php/phpunit.xml.dist $$GROUP
networks:
@@@ -137,11 -133,10 +143,12 @@@
- -c
- |
case "$$BDD_FEATURE" in
- basic_messaging) TAGS='--tags @basic-messaging' ;;
- raw_command) TAGS='--tags @raw-command' ;;
- stream_options) TAGS='--tags @stream-options' ;;
- basic_messaging) TAGS='--tags @basic-messaging' ;;
- raw_command) TAGS='--tags @raw-command' ;;
- session_contract) TAGS='--tags @session-contract' ;;
- *) TAGS='' ;;
++ basic_messaging) TAGS='--tags @basic-messaging' ;;
++ raw_command) TAGS='--tags @raw-command' ;;
++ session_contract) TAGS='--tags @session-contract' ;;
++ stream_options) TAGS='--tags @stream-options' ;;
+ resource_options_patch) TAGS='--tags @resource-options-patch' ;;
- *) TAGS='' ;;
++ *) TAGS='' ;;
esac
npm run test:bdd -- $$TAGS
networks:
@@@ -165,8 -160,7 +172,9 @@@
basic_messaging) FILTER='--filter-trait Category=basic-messaging' ;;
leader_redirection) FILTER='--filter-trait Category=requires-leader-awareness' ;;
raw_command) FILTER='--filter-trait Category=raw-command' ;;
+ stream_options) FILTER='--filter-trait Category=stream-options' ;;
+ resource_options_patch) FILTER='--filter-trait Category=resource-options-patch' ;;
+ session_contract) FILTER='--filter-trait Category=session-contract' ;;
esac
dotnet test $$FILTER
networks:
@@@ -185,9 -179,8 +193,10 @@@
- ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature
- ./scenarios/leader_redirection.feature:/app/features/leader_redirection.feature
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
+ - ./scenarios/session_contract.feature:/app/features/session_contract.feature
- ./scenarios/stream_crud.feature:/app/features/stream_crud.feature
+ - ./scenarios/stream_options.feature:/app/features/stream_options.feature
+ - ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature
command:
- sh
- -c
@@@ -196,9 -189,8 +205,10 @@@
basic_messaging) export CUCUMBER_FILTER_TAGS='@basic-messaging' ;;
leader_redirection) export CUCUMBER_FILTER_TAGS='@requires-leader-awareness' ;;
raw_command) export CUCUMBER_FILTER_TAGS='@raw-command' ;;
+ session_contract) export CUCUMBER_FILTER_TAGS='@session-contract' ;;
stream_crud) export CUCUMBER_FILTER_TAGS='@stream-crud' ;;
+ stream_options) export CUCUMBER_FILTER_TAGS='@stream-options' ;;
+ resource_options_patch) export CUCUMBER_FILTER_TAGS='@resource-options-patch' ;;
esac
gradle --no-daemon test
networks:
@@@ -216,8 -208,7 +226,9 @@@
volumes:
- ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
+ - ./scenarios/stream_options.feature:/app/features/stream_options.feature
+ - ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature
+ - ./scenarios/session_contract.feature:/app/features/session_contract.feature
networks:
- iggy-bdd-network
diff --cc bdd/go/tests/suite_test.go
index 2da4a2741,7841899a7..3adf9c1e8
--- a/bdd/go/tests/suite_test.go
+++ b/bdd/go/tests/suite_test.go
@@@ -64,30 -64,18 +64,41 @@@ func TestFeatures(t *testing.T)
},
})
}
+ if feature == "all" || feature == "session_contract" {
+ suites = append(suites, godog.TestSuite{
+ ScenarioInitializer: initSessionContractScenario,
+ Options: &godog.Options{
+ Format: "pretty",
+ Paths: []string{"../../scenarios/session_contract.feature"},
+ Strict: true,
+ TestingT: t,
+ },
+ })
+ }
+ if feature == "all" || feature == "stream_options" {
+ suites = append(suites, godog.TestSuite{
+ ScenarioInitializer: initResourceOptionsScenario,
+ Options: &godog.Options{
+ Format: "pretty",
+ Paths: []string{"../../scenarios/stream_options.feature"},
+ Strict: true,
+ TestingT: t,
+ },
+ })
+ }
+ if feature == "all" || feature == "resource_options_patch" {
+ suites = append(suites, godog.TestSuite{
+ ScenarioInitializer: initResourceOptionsScenario,
+ Options: &godog.Options{
+ Format: "pretty",
+ Paths: []string{"../../scenarios/resource_options_patch.feature"},
+ Strict: true,
+ TestingT: t,
+ },
+ })
+ }
+
if len(suites) == 0 {
t.Fatalf("unknown BDD_FEATURE=%q", feature)
}
diff --cc bdd/node/Dockerfile
index d8a8df22b,e11d1977d..a76c12b7f
--- a/bdd/node/Dockerfile
+++ b/bdd/node/Dockerfile
@@@ -20,8 -20,7 +20,9 @@@ FROM node:26-sli
COPY ./foreign/node .
COPY ./bdd/scenarios/basic_messaging.feature ./bdd/
COPY ./bdd/scenarios/raw_command.feature ./bdd/
+COPY ./bdd/scenarios/stream_options.feature ./bdd/
+COPY ./bdd/scenarios/resource_options_patch.feature ./bdd/
+ COPY ./bdd/scenarios/session_contract.feature ./bdd/
RUN npm ci
diff --cc bdd/php/tests/SharedFeatureParser.php
index 29d835662,9a0d07009..a25855645
--- a/bdd/php/tests/SharedFeatureParser.php
+++ b/bdd/php/tests/SharedFeatureParser.php
@@@ -88,11 -74,11 +88,11 @@@ final class SharedFeatureParse
}
}
- if ($currentScenario !== null) {
+ if ($currentScenario !== null && array_intersect($currentTags, $excludedTags) === []) {
$scenarios[$currentScenario] = $currentSteps;
}
- if ($backgroundSteps === [] || $scenarios === []) {
- throw new RuntimeException("feature must contain a background and at least one scenario: {$featureFile}");
+ if ($scenarios === []) {
+ throw new RuntimeException("feature must contain at least one scenario: {$featureFile}");
}
$cases = [];
diff --cc bdd/python/tests/conftest.py
index c094c639e,0b1ddcefa..9930e66a7
--- a/bdd/python/tests/conftest.py
+++ b/bdd/python/tests/conftest.py
@@@ -42,29 -42,9 +42,31 @@@ class GlobalContext
last_sent_message: str | None = None # Store message payload as string
last_raw_response: bytes | None = None
last_raw_error: RuntimeError | None = None
+ last_stream_key: bytes | None = None
+ last_update_error: RuntimeError | None = None
+ last_option_specs: list[OptionSpec] | None = None
+ last_user_id: int | None = None
+ last_call_error: RuntimeError | None = None
+def pytest_configure(config: pytest.Config) -> None:
+ """Register the shared feature tags as markers.
+
+ The Docker image runs pytest from /app, where bdd/python/pyproject.toml is
+ not the rootdir config, so markers declared there would never load.
+ """
+ for marker in (
+ "basic-messaging: shared basic messaging scenarios",
+ "raw-command: shared raw command scenarios",
++ "session-contract: shared session contract scenarios",
+ "stream-options: shared stream option scenarios",
+ "resource-options-patch: shared resource patch scenarios",
+ "message-headers: scenarios that assert message headers",
+ "describe-options: scenarios that assert the option catalog",
+ ):
+ config.addinivalue_line("markers", marker)
+
+
def required_env(name: str) -> str:
"""Read a variable the suite cannot run without.
diff --cc bdd/rust/tests/common/global_context.rs
index 5c2fedcc3,3313c003e..b7d36b86f
--- a/bdd/rust/tests/common/global_context.rs
+++ b/bdd/rust/tests/common/global_context.rs
@@@ -33,8 -33,5 +33,9 @@@ pub struct GlobalContext
pub last_polled_messages: Option<PolledMessages>,
pub last_sent_message: Option<IggyMessage>,
pub last_raw_result: Option<Result<Bytes, IggyError>>,
+ pub last_stream_key: Option<StreamEncryptionKey>,
+ pub last_update_error: Option<IggyError>,
+ pub last_option_specs: Vec<OptionSpec>,
+ pub last_user_id: Option<u32>,
+ pub last_call_result: Option<Result<(), IggyError>>,
}
diff --cc bdd/rust/tests/steps/mod.rs
index f3147ada5,e6c88141d..a091ca2cc
--- a/bdd/rust/tests/steps/mod.rs
+++ b/bdd/rust/tests/steps/mod.rs
@@@ -19,7 -19,7 +19,8 @@@ pub mod auth
pub mod leader_redirection;
pub mod messages;
pub mod raw_command;
+pub mod resource_options;
pub mod server;
+ pub mod session;
pub mod streams;
pub mod topics;
diff --cc core/binary_protocol/src/codes.rs
index 68bef7189,d90f4463d..ab8e8f9f4
--- a/core/binary_protocol/src/codes.rs
+++ b/core/binary_protocol/src/codes.rs
@@@ -112,7 -115,7 +113,7 @@@ pub const fn command_name(code: u32) -
mod tests {
use super::*;
- const RETIRED_CODES: &[u32] = &[14, 35, 102, 103, 104, 123, 204, 205, 304, 305];
- const RETIRED_CODES: &[u32] = &[14, 102, 103, 104, 123, 124, 205, 305];
++ const RETIRED_CODES: &[u32] = &[14, 35, 102, 103, 104, 123, 124, 204, 205, 304, 305];
const ALL_CODES: &[u32] = &[
PING_CODE,
diff --cc core/common/src/traits/binary_impls/mod.rs
index 4ec53ab62,80723f0b1..627bbca8e
--- a/core/common/src/traits/binary_impls/mod.rs
+++ b/core/common/src/traits/binary_impls/mod.rs
@@@ -34,15 -33,13 +34,16 @@@ use crate::OptionsPatch
use crate::http::users::defaults::{
MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH,
};
- use crate::{BinaryClient, ClientState, Credentials};
+ use crate::{BinaryClient, ClientState, Credentials, IdentityInfo};
use bytes::Bytes;
use iggy_binary_protocol::codec::WireEncode;
-use iggy_binary_protocol::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE};
+use iggy_binary_protocol::codes::{
+ LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE, UPDATE_RESOURCE_CODE,
+};
+use iggy_binary_protocol::requests::system::{ResourceTarget, UpdateResourceRequest};
use iggy_binary_protocol::requests::users::login_register::BindSecret;
use iggy_binary_protocol::requests::users::{LoginRegisterRequest, LoginRegisterWithPatRequest};
+ use iggy_binary_protocol::responses::users::LoginRegisterResponse;
use iggy_binary_protocol::{
ClientVersionInfo, IGGY_PROTOCOL_VERSION, MAX_WIRE_NAME_LENGTH, WireDecode, WireName,
};
diff --cc core/common/src/traits/binary_impls/users.rs
index eace4523e,9d873191c..7f9dd918a
--- a/core/common/src/traits/binary_impls/users.rs
+++ b/core/common/src/traits/binary_impls/users.rs
@@@ -18,23 -18,20 +18,21 @@@
use crate::traits::binary_auth::fail_if_not_authenticated;
use crate::wire_conversions::{identifier_to_wire, permissions_to_wire, users_from_wire};
use crate::{
- BinaryClient, ClientState, Credentials, DiagnosticEvent, Identifier, IdentityInfo, IggyError,
- Permissions, UserClient, UserCreateOptions, UserInfo, UserInfoDetails, UserStatus,
- UserUpdateOptions,
+ BinaryClient, Credentials, Identifier, IdentityInfo, IggyError, Permissions, UserClient,
- UserInfo, UserInfoDetails, UserStatus, UserUpdateOptions,
++ UserCreateOptions, UserInfo, UserInfoDetails, UserStatus, UserUpdateOptions,
};
+use iggy_binary_protocol::WireName;
use iggy_binary_protocol::codec::WireEncode;
use iggy_binary_protocol::codes::{
CHANGE_PASSWORD_CODE, CREATE_USER_CODE, DELETE_USER_CODE, GET_USER_CODE, GET_USERS_CODE,
- LOGOUT_USER_CODE, UPDATE_PERMISSIONS_CODE, UPDATE_USER_CODE,
+ LOGOUT_USER_CODE, UPDATE_PERMISSIONS_CODE,
};
+use iggy_binary_protocol::requests::system::ResourceTarget;
use iggy_binary_protocol::requests::users::{
ChangePasswordRequest, CreateUserRequest, DeleteUserRequest, GetUserRequest, GetUsersRequest,
- LogoutUserRequest, UpdatePermissionsRequest, UpdateUserRequest,
+ LogoutUserRequest, UpdatePermissionsRequest,
};
- use iggy_binary_protocol::responses::users::LoginRegisterResponse;
use iggy_binary_protocol::responses::users::{GetUsersResponse, UserDetailsResponse};
-use iggy_binary_protocol::{WireName, WireOptions};
use secrecy::SecretString;
#[async_trait::async_trait]
@@@ -211,150 -192,12 +180,160 @@@ impl<B: BinaryClient> UserClient for B
}
}
+ pub(super) async fn logout<B: BinaryClient>(client: &B, signing_in: bool) -> Result<(), IggyError> {
+ if !signing_in {
+ fail_if_not_authenticated(client).await?;
+ }
+ client
+ .send_raw_with_response(LOGOUT_USER_CODE, LogoutUserRequest.to_bytes())
+ .await?;
+ client.finish_sign_out(signing_in).await
+ }
++
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::{
- BinaryTransport, Client, ConsumerGroupClientState, NonZeroIggyDuration, OptionKey,
- PartitionContext, VsrSessionControl, VsrSessionSealed,
++ BinaryTransport, Client, ClientState, ConsumerGroupClientState, DiagnosticEvent,
++ NonZeroIggyDuration, OptionKey, PartitionContext, VsrSessionControl, VsrSessionSealed,
+ };
+ use async_broadcast::Receiver;
+ use bytes::Bytes;
+ use iggy_binary_protocol::requests::messages::PollMessagesRequest;
+ use iggy_binary_protocol::requests::system::SessionIdentity;
+ use iggy_binary_protocol::requests::users::login_register::BindSecret;
+ use std::collections::BTreeSet;
+ use std::sync::{Arc, Mutex};
+
+ #[tokio::test]
+ async fn given_short_username_and_status_reset_when_updating_user_should_report_admission_category()
+ {
+ let client = RecordingClient::default();
+ let outcome = client
+ .update_user(
+ &Identifier::numeric(5).expect("numeric id"),
+ &UserUpdateOptions {
+ username: Some("ab".to_owned()),
+ reset: BTreeSet::from([OptionKey::Status]),
+ ..UserUpdateOptions::default()
+ },
+ )
+ .await;
+ assert_eq!(
+ outcome,
+ Err(IggyError::OptionNotResettable(
+ OptionKey::Status.to_string()
+ ))
+ );
+ assert!(client.sent_codes().is_empty());
+ }
+
+ /// An authenticated binary client that records the codes it sends.
+ #[derive(Debug, Default)]
+ struct RecordingClient {
+ sent: Mutex<Vec<u32>>,
+ consumer_groups: Arc<ConsumerGroupClientState>,
+ }
+
+ impl RecordingClient {
+ fn sent_codes(&self) -> Vec<u32> {
+ self.sent.lock().expect("lock").clone()
+ }
+
+ fn record(&self, code: u32) -> Result<Bytes, IggyError> {
+ self.sent.lock().expect("lock").push(code);
+ Ok(Bytes::new())
+ }
+ }
+
+ #[async_trait::async_trait]
+ impl BinaryTransport for RecordingClient {
+ async fn get_state(&self) -> ClientState {
+ ClientState::Authenticated
+ }
+
+ async fn set_state(&self, _state: ClientState) {}
+
+ async fn publish_event(&self, _event: DiagnosticEvent) {}
+
+ async fn send_raw_with_context(
+ &self,
+ code: u32,
+ _payload: Bytes,
+ _context: PartitionContext,
+ ) -> Result<Bytes, IggyError> {
+ self.record(code)
+ }
+
+ async fn send_offset_write_with_response(
+ &self,
+ code: u32,
+ _payload: Bytes,
+ _context: Option<PartitionContext>,
+ ) -> Result<Bytes, IggyError> {
+ self.record(code)
+ }
+
+ async fn send_poll_with_response(
+ &self,
+ _request: &PollMessagesRequest,
+ _context: Option<PartitionContext>,
+ ) -> Result<Bytes, IggyError> {
+ Err(IggyError::FeatureUnavailable)
+ }
+
+ fn get_heartbeat_interval(&self) -> NonZeroIggyDuration {
+ NonZeroIggyDuration::ONE_SECOND
+ }
+
+ fn consumer_group_state(&self) -> Arc<ConsumerGroupClientState> {
+ Arc::clone(&self.consumer_groups)
+ }
+ }
+
+ #[async_trait::async_trait]
+ impl Client for RecordingClient {
+ async fn connect(&self) -> Result<(), IggyError> {
+ Ok(())
+ }
+
+ async fn disconnect(&self) -> Result<(), IggyError> {
+ Ok(())
+ }
+
+ async fn shutdown(&self) -> Result<(), IggyError> {
+ Ok(())
+ }
+
+ async fn subscribe_events(&self) -> Receiver<DiagnosticEvent> {
+ async_broadcast::broadcast(1).1
+ }
+ }
+
+ impl VsrSessionSealed for RecordingClient {}
+
+ #[async_trait::async_trait]
+ impl VsrSessionControl for RecordingClient {
+ async fn session_bind_secret(&self) -> Result<BindSecret, IggyError> {
+ Err(IggyError::FeatureUnavailable)
+ }
+
+ async fn session_identity(&self) -> Result<SessionIdentity, IggyError> {
+ Err(IggyError::FeatureUnavailable)
+ }
+
+ async fn bind_vsr_session(&self, _session: u64) -> Result<(), IggyError> {
+ Ok(())
+ }
+
+ async fn reset_vsr_session(&self) -> Result<(), IggyError> {
+ Ok(())
+ }
+
+ fn sdk_version(&self) -> &'static str {
+ env!("CARGO_PKG_VERSION")
+ }
+ }
+
+ impl BinaryClient for RecordingClient {}
+}
diff --cc core/common/src/wire_conversions.rs
index 7ae8788a6,c5d0f9353..193222944
--- a/core/common/src/wire_conversions.rs
+++ b/core/common/src/wire_conversions.rs
@@@ -23,14 -23,13 +23,14 @@@
use crate::{
CacheMetrics, CacheMetricsKey, ClientInfo, ClientInfoDetails, ClusterMetadata, ClusterNode,
- ClusterNodeRole, ClusterNodeStatus, CompressionAlgorithm, Consumer, ConsumerGroup,
- ConsumerGroupDetails, ConsumerGroupInfo, ConsumerGroupMember, ConsumerOffsetInfo,
- GlobalPermissions, HeaderKey, HeaderKind, HeaderValue, IdKind, Identifier, IdentityInfo,
- IggyByteSize, IggyError, IggyExpiry, MaxTopicSize, OptionSpec, OptionValue, OptionsProvenance,
- Partition, Permissions, PersonalAccessTokenInfo, RawPersonalAccessToken, ResourceOptions,
- Stats, Stream, StreamDetails, StreamPermissions, Topic, TopicDetails, TopicPermissions,
- TransportEndpoints, UserInfo, UserInfoDetails, UserStatus,
+ ClusterNodeRole, ClusterNodeStatus, Consumer, ConsumerGroup, ConsumerGroupDetails,
+ ConsumerGroupInfo, ConsumerGroupMember, ConsumerOffsetInfo, GlobalPermissions, HeaderKey,
- HeaderKind, HeaderValue, IdKind, IdentityInfo, IggyByteSize, IggyError, OPTION_FLAG_CRITICAL,
- OPTION_FLAG_READ_ONLY, OPTION_FLAG_RESETTABLE, OPTION_FLAG_SECRET, OPTION_FLAG_UPDATABLE,
- OptionSpec, OptionValue, OptionsProvenance, OptionsScope, Partition, Permissions,
- PersonalAccessTokenInfo, RawPersonalAccessToken, ResourceOptions, Stats, Stream, StreamDetails,
- StreamPermissions, Topic, TopicDetails, TopicPermissions, TransportEndpoints, UserInfo,
- UserInfoDetails, UserStatus, descriptors_for,
++ HeaderKind, HeaderValue, IdKind, Identifier, IdentityInfo, IggyByteSize, IggyError,
++ OPTION_FLAG_CRITICAL, OPTION_FLAG_READ_ONLY, OPTION_FLAG_RESETTABLE, OPTION_FLAG_SECRET,
++ OPTION_FLAG_UPDATABLE, OptionSpec, OptionValue, OptionsProvenance, OptionsScope, Partition,
++ Permissions, PersonalAccessTokenInfo, RawPersonalAccessToken, ResourceOptions, Stats, Stream,
++ StreamDetails, StreamPermissions, Topic, TopicDetails, TopicPermissions, TransportEndpoints,
++ UserInfo, UserInfoDetails, UserStatus, descriptors_for,
};
use bytes::{BufMut, BytesMut};
use iggy_binary_protocol::primitives::options::{MAX_OPTIONS, MAX_OPTIONS_BYTES, WireOptions};
diff --cc core/connectors/runtime/src/source.rs
index 26c225a0d,f6ab81a11..6f2c52892
--- a/core/connectors/runtime/src/source.rs
+++ b/core/connectors/runtime/src/source.rs
@@@ -18,11 -18,9 +18,10 @@@
use dashmap::DashMap;
use dlopen2::wrapper::Container;
use flume::{Receiver, Sender};
- use iggy::clients::MAX_BATCH_LENGTH;
+use iggy::prelude::StreamCreateOptions;
use iggy::prelude::{
Client, DirectConfig, HeaderKey, HeaderValue, Identifier, IggyClient, IggyDuration, IggyError,
- IggyMessage, IggyProducer, StreamClient, TopicClient, TopicCreateOptions,
+ IggyMessage, StreamClient, TopicClient, TopicCreateOptions,
};
use iggy_common::{BINARY_OPERATION_TIMEOUT, Durability, TopicRuntimeOptions};
use iggy_connector_sdk::encoders::avro::{AvroEncoderConfig, AvroStreamEncoder};
diff --cc core/consensus/src/client_table.rs
index d0d41c1fb,39a569a47..7fb462bd7
--- a/core/consensus/src/client_table.rs
+++ b/core/consensus/src/client_table.rs
@@@ -2287,73 -2296,80 +2298,147 @@@ mod tests
}
}
+ #[test]
+ fn given_self_deactivation_through_generic_patch_when_validating_protection_should_accept_resolved_operation()
+ {
+ const CLIENT: u128 = 7;
+ const EPOCH: u64 = 10;
+ const END: u64 = 40;
+ let secret = [0x5a; BIND_SECRET_BYTES];
+ for (operation, accepted) in [
+ (Operation::UpdateUserResolved, true),
+ (Operation::UpdateStreamResolved, false),
+ (Operation::UpdateTopicResolved, false),
+ ] {
+ let mut table = ClientTable::new(1);
+ table
+ .commit_register(
+ CLIENT,
+ TEST_USER_ID,
+ bind_verifier(CLIENT, TEST_USER_ID, &secret),
+ make_register_reply(CLIENT, EPOCH),
+ )
+ .unwrap();
+ table.commit_reply(CLIENT, TEST_USER_ID, make_reply_for(CLIENT, 5, END - 1));
+ assert_eq!(table.end_user_sessions(TEST_USER_ID, END).len(), 1);
+ let receipt = make_reply_for(CLIENT, 6, END).transmute_header(|old, header| {
+ *header = old;
+ header.operation = operation;
+ });
+ table.commit_reply(CLIENT, TEST_USER_ID, receipt);
+ assert_eq!(
+ ClientTable::from_snapshot(table.to_snapshot()).is_ok(),
+ accepted,
+ "{operation:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn given_resolved_update_receipt_when_retrying_client_operation_should_return_cached_reply() {
+ for resolved in [
+ Operation::UpdateStreamResolved,
+ Operation::UpdateTopicResolved,
+ Operation::UpdateUserResolved,
+ ] {
+ let (mut table, epoch) = table_with_client();
+ let reply = make_reply_for(1, 1, 11).transmute_header(|old, header| {
+ *header = old;
+ header.operation = resolved;
+ });
+ table.commit_reply(1, TEST_USER_ID, reply.clone());
+ for restored in [
+ ClientTable::from_snapshot(table.to_snapshot()).unwrap(),
+ ClientTable::decode(&table.encode()).unwrap(),
+ ] {
+ let RequestStatus::Duplicate(cached) =
+ restored.check_request(1, epoch, 1, Operation::UpdateResource)
+ else {
+ panic!("lost the receipt for {resolved:?}");
+ };
+ assert_eq!(cached.into_message().as_slice(), reply.as_slice());
+ assert!(matches!(
+ restored.check_request(1, epoch, 1, Operation::DeleteStream),
+ RequestStatus::OperationMismatch { .. }
+ ));
+ }
+ }
+ }
+
+ /// A request admitted before its user's revocation committed commits
+ /// right behind it. Its session already ended, so the op stays uncertain.
+ #[test]
+ fn given_revoked_user_when_a_pipelined_request_commits_should_keep_the_tombstone() {
+ const REVOKER: u128 = 7;
+ const PIPELINED: u128 = 8;
+ const REVOKER_EPOCH: u64 = 10;
+ const PIPELINED_EPOCH: u64 = 11;
+ const END: u64 = 40;
+ let secret = [0x5a; BIND_SECRET_BYTES];
+ for self_revocation in [
+ None,
- Some(Operation::UpdateUser),
++ Some(Operation::UpdateUserResolved),
+ Some(Operation::DeleteUser),
+ ] {
+ let mut table = ClientTable::new(2);
+ for (client, epoch) in [(REVOKER, REVOKER_EPOCH), (PIPELINED, PIPELINED_EPOCH)] {
+ table
+ .commit_register(
+ client,
+ TEST_USER_ID,
+ bind_verifier(client, TEST_USER_ID, &secret),
+ make_register_reply(client, epoch),
+ )
+ .unwrap();
+ }
+ let ended = table.end_user_sessions(TEST_USER_ID, END);
+ if let Some(operation) = self_revocation {
+ let revocation = make_reply_for(REVOKER, 1, END).transmute_header(|old, header| {
+ *header = old;
+ header.operation = operation;
+ });
+ assert_eq!(
+ table.commit_reply(REVOKER, TEST_USER_ID, revocation),
+ CommitReply::Cached,
+ "the revoking op commits at the end marker itself"
+ );
+ }
+ let at_end = table.encode();
+ assert_eq!(
+ table.commit_reply(
+ PIPELINED,
+ TEST_USER_ID,
+ make_reply_for(PIPELINED, 1, END + 1)
+ ),
+ CommitReply::SessionEnded { ended_op: END },
+ "{self_revocation:?}"
+ );
+ let transferred = table.encode();
+ assert!(
+ transferred == at_end,
+ "{self_revocation:?}: an op after the end must leave the table unchanged"
+ );
+ let revoker_receipt = if self_revocation.is_some() {
+ END
+ } else {
+ REVOKER_EPOCH
+ };
+ for restored in [
+ ClientTable::from_snapshot(table.to_snapshot()).expect("checkpoint must restore"),
+ ClientTable::decode(&transferred).expect("state transfer must decode"),
+ ] {
+ assert_eq!(restored.ended_sessions().collect::<Vec<_>>(), ended);
+ assert_eq!(
+ restored
+ .get_reply(REVOKER)
+ .map(|receipt| receipt.header().commit),
+ Some(revoker_receipt)
+ );
+ assert_eq!(restored.get_watermark(PIPELINED), Some(REGISTER_REQUEST_ID));
+ }
+ }
+ }
+
/// Register client 1 (register commit stamped at op 10). Returns
/// (table, epoch=1).
fn table_with_client() -> (ClientTable, u64) {
diff --cc core/integration/src/harness/helpers.rs
index 1a3096b3b,47bb1ba6a..eb60fb752
--- a/core/integration/src/harness/helpers.rs
+++ b/core/integration/src/harness/helpers.rs
@@@ -15,10 -15,11 +15,12 @@@
// specific language governing permissions and limitations
// under the License.
+ use crate::harness::{TestBinary, TestHarness};
+use iggy::prelude::UserCreateOptions;
use iggy::prelude::{
ConsumerGroupClient, ConsumerGroupDetails, DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME,
- GlobalPermissions, Identifier, IdentityInfo, Permissions, StreamClient, UserClient, UserStatus,
+ GlobalPermissions, Identifier, IdentityInfo, Permissions, StreamClient, TopicClient,
+ TopicDetails, UserClient, UserStatus,
};
use std::collections::HashSet;
use std::time::Duration;
diff --cc core/integration/tests/cli/topic/test_topic_get_command.rs
index 0bf93781d,ddaecb8c3..48a6a6d15
--- a/core/integration/tests/cli/topic/test_topic_get_command.rs
+++ b/core/integration/tests/cli/topic/test_topic_get_command.rs
@@@ -26,10 -26,8 +26,10 @@@ use iggy::prelude::Client
use iggy::prelude::Durability;
use iggy::prelude::IggyExpiry;
use iggy::prelude::defaults::{DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME};
-use iggy::prelude::{IggyMessage, Partitioning, StreamClient, TopicClient, TopicCreateOptions};
+use iggy::prelude::{
+ IggyMessage, Partitioning, StreamClient, StreamCreateOptions, TopicClient, TopicCreateOptions,
+};
- use integration::harness::{TestHarnessBuilder, TestServerConfig};
+ use integration::harness::{TestHarnessBuilder, TestServerConfig, wait_for_topic_messages_count};
use predicates::prelude::PredicateBooleanExt;
use predicates::str::{contains, starts_with};
use serial_test::parallel;
diff --cc core/integration/tests/cluster/client_table_restart.rs
index 53e177378,b243ac661..009e58e65
--- a/core/integration/tests/cluster/client_table_restart.rs
+++ b/core/integration/tests/cluster/client_table_restart.rs
@@@ -477,14 -622,6 +624,13 @@@ async fn resume_request_header
let mut last_failure = "the listener never came back".to_string();
let mut attempt = 0usize;
let mut authenticated_attempts = Vec::new();
- let mut header =
- RequestHeader::for_request(CREATE_STREAM_CODE, CLIENT_ID, request, session, body).unwrap();
++ let mut header = *header;
+ if let Some(original) = expect_replay_of {
+ let receipt =
+ bytemuck::checked::try_pod_read_unaligned::<ReplyHeader>(original.header.as_ref())
+ .unwrap();
+ header.request_checksum = receipt.request_checksum;
+ }
while Instant::now() < deadline {
let addr = addrs[attempt % addrs.len()];
attempt += 1;
@@@ -508,8 -644,7 +653,7 @@@
continue;
}
};
- header.session = resumed;
- match exchange(&mut stream, header, body).await.verdict() {
+ match exchange(&mut stream, &header, body).await.verdict() {
Verdict::Success(reply) => {
if let Some(original) = expect_replay_of {
assert_replayed_from_cache(original, &reply, request);
@@@ -1164,6 -1300,85 +1309,88 @@@ async fn given_primary_member_when_host
assert_eq!(after.members[0].id, expired.members[0].id);
}
+ /// A session needs no socket to the metadata primary. The partition primary
+ /// it still talks to reports its activity, which renews the lease.
+ #[iggy_harness(cluster_nodes = 3, server(
+ heartbeat.enabled = true,
+ heartbeat.interval = "7s",
+ consumer_group.heartbeat_interval = "500ms",
+ sharding.cpu_allocation = "0..1",
+ ))]
+ async fn given_a_cut_metadata_link_when_partition_links_stay_up_should_keep_sending_and_polling(
+ harness: &mut TestHarness,
+ ) {
+ let observer = harness.root_client_for_node(0).await.unwrap();
+ let partitions_count = u32::try_from(harness.cluster_size()).unwrap();
- observer.create_stream(LIVENESS_STREAM).await.unwrap();
++ observer
++ .create_stream(LIVENESS_STREAM, &StreamCreateOptions::default())
++ .await
++ .unwrap();
+ observer
+ .create_topic(
+ &Identifier::named(LIVENESS_STREAM).unwrap(),
+ LIVENESS_TOPIC,
+ &TopicCreateOptions {
+ partitions_count: Some(partitions_count),
+ durability: Durability::Persisted,
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ let metadata_primary = primary_index(harness, &observer).await;
+ let metadata_address = harness.node(metadata_primary).tcp_addr().unwrap();
+ let (mut metadata_link, session) = register(metadata_address).await;
+ let mut backup_route = None;
+ for partition_id in 0..partitions_count {
+ let route = partition_route(&mut metadata_link, session, partition_id).await;
+ if usize::from(route.primary_replica) != metadata_primary {
+ backup_route = Some((partition_id, route));
+ break;
+ }
+ }
+ let (partition_id, route) =
+ backup_route.expect("spread placement must give a metadata backup a partition primary");
+ let partition_address = harness
+ .node(usize::from(route.primary_replica))
+ .tcp_addr()
+ .unwrap();
+ let (mut partition_link, rebound) = register(partition_address).await;
+ assert_eq!(
+ rebound, session,
+ "the partition link must join the same logical session"
+ );
+ drop(metadata_link);
+
+ let context = route.context();
+ let deadline = Instant::now() + LIVENESS_OBSERVATION;
+ let mut offset = 0;
+ while Instant::now() < deadline {
+ let payload = format!("activity-{offset}");
+ send_payload(
+ &mut partition_link,
+ session,
+ context,
+ partition_id,
+ offset + 1,
+ payload.as_bytes(),
+ )
+ .await;
+ assert_eq!(
+ poll_payload(&mut partition_link, session, context, partition_id, offset).await,
+ payload.as_bytes(),
+ "the poll at offset {offset} must read the message just sent"
+ );
+ offset += 1;
+ sleep(RETRY_PAUSE).await;
+ }
+ let (_, resumed) = register(metadata_address).await;
+ assert_eq!(
+ resumed, session,
+ "the metadata primary must keep the session its partition link kept active"
+ );
+ }
+
async fn liveness_group(observer: &IggyClient) -> ConsumerGroupDetails {
observer
.get_consumer_group(
diff --cc core/integration/tests/cluster/crash_recovery_corruption.rs
index 560e2dca0,9d36ef34d..339b03a2d
--- a/core/integration/tests/cluster/crash_recovery_corruption.rs
+++ b/core/integration/tests/cluster/crash_recovery_corruption.rs
@@@ -409,6 -454,139 +454,139 @@@ fn assert_segment_logs_identical(data_p
assert!(problems.is_empty(), "{invariant}:\n{}", problems.join("\n"));
}
+ /// A graceful stop tears nothing. Below the default flush thresholds the
+ /// committed batches stay unindexed until the shutdown flush writes their
+ /// sparse index entry past the WAL checkpoint, and the next boot drops that
+ /// entry. The boot must keep every `.log` byte and must not report segment
+ /// damage, and every acked offset must still read back.
+ #[iggy_harness(cluster_nodes = 3)]
+ async fn given_a_graceful_stop_when_the_cluster_restarts_should_not_report_segment_damage(
+ harness: &mut TestHarness,
+ ) {
+ let client = harness.tcp_root_client().await.unwrap();
+ client
- .create_stream(STREAM_NAME)
++ .create_stream(STREAM_NAME, &StreamCreateOptions::default())
+ .await
+ .expect("create stream");
+ // Not `create_stream_and_topic`: its eager flush indexes every batch at
+ // commit, and this spec needs the shutdown flush to write the index.
+ client
+ .create_topic(
+ &Identifier::named(STREAM_NAME).unwrap(),
+ TOPIC_NAME,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..TopicCreateOptions::default()
+ },
+ )
+ .await
+ .expect("create topic");
+ let acked = produce_acked(&client, "graceful-stop", GRACEFUL_STOP_BATCHES).await;
+ drop(client);
+
+ let data_paths: Vec<PathBuf> = harness
+ .all_servers()
+ .iter()
+ .map(|server| server.data_path())
+ .collect();
+ harness
+ .stop_all_at_once()
+ .await
+ .expect("stop every node gracefully");
+ let stopped_indexes = segment_file_sizes(&data_paths, "index");
+ assert!(
+ stopped_indexes.values().any(|size| *size > 0),
+ "the graceful stop must index the committed batches; index files: {stopped_indexes:?}"
+ );
+ let stopped_logs = segment_file_sizes(&data_paths, "log");
+
+ harness
+ .start()
+ .await
+ .expect("start every node on its own data");
+ let restarted_logs = segment_file_sizes(&data_paths, "log");
+ let truncated: Vec<String> = stopped_logs
+ .iter()
+ .filter_map(|(path, &stopped_size)| {
+ let restarted_size = restarted_logs.get(path).copied().unwrap_or(0);
+ (restarted_size < stopped_size).then(|| {
+ format!(
+ "{}: {stopped_size} -> {restarted_size} bytes",
+ path.display()
+ )
+ })
+ })
+ .collect();
+ assert!(
+ truncated.is_empty(),
+ "the boot after a graceful stop truncated segment logs:\n{}",
+ truncated.join("\n")
+ );
+
+ let client = harness.tcp_root_client().await.unwrap();
+ wait_for_acked_readable(&client, &acked, CONVERGE_TIMEOUT)
+ .await
+ .unwrap_or_else(|state| {
+ panic!("every acked offset must poll back after the graceful restart: {state}")
+ });
+ for node in 0..harness.cluster_size() {
+ assert_no_segment_damage_reported(harness, node);
+ }
+ }
+
+ /// A SIGKILL without surgery tears nothing, yet it leaves the sparse index
+ /// entries that eager flushes wrote past the WAL checkpoint, and the reboot
+ /// drops them. The reboot must not report segment damage, and the node must
+ /// still converge on every ack.
+ #[iggy_harness(cluster_nodes = 3)]
+ async fn given_index_entries_past_the_checkpoint_when_a_killed_node_reboots_should_not_report_segment_damage(
+ harness: &mut TestHarness,
+ ) {
+ let client = harness.tcp_root_client().await.unwrap();
+ create_stream_and_topic(&client, Durability::Replicated).await;
+ let acked = produce_acked(&client, "killed-reboot", INDEX_AHEAD_BATCHES).await;
+ let payloads: Vec<String> = acked.iter().map(|(_, payload)| payload.clone()).collect();
+ for node in 0..harness.cluster_size() {
+ wait_until_node_holds_payloads(
+ harness,
+ node,
+ &payloads,
+ FLUSH_INSTALL_TIMEOUT,
+ "eager flush before the kill",
+ )
+ .await;
+ }
+ drop(client);
+
+ let (_, backup) = pick_backup(harness).await;
+ harness.kill_node(backup).expect("SIGKILL the backup");
+ let index_path = find_active_segment_file(&harness.node(backup).data_path(), "index");
+ assert!(
+ !index_positions(&index_path).is_empty(),
+ "the eager flushes must leave index entries for the reboot to drop"
+ );
+
+ harness
+ .restart_node(backup)
+ .unwrap_or_else(|error| panic!("the killed backup must reboot on its own data: {error}"));
+ wait_until_node_holds_payloads(
+ harness,
+ backup,
+ &payloads,
+ CONVERGE_TIMEOUT,
+ "rejoin after the reboot",
+ )
+ .await;
+ let nodes: Vec<usize> = (0..harness.cluster_size()).collect();
+ let client = wait_until_cluster_serves(harness, &nodes, CONVERGE_TIMEOUT).await;
+ wait_for_acked_readable(&client, &acked, CONVERGE_TIMEOUT)
+ .await
+ .unwrap_or_else(|state| {
+ panic!("every acked offset must poll back after the killed node reboots: {state}")
+ });
+ assert_no_segment_damage_reported(harness, backup);
+ }
+
/// A torn tail of garbage on the active segment `.log` must be truncated for
/// good by recovery: the walked bounds govern both the on-disk length and the
/// reopened write cursor, so post-recovery appends land exactly where their
diff --cc core/integration/tests/cluster/partition_dedup.rs
index a3bb32734,f1fb57c48..482edd149
--- a/core/integration/tests/cluster/partition_dedup.rs
+++ b/core/integration/tests/cluster/partition_dedup.rs
@@@ -290,6 -293,142 +293,143 @@@ async fn verify_lost_reply_restart(harn
);
}
+ #[iggy_harness(cluster_nodes = 1, server(sharding.cpu_allocation = "0..1"))]
+ async fn given_lost_send_reply_on_singleton_when_send_permission_is_revoked_should_deny_retries_until_regrant(
+ harness: &mut TestHarness,
+ ) {
+ verify_lost_reply_after_send_revocation(harness).await;
+ }
+
+ #[iggy_harness(cluster_nodes = 3, server(sharding.cpu_allocation = "0..1"))]
+ async fn given_lost_send_reply_on_cluster_when_send_permission_is_revoked_should_deny_retries_until_regrant(
+ harness: &mut TestHarness,
+ ) {
+ verify_lost_reply_after_send_revocation(harness).await;
+ }
+
+ /// The client retains a send whose reply was lost. While its user lacks the
+ /// send permission, the partition primary denies each retry before its dedup
+ /// reads the receipt, so the send stays uncertain. After a re-grant, the retry
+ /// returns the original receipt. Node 0 is the metadata primary and the
+ /// primary of the first topic's partition, so it applies a permission change
+ /// before the root client sees it commit.
+ async fn verify_lost_reply_after_send_revocation(harness: &mut TestHarness) {
+ const USERNAME: &str = "partition-dedup-producer";
+ const PASSWORD: &str = "partition-dedup-password";
+ // Long enough for the frame to reach the stopped node, far below the
+ // cluster heartbeat timeout.
+ const LOST_REPLY_DEADLINE: Duration = Duration::from_secs(1);
+ const DENIAL_MARKER: &str = "partition request denied by authorization";
+ let permissions = |send_messages| Permissions {
+ global: GlobalPermissions {
+ send_messages,
+ // The route lookup of a retry passes on this one alone.
+ poll_messages: true,
+ ..GlobalPermissions::default()
+ },
+ streams: None,
+ };
+ let observer = harness.root_client_for_node(0).await.unwrap();
+ seed_topic(&observer).await;
+ let route_client = connect_without_login(harness.node(0).tcp_addr().unwrap()).await;
+ route_client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ let route = partition_route(
+ &route_client,
+ &Identifier::named(STREAM_NAME).unwrap(),
+ &Identifier::named(TOPIC_NAME).unwrap(),
+ PARTITION_ID,
+ )
+ .await;
+ assert_eq!(
+ route.primary_replica, 0,
+ "the stopped node must own the partition whose reply is lost"
+ );
+ drop(route_client);
+ observer
+ .create_user(
+ USERNAME,
+ PASSWORD,
+ UserStatus::Active,
+ Some(permissions(true)),
++ &UserCreateOptions::default(),
+ )
+ .await
+ .expect("create the producer user");
+ let producer = harness.new_client().await.unwrap();
+ producer
+ .login_user(USERNAME, PASSWORD)
+ .await
+ .expect("sign in the producer");
+ // A committed send caches the route, so the next send needs no lookup.
+ send_one(&producer, b"before-lost-reply")
+ .await
+ .expect("send while every node runs");
+
+ pause_node(harness, 0, true);
+ let lost = with_binary_deadline(
+ Instant::now() + LOST_REPLY_DEADLINE,
+ send_one(&producer, b"lost-reply"),
+ )
+ .await;
+ pause_node(harness, 0, false);
+ let Err(IggyError::OperationUncertain(op)) = lost else {
+ panic!("a send without a reply must end uncertain, got {lost:?}");
+ };
+ let deadline = Instant::now() + COMMIT_BUDGET;
+ while poll_all(&observer).await != 2 {
+ assert!(
+ Instant::now() < deadline,
+ "the unanswered send must commit after the node resumes"
+ );
+ sleep(RETRY_PAUSE).await;
+ }
+
+ let user = Identifier::named(USERNAME).unwrap();
+ observer
+ .update_permissions(&user, Some(permissions(false)))
+ .await
+ .expect("revoke the send permission");
+ assert_eq!(
+ producer
+ .resolve_operation(op, std::time::Instant::now() + REPLY_WAIT)
+ .await,
+ Err(IggyError::OperationUncertain(op)),
+ "a retry without the send permission must not return the receipt"
+ );
+ await_marker(harness, 0, DENIAL_MARKER).await;
+ let unauthorized = format!("status={}", IggyError::Unauthorized.as_code());
+ assert!(
+ harness
+ .node(0)
+ .stdout_plain()
+ .lines()
+ .any(|line| line.contains(DENIAL_MARKER) && line.contains(&unauthorized)),
+ "the partition primary must deny the retry with {unauthorized}"
+ );
+ assert_eq!(
+ send_one(&producer, b"behind-uncertain-send").await,
+ Err(IggyError::LaneBlocked(op)),
+ "the client must keep the uncertain send"
+ );
+
+ observer
+ .update_permissions(&user, Some(permissions(true)))
+ .await
+ .expect("grant the send permission again");
+ producer
+ .resolve_operation(op, std::time::Instant::now() + REPLY_WAIT)
+ .await
+ .expect("after a re-grant the retry must return the original receipt");
+ assert_eq!(
+ poll_all(&observer).await,
+ 2,
+ "the resolved retry must not append again"
+ );
+ }
+
#[iggy_harness(cluster_nodes = 3, server(partition.wal_bytes_max = "134225920 B", sharding.cpu_allocation = "0..1"))]
async fn given_reclaimed_wal_when_recovered_quorum_replays_should_return_the_original_receipt(
harness: &mut TestHarness,
diff --cc core/integration/tests/cluster/partition_primary_routing.rs
index 7d8f33c56,408c4b79b..7dd60a161
--- a/core/integration/tests/cluster/partition_primary_routing.rs
+++ b/core/integration/tests/cluster/partition_primary_routing.rs
@@@ -119,8 -200,864 +200,876 @@@ fn send_body(payload: &str) -> SendMess
}
}
+ pub(super) async fn partition_route(
+ client: &TcpClient,
+ stream: &Identifier,
+ topic: &Identifier,
+ partition_id: u32,
+ ) -> PartitionRouteResponse {
+ let request = GetPartitionRouteRequest {
+ stream_id: identifier_to_wire(stream).unwrap(),
+ topic_id: identifier_to_wire(topic).unwrap(),
+ partition_id,
+ }
+ .to_bytes();
+ timeout(PRECONDITION_BUDGET, async {
+ loop {
+ match client
+ .send_raw_with_response(GET_PARTITION_ROUTE_CODE, request.clone())
+ .await
+ {
+ Ok(reply) => break PartitionRouteResponse::decode_from(&reply).unwrap(),
+ Err(IggyError::TransientNotAccepted) => sleep(PRECONDITION_POLL).await,
+ Err(error) => panic!("route {stream}/{topic}/{partition_id}: {error}"),
+ }
+ }
+ })
+ .await
+ .unwrap_or_else(|_| panic!("partition {stream}/{topic}/{partition_id} never became routable"))
+ }
+
+ #[iggy_harness(cluster_nodes = 3, server(sharding.cpu_allocation = "2"))]
+ async fn given_concurrent_creates_when_growing_partitions_should_spread_and_serve_every_partition(
+ harness: &mut TestHarness,
+ ) {
+ const FIRST_TOPIC: &str = "spread-first";
+ const SECOND_TOPIC: &str = "spread-second";
+ const THIRD_TOPIC: &str = "spread-third";
+ const INITIAL_PARTITIONS: u32 = 3;
+ const ADDED_PARTITIONS: u32 = 3;
+ const SECOND_PARTITIONS: u32 = 2;
+ const THIRD_PARTITIONS: u32 = 4;
+
+ let first = connect_without_login(harness.node(0).tcp_addr().unwrap()).await;
+ first
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ let second = connect_without_login(harness.node(0).tcp_addr().unwrap()).await;
+ second
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
- first.create_stream(STREAM_NAME).await.unwrap();
++ first
++ .create_stream(STREAM_NAME, &StreamCreateOptions::default())
++ .await
++ .unwrap();
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let options = TopicCreateOptions {
+ partitions_count: Some(INITIAL_PARTITIONS),
+ durability: Durability::Persisted,
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ };
+ first
+ .create_topic(&stream, FIRST_TOPIC, &options)
+ .await
+ .unwrap();
+ let first_topic = Identifier::named(FIRST_TOPIC).unwrap();
+ for partition_id in 0..INITIAL_PARTITIONS {
+ let route = partition_route(&first, &stream, &first_topic, partition_id).await;
+ assert_eq!(u32::from(route.primary_replica), partition_id);
+ assert_eq!(route.view, partition_id);
+ }
+
+ let second_options = TopicCreateOptions {
+ partitions_count: Some(SECOND_PARTITIONS),
+ ..options.clone()
+ };
+ let third_options = TopicCreateOptions {
+ partitions_count: Some(THIRD_PARTITIONS),
+ ..options
+ };
+ let (second_result, third_result) = futures::join!(
+ first.create_topic(&stream, SECOND_TOPIC, &second_options),
+ second.create_topic(&stream, THIRD_TOPIC, &third_options),
+ );
+ second_result.unwrap();
+ third_result.unwrap();
+ let mut topics = [
+ (FIRST_TOPIC, INITIAL_PARTITIONS),
+ (SECOND_TOPIC, SECOND_PARTITIONS),
+ (THIRD_TOPIC, THIRD_PARTITIONS),
+ ];
+ assert_initial_spread(&first, &stream, &topics, harness.cluster_size()).await;
+ first
+ .create_partitions(&stream, &first_topic, ADDED_PARTITIONS)
+ .await
+ .unwrap();
+ topics[0].1 += ADDED_PARTITIONS;
+ assert_initial_spread(&first, &stream, &topics, harness.cluster_size()).await;
+
+ let mut original_routes = Vec::new();
+ for (name, count) in topics {
+ let topic = Identifier::named(name).unwrap();
+ for partition_id in 0..count {
+ let route = partition_route(&first, &stream, &topic, partition_id).await;
+ original_routes.push((name, partition_id, route));
+ let payload = format!("{name}-{partition_id}");
+ let mut messages = vec![message(&payload)];
+ first
+ .send_messages(
+ &stream,
+ &topic,
+ &Partitioning::partition_id(partition_id),
+ &mut messages,
+ )
+ .await
+ .unwrap();
+ assert_partition_payload(&second, &stream, &topic, partition_id, &payload).await;
+ }
+ }
+ drop(first);
+ drop(second);
+ for node in 0..harness.cluster_size() {
+ harness.stop_node(node).unwrap();
+ }
+ for node in 0..harness.cluster_size() {
+ harness.restart_node(node).unwrap();
+ }
+ let recovered = connect_without_login(harness.node(0).tcp_addr().unwrap()).await;
+ recovered
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ for (name, partition_id, original) in original_routes {
+ let topic = Identifier::named(name).unwrap();
+ let route = partition_route(&recovered, &stream, &topic, partition_id).await;
+ assert_eq!(route.incarnation, original.incarnation);
+ assert_eq!(route.partitions_count, original.partitions_count);
+ assert!(route.view >= original.view, "{name}/{partition_id}");
+ let payload = format!("{name}-{partition_id}");
+ assert_partition_payload(&recovered, &stream, &topic, partition_id, &payload).await;
+ }
+ }
+
+ async fn assert_initial_spread(
+ client: &TcpClient,
+ stream: &Identifier,
+ topics: &[(&str, u32)],
+ replicas: usize,
+ ) {
+ let mut primary_counts = vec![0_u32; replicas];
+ for &(name, count) in topics {
+ let topic = Identifier::named(name).unwrap();
+ for partition_id in 0..count {
+ let route = partition_route(client, stream, &topic, partition_id).await;
+ assert_eq!(route.view, u32::from(route.primary_replica));
+ primary_counts[usize::from(route.primary_replica)] += 1;
+ }
+ }
+ let total: u32 = topics.iter().map(|(_, count)| count).sum();
+ assert_eq!(
+ primary_counts,
+ vec![total / u32::try_from(replicas).unwrap(); replicas],
+ "create batches and growth must count already admitted assignments"
+ );
+ }
+
+ async fn assert_partition_payload(
+ client: &TcpClient,
+ stream: &Identifier,
+ topic: &Identifier,
+ partition_id: u32,
+ expected: &str,
+ ) {
+ let polled = client
+ .poll_messages(
+ stream,
+ topic,
+ Some(partition_id),
+ &Consumer::default(),
+ &PollingStrategy::offset(0),
+ 1,
+ false,
+ )
+ .await
+ .unwrap();
+ assert_eq!(polled.messages.len(), 1, "{topic}/{partition_id}");
+ assert_eq!(polled.messages[0].payload.as_ref(), expected.as_bytes());
+ }
+
+ #[iggy_harness(cluster_nodes = 5, server(sharding.cpu_allocation = "2"))]
+ async fn given_balanced_primaries_when_a_replica_fails_and_returns_should_rebalance_without_data_loss(
+ harness: &mut TestHarness,
+ ) {
+ const TOPIC_COUNT: usize = 100;
+ const PHASES: [&str; 3] = ["before-failure", "while-absent", "after-return"];
+
+ let client = connect_without_login(harness.node(1).tcp_addr().unwrap()).await;
+ client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ let identity = client.session_identity().await.unwrap();
- client.create_stream(STREAM_NAME).await.unwrap();
++ client
++ .create_stream(STREAM_NAME, &StreamCreateOptions::default())
++ .await
++ .unwrap();
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let options = TopicCreateOptions {
+ partitions_count: Some(1),
+ durability: Durability::Persisted,
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ };
+ let mut topics = Vec::with_capacity(TOPIC_COUNT);
+ for index in 0..TOPIC_COUNT {
+ let name = format!("rebalance-{index}");
+ client.create_topic(&stream, &name, &options).await.unwrap();
+ topics.push(Identifier::named(&name).unwrap());
+ }
+ let mut request = RequestHeader {
+ client: identity.client_id,
+ session: identity.session,
+ request: RAW_REQUEST_START,
+ ..Default::default()
+ };
+ let initial =
+ wait_for_balanced_routes(harness, &client, &stream, &topics, &[20; 5], &mut request).await;
+ append_and_assert_partition_sentinels(&client, &stream, &topics, &PHASES[..1]).await;
+
+ harness.kill_node(0).unwrap();
+ let surviving = wait_for_balanced_routes(
+ harness,
+ &client,
+ &stream,
+ &topics,
+ &[0, 25, 25, 25, 25],
+ &mut request,
+ )
+ .await;
+ append_and_assert_partition_sentinels(&client, &stream, &topics, &PHASES[..2]).await;
+
+ harness.restart_node(0).unwrap();
+ let returned =
+ wait_for_balanced_routes(harness, &client, &stream, &topics, &[20; 5], &mut request).await;
+ for (index, original) in initial.iter().enumerate() {
+ assert_ne!(original.incarnation, 0);
+ for current in [&surviving[index], &returned[index]] {
+ assert_eq!(
+ current.incarnation, original.incarnation,
+ "{}",
+ topics[index]
+ );
+ assert_eq!(current.partitions_count, 1, "{}", topics[index]);
+ assert!(current.view >= original.view, "{}", topics[index]);
+ }
+ assert!(
+ returned[index].view >= surviving[index].view,
+ "{}",
+ topics[index]
+ );
+ }
+ append_and_assert_partition_sentinels(&client, &stream, &topics, &PHASES).await;
+ let current_identity = client.session_identity().await.unwrap();
+ assert_eq!(current_identity.client_id, identity.client_id);
+ assert_eq!(current_identity.session, identity.session);
+ }
+
+ async fn wait_for_balanced_routes(
+ harness: &TestHarness,
+ client: &TcpClient,
+ stream: &Identifier,
+ topics: &[Identifier],
+ expected_counts: &[usize],
+ request: &mut RequestHeader,
+ ) -> Vec<PartitionRouteResponse> {
+ let mut last_counts = vec![vec![0; expected_counts.len()]; expected_counts.len()];
+ timeout(REBALANCE_BUDGET, async {
+ let binding = BindSessionRequest {
+ version_info: ClientVersionInfo {
+ protocol_version: IGGY_PROTOCOL_VERSION,
+ sdk_name: WireName::new("routing-test").unwrap(),
+ sdk_version: WireName::new("1.0.0").unwrap(),
+ },
+ identity: client.session_identity().await.unwrap(),
+ bind_secret: client.session_bind_secret().await.unwrap(),
+ }
+ .to_bytes();
+ let route_requests: Vec<_> = topics
+ .iter()
+ .map(|topic| {
+ GetPartitionRouteRequest {
+ stream_id: identifier_to_wire(stream).unwrap(),
+ topic_id: identifier_to_wire(topic).unwrap(),
+ partition_id: PARTITION_ID,
+ }
+ .to_bytes()
+ })
+ .collect();
+ let mut observers = Vec::with_capacity(expected_counts.len());
+ for (node, count) in expected_counts.iter().enumerate() {
+ if *count == 0 {
+ continue;
+ }
+ let address = harness.node(node).tcp_addr().unwrap();
+ let mut socket = raw_tcp::connect_to(address).await;
+ loop {
+ match data_request(&mut socket, request, BIND_SESSION_CODE, &binding, None).await {
+ Ok(_) => break,
+ Err(IggyError::TransientNotAccepted) => sleep(PRECONDITION_POLL).await,
+ Err(error) => panic!("node {node} refused the existing session: {error:?}"),
+ }
+ }
+ assert_eq!(socket.peer_addr().unwrap(), address);
+ observers.push((node, socket));
+ }
+ loop {
+ let mut snapshots = Vec::with_capacity(observers.len());
+ for (node, socket) in &mut observers {
+ let mut routes = Vec::with_capacity(topics.len());
+ for body in &route_requests {
+ match data_request(socket, request, GET_PARTITION_ROUTE_CODE, body, None).await {
+ Ok(body) => routes.push(PartitionRouteResponse::decode_from(&body).unwrap()),
+ Err(IggyError::TransientNotAccepted) => break,
+ Err(error) => panic!("node {node} refused a partition route: {error:?}"),
+ }
+ }
+ if routes.len() != topics.len() {
+ break;
+ }
+ let mut counts = vec![0; expected_counts.len()];
+ for route in &routes {
+ counts[usize::from(route.primary_replica)] += 1;
+ }
+ last_counts[*node].clone_from(&counts);
+ if counts != expected_counts {
+ break;
+ }
+ snapshots.push(routes);
+ }
+ if snapshots.len() == observers.len()
+ && snapshots.windows(2).all(|pair| {
+ pair[0].iter().zip(&pair[1]).all(|(left, right)| {
+ (left.incarnation, left.partitions_count, left.view, left.primary_replica)
+ == (right.incarnation, right.partitions_count, right.view, right.primary_replica)
+ })
+ })
+ {
+ return snapshots.remove(0);
+ }
+ sleep(PRECONDITION_POLL).await;
+ }
+ })
+ .await
+ .unwrap_or_else(|_| {
+ panic!(
+ "partition primaries did not converge to {expected_counts:?} within {REBALANCE_BUDGET:?}; \
+ physical-node observations: {last_counts:?}"
+ )
+ })
+ }
+
+ async fn append_and_assert_partition_sentinels(
+ client: &TcpClient,
+ stream: &Identifier,
+ topics: &[Identifier],
+ phases: &[&str],
+ ) {
+ timeout(REBALANCE_BUDGET, async {
+ for topic in topics {
+ let payload = format!("{topic}:{}", phases.last().unwrap());
+ client
+ .send_messages(
+ stream,
+ topic,
+ &Partitioning::partition_id(PARTITION_ID),
+ &mut [message(&payload)],
+ )
+ .await
+ .unwrap();
+ let polled = client
+ .poll_messages(
+ stream,
+ topic,
+ Some(PARTITION_ID),
+ &Consumer::default(),
+ &PollingStrategy::offset(0),
+ u32::try_from(phases.len() + 1).unwrap(),
+ false,
+ )
+ .await
+ .unwrap();
+ assert_eq!(polled.messages.len(), phases.len(), "{topic}");
+ for (message, phase) in polled.messages.iter().zip(phases) {
+ assert_eq!(
+ message.payload.as_ref(),
+ format!("{topic}:{phase}").as_bytes()
+ );
+ }
+ }
+ })
+ .await
+ .expect("every partition must preserve and serve its history across rebalancing");
+ }
+
+ /// Producers write every partition back to back while the rebalancer moves
+ /// primaries to a returning replica. Every acknowledged send stays readable in
+ /// send order. A send that ends uncertain is never sent again, so no message
+ /// may appear twice. Each send ends acknowledged, uncertain or with the refusal
+ /// the SDK retries, and none waits longer than a move can fence its partition.
+ #[iggy_harness(cluster_nodes = 3, server(sharding.cpu_allocation = "2"))]
+ async fn given_producers_on_every_partition_when_primaries_move_should_keep_acknowledged_order_without_stalls(
+ harness: &mut TestHarness,
+ ) {
+ const TOPIC_COUNT: usize = 12;
+ const RETURNING_MOVES: usize = 4;
+
+ let client = connect_without_login(harness.node(1).tcp_addr().unwrap()).await;
+ client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ let identity = client.session_identity().await.unwrap();
- client.create_stream(STREAM_NAME).await.unwrap();
++ client
++ .create_stream(STREAM_NAME, &StreamCreateOptions::default())
++ .await
++ .unwrap();
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let options = TopicCreateOptions {
+ partitions_count: Some(1),
+ durability: Durability::Persisted,
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ };
+ let mut topics = Vec::with_capacity(TOPIC_COUNT);
+ for index in 0..TOPIC_COUNT {
+ let name = format!("moving-{index}");
+ client.create_topic(&stream, &name, &options).await.unwrap();
+ topics.push(Identifier::named(&name).unwrap());
+ }
+ let mut request = RequestHeader {
+ client: identity.client_id,
+ session: identity.session,
+ request: RAW_REQUEST_START,
+ ..Default::default()
+ };
+ wait_for_balanced_routes(harness, &client, &stream, &topics, &[4; 3], &mut request).await;
+ harness.kill_node(0).unwrap();
+ wait_for_balanced_routes(harness, &client, &stream, &topics, &[0, 6, 6], &mut request).await;
+
+ let producing = Arc::new(AtomicUsize::new(0));
+ let stop = Arc::new(AtomicBool::new(false));
+ let producers = spawn_producers(
+ harness.node(1).tcp_addr().unwrap(),
+ topics.clone(),
+ Arc::clone(&producing),
+ Arc::clone(&stop),
+ );
+ timeout(PRECONDITION_BUDGET, async {
+ while producing.load(Ordering::Acquire) < topics.len() {
+ sleep(PRECONDITION_POLL).await;
+ }
+ })
+ .await
+ .expect("every producer must acknowledge a send before the replica returns");
+ let transfers_before = transfers_started(harness);
+ harness.restart_node(0).unwrap();
+ wait_for_balanced_routes(harness, &client, &stream, &topics, &[4; 3], &mut request).await;
+ let transfers = transfers_started(harness) - transfers_before;
+ stop.store(true, Ordering::Release);
+ let logs = producers.join().expect("producer thread");
+
+ let mut latencies: Vec<Duration> = logs
+ .iter()
+ .flat_map(|log| log.latencies.iter().copied())
+ .collect();
+ latencies.sort_unstable();
+ println!(
+ "{} sends across {transfers} primary moves: {} acknowledged, {} uncertain, {} refused, \
+ {} unexpected; latency p50 {:?}, p99 {:?}, max {:?}",
+ latencies.len(),
+ logs.iter().map(|log| log.acknowledged.len()).sum::<usize>(),
+ logs.iter().map(|log| log.uncertain.len()).sum::<usize>(),
+ logs.iter().map(|log| log.refused.len()).sum::<usize>(),
+ logs.iter().map(|log| log.unexpected.len()).sum::<usize>(),
+ latencies[latencies.len() / 2],
+ latencies[latencies.len() * 99 / 100],
+ latencies[latencies.len() - 1],
+ );
+ assert!(
+ transfers >= RETURNING_MOVES,
+ "the producers ran across {transfers} moves, expected at least {RETURNING_MOVES}"
+ );
+ for (topic, log) in topics.iter().zip(&logs) {
+ let sequences = read_sequences(&client, &stream, topic).await;
+ assert_produced_history(topic, log, &sequences);
+ }
+ }
+
+ /// What one producer saw of its sends. Sequence `n` is the `n`th send.
+ #[derive(Default)]
+ struct ProducerLog {
+ acknowledged: Vec<u64>,
+ uncertain: Vec<u64>,
+ refused: Vec<u64>,
+ unexpected: Vec<(u64, IggyError)>,
+ latencies: Vec<Duration>,
+ }
+
+ /// The producers run on a thread of their own, so a blocking harness call on
+ /// the test thread does not stretch their latencies.
+ fn spawn_producers(
+ address: SocketAddr,
+ topics: Vec<Identifier>,
+ producing: Arc<AtomicUsize>,
+ stop: Arc<AtomicBool>,
+ ) -> JoinHandle<Vec<ProducerLog>> {
+ thread::spawn(move || {
+ runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .expect("producer runtime")
+ .block_on(join_all(topics.iter().map(|topic| {
+ produce_until_stopped(address, topic, &producing, &stop)
+ })))
+ })
+ }
+
+ async fn produce_until_stopped(
+ address: SocketAddr,
+ topic: &Identifier,
+ producing: &AtomicUsize,
+ stop: &AtomicBool,
+ ) -> ProducerLog {
+ let client = connect_without_login(address).await;
+ client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let partitioning = Partitioning::partition_id(PARTITION_ID);
+ let mut log = ProducerLog::default();
+ let mut sequence = 0;
+ while !stop.load(Ordering::Acquire) {
+ let payload = format!("{topic}:{sequence}");
+ let started = Instant::now();
+ let outcome = client
+ .send_messages(&stream, topic, &partitioning, &mut [message(&payload)])
+ .await;
+ log.latencies.push(started.elapsed());
+ match outcome {
+ Ok(_) => {
+ if log.acknowledged.is_empty() {
+ producing.fetch_add(1, Ordering::Release);
+ }
+ log.acknowledged.push(sequence);
+ }
+ Err(IggyError::OperationUncertain(operation)) => {
+ client
+ .abandon_operation(operation)
+ .await
+ .expect("free the lane of an uncertain send");
+ log.uncertain.push(sequence);
+ }
+ Err(IggyError::TransientNotAccepted) => log.refused.push(sequence),
+ Err(error) => log.unexpected.push((sequence, error)),
+ }
+ sequence += 1;
+ }
+ log
+ }
+
+ fn transfers_started(harness: &TestHarness) -> usize {
+ // Node 0 restarts, which truncates its log, and it never moves a primary away.
+ (1..harness.cluster_size())
+ .map(|node| harness.node(node).stdout_occurrences(TRANSFER_MARKER))
+ .sum()
+ }
+
+ async fn read_sequences(client: &TcpClient, stream: &Identifier, topic: &Identifier) -> Vec<u64> {
+ let mut sequences = Vec::new();
+ let mut offset = 0;
+ loop {
+ let polled = client
+ .poll_messages(
+ stream,
+ topic,
+ Some(PARTITION_ID),
+ &Consumer::default(),
+ &PollingStrategy::offset(offset),
+ READBACK_BATCH,
+ false,
+ )
+ .await
+ .unwrap();
+ let Some(last) = polled.messages.last() else {
+ return sequences;
+ };
+ offset = last.header.offset + 1;
+ for message in &polled.messages {
+ let payload = std::str::from_utf8(&message.payload).expect("a produced payload");
+ let (name, sequence) = payload.rsplit_once(':').expect("a produced payload");
+ assert_eq!(name, topic.to_string());
+ sequences.push(sequence.parse().expect("a produced sequence"));
+ }
+ }
+ }
+
+ fn assert_produced_history(topic: &Identifier, log: &ProducerLog, sequences: &[u64]) {
+ assert!(
+ log.unexpected.is_empty(),
+ "{topic}: sends ended with errors the SDK does not retry: {:?}",
+ log.unexpected
+ );
+ let stalled: Vec<_> = log
+ .latencies
+ .iter()
+ .enumerate()
+ .filter(|(_, latency)| **latency > MOVE_STALL_BOUND)
+ .collect();
+ assert!(
+ stalled.is_empty(),
+ "{topic}: sends stalled past {MOVE_STALL_BOUND:?}: {stalled:?}"
+ );
+ if let Some(pair) = sequences.windows(2).find(|pair| pair[0] >= pair[1]) {
+ panic!(
+ "{topic}: sequence {} follows {} in the log",
+ pair[1], pair[0]
+ );
+ }
+ let read: HashSet<u64> = sequences.iter().copied().collect();
+ let lost: Vec<_> = log
+ .acknowledged
+ .iter()
+ .filter(|sequence| !read.contains(sequence))
+ .collect();
+ assert!(
+ lost.is_empty(),
+ "{topic}: acknowledged sends lost: {lost:?}"
+ );
+ let sent: HashSet<u64> = log
+ .acknowledged
+ .iter()
+ .chain(&log.uncertain)
+ .copied()
+ .collect();
+ let refused: Vec<_> = sequences
+ .iter()
+ .filter(|sequence| !sent.contains(sequence))
+ .collect();
+ assert!(
+ refused.is_empty(),
+ "{topic}: refused sends reached the log: {refused:?}"
+ );
+ }
+
+ #[iggy_harness(cluster_nodes = 3)]
+ async fn given_a_partition_primary_leading_metadata_when_its_quorum_freezes_should_refuse_writes_until_it_resumes(
+ harness: &mut TestHarness,
+ ) {
+ assert_isolated_primary_fence(harness, true).await;
+ }
+
+ #[iggy_harness(cluster_nodes = 3)]
+ async fn given_a_partition_primary_backing_up_metadata_when_its_quorum_freezes_should_refuse_writes_until_it_resumes(
+ harness: &mut TestHarness,
+ ) {
+ assert_isolated_primary_fence(harness, false).await;
+ }
+
+ /// A partition primary whose quorum peers are frozen gets no fresh metadata
+ /// quorum evidence, whether it credits answered probes as the metadata primary
+ /// or probes as a backup. It must refuse partition writes with 58 once that
+ /// evidence is `heartbeat_timeout + 3s` old, admit nothing it refused, and
+ /// serve the partition again after the peers resume.
+ async fn assert_isolated_primary_fence(harness: &TestHarness, leads_metadata: bool) {
+ const ADMITTED: [&str; 2] = ["before-isolation", "after-resume"];
+ const REFUSED: &str = "refused-while-isolated";
+
+ let client = connect_without_login(harness.node(0).tcp_addr().unwrap()).await;
+ client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
- client.create_stream(STREAM_NAME).await.unwrap();
++ client
++ .create_stream(STREAM_NAME, &StreamCreateOptions::default())
++ .await
++ .unwrap();
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let topic = Identifier::named(TOPIC_NAME).unwrap();
+ let partitions = u32::try_from(harness.cluster_size()).unwrap();
+ let options = TopicCreateOptions {
+ partitions_count: Some(partitions),
+ ..TopicCreateOptions::default()
+ };
+ client
+ .create_topic(&stream, TOPIC_NAME, &options)
+ .await
+ .unwrap();
+ let leader = leader_node_index_via(harness, 0).await;
+ let mut routes = Vec::with_capacity(harness.cluster_size());
+ for partition_id in 0..partitions {
+ routes.push(partition_route(&client, &stream, &topic, partition_id).await);
+ }
+ let (partition_id, route) = (0..partitions)
+ .zip(routes)
+ .find(|(_, route)| (usize::from(route.primary_replica) == leader) == leads_metadata)
+ .expect("one partition primary on every node, the metadata primary among them");
+ let primary = usize::from(route.primary_replica);
+ let address = harness.node(primary).tcp_addr().unwrap();
+ let partitioning = Partitioning::partition_id(partition_id);
+ client
+ .send_messages(&stream, &topic, &partitioning, &mut [message(ADMITTED[0])])
+ .await
+ .unwrap();
+
+ let probe = connect_without_login(address).await;
+ probe
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ let identity = probe.session_identity().await.unwrap();
+ let binding = BindSessionRequest {
+ version_info: ClientVersionInfo {
+ protocol_version: IGGY_PROTOCOL_VERSION,
+ sdk_name: WireName::new("routing-test").unwrap(),
+ sdk_version: WireName::new("1.0.0").unwrap(),
+ },
+ identity,
+ bind_secret: probe.session_bind_secret().await.unwrap(),
+ }
+ .to_bytes();
+ let mut request = RequestHeader {
+ client: identity.client_id,
+ session: identity.session,
+ request: RAW_REQUEST_START,
+ ..Default::default()
+ };
+ let mut socket = raw_tcp::connect_to(address).await;
+ data_request(&mut socket, &mut request, BIND_SESSION_CODE, &binding, None)
+ .await
+ .expect("bind the probe session on the partition primary");
+ let poll = PollMessagesRequest {
+ consumer: consumer_to_wire(&Consumer::default()).unwrap(),
+ stream_id: identifier_to_wire(&stream).unwrap(),
+ topic_id: identifier_to_wire(&topic).unwrap(),
+ partition_id: Some(partition_id),
+ strategy: polling_strategy_to_wire(&PollingStrategy::offset(0)),
+ count: 1,
+ auto_commit: false,
+ }
+ .to_bytes();
+ data_request(
+ &mut socket,
+ &mut request,
+ POLL_MESSAGES_CODE,
+ &poll,
+ Some(route.context()),
+ )
+ .await
+ .expect("a primary with its quorum must serve the partition");
+
+ let peers: Vec<usize> = (0..harness.cluster_size())
+ .filter(|node| *node != primary)
+ .collect();
+ let isolated_at = Instant::now();
+ for &peer in &peers {
+ pause_node(harness, peer, true);
+ }
+ let fenced = timeout(FENCE_BOUND, async {
+ loop {
+ match data_request(
+ &mut socket,
+ &mut request,
+ POLL_MESSAGES_CODE,
+ &poll,
+ Some(route.context()),
+ )
+ .await
+ {
+ Ok(_) => sleep(FENCE_PROBE_INTERVAL).await,
+ Err(error) => break error,
+ }
+ }
+ })
+ .await;
+ let write = data_request(
+ &mut socket,
+ &mut request,
+ SEND_MESSAGES_CODE,
+ &raw_send_body(partition_id, REFUSED),
+ Some(route.context()),
+ )
+ .await;
+ let refused_after = isolated_at.elapsed();
+ for &peer in &peers {
+ pause_node(harness, peer, false);
+ }
+ println!(
+ "node {primary}, metadata leader {leader}, refused a write to partition {partition_id} \
+ {refused_after:?} after isolation"
+ );
+ assert_eq!(
+ fenced,
+ Ok(IggyError::TransientNotAccepted),
+ "the isolated primary must close partition admission within {FENCE_BOUND:?}"
+ );
+ assert_eq!(
+ write.err(),
+ Some(IggyError::TransientNotAccepted),
+ "the isolated primary must refuse a write with 58"
+ );
+ assert!(
+ refused_after <= FENCE_BOUND,
+ "the write was refused {refused_after:?} after isolation, past {FENCE_BOUND:?}"
+ );
+
+ timeout(
+ DATA_FAILOVER_BUDGET,
+ client.send_messages(&stream, &topic, &partitioning, &mut [message(ADMITTED[1])]),
+ )
+ .await
+ .expect("the cluster must answer a write after the peers resume")
+ .expect("the cluster must accept a write after the peers resume");
+ let polled = client
+ .poll_messages(
+ &stream,
+ &topic,
+ Some(partition_id),
+ &Consumer::default(),
+ &PollingStrategy::offset(0),
+ u32::try_from(ADMITTED.len() + 1).unwrap(),
+ false,
+ )
+ .await
+ .unwrap();
+ let payloads: Vec<&[u8]> = polled
+ .messages
+ .iter()
+ .map(|message| message.payload.as_ref())
+ .collect();
+ assert_eq!(
+ payloads,
+ ADMITTED.map(str::as_bytes),
+ "a write refused with 58 must not reach the log"
+ );
+ }
+
+ fn raw_send_body(partition_id: u32, payload: &str) -> Bytes {
+ let stream_id = WireIdentifier::named(STREAM_NAME).unwrap();
+ let topic_id = WireIdentifier::named(TOPIC_NAME).unwrap();
+ let partitioning = WirePartitioning::PartitionId(partition_id);
+ let messages = [RawMessage {
+ id: 0,
+ origin_timestamp: 0,
+ headers: None,
+ payload: payload.as_bytes(),
+ }];
+ let mut body = BytesMut::with_capacity(SendMessagesEncoder::encoded_size(
+ &stream_id,
+ &topic_id,
+ &partitioning,
+ &messages,
+ ));
+ SendMessagesEncoder::encode(&mut body, &stream_id, &topic_id, &partitioning, &messages)
+ .unwrap();
+ body.freeze()
+ }
+
#[iggy_harness(cluster_nodes = 3)]
- async fn given_metadata_view_moved_when_producing_to_a_fresh_topic_should_reach_the_advertised_leader(
+ async fn given_metadata_view_moved_when_creating_topic_should_route_to_its_assigned_primary(
harness: &mut TestHarness,
) {
// Kill node 0: it is the view-0 primary of BOTH planes, so the metadata
@@@ -1766,3 -3041,578 +3068,582 @@@ async fn data_request
Ok(body)
}
}
+
+ /// A hop target serves relays from a budget of their own. Stalled relayed
+ /// uploads that hold every slot leave the node's own relays flowing, a caller
+ /// that hangs up frees its slots at once, and the rest end at the deadline.
+ #[iggy_harness(cluster_nodes = 3, server(
+ http.jwt.encoding_secret = "0123456789abcdef0123456789abcdef",
+ http.jwt.decoding_secret = "0123456789abcdef0123456789abcdef"
+ ))]
+ async fn given_stalled_http_relay_uploads_when_every_target_slot_is_held_should_keep_relaying_and_free_them(
+ harness: &mut TestHarness,
+ ) {
+ let route = relay_route(spread_relay_target(harness).await, 0);
+ let http = HttpClient::login_root_no_redirect(node_url(harness, RELAY_TARGET_NODE)).await;
+ let mut tokens = Vec::new();
+ for _ in 0..=RELAY_TARGET_BUDGET / RELAY_TARGET_SHARE {
+ let session = http
+ .login(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await;
+ tokens.push(session.token);
+ }
+ let (fresh, stalling) = tokens
+ .split_last()
+ .expect("one credential per share, plus one");
+ let (cancelled, waiting) = stalling.split_last().expect("credentials that stall");
+ let address = harness
+ .node(RELAY_TARGET_NODE)
+ .http_addr()
+ .expect("hop target HTTP address");
+ // The credential that hangs up opens last, so its deadline comes last too.
+ let opened = Instant::now();
+ let mut uploads = Vec::with_capacity(RELAY_TARGET_BUDGET);
+ for token in waiting.iter().chain([cancelled]) {
+ for _ in 0..RELAY_TARGET_SHARE {
+ uploads.push(stalled_relay_upload(address, token, &route).await);
+ }
+ }
+
+ timeout(PRECONDITION_BUDGET, async {
+ loop {
+ let (status, body) = relayed_poll(&http, fresh, &route).await;
+ if status == StatusCode::SERVICE_UNAVAILABLE && body.contains("relay_target_busy") {
+ break;
+ }
+ assert_eq!(
+ status,
+ StatusCode::OK,
+ "relayed poll before the budget fills: {body}"
+ );
+ sleep(RELAY_PROBE_INTERVAL).await;
+ }
+ })
+ .await
+ .expect("the stalled uploads must hold every hop-target slot");
+ let (status, body) = relayed_poll(&http, cancelled, &route).await;
+ assert_eq!(
+ status,
+ StatusCode::TOO_MANY_REQUESTS,
+ "saturated credential: {body}"
+ );
+ assert!(body.contains("too_many_relayed_requests"), "{body}");
+
+ let response = http
+ .post_json(
+ "/streams",
+ &serde_json::json!({ "name": "created-while-every-target-slot-is-held" }),
+ )
+ .await;
+ assert_eq!(
+ response.status(),
+ StatusCode::OK,
+ "control-plane relay from the hop target"
+ );
+ let response = http
+ .client
+ .post(http.url(&format!(
+ "/streams/{STREAM_NAME}/topics/{TOPIC_NAME}/messages"
+ )))
+ .bearer_auth(&http.token)
+ .json(&send_body(RELAYED_PAYLOADS[0]))
+ .send()
+ .await
+ .expect("partition relay from the hop target");
+ assert_eq!(
+ response.status(),
+ StatusCode::CREATED,
+ "partition relay from the hop target"
+ );
+
+ drop(uploads.split_off(uploads.len() - RELAY_TARGET_SHARE));
+ timeout(PRECONDITION_BUDGET, async {
+ loop {
+ let (status, body) = relayed_poll(&http, cancelled, &route).await;
+ if status == StatusCode::OK {
+ break;
+ }
+ assert_eq!(status, StatusCode::TOO_MANY_REQUESTS, "{body}");
+ sleep(RELAY_PROBE_INTERVAL).await;
+ }
+ })
+ .await
+ .expect("a caller hanging up mid-body must free its slots");
+ let mut unread = [0; 1];
+ assert!(
+ matches!(
+ uploads[0].try_read(&mut unread),
+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
+ ),
+ "the slots must be freed by the hang-up, before any deadline"
+ );
+
+ for upload in uploads {
+ let answer = timeout(
+ (opened + RELAY_BUDGET + DEADLINE_SLACK).saturating_duration_since(Instant::now()),
+ stalled_answer(upload),
+ )
+ .await
+ .expect("a stalled upload must end at its deadline");
+ assert!(answer.starts_with("HTTP/1.1 503"), "{answer}");
+ assert!(
+ answer.contains(&format!(
+ "\"id\":{}",
+ IggyError::TransientNotAccepted.as_code()
+ )),
+ "{answer}"
+ );
+ }
+ let (status, body) = relayed_poll(&http, fresh, &route).await;
+ assert_eq!(
+ status,
+ StatusCode::OK,
+ "every slot must be free again: {body}"
+ );
+ }
+
+ /// A metadata backup relays control-plane writes and partition requests from
+ /// one pool of outbound slots. Control-plane uploads that stall mid-body take
+ /// no slot, so the node's partition relays keep flowing, and each stalled
+ /// upload ends at the relay deadline.
+ #[iggy_harness(cluster_nodes = 3, server(
+ http.jwt.encoding_secret = "0123456789abcdef0123456789abcdef",
+ http.jwt.decoding_secret = "0123456789abcdef0123456789abcdef"
+ ))]
+ async fn given_stalled_http_metadata_uploads_when_they_match_the_forward_slots_should_keep_relaying_and_end_them(
+ harness: &mut TestHarness,
+ ) {
+ spread_relay_target(harness).await;
+ let http = HttpClient::login_root_no_redirect(node_url(harness, RELAY_TARGET_NODE)).await;
+ let address = harness
+ .node(RELAY_TARGET_NODE)
+ .http_addr()
+ .expect("metadata backup HTTP address");
+ let counted = counter(&metrics(&http).await, REQUESTS_METRIC);
+ let opened = Instant::now();
+ let mut uploads = Vec::new();
+ for _ in 0..FORWARD_SLOTS {
+ uploads.push(
+ stalled_upload(
+ address,
+ "/streams",
+ &format!("authorization: Bearer {}\r\n", http.token),
+ )
+ .await,
+ );
+ }
+ // Each scrape counts itself too.
+ timeout(PRECONDITION_BUDGET, async {
+ for scrapes in 1.. {
+ if counter(&metrics(&http).await, REQUESTS_METRIC) >= counted + FORWARD_SLOTS + scrapes
+ {
+ break;
+ }
+ sleep(RELAY_PROBE_INTERVAL).await;
+ }
+ })
+ .await
+ .expect("the metadata backup must take the head of every stalled upload");
+
+ let response = http
+ .client
+ .post(http.url(&format!(
+ "/streams/{STREAM_NAME}/topics/{TOPIC_NAME}/messages"
+ )))
+ .bearer_auth(&http.token)
+ .json(&send_body(RELAYED_PAYLOADS[0]))
+ .send()
+ .await
+ .expect("partition relay from the metadata backup");
+ let (status, body) = status_and_body(response).await;
+ assert_eq!(
+ status,
+ StatusCode::CREATED,
+ "partition relay while control-plane uploads stall: {body}"
+ );
+
+ for upload in uploads {
+ let answer = timeout(
+ (opened + FORWARD_DEADLINE + DEADLINE_SLACK).saturating_duration_since(Instant::now()),
+ stalled_answer(upload),
+ )
+ .await
+ .expect("a stalled upload must end at its deadline");
+ assert!(answer.starts_with("HTTP/1.1 503"), "{answer}");
+ assert!(
+ answer.contains(&format!(
+ "\"id\":{}",
+ IggyError::TransientNotAccepted.as_code()
+ )),
+ "{answer}"
+ );
+ }
+ }
+
+ /// A client cannot pose as a relay. Forged relay fields that ask the hop
+ /// target to hold for a metadata op the cluster never reaches take no slot and
+ /// no hold, so the relays to and from the node keep flowing. A genuine relay
+ /// for an op the cluster has not committed yet still holds until the hop
+ /// target applies it, then is served.
+ #[iggy_harness(cluster_nodes = 3, server(
+ http.jwt.encoding_secret = "0123456789abcdef0123456789abcdef",
+ http.jwt.decoding_secret = "0123456789abcdef0123456789abcdef"
+ ))]
+ async fn given_forged_http_relays_when_flooding_a_partition_primary_should_refuse_them_without_a_hold(
+ harness: &mut TestHarness,
+ ) {
+ let incarnation = spread_relay_target(harness).await;
+ let admin = connect_without_login(harness.node(0).tcp_addr().unwrap()).await;
+ admin
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ admin
+ .create_user(
+ RELAY_FORGER,
+ RELAY_FORGER_PASSWORD,
+ UserStatus::Active,
+ None,
++ &UserCreateOptions::default(),
+ )
+ .await
+ .expect("create a user without permissions");
+ let root = HttpClient::login_root_no_redirect(node_url(harness, RELAY_TARGET_NODE)).await;
+ // The hop target verifies the forger's bearer only once it applied the user.
+ let forger = timeout(PRECONDITION_BUDGET, async {
+ loop {
+ let response = root
+ .client
+ .post(root.url("/users/login"))
+ .json(&serde_json::json!({
+ "username": RELAY_FORGER,
+ "password": RELAY_FORGER_PASSWORD,
+ }))
+ .send()
+ .await
+ .expect("login request");
+ if response.status().is_success() {
+ break root.with_token(access_token(response).await);
+ }
+ sleep(PRECONDITION_POLL).await;
+ }
+ })
+ .await
+ .expect("the hop target must apply the new user");
+ let entry = HttpClient::login_root_no_redirect(node_url(harness, RELAY_ENTRY_NODE)).await;
+ let send_path = format!("/streams/{STREAM_NAME}/topics/{TOPIC_NAME}/messages");
+ let unreachable_frontier = relay_route(incarnation, u64::MAX);
+ let proof_of_another_route =
+ fresh_relay_proof("POST", &send_path, &relay_route(incarnation, 0));
+ let forged = (0..RELAY_TARGET_BUDGET).map(|index| {
+ let request = forger
+ .client
+ .post(forger.url(&send_path))
+ .bearer_auth(&forger.token)
+ .header(FORWARDED_HEADER, "1")
+ .header(PARTITION_ROUTE_HEADER, &unreachable_frontier)
+ .json(&send_body_to(RELAY_TARGET_PARTITION, "forged"));
+ if index % 2 == 0 {
+ request.send()
+ } else {
+ request
+ .header(RELAY_PROOF_HEADER, &proof_of_another_route)
+ .send()
+ }
+ });
+
+ let started = Instant::now();
+ let (forged, (to_target, from_target)) =
+ tokio::join!(futures::future::join_all(forged), async {
+ sleep(FORGED_HEAD_START).await;
+ let to_target = entry
+ .client
+ .post(entry.url(&send_path))
+ .bearer_auth(&entry.token)
+ .json(&send_body_to(RELAY_TARGET_PARTITION, RELAYED_PAYLOADS[0]))
+ .send()
+ .await
+ .expect("relay to the hop target");
+ let from_target = root
+ .client
+ .post(root.url(&send_path))
+ .bearer_auth(&root.token)
+ .json(&send_body(RELAYED_PAYLOADS[1]))
+ .send()
+ .await
+ .expect("relay from the hop target");
+ (to_target, from_target)
+ },);
+ for response in forged {
+ let (status, body) = status_and_body(response.expect("forged relay")).await;
+ assert_eq!(status, StatusCode::FORBIDDEN, "forged relay: {body}");
+ }
+ assert!(
+ started.elapsed() < RELAY_BUDGET,
+ "no forged relay may hold for its metadata op"
+ );
+ let (status, body) = status_and_body(to_target).await;
+ assert_eq!(
+ status,
+ StatusCode::CREATED,
+ "relay to the hop target: {body}"
+ );
+ let (status, body) = status_and_body(from_target).await;
+ assert_eq!(
+ status,
+ StatusCode::CREATED,
+ "relay from the hop target: {body}"
+ );
+
+ let leader = leader_node_index_via(harness, RELAY_TARGET_NODE).await;
+ let leader_http = HttpClient::login_root_no_redirect(node_url(harness, leader)).await;
+ let committed = applied_op(&leader_http.get("/streams").await);
+ let next_op = relay_route(incarnation, committed + 1);
+ let held = tokio::spawn(
+ root.client
+ .post(root.url(&send_path))
+ .bearer_auth(&root.token)
+ .header(FORWARDED_HEADER, "1")
+ .header(PARTITION_ROUTE_HEADER, &next_op)
+ .header(
+ RELAY_PROOF_HEADER,
+ fresh_relay_proof("POST", &send_path, &next_op),
+ )
+ .json(&send_body_to(RELAY_TARGET_PARTITION, HELD_PAYLOAD))
+ .send(),
+ );
+ sleep(HOLD_PROBE).await;
+ assert!(
+ !held.is_finished(),
+ "a genuine relay must hold for an op the cluster has not committed"
+ );
+ let response = leader_http
+ .post_json("/streams", &serde_json::json!({ "name": FRONTIER_STREAM }))
+ .await;
+ assert_eq!(
+ response.status(),
+ StatusCode::OK,
+ "commit the op the held relay waits for"
+ );
+ let response = held.await.expect("held relay task").expect("held relay");
+ let served_at = response
+ .status()
+ .is_success()
+ .then(|| applied_op(&response));
+ let (status, body) = status_and_body(response).await;
+ assert_eq!(status, StatusCode::CREATED, "held relay: {body}");
+ assert!(
+ served_at > Some(committed),
+ "the hop target must serve the held relay only after applying its op"
+ );
+ }
+
+ /// A caller can name any metadata op in the context of its request, and a hop
+ /// target holds a relay until it applied the op of its route. The contacted
+ /// node relays only an op it applied, so requests that name an op the cluster
+ /// never reaches wait there, relay nothing and take no hop-target slot: the
+ /// hop target still admits a relay of the same credential.
+ #[iggy_harness(cluster_nodes = 3, server(
+ http.jwt.encoding_secret = "0123456789abcdef0123456789abcdef",
+ http.jwt.decoding_secret = "0123456789abcdef0123456789abcdef"
+ ))]
+ async fn given_contexts_naming_an_unreached_op_when_relaying_should_hold_them_on_the_contacted_node(
+ harness: &mut TestHarness,
+ ) {
+ let incarnation = spread_relay_target(harness).await;
+ let entry = HttpClient::login_root_no_redirect(node_url(harness, RELAY_ENTRY_NODE)).await;
+ let send_url = entry.url(&format!(
+ "/streams/{STREAM_NAME}/topics/{TOPIC_NAME}/messages"
+ ));
+ let unreached_body = SendMessages {
+ context: Some(PartitionContext {
+ incarnation,
+ owner_generation: 0,
+ metadata_op: u64::MAX,
+ }),
+ ..send_body_to(RELAY_TARGET_PARTITION, "names-an-unreached-op")
+ };
+ let send = |body: &SendMessages| {
+ entry
+ .client
+ .post(&send_url)
+ .bearer_auth(&entry.token)
+ .json(body)
+ .send()
+ };
+
+ let before = RoutingCost::of(&entry).await;
+ let (unreached, admitted) = tokio::join!(
+ futures::future::join_all((0..RELAY_TARGET_SHARE).map(|_| send(&unreached_body))),
+ async {
+ sleep(FORGED_HEAD_START).await;
+ send(&send_body_to(RELAY_TARGET_PARTITION, RELAYED_PAYLOADS[0]))
+ .await
+ .expect("relay of the same credential")
+ },
+ );
+ let (status, body) = status_and_body(admitted).await;
+ assert_eq!(
+ status,
+ StatusCode::CREATED,
+ "relay of the same credential: {body}"
+ );
+ for response in unreached {
+ let (status, body) = status_and_body(response.expect("send naming an unreached op")).await;
+ assert_eq!(
+ status,
+ StatusCode::SERVICE_UNAVAILABLE,
+ "send naming an unreached op: {body}"
+ );
+ let error: serde_json::Value = serde_json::from_str(&body).expect("decode the error body");
+ assert_eq!(error["id"], IggyError::TransientNotAccepted.as_code());
+ }
+ assert_eq!(
+ RoutingCost::of(&entry).await.since(before).relays,
+ 1,
+ "the contacted node must not relay an op it has not applied"
+ );
+ }
+
+ /// A three-partition topic places partition N on node N while node 0 stays
+ /// the metadata primary, so node 1 serves partition 1 and relays partition 0
+ /// and every control-plane write. Returns the incarnation of partition 1.
+ async fn spread_relay_target(harness: &TestHarness) -> u64 {
+ let client = connect_without_login(harness.node(0).tcp_addr().unwrap()).await;
+ client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
- client.create_stream(STREAM_NAME).await.unwrap();
++ client
++ .create_stream(STREAM_NAME, &StreamCreateOptions::default())
++ .await
++ .unwrap();
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let topic = Identifier::named(TOPIC_NAME).unwrap();
+ let options = TopicCreateOptions {
+ partitions_count: Some(SPREAD_PARTITIONS),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ };
+ client
+ .create_topic(&stream, TOPIC_NAME, &options)
+ .await
+ .unwrap();
+ let relayed = partition_route(&client, &stream, &topic, PARTITION_ID).await;
+ assert_eq!(relayed.primary_replica, 0);
+ let served = partition_route(&client, &stream, &topic, RELAY_TARGET_PARTITION).await;
+ assert_eq!(usize::from(served.primary_replica), RELAY_TARGET_NODE);
+ assert_ne!(
+ leader_node_index_via(harness, RELAY_TARGET_NODE).await,
+ RELAY_TARGET_NODE
+ );
+ served.incarnation
+ }
+
+ /// The route a relay of partition 1 carries, with the longest budget a hop
+ /// target grants.
+ fn relay_route(incarnation: u64, metadata_op: u64) -> String {
+ format!(
+ "{RELAY_TARGET_PARTITION},{incarnation},0,{metadata_op},{}",
+ RELAY_BUDGET.as_millis()
+ )
+ }
+
+ fn send_body_to(partition_id: u32, payload: &str) -> SendMessages {
+ SendMessages {
+ partitioning: Partitioning::partition_id(partition_id),
+ ..send_body(payload)
+ }
+ }
+
+ /// Prove relay fields the way a cluster node does: HMAC-SHA256 under the key
+ /// BLAKE3 derives from the cluster key material, over the length-prefixed
+ /// method, path and query, and route, then the expiry in unix microseconds.
+ fn relay_proof(method: &str, path_and_query: &str, route: &str, expires_at: u64) -> String {
+ let key = ring::hmac::Key::new(
+ ring::hmac::HMAC_SHA256,
+ &blake3::derive_key(RELAY_KEY_CONTEXT, RELAY_KEY_MATERIAL.as_bytes()),
+ );
+ let mut signed = Vec::new();
+ for field in [method, path_and_query, route] {
+ signed.extend_from_slice(&(field.len() as u64).to_le_bytes());
+ signed.extend_from_slice(field.as_bytes());
+ }
+ signed.extend_from_slice(&expires_at.to_le_bytes());
+ format!(
+ "{expires_at},{}",
+ hex::encode(ring::hmac::sign(&key, &signed))
+ )
+ }
+
+ fn fresh_relay_proof(method: &str, path_and_query: &str, route: &str) -> String {
+ relay_proof(
+ method,
+ path_and_query,
+ route,
+ IggyTimestamp::now().as_micros() + RELAY_PROOF_LIFETIME_MICROS,
+ )
+ }
+
+ /// Poll partition 1 on node 1 as a relay carrying `route`.
+ async fn relayed_poll(http: &HttpClient, token: &str, route: &str) -> (StatusCode, String) {
+ let path_and_query = format!(
+ "/streams/{STREAM_NAME}/topics/{TOPIC_NAME}/messages?consumer_id={RELAY_CONSUMER_ID}\
+ &partition_id={RELAY_TARGET_PARTITION}&kind=offset&value=0&count=1&auto_commit=false"
+ );
+ let response = http
+ .client
+ .get(http.url(&path_and_query))
+ .bearer_auth(token)
+ .header(FORWARDED_HEADER, "1")
+ .header(PARTITION_ROUTE_HEADER, route)
+ .header(
+ RELAY_PROOF_HEADER,
+ fresh_relay_proof("GET", &path_and_query, route),
+ )
+ .send()
+ .await
+ .expect("relayed poll");
+ status_and_body(response).await
+ }
+
+ /// Send the head of a relayed send and the first byte of its body, then stall.
+ async fn stalled_relay_upload(address: SocketAddr, token: &str, route: &str) -> TcpStream {
+ let path = format!("/streams/{STREAM_NAME}/topics/{TOPIC_NAME}/messages");
+ let proof = fresh_relay_proof("POST", &path, route);
+ stalled_upload(
+ address,
+ &path,
+ &format!(
+ "authorization: Bearer {token}\r\n{FORWARDED_HEADER}: 1\r\n\
+ {PARTITION_ROUTE_HEADER}: {route}\r\n{RELAY_PROOF_HEADER}: {proof}\r\n"
+ ),
+ )
+ .await
+ }
+
+ /// Send the head of a POST to `path` with `headers`, each line ended by CRLF,
+ /// and the first byte of its body, then stall.
+ async fn stalled_upload(address: SocketAddr, path: &str, headers: &str) -> TcpStream {
+ let mut upload = TcpStream::connect(address)
+ .await
+ .expect("connect to the node");
+ let head = format!(
+ "POST {path} HTTP/1.1\r\nhost: {address}\r\n{headers}\
+ content-type: application/json\r\ncontent-length: {STALLED_BODY_LENGTH}\r\n\r\n{{"
+ );
+ upload
+ .write_all(head.as_bytes())
+ .await
+ .expect("send the head of an upload");
+ upload
+ }
+
+ /// The hop target's answer to a stalled upload. It closes the connection after
+ /// answering, because the rest of the body never comes; a reset after the
+ /// answer leaves what was read.
+ async fn stalled_answer(mut upload: TcpStream) -> String {
+ let mut answer = Vec::new();
+ let _ = upload.read_to_end(&mut answer).await;
+ String::from_utf8_lossy(&answer).into_owned()
+ }
diff --cc core/integration/tests/sdk/options.rs
index acb103a6c,1095c59f9..44959e09c
--- a/core/integration/tests/sdk/options.rs
+++ b/core/integration/tests/sdk/options.rs
@@@ -16,64 -16,12 +16,65 @@@
// under the License.
use iggy::prelude::*;
-use iggy_common::{OptionsScope, TOPIC_OPTION_KEYS, topic_option_keys};
+use iggy_common::{OptionsScope, descriptors_for, topic_option_keys};
+ use integration::harness::{backup_commit_budget, wait_for_topic};
use integration::iggy_harness;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
use std::str::FromStr;
+#[iggy_harness(test_client_transport = [Tcp, Http])]
+async fn given_manage_users_caller_when_updating_root_should_return_4108_and_keep_root_active(
+ harness: &TestHarness,
+) {
+ let root = harness.root_client().await.expect("root client");
+ let root_id = Identifier::numeric(0).expect("root id");
+ root.create_user(
+ "manager",
+ "secret123",
+ UserStatus::Active,
+ Some(Permissions {
+ global: GlobalPermissions {
+ manage_users: true,
+ ..GlobalPermissions::default()
+ },
+ streams: None,
+ }),
+ &UserCreateOptions::default(),
+ )
+ .await
+ .expect("create manager");
+ let manager = harness.new_client().await.expect("manager client");
+ manager
+ .login_user("manager", "secret123")
+ .await
+ .expect("manager login");
+
+ for patch in [
+ UserUpdateOptions {
+ username: Some("root-renamed".to_owned()),
+ ..UserUpdateOptions::default()
+ },
+ UserUpdateOptions {
+ status: Some(UserStatus::Inactive),
+ ..UserUpdateOptions::default()
+ },
+ ] {
+ let error = manager
+ .update_user(&root_id, &patch)
+ .await
+ .expect_err("manager cannot rename or deactivate root");
+ assert_policy_error(error, IggyError::CannotModifyRootUser);
+ }
+
+ let unchanged = root
+ .get_user(&root_id)
+ .await
+ .expect("read root")
+ .expect("root exists");
+ assert_eq!(unchanged.username, "iggy");
+ assert_eq!(unchanged.status, UserStatus::Active);
+}
+
#[iggy_harness]
async fn given_topic_scope_when_describing_options_should_list_the_catalog(harness: &TestHarness) {
let client = harness.root_client().await.unwrap();
diff --cc core/metadata/src/impls/metadata.rs
index 5494fc503,47cfbaffe..b328ffbc6
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@@ -4327,7 -4660,33 +4736,8 @@@ wher
topic_option_keys::MAX_TOPIC_SIZE.to_string(),
));
}
- let derived_options = explicit.derived_block(
- explicit.compression_algorithm.unwrap_or_default(),
- explicit
- .message_expiry
- .unwrap_or_else(|| IggyExpiry::from(iggy_common::DEFAULT_MESSAGE_EXPIRY)),
- resolved_max_topic_size,
- TopicRuntimeDefaults {
- segment_size: resolved_segment_size,
- durability: iggy_common::Durability::default(),
- consumer_offset_durability: iggy_common::Durability::default(),
- messages_required_to_save: explicit
- .messages_required_to_save
- .unwrap_or(iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE),
- size_of_messages_required_to_save: explicit
- .size_of_messages_required_to_save
- .unwrap_or_else(|| {
- IggyByteSize::from(
- iggy_common::DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE,
- )
- }),
- preallocate_segments: explicit
- .preallocate_segments
- .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS),
- },
- &supplied_options,
- )?;
+ let derived_options = resource_options_to_wire(&derived, OptionsProvenance::All)?;
+ let mut counts = self.placement_counts()?;
let partitions = self
.allocator
.allocate_many(request.partitions_count as usize)
@@@ -4394,39 -4755,20 +4806,43 @@@
&body,
))
}
- // `UpdateTopic` deliberately takes the default arm: unlike create,
- // an update stores `ServerDefault` sentinels verbatim (legacy
- // parity), so a later get echoes `ServerDefault` instead of the
- // node default frozen at update time.
+ Operation::CreateStream => {
+ let request = WireCreateStreamRequest::decode_from(body)
+ .map_err(|_| IggyError::InvalidCommand)?;
+ let options = resource_options_from_wire(&request.options, true)?;
+ // Apply refuses a plaintext key too, but only after every replica journaled it.
+ if let Some(secret) = options
+ .keys()
+ .filter_map(|key| OptionKey::try_from(key).ok())
+ .find(|key| key.descriptor().visibility == OptionVisibility::Secret)
+ {
+ return Err(IggyError::InvalidOptionValue(secret.to_string()));
+ }
+ // An edge whose keys no peer holds wraps only under its own, and
+ // data committed under such a wrap opens nowhere else.
+ if let Some(set) = stream_wrap_set(&options)?
+ && self.stream_key_context.open(&set).is_err()
+ {
+ return Err(IggyError::StreamEncryptionUnavailable);
+ }
+ Ok(message.project(consensus))
+ }
+ Operation::UpdateResource => {
+ let request = UpdateResourceRequest::decode_from(body)
+ .map_err(|_| IggyError::InvalidCommand)?;
+ let (operation, body) = resolve_update_resource(&request)?;
+ Ok(build_prepare_message(consensus, &header, operation, &body))
+ }
_ => Ok(message.project(consensus)),
}?;
- Ok(consensus::seal_prepare_capacity(
+ let prepare = consensus::seal_prepare_capacity(
prepare,
consensus.retry_capacity(&self.client_table.borrow()),
- ))
+ );
+ self.placement
+ .borrow_mut()
+ .remember(&prepare, consensus.replica_count())?;
+ Ok(prepare)
}
/// Replicate a prepare message to the next replica in the chain.
@@@ -5115,30 -5447,26 +5619,30 @@@ mod tests
use crate::stm::StateHandler;
use crate::stm::authz::GatedApply;
use crate::stm::consumer_group::JoinConsumerGroupRequest;
- use crate::stm::lifecycle::{CompleteLifecycleRequest, LifecycleCompletion};
+ use crate::stm::lifecycle::{
+ CompleteLifecycleRequest, LifecycleCompletion, apply_with_lifecycle_completion,
+ };
use crate::stm::stream::{Streams, StreamsInner};
use crate::stm::user::{PAT_TOKEN_HASH_BYTES, Users, UsersInner};
- use consensus::LocalPipeline;
+ use consensus::{HistoryRecovery, LocalPipeline};
+ use iggy_binary_protocol::codes::CREATE_USER_CODE;
use iggy_binary_protocol::primitives::permissions::{
WireGlobalPermissions, WirePermissions, WireStreamPermissions,
};
use iggy_binary_protocol::requests::consumer_groups::CreateConsumerGroupRequest;
use iggy_binary_protocol::requests::streams::CreateStreamRequest;
use iggy_binary_protocol::requests::topics::{CreateTopicRequest, DeleteTopicRequest};
- use iggy_binary_protocol::requests::users::{
- CreateUserRequest, DeleteUserRequest, UpdateUserRequest,
+ use iggy_binary_protocol::requests::users::{CreateUserRequest, DeleteUserRequest};
+ use iggy_binary_protocol::{RequestHeader, StartViewHeader, WireOptions};
+ use iggy_common::{
+ HeaderValue, IggyExpiry, IggyTimestamp, OptionKey, OptionStorage, OptionValueCodec,
+ ServerKeyContext, StreamEncryptionKey, StreamKeyOrigin, TopicUpdateOptions, UserStatus,
+ UserUpdateOptions, WrapSet, critical_keys_marker, descriptors_for, stream_wrap_set,
+ variadic,
};
- use journal::prepare_journal::PrepareJournal;
- use iggy_binary_protocol::{StartViewHeader, WireOptions};
- use iggy_common::{IggyTimestamp, UserStatus, variadic};
+ use journal::prepare_journal::{DEFAULT_SLOT_COUNT, PrepareJournal};
use journal::superblock::SuperblockContents;
- use message_bus::{
- BusMessage, ClientForwardFn, ConnectionLostFn, JoinHandle, ReplicaForwardFn, SendError,
- };
+ use message_bus::{BusMessage, ConnectionLostFn, JoinHandle, ReplicaForwardFn, SendError};
use server_common::MESSAGE_ALIGN;
use server_common::iobuf::Frozen;
use std::cell::RefCell;
@@@ -8588,6 -9759,162 +10295,169 @@@
}
}
+ /// The primary admits a request of a live session while a revocation of
+ /// its user is still uncommitted, so the request commits right behind it.
+ /// A checkpoint of the resulting table must restart, and a state transfer
+ /// of it must install on a fresh replica.
+ #[compio::test]
+ async fn given_request_pipelined_behind_user_revocation_when_checkpointed_should_restart_and_transfer()
+ {
+ const ADMIN: u128 = 1;
+ const VICTIM: u128 = 2;
+ const ROOT_USER: u32 = 0;
+ const VICTIM_USER: u32 = 7;
+ const REQUEST: u64 = 1;
+ let registration = || RegisterRequest {
+ user_created_at: 1,
+ bind_verifier: [0x5a; 32],
+ };
+ let with_session = |request: Message<RoutedRequestHeader>, session: u64| {
+ request.transmute_header(|old, header: &mut RoutedRequestHeader| {
+ *header = old;
+ header.session = session;
+ })
+ };
- for revocation in [Operation::DeleteUser, Operation::UpdateUser] {
++ for revocation in [Operation::DeleteUser, Operation::UpdateResource] {
+ let directory = tempfile::tempdir().unwrap();
+ let metadata_directory = directory.path().join(crate::impls::METADATA_DIR);
+ std::fs::create_dir_all(&metadata_directory).unwrap();
+ let journal = PrepareJournal::open(&metadata_directory.join("journal.wal"), 0)
+ .await
+ .unwrap();
+ let consensus = VsrConsensus::new(
+ 1,
+ 0,
+ 1,
+ server_common::sharding::METADATA_GROUP,
+ NoopBus,
+ LocalPipeline::new(),
+ );
+ consensus.init();
+ let metadata: IggyMetadata<_, PrepareJournal, (), TestMux> = IggyMetadata::new(
+ Some(consensus),
+ Some(journal),
+ None,
+ None,
+ TestMux::new((session_users(), (StreamsInner::new().into(), ()))),
+ Some(directory.path().to_path_buf()),
+ );
+ let admin = metadata
+ .submit_register_in_process(ADMIN, ROOT_USER, registration())
+ .await
+ .unwrap();
+ let victim = metadata
+ .submit_register_in_process(VICTIM, VICTIM_USER, registration())
+ .await
+ .unwrap();
+ let body = if revocation == Operation::DeleteUser {
+ DeleteUserRequest {
+ user_id: WireIdentifier::numeric(VICTIM_USER),
+ }
+ .to_bytes()
+ } else {
- UpdateUserRequest {
- user_id: WireIdentifier::numeric(VICTIM_USER),
- username: None,
- status: Some(UserStatus::Inactive.as_code()),
- options: WireOptions::empty(),
++ let patch = UserUpdateOptions {
++ status: Some(UserStatus::Inactive),
++ ..UserUpdateOptions::default()
++ }
++ .to_patch()
++ .unwrap();
++ UpdateResourceRequest {
++ target: ResourceTarget::User {
++ user_id: WireIdentifier::numeric(VICTIM_USER),
++ },
++ reset: patch.reset,
++ options: patch.set,
+ }
+ .to_bytes()
+ };
+ let consensus = metadata.consensus.as_ref().unwrap();
+ let journal = metadata.journal.as_ref().unwrap();
+ let revocation_op = consensus.commit_min() + 1;
+ for request in [
+ with_session(pat_request(ADMIN, REQUEST, revocation, &body), admin.epoch),
+ with_session(
+ create_stream_request(VICTIM, REQUEST, "pipelined"),
+ victim.epoch,
+ ),
+ ] {
+ let prepare = metadata.prepare_request(request).unwrap();
+ let op = prepare.header().op;
+ consensus.pipeline_message(PlaneKind::Metadata, &prepare);
+ journal.handle().append(prepare).await.unwrap();
+ consensus.advance_commit_max(op);
+ }
+ metadata.resume_stranded_commits().await;
+ let pipelined_op = revocation_op + 1;
+ assert_eq!(consensus.commit_min(), pipelined_op);
+ assert!(
+ metadata
+ .mux_stm
+ .streams()
+ .read(|streams| streams.items.is_empty()),
+ "{revocation:?}: the op behind the revocation must not apply"
+ );
+ let ended = vec![SessionIdentity {
+ client_id: VICTIM,
+ session: victim.epoch,
+ metadata_watermark: revocation_op,
+ }];
+ assert_eq!(
+ metadata
+ .client_table
+ .borrow()
+ .ended_sessions()
+ .collect::<Vec<_>>(),
+ ended
+ );
+ let table = metadata.client_table.borrow().encode();
+ let transferred =
+ ClientTable::decode(&table).expect("the transferred table must decode");
+
+ metadata.set_checkpoint_margin(usize::MAX);
+ metadata.checkpoint_if_needed(consensus, journal).await;
+ let checkpoint =
+ IggySnapshot::load(&metadata.coordinator.as_ref().unwrap().snapshot_path())
+ .unwrap()
+ .0;
+ assert_eq!(checkpoint.sequence_number(), pipelined_op);
+
+ let receiver = metadata_plane();
+ receiver
+ .install_state_transfer(
+ &checkpoint.encode().unwrap(),
+ transferred,
+ pipelined_op,
+ pipelined_op,
+ )
+ .await
+ .expect("the transferred checkpoint must install");
+ assert!(
+ receiver.client_table.borrow().encode() == table,
+ "{revocation:?}: the receiver must hold the transferred table"
+ );
+
+ drop(metadata);
+ let recovered = crate::impls::recovery::recover::<TestMux>(
+ directory.path(),
+ crate::impls::recovery::ReplicaIdentity {
+ cluster: 1,
+ replica_id: 0,
+ replica_count: 1,
+ },
+ journal::prepare_journal::DEFAULT_SLOT_COUNT,
+ CLIENTS_TABLE_MAX,
+ |_| {},
+ |_, _| {},
+ )
+ .await
+ .expect("restart must accept the checkpointed table");
+ assert_eq!(
+ recovered.client_table.ended_sessions().collect::<Vec<_>>(),
+ ended
+ );
+ }
+ }
+
/// All owner decisions of one revocation share its op as generation, and a
/// reconciler on another shard installs what it reads. A reader that holds
/// a guard across the revocation stalls any second publish, so a racing
diff --cc core/metadata/src/stm/stream.rs
index 368c1a1cd,6416bb5ad..c1ae327ac
--- a/core/metadata/src/stm/stream.rs
+++ b/core/metadata/src/stm/stream.rs
@@@ -45,9 -50,8 +45,9 @@@ use iggy_binary_protocol::requests::par
CreatePartitionsWithAssignmentsRequest, DeletePartitionsRequest,
};
use iggy_binary_protocol::requests::streams::{
- CreateStreamRequest, DeleteStreamRequest, UpdateStreamRequest,
+ CreateStreamRequest, DeleteStreamRequest, RewrapStreamKeysRequest, UpdateStreamResolvedRequest,
};
- use std::collections::{BTreeMap, BTreeSet};
++use std::collections::{BTreeMap, BTreeSet, VecDeque};
// Only the slab-seeding helpers build a bare `CreateTopicRequest`; without
// their cfg the import is dead and `-p <crate>` clippy (which skips the
// simulator feature) rejects it.
@@@ -3633,9 -3347,7 +3763,8 @@@ mod tests
create_stream(&mut inner, "stream");
let message_expiry = IggyExpiry::from(5_000_000u64);
let max_topic_size = MaxTopicSize::from(10_000_000_000u64);
+ let segment_size = IggyByteSize::from(64 * 1024 * 1024);
let create = CreateTopicWithAssignmentsRequest {
- created_view: 0,
request: WireCreateTopicRequest {
stream_id: WireIdentifier::numeric(0),
partitions_count: 1,
@@@ -3663,436 -3375,53 +3793,436 @@@
StateHandler::apply(&create, &mut inner, IggyTimestamp::from(1)).code,
0
);
- let original_options = inner
- .items
- .get(0)
- .unwrap()
- .topics
- .get(0)
- .unwrap()
+ let streams: Streams = inner.into();
+ let mut snapshot = MetadataSnapshot::new(1);
+ snapshot.streams = Some(streams.to_snapshot());
+ let decoded = MetadataSnapshot::decode(&snapshot.encode().unwrap()).unwrap();
+
+ let restored = Streams::from_snapshot(decoded.streams.unwrap()).unwrap();
+
+ assert_eq!(
+ restored.topic_retention_config(0, 0).unwrap(),
+ (message_expiry, max_topic_size, 1, Some(segment_size))
+ );
+ assert_eq!(
+ restored.topic_segment_size(&WireIdentifier::numeric(0), &WireIdentifier::numeric(0)),
+ Some(segment_size)
+ );
+ }
+
+ #[test]
+ fn given_stored_critical_marker_when_patch_declares_more_keys_should_store_the_bounded_union() {
+ let marker_key = HeaderKey::from(OptionKey::CriticalKeys);
+ let marker = |text: &str| OptionValue::explicit(HeaderValue::try_from(text).unwrap());
+ let update = |names: &str| {
+ let mut explicit = ResourceOptions::new();
+ explicit.insert(marker_key.clone(), marker(names));
+ UpdateStreamResolvedRequest {
+ stream_id: WireIdentifier::numeric(0),
+ name: None,
+ explicit_options: resource_options_to_wire(&explicit, OptionsProvenance::All)
+ .unwrap(),
+ derived_options: WireOptions::empty(),
+ }
+ };
+ let mut inner = StreamsInner::new();
+ create_stream(&mut inner, "stream");
+ inner.items[0]
.options
- .clone();
-
- for options in [
- TopicUpdateOptions::default(),
- TopicUpdateOptions {
- message_expiry: Some(IggyExpiry::ServerDefault),
- max_topic_size: Some(MaxTopicSize::ServerDefault),
- ..TopicUpdateOptions::default()
- },
- TopicUpdateOptions {
- raw: [
- topic_option_keys::MESSAGE_EXPIRY,
- topic_option_keys::MAX_TOPIC_SIZE,
- ]
- .into_iter()
- .map(|key| (key.to_owned(), "server_default".to_owned()))
- .collect(),
- ..TopicUpdateOptions::default()
+ .insert(marker_key.clone(), marker("encryption_key_wrapped"));
+
+ let reply = StateHandler::apply(&update("a_future_key"), &mut inner, IggyTimestamp::now());
+ assert_eq!(reply.code, 0);
+ let merged = marker("a_future_key,encryption_key_wrapped");
+ assert_eq!(inner.items[0].options.get(&marker_key), Some(&merged));
+
+ let reply =
+ StateHandler::apply(&update(&"z".repeat(230)), &mut inner, IggyTimestamp::now());
+ assert_eq!(
+ reply.code,
+ u32::from(UpdateStreamResult::OptionsBlockTooLarge)
+ );
+ assert_eq!(inner.items[0].options.get(&marker_key), Some(&merged));
+ }
+
+ #[test]
+ fn given_internal_or_unservable_topic_options_when_creating_should_not_echo_them() {
+ let marker_key = HeaderKey::from(OptionKey::CriticalKeys);
+ let wrapped_key = HeaderKey::from(OptionKey::EncryptionKeyWrapped);
+ let future_key = HeaderKey::try_from("a_future_key").unwrap();
+ let marker = |text: &str| OptionValue::explicit(HeaderValue::try_from(text).unwrap());
+ let set = WrapSet::from(envelope(
+ &key_context(OLD_CONFIG_KEY, &[]),
+ 1,
+ StreamKeyOrigin::Server,
+ ));
+ let mut servable = ResourceOptions::new();
+ servable.insert(marker_key.clone(), marker("encryption_key_wrapped"));
+ servable.insert(
+ wrapped_key.clone(),
+ OptionValue::explicit(HeaderValue::try_from(set.as_ref()).unwrap()),
+ );
+ let mut unservable = ResourceOptions::new();
+ unservable.insert(marker_key.clone(), marker("a_future_key"));
+ unservable.insert(
+ future_key.clone(),
+ OptionValue::explicit(HeaderValue::from(7u64)),
+ );
+
+ for stored in [servable, unservable] {
+ let mut inner = StreamsInner::new();
+ create_stream(&mut inner, "stream");
+ let create = CreateTopicWithAssignmentsRequest {
- created_view: 0,
+ request: make_topic_request(0, 1, "topic"),
+ derived_options: resource_options_to_wire(&stored, OptionsProvenance::All).unwrap(),
+ partitions: vec![CreatedPartitionAssignment {
+ partition_id: 0,
+ consensus_group_id: 1,
++ initial_view: 0,
+ }],
+ };
+
+ let apply = StateHandler::apply(&create, &mut inner, IggyTimestamp::now());
+
+ assert_eq!(apply.code, 0);
+ assert!(inner.items[0].topics[0].options.contains_key(&marker_key));
+ let (reply, _) = GetTopicResponse::decode(&apply.body).unwrap();
+ let echoed: Vec<HeaderKey> = resource_options_from_wire(&reply.topic.options, true)
+ .unwrap()
+ .into_keys()
+ .chain(
+ resource_options_from_wire(&reply.topic.derived_options, false)
+ .unwrap()
+ .into_keys(),
+ )
+ .collect();
+ for hidden in [&marker_key, &wrapped_key, &future_key] {
+ assert!(!echoed.contains(hidden), "{echoed:?}");
+ }
+ }
+ }
+
+ #[test]
+ fn given_unknown_critical_key_when_applying_create_stream_should_store_it_without_echoing_options()
+ {
+ let mut options = ResourceOptions::new();
+ options.insert(
+ HeaderKey::from(OptionKey::CriticalKeys),
+ OptionValue::explicit(HeaderValue::try_from("a_future_key").unwrap()),
+ );
+ options.insert(
+ HeaderKey::try_from("a_future_key").unwrap(),
+ OptionValue::explicit(HeaderValue::from(7u64)),
+ );
+ let request = CreateStreamRequest {
+ name: WireName::new("future").unwrap(),
+ options: resource_options_to_wire(&options, OptionsProvenance::All).unwrap(),
+ };
+ let mut inner = StreamsInner::new();
+
+ let reply = StateHandler::apply(&request, &mut inner, IggyTimestamp::now());
+
+ assert_eq!(reply.code, 0);
+ assert_eq!(inner.items[0].options, options);
+ let (response, _) = GetStreamResponse::decode(&reply.body).unwrap();
+ assert_eq!(
+ (response.stream.options, response.stream.derived_options),
+ (WireOptions::empty(), WireOptions::empty())
+ );
+ }
+
+ #[test]
+ fn given_reused_stream_id_when_creating_should_have_distinct_created_op() {
+ let mut inner = StreamsInner::new();
+ let timestamp = IggyTimestamp::from(1);
+ let create = CreateStreamRequest {
+ name: WireName::new("orders").unwrap(),
+ options: WireOptions::empty(),
+ };
+ inner.apply_context.metadata_op = 5;
+ assert_eq!(StateHandler::apply(&create, &mut inner, timestamp).code, 0);
+ let first = inner.items[0].created_op;
+
+ inner.apply_context.metadata_op = 6;
+ let delete = DeleteStreamRequest {
+ stream_id: WireIdentifier::numeric(0),
+ };
+ assert_eq!(
+ crate::stm::lifecycle::apply_with_lifecycle_completion(&delete, &mut inner, timestamp)
+ .code,
+ 0
+ );
+ assert!(inner.items.get(0).is_none());
+
+ inner.apply_context.metadata_op = 7;
+ assert_eq!(StateHandler::apply(&create, &mut inner, timestamp).code, 0);
+ let recreated = &inner.items[0];
+ assert_eq!(recreated.created_at, timestamp);
+ assert_eq!((first, recreated.created_op), (5, 7));
+ }
+
+ #[test]
+ fn given_rename_only_patch_when_applying_should_preserve_options_and_provenance() {
+ let mut inner = StreamsInner::new();
+ create_stream(&mut inner, "stream");
+ let message_expiry = IggyExpiry::from(5_000_000u64);
+ let max_topic_size = MaxTopicSize::from(10_000_000_000u64);
+ let create = CreateTopicWithAssignmentsRequest {
- created_view: 0,
+ request: WireCreateTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ partitions_count: 1,
+ name: WireName::new("topic").unwrap(),
+ options: TopicCreateOptions {
+ message_expiry: Some(message_expiry),
+ ..TopicCreateOptions::default()
+ }
+ .to_explicit_wire(|key| key == topic_option_keys::MESSAGE_EXPIRY)
+ .unwrap(),
},
- ] {
- let update = UpdateTopicRequest {
+ derived_options: TopicCreateOptions {
+ max_topic_size: Some(max_topic_size),
+ ..TopicCreateOptions::default()
+ }
+ .to_wire()
+ .unwrap(),
+ partitions: vec![CreatedPartitionAssignment {
+ partition_id: 0,
+ consensus_group_id: 1,
++ initial_view: 0,
+ }],
+ };
+ assert_eq!(
+ StateHandler::apply(&create, &mut inner, IggyTimestamp::from(1)).code,
+ 0
+ );
+ let original_options = inner.items[0].topics[0].options.clone();
+
+ let update = UpdateTopicResolvedRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(0),
+ name: Some(WireName::new("renamed").unwrap()),
+ explicit_options: WireOptions::empty(),
+ derived_options: WireOptions::empty(),
+ };
+ assert_eq!(
+ StateHandler::apply(&update, &mut inner, IggyTimestamp::from(2)).code,
+ 0
+ );
+ let topic = &inner.items[0].topics[0];
+ assert_eq!(topic.name.as_ref(), "renamed");
+ assert_eq!(topic.resolved.message_expiry, message_expiry);
+ assert_eq!(topic.resolved.max_topic_size, max_topic_size);
+ assert_eq!(topic.options, original_options);
+ }
+
+ #[test]
+ fn given_topic_patch_without_name_when_applying_should_preserve_name_and_other_keys() {
+ let mut inner = streams_with_resize_policy(Some(PartitionResizePolicy::Mutable));
+ let before = inner.items[0].topics[0].options.clone();
+ let expiry = IggyExpiry::from(9_000_000u64);
+ let mut explicit = ResourceOptions::new();
+ explicit.insert(
+ HeaderKey::from(OptionKey::MessageExpiry),
+ OptionValue::explicit(expiry.encode().unwrap()),
+ );
+ let update = UpdateTopicResolvedRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(0),
+ name: None,
+ explicit_options: resource_options_to_wire(&explicit, OptionsProvenance::All).unwrap(),
+ derived_options: WireOptions::empty(),
+ };
+ assert_eq!(
+ StateHandler::apply(&update, &mut inner, IggyTimestamp::now()).code,
+ 0
+ );
+ let topic = &inner.items[0].topics[0];
+ assert_eq!(topic.name.as_ref(), "topic");
+ assert_eq!(topic.resolved.message_expiry, expiry);
+ let mut expected = before;
+ expected.extend(explicit);
+ assert_eq!(topic.options, expected);
+ }
+
+ #[test]
+ fn given_explicit_topic_key_when_resetting_should_store_primary_default_as_derived() {
+ let mut inner = StreamsInner::new();
+ create_stream(&mut inner, "stream");
+ let create = CreateTopicWithAssignmentsRequest {
- created_view: 0,
+ request: WireCreateTopicRequest {
stream_id: WireIdentifier::numeric(0),
- topic_id: WireIdentifier::numeric(0),
- name: WireName::new("renamed").unwrap(),
- options: options.to_wire().unwrap(),
+ partitions_count: 1,
+ name: WireName::new("topic").unwrap(),
+ options: TopicCreateOptions {
+ max_topic_size: Some(MaxTopicSize::from(10_000_000_000u64)),
+ ..TopicCreateOptions::default()
+ }
+ .to_explicit_wire(|key| key == topic_option_keys::MAX_TOPIC_SIZE)
+ .unwrap(),
+ },
+ derived_options: WireOptions::empty(),
+ partitions: vec![CreatedPartitionAssignment {
+ partition_id: 0,
+ consensus_group_id: 1,
++ initial_view: 0,
+ }],
+ };
+ assert_eq!(
+ StateHandler::apply(&create, &mut inner, IggyTimestamp::from(1)).code,
+ 0
+ );
+ let default = OptionKey::MaxTopicSize
+ .descriptor()
+ .default_value()
+ .unwrap()
+ .unwrap();
+ let mut derived = ResourceOptions::new();
+ derived.insert(
+ HeaderKey::from(OptionKey::MaxTopicSize),
+ OptionValue::derived(default.clone()),
+ );
+ let update = UpdateTopicResolvedRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(0),
+ name: None,
+ explicit_options: WireOptions::empty(),
+ derived_options: resource_options_to_wire(&derived, OptionsProvenance::All).unwrap(),
+ };
+ assert_eq!(
+ StateHandler::apply(&update, &mut inner, IggyTimestamp::from(2)).code,
+ 0
+ );
+ let topic = &inner.items[0].topics[0];
+ let stored = &topic.options[&HeaderKey::from(OptionKey::MaxTopicSize)];
+ assert_eq!(stored, &OptionValue::derived(default));
+ assert_eq!(
+ topic.resolved.max_topic_size,
+ MaxTopicSize::from(iggy_common::DEFAULT_MAX_TOPIC_SIZE)
+ );
+ }
+
+ #[test]
+ fn given_conflicting_rename_with_patch_when_applying_should_leave_state_unchanged() {
+ let mut inner = StreamsInner::new();
+ create_stream(&mut inner, "stream");
+ for name in ["first", "second"] {
+ let create = CreateTopicWithAssignmentsRequest {
- created_view: 0,
+ request: make_topic_request(0, 1, name),
+ derived_options: WireOptions::empty(),
+ partitions: vec![CreatedPartitionAssignment {
+ partition_id: 0,
+ consensus_group_id: 1,
++ initial_view: 0,
+ }],
};
assert_eq!(
- StateHandler::apply(&update, &mut inner, IggyTimestamp::from(2)).code,
+ StateHandler::apply(&create, &mut inner, IggyTimestamp::now()).code,
0
);
- let topic = inner.items.get(0).unwrap().topics.get(0).unwrap();
- assert_eq!(topic.name.as_ref(), "renamed");
- assert_eq!(topic.message_expiry, message_expiry);
- assert_eq!(topic.max_topic_size, max_topic_size);
- assert_eq!(
- topic.options, original_options,
- "update {options:?} must preserve values and provenance"
- );
}
+ let before = inner.items[0].topics[0].options.clone();
+ let index_before = inner.items[0].topic_index.clone();
+ let mut explicit = ResourceOptions::new();
+ explicit.insert(
+ HeaderKey::from(OptionKey::MessageExpiry),
+ OptionValue::explicit(IggyExpiry::from(1_000_000u64).encode().unwrap()),
+ );
+ let update = UpdateTopicResolvedRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(0),
+ name: Some(WireName::new("second").unwrap()),
+ explicit_options: resource_options_to_wire(&explicit, OptionsProvenance::All).unwrap(),
+ derived_options: WireOptions::empty(),
+ };
+ assert_eq!(
+ StateHandler::apply(&update, &mut inner, IggyTimestamp::now()).code,
+ u32::from(UpdateTopicResult::NameAlreadyExists)
+ );
+ let topic = &inner.items[0].topics[0];
+ assert_eq!(topic.name.as_ref(), "first");
+ assert_eq!(topic.options, before);
+ assert_eq!(inner.items[0].topic_index, index_before);
+
+ let rename_stream = UpdateStreamResolvedRequest {
+ stream_id: WireIdentifier::numeric(0),
+ name: Some(WireName::new("renamed-stream").unwrap()),
+ explicit_options: WireOptions::empty(),
+ derived_options: WireOptions::empty(),
+ };
+ assert_eq!(
+ StateHandler::apply(&rename_stream, &mut inner, IggyTimestamp::now()).code,
+ 0
+ );
+ assert_eq!(inner.items[0].name.as_ref(), "renamed-stream");
+ assert_eq!(
+ inner.resolve_stream_id(&WireIdentifier::named("renamed-stream").unwrap()),
+ Some(0)
+ );
+ assert_eq!(
+ inner.resolve_stream_id(&WireIdentifier::named("stream").unwrap()),
+ None
+ );
+ }
+
+ #[test]
+ fn given_cap_below_topic_segment_size_when_applying_update_should_refuse_it_unchanged() {
+ let segment_size = 64 * 1024 * 1024;
+ let mut inner = StreamsInner::new();
+ create_stream(&mut inner, "stream");
+ let mut request = make_topic_request(0, 1, "topic");
+ request.options = TopicCreateOptions {
+ segment_size: Some(IggyByteSize::from(segment_size)),
+ ..TopicCreateOptions::default()
+ }
+ .to_explicit_wire(|key| key == topic_option_keys::SEGMENT_SIZE)
+ .unwrap();
+ let create = CreateTopicWithAssignmentsRequest {
- created_view: 0,
+ request,
+ derived_options: WireOptions::empty(),
+ partitions: vec![CreatedPartitionAssignment {
+ partition_id: 0,
+ consensus_group_id: 1,
++ initial_view: 0,
+ }],
+ };
+ assert_eq!(
+ StateHandler::apply(&create, &mut inner, IggyTimestamp::from(1)).code,
+ 0
+ );
+ let before = inner.items[0].topics[0].options.clone();
+
+ let below = topic_cap_update(segment_size - 1, Some("renamed"));
+ assert_eq!(
+ StateHandler::apply(&below, &mut inner, IggyTimestamp::from(2)).code,
+ u32::from(UpdateTopicResult::InvalidTopicSize)
+ );
+ let topic = &inner.items[0].topics[0];
+ assert_eq!(topic.name.as_ref(), "topic");
+ assert_eq!(topic.options, before);
+
+ let at_floor = topic_cap_update(segment_size, Some("renamed"));
+ assert_eq!(
+ StateHandler::apply(&at_floor, &mut inner, IggyTimestamp::from(3)).code,
+ 0
+ );
+ let topic = &inner.items[0].topics[0];
+ assert_eq!(topic.name.as_ref(), "renamed");
+ assert_eq!(
+ topic.resolved.max_topic_size,
+ MaxTopicSize::from(segment_size)
+ );
+ }
+
+ #[test]
+ fn given_topic_without_segment_size_when_applying_cap_below_default_should_refuse_it() {
+ let mut inner = streams_with_resize_policy(None);
+ let before = inner.items[0].topics[0].options.clone();
+ let update = topic_cap_update(DEFAULT_SEGMENT_SIZE - 1, None);
+ assert_eq!(
+ StateHandler::apply(&update, &mut inner, IggyTimestamp::now()).code,
+ u32::from(UpdateTopicResult::InvalidTopicSize)
+ );
+ assert_eq!(inner.items[0].topics[0].options, before);
}
#[test]
diff --cc core/partitions/src/iggy_partitions.rs
index 954a766ba,f4901ab1c..ab103a58d
--- a/core/partitions/src/iggy_partitions.rs
+++ b/core/partitions/src/iggy_partitions.rs
@@@ -1417,4 -1379,64 +1421,64 @@@ mod tests
.replicate_poll_completion(&namespace, replication)
.await;
}
+
+ /// Under disk-bound load a primary's quorum commits wait for the writer,
+ /// so `commit_min` stays below `commit_max`. Those commits can seal the
+ /// active segment but never change a sealed one.
+ #[test]
+ fn given_commits_waiting_for_the_writer_when_resolving_segment_delete_should_settle_sealed_offsets()
+ {
+ const SEALED_END: u64 = 9;
+ let namespace = IggyNamespace::new(1, 1, 0);
+ let segment_size = IggyByteSize::from(1024 * 1024);
+ let config = PartitionsConfig {
+ messages_required_to_save: 1,
+ size_of_messages_required_to_save: segment_size,
+ validate_checksum: false,
+ segment_size,
+ preallocate_segments: false,
- encryptor: None,
++ key_context: Arc::default(),
+ path_layout: crate::PartitionPathLayout::default(),
+ };
+ let partitions = IggyPartitions::new(ShardId::new(0), config);
+ partitions.insert(namespace, build_replicated_partition());
+ let resolve = |count| {
+ partitions
+ .segment_delete_resolution(&namespace, count)
+ .map(|(up_to_offset, lagging, _)| (up_to_offset, lagging))
+ };
+ assert_eq!(
+ resolve(1),
+ Some((None, false)),
+ "a converged primary settles the no-op"
+ );
+
+ {
+ let consensus = partitions.get_by_ns(&namespace).unwrap().consensus();
+ consensus.advance_commit_max(consensus.commit_min() + 1);
+ }
+ assert_eq!(
+ resolve(1),
+ Some((None, true)),
+ "the waiting commits can still seal the first segment"
+ );
+
+ let partition = partitions.get_mut_by_ns(&namespace).unwrap();
+ let first = &mut partition.log.segments_mut()[0];
+ first.end_offset = SEALED_END;
+ first.sealed = true;
+ partition.log.add_persisted_segment(
+ Segment::new(SEALED_END + 1, segment_size),
+ SegmentStorage::default(),
+ None,
+ None,
+ );
+ for count in [1, 2] {
+ assert_eq!(
+ resolve(count),
+ Some((Some(SEALED_END), false)),
+ "count {count} resolves to a sealed segment, which the waiting commits cannot change"
+ );
+ }
+ }
}
diff --cc core/partitions/src/lib.rs
index 657d25a7b,3e7545172..29268f238
--- a/core/partitions/src/lib.rs
+++ b/core/partitions/src/lib.rs
@@@ -42,8 -42,15 +42,16 @@@ mod segment
pub mod segment_anchor;
mod segment_recovery;
pub mod state_transfer;
+mod stream_keyring;
mod types;
+ #[cfg(feature = "simulator")]
+ mod wal_storage;
+ #[cfg(feature = "simulator")]
+ pub use wal_storage::{InjectedFile, InjectedStorage, WalFile, WalStorage};
+
+ /// The storage under every partition WAL.
+ #[cfg(not(feature = "simulator"))]
+ pub type WalStorage = ::journal::durable_storage::DiskStorage;
pub use consumer_offset_capacity::{AutoCommitReservation, ConsumerOffsetCapacityError};
use iggy_binary_protocol::PrepareHeader;
diff --cc core/sdk/src/clients/client.rs
index 8f4815c32,46f7a931d..f1b50334f
--- a/core/sdk/src/clients/client.rs
+++ b/core/sdk/src/clients/client.rs
@@@ -1093,13 -1104,19 +1104,21 @@@ impl Client for IggyClient
#[cfg(test)]
mod tests {
use super::*;
- use crate::links::OPERATION_TIMEOUT;
- use crate::links::fake_cluster::{Answer, FakeCluster, connected, endpoint};
+ use crate::links::fake_cluster::{
+ Answer, FakeCluster, connected, connected_with, endpoint, eventually,
+ };
use crate::links::test_support::LOCALHOST;
- use iggy_binary_protocol::codes::{CREATE_STREAM_CODE, GET_STATS_CODE};
- use iggy_common::{StreamClient, StreamCreateOptions, TcpClientConfig};
+ use crate::links::{OPERATION_TIMEOUT, ReconnectPolicy};
+ use iggy_binary_protocol::codes::{
+ CREATE_STREAM_CODE, GET_STATS_CODE, SYNC_CONSUMER_GROUP_CODE,
+ };
- use iggy_common::{StreamClient, TcpClientConfig, TcpClientReconnectionConfig};
++ use iggy_common::{
++ StreamClient, StreamCreateOptions, TcpClientConfig, TcpClientReconnectionConfig,
++ };
+ use std::iter::repeat_n;
use std::time::Duration;
use tokio::net::TcpListener;
+ use tokio::time::sleep;
#[tokio::test(start_paused = true)]
async fn given_nested_client_locks_when_reading_should_share_one_deadline() {
diff --cc core/sdk/src/clients/producer.rs
index 5233b5637,44e7b059e..b68bf67c9
--- a/core/sdk/src/clients/producer.rs
+++ b/core/sdk/src/clients/producer.rs
@@@ -21,14 -21,10 +21,13 @@@ use crate::clients::MAX_BATCH_LENGTH
use crate::clients::producer_builder::SendMode;
use crate::clients::producer_config::DirectConfig;
use crate::clients::producer_dispatcher::ProducerDispatcher;
- use crate::driver::{operation_deadline, with_deadline};
- use crate::links::OPERATION_TIMEOUT;
+ use crate::driver::operation_deadline;
use bytes::Bytes;
use iggy_common::locking::{IggyRwLock, IggyRwLockFn};
-use iggy_common::{BinarySendPartitioning, Client, StreamClient, TopicClient, TopicCreateOptions};
+use iggy_common::{
+ BinarySendPartitioning, Client, StreamClient, StreamCreateOptions, TopicClient,
+ TopicCreateOptions,
+};
use iggy_common::{
Durability, EncryptorKind, IdKind, Identifier, IggyError, IggyExpiry, IggyMessage,
IggyTimestamp, MaxTopicSize, Partitioner, Partitioning, SendMessagesConfirmationResponse,
@@@ -98,74 -94,74 +97,76 @@@ impl ProducerCore
let stream_id = self.stream_id.clone();
let topic_id = self.topic_id.clone();
info!("Initializing producer for stream: {stream_id} and topic: {topic_id}...");
- let client = self.client.clone();
- let client = client.read().await;
- if client.get_stream(&stream_id).await?.is_none() {
- let name = match stream_id.kind {
- IdKind::String => stream_id.get_string_value()?,
- IdKind::Numeric => {
- error!(
- "Stream: {stream_id} does not exist and cannot be created from a numeric identifier."
- );
- return Err(IggyError::StreamIdNotFound(Identifier::from_identifier(
- &stream_id,
- )));
+ let client = self.client.read().await;
+ with_binary_lineage(client.operation_lineage().await, async {
+ if client.get_stream(&stream_id).await?.is_none() {
+ let name = match stream_id.kind {
+ IdKind::String => stream_id.get_string_value()?,
+ IdKind::Numeric => {
+ error!(
+ "Stream: {stream_id} does not exist and cannot be created from a numeric identifier."
+ );
+ return Err(IggyError::StreamIdNotFound(Identifier::from_identifier(
+ &stream_id,
+ )));
+ }
+ };
+
+ if !self.create_stream_if_not_exists {
+ error!("Stream does not exist and auto-creation is disabled.");
+ return Err(IggyError::StreamNameNotFound(name));
}
- };
- if !self.create_stream_if_not_exists {
- error!("Stream does not exist and auto-creation is disabled.");
- return Err(IggyError::StreamNameNotFound(name));
+ info!("Creating stream: {name}");
- client.create_stream(&name).await?;
++ client
++ .create_stream(&name, &StreamCreateOptions::default())
++ .await?;
}
- info!("Creating stream: {name}");
- client
- .create_stream(&name, &StreamCreateOptions::default())
- .await?;
- }
+ if client.get_topic(&stream_id, &topic_id).await?.is_none() {
+ let name = match topic_id.kind {
+ IdKind::String => topic_id.get_string_value()?,
+ IdKind::Numeric => {
+ error!(
+ "Topic: {topic_id} does not exist and cannot be created from a numeric identifier."
+ );
+ return Err(IggyError::TopicIdNotFound(
+ Identifier::from_identifier(&topic_id),
+ Identifier::from_identifier(&stream_id),
+ ));
+ }
+ };
- if client.get_topic(&stream_id, &topic_id).await?.is_none() {
- let name = match topic_id.kind {
- IdKind::String => topic_id.get_string_value()?,
- IdKind::Numeric => {
- error!(
- "Topic: {topic_id} does not exist and cannot be created from a numeric identifier."
- );
- return Err(IggyError::TopicIdNotFound(
- Identifier::from_identifier(&topic_id),
- Identifier::from_identifier(&stream_id),
- ));
+ if !self.create_topic_if_not_exists {
+ error!("Topic does not exist and auto-creation is disabled.");
+ return Err(IggyError::TopicNameNotFound(name, stream_id.to_string()));
}
- };
- if !self.create_topic_if_not_exists {
- error!("Topic does not exist and auto-creation is disabled.");
- return Err(IggyError::TopicNameNotFound(name, stream_id.to_string()));
+ info!("Creating topic: {name} for stream: {stream_id}");
+ client
+ .create_topic(
+ &stream_id,
+ &name,
+ &TopicCreateOptions {
+ partitions_count: Some(self.topic_partitions_count),
+ message_expiry: (self.topic_message_expiry != IggyExpiry::ServerDefault)
+ .then_some(self.topic_message_expiry),
+ max_topic_size: (self.topic_max_size != MaxTopicSize::ServerDefault)
+ .then_some(self.topic_max_size),
+ durability: self.topic_durability,
+ ..TopicCreateOptions::default()
+ },
+ )
+ .await?;
}
- info!("Creating topic: {name} for stream: {stream_id}");
- client
- .create_topic(
- &stream_id,
- &name,
- &TopicCreateOptions {
- partitions_count: Some(self.topic_partitions_count),
- message_expiry: (self.topic_message_expiry != IggyExpiry::ServerDefault)
- .then_some(self.topic_message_expiry),
- max_topic_size: (self.topic_max_size != MaxTopicSize::ServerDefault)
- .then_some(self.topic_max_size),
- durability: self.topic_durability,
- ..TopicCreateOptions::default()
- },
- )
- .await?;
- }
-
- let _ = self
- .initialized
- .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst);
- info!("Producer has been initialized for stream: {stream_id} and topic: {topic_id}.");
- Ok(())
+ let _ = self
+ .initialized
+ .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst);
+ info!("Producer has been initialized for stream: {stream_id} and topic: {topic_id}.");
+ Ok(())
+ })
+ .await
}
async fn try_send_messages(
@@@ -1046,6 -1062,293 +1067,294 @@@ mod tests
.await
}
+ #[derive(Debug)]
+ struct FailedBatches(flume::Sender<ErrorCtx>);
+
+ impl ErrorCallback for FailedBatches {
+ fn call(&self, ctx: ErrorCtx) -> Pin<Box<dyn Future<Output = ()> + Send + 'static>> {
+ let sender = self.0.clone();
+ Box::pin(async move {
+ sender.send(ctx).unwrap();
+ })
+ }
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn given_a_confirmed_chunk_when_signing_in_should_end_the_unsent_tail_lineage() {
+ let (cluster, producer) = connected_producer().await;
+ let producer = Arc::new(producer);
+ cluster.script(
+ &endpoint(0),
+ SEND_MESSAGES_CODE,
+ [confirmation(0), confirmation(1)],
+ );
+ cluster.hold(SEND_MESSAGES_CODE);
+ let pending = tokio::spawn({
+ let producer = Arc::clone(&producer);
+ async move {
+ producer
+ .send(vec![message(b"first"), message(b"tail")])
+ .await
+ }
+ });
+ eventually("the first chunk arrives", || {
+ cluster.count(SEND_MESSAGES_CODE) == 1
+ })
+ .await;
+ producer
+ .core
+ .client
+ .read()
+ .await
+ .login_user("iggy", "iggy")
+ .await
+ .unwrap();
+ cluster.release(SEND_MESSAGES_CODE);
+
+ let Err(IggyError::ProducerSendFailed {
+ cause,
+ failed,
+ uncertain,
+ committed,
+ ..
+ }) = pending.await.unwrap()
+ else {
+ panic!("the old operation must return its unsent tail");
+ };
+ assert_eq!(*cause, IggyError::Unauthenticated);
+ assert_eq!(uncertain, 0);
+ assert_eq!(failed.len(), 1);
+ assert_eq!(failed[0].payload, Bytes::from_static(b"tail"));
+ assert_eq!(committed.len(), 1);
+ assert_eq!(committed[0].base_offset, 0);
+ assert_eq!(cluster.count(SEND_MESSAGES_CODE), 1);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn given_initialization_when_signing_in_should_end_the_creation_lineage() {
+ let (cluster, mut producer) = connected_producer().await;
+ Arc::get_mut(&mut producer.core).unwrap().stream_id =
+ Arc::new(Identifier::named("stream").unwrap());
+ cluster.script(
+ &endpoint(0),
+ CREATE_STREAM_CODE,
+ [Answer::CommitBody(
+ 1,
+ StreamResponse {
+ id: 1,
+ created_at: 0,
+ topics_count: 0,
+ size_bytes: 0,
+ messages_count: 0,
+ name: WireName::new("stream").unwrap(),
+ options: WireOptions::empty(),
++ derived_options: WireOptions::empty(),
+ }
+ .to_bytes()
+ .to_vec(),
+ )],
+ );
+ cluster.script(&endpoint(0), GET_TOPIC_CODE, [Answer::Body(topic_body(1))]);
+ cluster.hold(GET_STREAM_CODE);
+ let producer = Arc::new(producer);
+ let pending = tokio::spawn({
+ let producer = Arc::clone(&producer);
+ async move { producer.init().await }
+ });
+ eventually("the stream lookup arrives", || {
+ cluster.count(GET_STREAM_CODE) == 1
+ })
+ .await;
+ producer
+ .core
+ .client
+ .read()
+ .await
+ .login_user("iggy", "iggy")
+ .await
+ .unwrap();
+ cluster.release(GET_STREAM_CODE);
+
+ assert_eq!(
+ pending.await.unwrap().unwrap_err(),
+ IggyError::Unauthenticated
+ );
+ assert_eq!(cluster.count(CREATE_STREAM_CODE), 0);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn given_queued_batches_when_signing_in_should_never_merge_different_lineages() {
+ let (cluster, mut producer) = connected_producer().await;
+ Arc::get_mut(&mut producer.core).unwrap().direct_config = None;
+ let (failures, received) = flume::unbounded();
+ producer.dispatcher = Some(super::ProducerDispatcher::new(
+ Arc::clone(&producer.core),
+ BackgroundConfig::builder()
+ .batch_length(2)
+ .batch_size(0)
+ .linger_time(IggyDuration::new(Duration::from_secs(60)))
+ .error_callback(Arc::new(Box::new(FailedBatches(failures))))
+ .build(),
+ ));
+ cluster.script(&endpoint(0), SEND_MESSAGES_CODE, [confirmation(0)]);
+ producer.send(vec![message(b"old")]).await.unwrap();
+ producer
+ .core
+ .client
+ .read()
+ .await
+ .login_user("iggy", "iggy")
+ .await
+ .unwrap();
+ producer.send(vec![message(b"new")]).await.unwrap();
+ producer.shutdown().await;
+
+ let failure = received.try_recv().expect("the old queued batch must fail");
+ assert_eq!(*failure.cause, IggyError::Unauthenticated);
+ assert_eq!(failure.messages.len(), 1);
+ assert_eq!(failure.messages[0].payload, Bytes::from_static(b"old"));
+ assert!(received.try_recv().is_err());
+ let sends = cluster.received(SEND_MESSAGES_CODE);
+ assert_eq!(sends.len(), 1);
+ let (size, body) = sends[0].body.split_first_chunk().unwrap();
+ assert_eq!(
+ SendMessagesHeader::decode_from(&body[..u32::from_le_bytes(*size) as usize])
+ .unwrap()
+ .messages_count,
+ 1
+ );
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn given_backpressure_when_signing_in_should_keep_the_waiting_batch_lineage() {
+ let (cluster, mut producer) = connected_producer().await;
+ Arc::get_mut(&mut producer.core).unwrap().direct_config = None;
+ let batch = ShardMessage {
+ stream: Arc::clone(&producer.core.stream_id),
+ topic: Arc::clone(&producer.core.topic_id),
+ messages: vec![message(b"old")],
+ partitioning: None,
+ };
+ let (failures, received) = flume::unbounded();
+ producer.dispatcher = Some(super::ProducerDispatcher::new(
+ Arc::clone(&producer.core),
+ BackgroundConfig::builder()
+ .batch_length(1)
+ .max_buffer_size(batch.get_size_bytes())
+ .error_callback(Arc::new(Box::new(FailedBatches(failures))))
+ .build(),
+ ));
+ cluster.script(
+ &endpoint(0),
+ SEND_MESSAGES_CODE,
+ [confirmation(0), confirmation(1)],
+ );
+ cluster.hold(SEND_MESSAGES_CODE);
+ producer.send(vec![message(b"old")]).await.unwrap();
+ eventually("the charged batch arrives", || {
+ cluster.count(SEND_MESSAGES_CODE) == 1
+ })
+ .await;
+ let waiting = producer.send(vec![message(b"old")]);
+ tokio::pin!(waiting);
+ assert!(futures::poll!(&mut waiting).is_pending());
+ producer
+ .core
+ .client
+ .read()
+ .await
+ .login_user("iggy", "iggy")
+ .await
+ .unwrap();
+ cluster.release(SEND_MESSAGES_CODE);
+ waiting.await.unwrap();
+ let failure = tokio::time::timeout(
+ crate::links::fake_cluster::TEST_BUDGET,
+ received.recv_async(),
+ )
+ .await
+ .unwrap()
+ .unwrap();
+
+ assert_eq!(*failure.cause, IggyError::Unauthenticated);
+ assert_eq!(failure.messages.len(), 1);
+ assert_eq!(cluster.count(SEND_MESSAGES_CODE), 1);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn given_queued_batches_when_recovering_should_preserve_their_lineage() {
+ let (cluster, mut producer) = connected_producer().await;
+ Arc::get_mut(&mut producer.core).unwrap().direct_config = None;
+ let (failures, received) = flume::unbounded();
+ producer.dispatcher = Some(super::ProducerDispatcher::new(
+ Arc::clone(&producer.core),
+ BackgroundConfig::builder()
+ .batch_length(2)
+ .batch_size(0)
+ .linger_time(IggyDuration::new(crate::links::fake_cluster::TEST_BUDGET))
+ .error_callback(Arc::new(Box::new(FailedBatches(failures))))
+ .build(),
+ ));
+ producer.send(vec![message(b"old")]).await.unwrap();
+ *cluster.session.lock().unwrap() = None;
+ cluster.drop_connections(&endpoint(0));
+ eventually("automatic sign-in restores the session", || {
+ cluster.count(LOGIN_REGISTER_CODE) == 2
+ })
+ .await;
+ producer.send(vec![message(b"new")]).await.unwrap();
+ producer.shutdown().await;
+
+ assert!(received.try_recv().is_err());
+ let sends = cluster.received(SEND_MESSAGES_CODE);
+ assert_eq!(sends.len(), 1);
+ let (size, body) = sends[0].body.split_first_chunk().unwrap();
+ assert_eq!(
+ SendMessagesHeader::decode_from(&body[..u32::from_le_bytes(*size) as usize])
+ .unwrap()
+ .messages_count,
+ 2
+ );
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn given_an_unanswered_background_batch_when_signing_in_should_keep_its_lineage_uncertain()
+ {
+ let (cluster, mut producer) = connected_producer().await;
+ Arc::get_mut(&mut producer.core).unwrap().direct_config = None;
+ let (failures, received) = flume::unbounded();
+ producer.dispatcher = Some(super::ProducerDispatcher::new(
+ Arc::clone(&producer.core),
+ BackgroundConfig::builder()
+ .batch_length(1)
+ .error_callback(Arc::new(Box::new(FailedBatches(failures))))
+ .build(),
+ ));
+ cluster.script(&endpoint(0), SEND_MESSAGES_CODE, [Answer::Silence]);
+ producer.send(vec![message(b"old")]).await.unwrap();
+ eventually("the batch may have applied", || {
+ cluster.count(SEND_MESSAGES_CODE) == 1
+ })
+ .await;
+ producer
+ .core
+ .client
+ .read()
+ .await
+ .login_user("iggy", "iggy")
+ .await
+ .unwrap();
+ producer.shutdown().await;
+
+ let failure = received.try_recv().unwrap();
+ assert!(
+ matches!(*failure.cause, IggyError::OperationUncertain(_)),
+ "{failure:?}"
+ );
+ assert_eq!(failure.messages.len(), 1);
+ assert_eq!(cluster.count(SEND_MESSAGES_CODE), 1);
+ }
+
#[tokio::test(start_paused = true)]
async fn given_a_missing_chosen_partition_when_sending_should_refresh_and_reselect_once() {
for succeeds in [true, false] {
diff --cc core/sdk/src/driver.rs
index 09b95485f,4977ecac8..6c630efa0
--- a/core/sdk/src/driver.rs
+++ b/core/sdk/src/driver.rs
@@@ -37,14 -37,13 +37,17 @@@ use iggy_binary_protocol::requests::con
use iggy_binary_protocol::requests::messages::{
GetPartitionRouteRequest, PollMessagesRequest, SendMessagesHeader,
};
-use iggy_binary_protocol::requests::users::{ChangePasswordRequest, UpdateUserRequest};
+use iggy_binary_protocol::requests::system::{ResourceTarget, UpdateResourceRequest};
+use iggy_binary_protocol::requests::users::ChangePasswordRequest;
use iggy_binary_protocol::responses::messages::PartitionRouteResponse;
use iggy_binary_protocol::responses::users::LoginRegisterResponse;
-use iggy_binary_protocol::{WireDecode, WireEncode, WirePartitioning, WirePollingStrategy};
+use iggy_binary_protocol::{
+ WireDecode, WireEncode, WireOptions, WirePartitioning, WirePollingStrategy,
+};
- use iggy_common::{ClientState, Credentials, HeaderKind, Identifier, IggyError, OptionKey};
+ use iggy_common::{
- BinaryOperationLineage, ClientState, Credentials, DiagnosticEvent, Identifier, IggyError,
++ BinaryOperationLineage, ClientState, Credentials, DiagnosticEvent, HeaderKind, Identifier,
++ IggyError, OptionKey,
+ };
use route_cache::RouteCache;
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use tokio::time::{Instant, timeout_at};
diff --cc core/sdk/src/tcp/tcp_client.rs
index a9dc6defd,e89243c15..25dd57033
--- a/core/sdk/src/tcp/tcp_client.rs
+++ b/core/sdk/src/tcp/tcp_client.rs
@@@ -116,21 -116,29 +116,29 @@@ mod tests
use super::*;
use crate::links::OPERATION_TIMEOUT;
use crate::links::fake_cluster::{
- Answer, FakeCluster, TEST_BUDGET, endpoint, eventually, links, policy,
+ Answer, FakeCluster, TEST_BUDGET, endpoint, eventually, links, policy, topic_body,
};
+ use bytes::Bytes;
use iggy_binary_protocol::codes::{
- BIND_SESSION_CODE, CHANGE_PASSWORD_CODE, LOGIN_REGISTER_CODE, LOGOUT_USER_CODE, PING_CODE,
+ BIND_SESSION_CODE, CHANGE_PASSWORD_CODE, GET_CLUSTER_METADATA_CODE, GET_STREAMS_CODE,
+ GET_TOPIC_CODE, LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE, LOGOUT_USER_CODE,
+ PING_CODE, POLL_MESSAGES_CODE, SEND_MESSAGES_CODE, SYNC_CONSUMER_GROUP_CODE,
- UPDATE_USER_CODE,
+ UPDATE_RESOURCE_CODE,
};
use iggy_binary_protocol::consensus::EvictionReason;
use iggy_binary_protocol::requests::users::{
ChangePasswordRequest, LoginRegisterRequest, LoginRegisterWithPatRequest,
};
+ use iggy_binary_protocol::responses::consumer_groups::SyncConsumerGroupResponse;
+ use iggy_binary_protocol::responses::messages::{
+ PartitionRouteResponse, PollMessagesResponseHeader,
+ };
use iggy_binary_protocol::{WireDecode, WireEncode, WireIdentifier};
use iggy_common::{
- BinaryTransport, Client, ClientState, Credentials, Identifier, IggyDuration,
- PersonalAccessTokenClient, SystemClient, TcpClientReconnectionConfig, TopicClient,
- TopicUpdateOptions, UserClient, UserUpdateOptions, with_binary_deadline,
+ BinaryTransport, Client, ClientState, Consumer, Credentials, Identifier, IggyDuration,
+ IggyMessage, MessageClient, Partitioning, PersonalAccessTokenClient, PollingStrategy,
- SystemClient, TcpClientReconnectionConfig, UserClient, UserUpdateOptions,
- with_binary_deadline,
++ SystemClient, TcpClientReconnectionConfig, TopicClient, TopicUpdateOptions, UserClient,
++ UserUpdateOptions, with_binary_deadline,
};
use secrecy::{ExposeSecret, SecretString};
use std::time::Duration;
diff --cc core/server/README.md
index fdd9594c6,ab38fb2b6..20b4cdecb
--- a/core/server/README.md
+++ b/core/server/README.md
@@@ -47,11 -47,35 +47,36 @@@ Command-line arguments include `--repli
Deploy matching server and SDK releases together. Rolling upgrades and mixed
server versions are unsupported. Partition control frames from older servers
carry incarnation zero and are rejected, so a mixed-version cluster cannot
-complete partition view changes. The `IGGY-BOUND-GRANTS-5` storage gate
+complete partition view changes. The `IGGY-RESOURCE-OPTIONS-1` storage gate
requires fresh storage and rejects older data directories without modifying them.
Cached replies now preserve a committed-result marker, which older receipts lack.
- No in-place upgrade or implicit migration is supported. A separately validated
- offline migration or restore procedure would be required to reuse existing data.
+ Partition creation records carry an initial view in each 16-byte assignment,
+ replacing the 12-byte assignments and single request-wide view in older metadata
-WAL entries. This boundary also rejects `IGGY-BOUND-GRANTS-4` directories from
-upstream and earlier development builds, unchanged before replay.
++WAL entries. This boundary also rejects `IGGY-BOUND-GRANTS-4` and
++`IGGY-BOUND-GRANTS-5` directories from upstream and earlier development builds,
++unchanged before replay.
+ No in-place upgrade, storage conversion tool or retained-data migration is
+ provided. Use an offline cutover with logical re-ingestion:
+
+ 1. Stop application writes, consumers that advance offsets, and administrative
+ changes. While the old cluster is still readable, export the required messages
+ with an application-specific reader using its compatible SDK. Record payloads,
+ headers and per-partition order, topic policies, permissions, and the intended
+ consumer restart positions. Check the export against the source before stopping it.
+ 2. Stop every old replica. Preserve a consistent backup of all metadata, partition
+ superblocks, WALs, segments and initialization records, with the old binary and
+ configuration. Keep these originals unchanged for rollback.
+ 3. Deploy matching new servers and SDKs with separate, empty storage paths on every
+ replica. Recreate the required accounts, permissions, streams, topics and groups,
+ then re-ingest through the new SDK. New sessions, receipts, partition incarnations
+ and offsets do not continue the old ones. Map consumer progress to the re-ingested
+ data or choose an explicit new starting position.
+ 4. Verify the restored configuration, permissions, message counts and contents
+ before resuming applications. Rollback uses the old binary and its consistent
+ backup; it excludes writes accepted by the new cluster unless reconciled separately.
+
+ This procedure copies logical application data. It does not convert old storage
+ files or recover data that is absent from both the source and its backups.
The server no longer generates a root password when `IGGY_ROOT_USERNAME` and `IGGY_ROOT_PASSWORD` are unset. Supply both variables on every replica before the first cluster start so any promoted node can initialize root. Use `--with-default-root-credentials` only in development. A cluster node without credentials waits for root to be replicated. Once root is persisted, restarts need neither.
@@@ -94,8 -118,113 +119,113 @@@ IGGY_TCP_ADDRESS=127.0.0.1:8090 IGGY_HT
Cluster membership, quorum and replica addressing live under `[cluster]`.
+ New partition primaries are spread across the configured replicas using each
+ partition's last observed primary, or its committed initial view before an
+ observation arrives. Concurrent creates count pending assignments too. Once
+ readiness probes identify responsive replicas, new primaries are assigned only
+ to those replicas. Before readiness is known, all configured replicas remain
+ eligible. After the responsive replica set
+ has been stable for five seconds, the metadata primary balances existing primary
+ counts one partition at a time. It confirms the source's committed frontier and
+ the target's applied history, then grants the move to the source primary. The
+ source finishes its accepted writes before it requests a persisted consensus
+ view change.
+ Recovering partitions are skipped without delaying transfers of other partitions.
+ Every replica still stores every partition.
+ Balancing primary counts does not balance bytes, throughput or CPU load. A hot
+ partition still has one ordering primary.
+
+ A create whose namespace is invalid can receive 58 (`TransientNotAccepted`) while
+ any metadata write is pending, even an unrelated one. Under steady unrelated
+ traffic its retries can reach the deadline with 58 instead of the namespace
+ error. A valid create that exceeds its configured capacity with the committed
+ partitions alone still gets the capacity error. If only the creates in flight
+ leave no room, it gets 58, and under steady create traffic its retries can reach
+ the deadline with 58. A personal access token create follows the same rule for
+ the user's tokens and `max_tokens_per_user`.
+
+ ### HTTP routing and backpressure
+
+ An HTTP client therefore often calls a node that is not the partition primary.
+ If every node accepts the caller's bearer, that node relays message sends, polls
+ and consumer offset calls to the primary. This needs identical `http.jwt`
+ secrets on every node, or `cluster.auth`. Without that key material, these calls
+ answer a transient 503 with code 58 (`TransientNotAccepted`).
+
+ A partition relay carries an HMAC-SHA256 proof over its method, path and query,
+ route, and expiry. The target still authenticates and authorizes the caller.
+ The relay key derives from the first configured `http.jwt.encoding_secret` or
+ `http.jwt.decoding_secret`, otherwise from `cluster.auth.shared_secret`.
+ Keep that material identical on every node. Rotating the material used for
+ bearers also changes the relay key, so nodes with different keys refuse each
+ other's relays. Keep node wall clocks synchronized within a few seconds: a proof
+ has a 10-second lifetime. Relay numeric fields use unsigned decimal with no
+ leading zeros, and proof tags use lowercase hexadecimal; noncanonical fields
+ are refused. Proxies must preserve these internal headers unchanged.
+
+ A target refuses a missing, invalid or expired proof with 403. When a node's own
+ relay proof is refused, that node logs a warning and answers the application with
+ 503, code `relay_refused`, and `Retry-After: 1`. Nothing was admitted by that
+ attempt. Check shared keys and clocks; this response is not a denial of the
+ application's permission. SDKs with HTTP transports use their ordinary 503 retry
+ policy. Rust and Python use their configured retry schedule rather than
+ `Retry-After`; C# honors its delta value; Java uses Apache HttpClient's ordinary
+ 429/503 retry strategy.
+
+ A 503 answer to a write means that nothing was admitted, so the write can be sent
+ again. A 502 or 504 answer to a write means that its outcome is unknown, and the
+ write can still commit. The body `id` 57 (`TransientNotCommitted`) or the body
+ `code`, such as `produce_timeout`, names the cause. Read the state before writing
+ again, because a resend is a new request that can apply twice. The Rust, C# and
+ Java SDKs never resend such a write, and the Rust SDK reports it as 57.
+
+ Metadata forwards and partition relays share a limit of
+ 128 concurrent forwards per node. New forwards at capacity return 503 `forward_busy` with
+ `Retry-After: 1`. Metadata forwarding has one 30-second retry budget beginning
+ before the request body is buffered, so a slow upload leaves less time for
+ forwarding. Partition requests share a 10-second deadline across their metadata
+ wait, local attempts, relays and reply waits.
+
+ A node sends one metadata-floor read at a time and holds no forward slot for it. The read serves every linearizable read that arrived before it was sent, and the reads that arrive while it is in flight share the next one.
+
+ Socket replies use `message_bus.client_queue_capacity` per connection. A reply that cannot enter the non-waiting queue closes that connection, leaving the caller to recover any uncertain operation. Paths that wait for reply space hold their bounded permit until delivery or connection teardown.
+
+ A reply that waits longer than `message_bus.client_write_stall_timeout` (30 seconds by default) closes the connection, so a client that stops reading its socket, an administrative one included, holds its queue and request tasks for at most that long.
+
+ Requests queued behind the ones a connection runs are capped at 1024 frames and at `message_bus.client_request_queue_bytes_max` (256 MiB by default). A request past either cap gets 58 (`TransientNotAccepted`), unless the queue is empty.
+
+ ### Metadata freshness and counters
+
+ On a multi-replica node, client partition requests require recent metadata quorum
+ evidence before namespace lookup and authorization. The evidence lasts
+ `K = cluster.heartbeat_timeout + 3 seconds` (8 seconds by default); expired or
+ absent evidence yields 58. A primary credits answered probes while ready. A
+ backup credits only a probe from its current metadata primary whose advertised
+ commit covers the adopted view's head and is already applied locally. Commit
+ messages alone do not renew this evidence.
+
+ The gate uses elapsed time, so a brief view or role change preserves existing
+ credit without restarting its lifetime. A restarted node has no credit until
+ fresh evidence arrives. Single-replica nodes do not require network evidence.
+ A primary advertises evidence to a backup only after that backup answered its
+ previous probe. The backup ages the evidence from the arrival of that previous
+ probe. A delayed probe therefore cannot extend backup credit past K. The one
+ exception is a primary restart in the same view: a late probe of the old
+ incarnation can then add its own transit time. The aging costs up to one probe
+ interval of credit. If the primary stops probing, a backup therefore stops
+ admitting up to one probe interval before the primary does.
+ K is a lease, not an instantaneous revocation guarantee.
+
+ Message and size counters in stream, topic and system stats (`get_stream`,
+ `get_topic`, `get_stats` and the list forms) are counted per node. A node that
+ is not the partition primary learns of a commit from the next prepare or from
+ the periodic commit broadcast. It counts the message only then. While the
+ cluster is idle, its counters can lag by up to `cluster.commit_broadcast_interval`.
+
+ ### Sessions and retry receipts
+
-Protocol 0.11.1 uses client-owned registration proofs and `BindSession` (15) to
+Protocol 0.11.1 and later use client-owned registration proofs and `BindSession` (15) to
- share a logical session across connections. Commands 14, 103, 104 and 123 are
+ share a logical session across connections. Commands 14, 103, 104, 123 and 124 are
retired. Partition routing uses 105, polling uses 100 and offset reads use 120. Disconnecting a
connection does not log out its logical session. Server-observed activity renews
the session lease, and expiry or explicit logout retires it before capacity is
@@@ -209,9 -328,25 +373,25 @@@ explicitly supported. Edge builds requi
when their protocol numbers match.
An unsupported data directory is refused before WAL scanning or file changes.
- Replacing the binary does not migrate old data. Restore or migrate retained data
- only through a separately verified procedure; copying a format marker is not
- a migration.
+ Replacing the binary does not migrate old data. Use the offline re-ingestion
+ procedure above for an upgrade. A recovery from a backup must use a separately
+ verified, compatible backup; copying a format marker is not a migration.
+
-Within compatible `IGGY-BOUND-GRANTS-5` storage, this release reads superblock
++Within compatible `IGGY-RESOURCE-OPTIONS-1` storage, this release reads superblock
+ envelope versions 1 and 2 and writes version 2. Legacy 66- and 74-byte consensus
+ records remain readable. New records atomically retain an adopted view, its
+ advertised head and exact prepare headers alongside the existing consensus state.
+ These headers preserve log identity across restart; they do not prove that a
+ message body is present or safe to acknowledge. Recovery still repairs missing
+ bodies before serving them.
+
+ A binary that only understands version 1 refuses a version-2 slot, even if the
+ other slot still has version 1. After the first new superblock write, replacing
+ the binary with an older one is not a supported rollback. Restore a consistent
+ pre-upgrade backup with its matching binary, or use fresh storage and the logical
+ re-ingestion procedure. Never remove one slot, edit a version or view, or combine
+ superblocks from different backups to bypass the refusal. This boundary applies
+ to both metadata and partition superblocks and does not relax the storage gate.
Partition recovery refuses a superblock whose nonzero `log_view`
is below the partition's committed `created_view`. It also refuses a WAL
diff --cc core/server/src/boot/mod.rs
index 39778d061,e6d6356c3..aa10cb575
--- a/core/server/src/boot/mod.rs
+++ b/core/server/src/boot/mod.rs
@@@ -1277,6 -1251,6 +1287,7 @@@ mod tests
b"IGGY-BOUND-GRANTS-2\n",
b"IGGY-BOUND-GRANTS-3\n",
b"IGGY-BOUND-GRANTS-4\n",
++ b"IGGY-BOUND-GRANTS-5\n",
b"IGGY-DURABLE-SESSIONS-3\n",
&STORAGE_FORMAT[..STORAGE_FORMAT.len() - 1],
] {
@@@ -1306,14 -1280,13 +1317,17 @@@
}
#[test]
- fn given_different_root_bootstrap_formats_when_identifying_replicas_should_reject_mixing() {
- assert_ne!(
- message_bus::replica::handshake::binary_identity(crate::VERSION, STORAGE_FORMAT),
- message_bus::replica::handshake::binary_identity(
- crate::VERSION,
- b"IGGY-BOUND-GRANTS-3\n"
- ),
- );
+ fn given_different_storage_formats_when_identifying_replicas_should_reject_mixing() {
- for older_format in [b"IGGY-BOUND-GRANTS-3\n", b"IGGY-BOUND-GRANTS-4\n"] {
++ for older_format in [
++ b"IGGY-BOUND-GRANTS-3\n".as_slice(),
++ b"IGGY-BOUND-GRANTS-4\n",
++ b"IGGY-BOUND-GRANTS-5\n",
++ ] {
+ assert_ne!(
+ message_bus::replica::handshake::binary_identity(crate::VERSION, STORAGE_FORMAT),
+ message_bus::replica::handshake::binary_identity(crate::VERSION, older_format),
+ );
+ }
}
#[test]
diff --cc core/server/src/boot/recovery.rs
index e70f4afe8,0e5bac6a8..a7f33bb22
--- a/core/server/src/boot/recovery.rs
+++ b/core/server/src/boot/recovery.rs
@@@ -29,13 -29,10 +29,13 @@@ use crate::shell::
};
use configs::server::ServerConfig;
use consensus::{
- ClientTable, JoinMode, LocalPipeline, PipelineEntry, Sequencer, VsrConsensus, VsrRestore,
- VsrState,
+ ClientTable, HistoryRecovery, JoinMode, LocalPipeline, PipelineEntry, Sequencer, VsrConsensus,
+ VsrRestore, VsrState,
};
-use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyByteSize, TopicRuntimeOptions};
+use iggy_common::{
+ EncryptorKind, IggyByteSize, IggyError, PartitionStats, ServerKeyContext, StreamEncryption,
+ TopicRuntimeOptions, check_critical_options,
+};
use journal::Journal;
use journal::prepare_journal::PrepareJournal;
use journal::superblock::PingPongSuperblock;
@@@ -872,225 -695,6 +888,225 @@@ pub(in crate::boot) fn restore_metadata
#[cfg(test)]
mod tests {
use super::*;
+ use crate::dispatch::test_support::{TestMux, prepare_message};
+ use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment;
+ use iggy_binary_protocol::requests::streams::CreateStreamRequest;
+ use iggy_binary_protocol::requests::topics::{
+ CreateTopicRequest, CreateTopicWithAssignmentsRequest,
+ };
+ use iggy_binary_protocol::{Operation, WireEncode, WireIdentifier, WireName, WireOptions};
+ use iggy_common::wire_conversions::resource_options_to_wire;
+ use iggy_common::{
+ HeaderKey, HeaderValue, OptionKey, OptionValue, OptionsProvenance, ResourceOptions,
+ StreamEncryptionKey, StreamKeyOrigin, WrapSet,
+ };
+ use metadata::stm::StateMachine as _;
+
+ const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=";
+ const OTHER_CONFIG_KEY: &str = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=";
+
+ #[test]
+ fn given_topic_with_unknown_critical_option_when_booting_should_refuse_it() {
+ let future_key = HeaderKey::try_from("a_future_key").expect("key fits");
+ let unreadable = ResourceOptions::from([
+ (
+ HeaderKey::from(OptionKey::CriticalKeys),
+ OptionValue::explicit(HeaderValue::try_from("a_future_key").expect("marker fits")),
+ ),
+ (future_key, OptionValue::explicit(HeaderValue::from(7u64))),
+ ]);
+ for (derived, refused) in [(ResourceOptions::new(), false), (unreadable, true)] {
+ let mux = TestMux::default();
+ mux.users().ensure_root_user("iggy", "hash");
+ mux.update(prepare_message(
+ Operation::CreateStream,
+ 1,
+ 1,
+ &CreateStreamRequest {
+ name: WireName::new("stream").expect("name fits"),
+ options: WireOptions::empty(),
+ }
+ .to_bytes(),
+ ))
+ .expect("stream create applies");
+ mux.update(prepare_message(
+ Operation::CreateTopicWithAssignments,
+ 1,
+ 2,
+ &CreateTopicWithAssignmentsRequest {
+ request: CreateTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ partitions_count: 1,
+ name: WireName::new("topic").expect("name fits"),
+ options: WireOptions::empty(),
+ },
+ derived_options: resource_options_to_wire(&derived, OptionsProvenance::All)
+ .expect("options encode"),
+ partitions: vec![CreatedPartitionAssignment {
+ partition_id: 0,
+ consensus_group_id: 1,
++ initial_view: 0,
+ }],
- created_view: 0,
+ }
+ .to_bytes(),
+ ))
+ .expect("topic create applies");
+
+ let result = refuse_unreadable_topics(mux.streams());
+
+ assert_eq!(
+ matches!(
+ result,
+ Err(ServerError::TopicOptionsUnavailable {
+ stream_id: 0,
+ topic_id: 0,
+ ..
+ })
+ ),
+ refused,
+ "{result:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn given_stream_key_that_does_not_open_when_booting_should_refuse_only_a_complete_replay() {
+ let stream_context = ServerKeyContext::from_config(CONFIG_KEY, &[], true).expect("key");
+ let key = StreamEncryptionKey::try_from([1; 32].as_slice()).expect("32 bytes");
+ let set = WrapSet::from(
+ stream_context
+ .wrap(&key, StreamKeyOrigin::Server)
+ .expect("wraps"),
+ );
+ let wrapped = ResourceOptions::from([(
+ HeaderKey::from(OptionKey::EncryptionKeyWrapped),
+ OptionValue::explicit(HeaderValue::try_from(set.as_ref()).expect("set fits")),
+ )]);
+ let unreadable = ResourceOptions::from([(
+ HeaderKey::from(OptionKey::CriticalKeys),
+ OptionValue::explicit(HeaderValue::try_from("a_future_key").expect("marker fits")),
+ )]);
+ let plaintext = ResourceOptions::new();
+ let same_key = ServerKeyContext::from_config(CONFIG_KEY, &[], true).expect("key");
+ let other_key = ServerKeyContext::from_config(OTHER_CONFIG_KEY, &[], true).expect("key");
+ let no_key = ServerKeyContext::disabled();
+ for (options, node_context, solo, refused) in [
+ (&wrapped, &same_key, true, false),
+ (&wrapped, &same_key, false, false),
+ (&wrapped, &other_key, true, true),
+ (&wrapped, &other_key, false, false),
+ (&wrapped, &no_key, true, true),
+ (&wrapped, &no_key, false, true),
+ (&plaintext, &no_key, true, false),
+ (&plaintext, &no_key, false, false),
+ (&unreadable, &same_key, true, true),
+ (&unreadable, &same_key, false, true),
+ ] {
+ let mux = TestMux::default();
+ mux.users().ensure_root_user("iggy", "hash");
+ mux.update(prepare_message(
+ Operation::CreateStream,
+ 1,
+ 1,
+ &CreateStreamRequest {
+ name: WireName::new("stream").expect("name fits"),
+ options: resource_options_to_wire(options, OptionsProvenance::All)
+ .expect("options encode"),
+ }
+ .to_bytes(),
+ ))
+ .expect("stream create applies");
+
+ let result = refuse_partitionless_stream_keys(mux.streams(), node_context, solo);
+
+ assert_eq!(
+ matches!(
+ result,
+ Err(ServerError::StreamKeyUnavailable { stream_id: 0, .. })
+ ),
+ refused,
+ "node keys {:?}, solo {solo}: {result:?}",
+ node_context.kek_fingerprints()
+ );
+ }
+ }
+
+ #[test]
+ fn given_stream_key_that_does_not_open_when_deciding_owned_partitions_should_leave_them_unbuilt()
+ {
+ let stream_context = ServerKeyContext::from_config(CONFIG_KEY, &[], true).expect("key");
+ let key = StreamEncryptionKey::try_from([1; 32].as_slice()).expect("32 bytes");
+ let set = WrapSet::from(
+ stream_context
+ .wrap(&key, StreamKeyOrigin::Server)
+ .expect("wraps"),
+ );
+ let wrapped = StreamEncryption::from(&ResourceOptions::from([(
+ HeaderKey::from(OptionKey::EncryptionKeyWrapped),
+ OptionValue::explicit(HeaderValue::try_from(set.as_ref()).expect("set fits")),
+ )]));
+ let identity = StreamKeyIdentity {
+ stream_id: 0,
+ created_op: 1,
+ };
+ let node_partitions = |config_key: &str| {
+ IggyPartitions::new(
+ ShardId::new(0),
+ PartitionsConfig {
+ messages_required_to_save: 1,
+ size_of_messages_required_to_save: IggyByteSize::from(1024_u64),
+ validate_checksum: true,
+ segment_size: IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE),
+ preallocate_segments: false,
+ key_context: Arc::new(
+ ServerKeyContext::from_config(config_key, &[], true).expect("key"),
+ ),
+ path_layout: partitions::PartitionPathLayout {
+ streams_root: "/var/lib/iggy/streams".to_owned(),
+ },
+ },
+ )
+ };
+
+ let other_key = node_partitions(OTHER_CONFIG_KEY);
+ let mut deferred = BTreeSet::new();
+ for _ in 0..2 {
+ let decision =
+ owned_partition_key(&other_key, &mut deferred, 0, identity, &wrapped, false);
+ assert!(matches!(decision, Ok(OwnedPartitionKey::Defer)));
+ }
+ assert_eq!(deferred, BTreeSet::from([0]));
+ assert!(matches!(
+ owned_partition_key(
+ &other_key,
+ &mut BTreeSet::new(),
+ 0,
+ identity,
+ &wrapped,
+ true
+ ),
+ Err(ServerError::StreamKeyUnavailable { stream_id: 0, .. })
+ ));
+
+ let same_key = node_partitions(CONFIG_KEY);
+ let mut deferred = BTreeSet::new();
+ assert!(matches!(
+ owned_partition_key(&same_key, &mut deferred, 0, identity, &wrapped, false),
+ Ok(OwnedPartitionKey::Build(Some(_)))
+ ));
+ assert!(matches!(
+ owned_partition_key(
+ &same_key,
+ &mut deferred,
+ 1,
+ identity,
+ &StreamEncryption::Plaintext,
+ false
+ ),
+ Ok(OwnedPartitionKey::Build(None))
+ ));
+ assert!(deferred.is_empty());
+ }
#[test]
fn partition_runtime_and_server_use_the_same_fixed_directory_layout() {
diff --cc core/server/src/dispatch/mod.rs
index dd999a2e1,037f7e74a..f8b75b596
--- a/core/server/src/dispatch/mod.rs
+++ b/core/server/src/dispatch/mod.rs
@@@ -1142,14 -1245,11 +1231,14 @@@ mod tests
};
use iggy_binary_protocol::requests::messages::{GetPartitionRouteRequest, PollMessagesRequest};
use iggy_binary_protocol::requests::personal_access_tokens::CreatePersonalAccessTokenRequest as WireCreatePersonalAccessTokenRequest;
- use iggy_binary_protocol::requests::streams::CreateStreamRequest;
- use iggy_binary_protocol::requests::streams::{CreateStreamRequest, UpdateStreamRequest};
++ use iggy_binary_protocol::requests::streams::{
++ CreateStreamRequest, UpdateStreamResolvedRequest,
++ };
use iggy_binary_protocol::requests::system::{
- BindSessionRequest, GetClientRequest, GetClientsRequest, GetStatsRequest,
+ BindSessionRequest, GetClientRequest, GetClientsRequest, GetStatsRequest, ResourceTarget,
+ UpdateResourceRequest,
};
- use iggy_binary_protocol::requests::topics::{
- CreateTopicRequest, CreateTopicWithAssignmentsRequest, DeleteTopicRequest,
- };
- use iggy_binary_protocol::requests::topics::{DeleteTopicRequest, UpdateTopicRequest};
++ use iggy_binary_protocol::requests::topics::{DeleteTopicRequest, UpdateTopicResolvedRequest};
use iggy_binary_protocol::requests::users::login_register::{
LoginRegisterRequest, RegisterRequest,
};
@@@ -1638,14 -1741,59 +1734,62 @@@
}
}
+ /// After the rename the names in the request resolve to nothing, so the
+ /// check after the owner answers must not resolve them again.
#[compio::test]
- async fn given_a_route_waiting_on_a_shard_when_the_topic_is_recreated_should_refuse_stale_route()
+ async fn given_a_read_by_name_waiting_on_a_shard_when_renamed_and_revoked_should_refuse_unauthorized()
{
- let (code, body) = shard_waiting_reads()
- .into_iter()
- .find(|(code, _)| *code == GET_PARTITION_ROUTE_CODE)
- .unwrap();
- assert_read_refused_after_shard_wait(code, &body, ReadRevocation::Topic).await;
+ let stream_id = WireIdentifier::named("stream").unwrap();
+ let topic_id = WireIdentifier::named("topic").unwrap();
+ for (code, body) in partition_namespace_reads(&stream_id, &topic_id) {
- for rename in [Operation::UpdateStream, Operation::UpdateTopic] {
++ for rename in [
++ Operation::UpdateStreamResolved,
++ Operation::UpdateTopicResolved,
++ ] {
+ assert_read_refused_after_shard_wait(
+ code,
+ &body,
+ ReadRevocation::Grants(Some(rename)),
+ )
+ .await;
+ }
+ }
+ }
+
+ /// A read waiting on a shard belongs to the connection that sent it, so a
+ /// connection that took the id during the wait must not get its reply.
+ #[compio::test]
+ async fn given_a_read_waiting_on_a_shard_when_the_connection_is_replaced_should_not_answer_the_new_one()
+ {
+ for (code, body) in shard_waiting_reads() {
+ assert_read_refused_after_shard_wait(code, &body, ReadRevocation::Connection).await;
+ }
+ }
+
+ #[compio::test]
+ async fn given_a_partition_read_waiting_on_a_shard_when_the_topic_is_recreated_should_refuse_not_accepted()
+ {
+ for (code, body) in shard_waiting_reads().into_iter().filter(|(code, _)| {
+ matches!(
+ *code,
+ GET_PARTITION_ROUTE_CODE | POLL_MESSAGES_CODE | GET_CONSUMER_OFFSET_CODE
+ )
+ }) {
+ assert_read_refused_after_shard_wait(code, &body, ReadRevocation::Topic).await;
+ }
+ }
+
+ #[compio::test]
+ async fn given_a_route_read_waiting_on_a_shard_when_topic_deletion_starts_should_refuse() {
+ let (_, body) =
+ partition_namespace_reads(&WireIdentifier::numeric(0), &WireIdentifier::numeric(0))[2]
+ .clone();
+ assert_read_refused_after_shard_wait(
+ GET_PARTITION_ROUTE_CODE,
+ &body,
+ ReadRevocation::TopicDeleting,
+ )
+ .await;
}
/// Every read that asks another shard before it answers: the client reads
@@@ -1751,9 -2004,37 +2000,39 @@@
READER_SESSION,
READER_SESSION + 1
));
- IggyError::StaleClient
+ Some(IggyError::StaleClient)
}
- ReadRevocation::Grants => {
+ ReadRevocation::Grants(rename) => {
+ if let Some(operation) = rename {
+ let stream_id =
+ WireIdentifier::numeric(namespace.stream_id().try_into().unwrap());
+ let name = WireName::new("renamed").unwrap();
+ let body = match operation {
- Operation::UpdateStream => UpdateStreamRequest {
++ Operation::UpdateStreamResolved => UpdateStreamResolvedRequest {
+ stream_id,
- name,
- options: WireOptions::empty(),
++ name: Some(name),
++ explicit_options: WireOptions::empty(),
++ derived_options: WireOptions::empty(),
+ }
+ .to_bytes(),
- Operation::UpdateTopic => UpdateTopicRequest {
++ Operation::UpdateTopicResolved => UpdateTopicResolvedRequest {
+ stream_id,
+ topic_id: WireIdentifier::numeric(
+ namespace.topic_id().try_into().unwrap(),
+ ),
- name,
- options: WireOptions::empty(),
++ name: Some(name),
++ explicit_options: WireOptions::empty(),
++ derived_options: WireOptions::empty(),
+ }
+ .to_bytes(),
+ _ => panic!("{operation:?} renames neither a stream nor a topic"),
+ };
+ let renamed = metadata
+ .mux_stm
+ .update(prepare_message(operation, 0, 4, &body))
+ .unwrap();
+ assert_eq!(renamed.code, 0);
+ }
let revoke = UpdatePermissionsRequest {
user_id: WireIdentifier::numeric(READER),
permissions: None,
diff --cc core/server/src/dispatch/partition.rs
index 1dac91293,d33c30314..0c067be82
--- a/core/server/src/dispatch/partition.rs
+++ b/core/server/src/dispatch/partition.rs
@@@ -1533,12 -1514,22 +1517,23 @@@ wher
mod tests {
use super::*;
use crate::dispatch::test_support::{
- SpyBus, TestMux, TestShard, create_one_partition, prepare_message, register_reply,
- request_message, stored_batch_record, test_shard,
+ SpyBus, TestMux, TestShard, create_consumer_group, create_one_partition,
- create_reader_request, prepare_message, register_reply, request_message, test_shard,
- test_shard_with_frontier,
++ create_reader_request, prepare_message, register_reply, request_message,
++ stored_batch_record, test_shard, test_shard_with_frontier,
+ };
+ use crate::dispatch::tests::non_replicated_request;
+ use crate::dispatch::{
+ ActiveClientRequests, Admission, ClientDrain, ClientRequestQueues, enqueue_client_request,
};
+ use crate::reply_frame::build_deny_reply;
+ use crate::session_manager::SessionManager;
+ use crate::shell::ShellShardHandle;
+ use bytes::BytesMut;
+ use configs::server::ServerConfig;
+ use consensus::PartitionsHandle;
#[cfg(target_os = "linux")]
use consensus::Sequencer;
+ use iggy_binary_protocol::codes::PING_CODE;
use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment;
use iggy_binary_protocol::requests::consumer_groups::CreateConsumerGroupRequest;
use iggy_binary_protocol::requests::consumer_offsets::DeleteConsumerOffsetRequest;
@@@ -1549,12 -1542,15 +1546,15 @@@
CreateTopicRequest, CreateTopicWithAssignmentsRequest, DeleteTopicRequest,
};
use iggy_binary_protocol::{
- Command, PrepareOkHeader, ReplyHeader, WireEncode, WireIdentifier, WireName, WireOptions,
- WirePartitioning,
+ Command, ConsensusHeader, GenericHeader, PrepareOkHeader, ReplyHeader, SegmentDeleteHeader,
+ SegmentDeleteKind, WireEncode, WireIdentifier, WireName, WireOptions, WirePartitioning,
};
- use iggy_common::Identifier;
use iggy_common::defaults::DEFAULT_ROOT_USER_ID;
+ use iggy_common::{Aes256GcmEncryptor, EncryptorKind, Identifier};
+ use journal::prepare_journal::PrepareJournal;
+ use message_bus::{BusReceiver, IggyMessageBus, Shutdown, ShutdownToken};
use metadata::IggyMetadata;
+ use metadata::impls::metadata::IggySnapshot;
use metadata::stm::StateMachine as _;
use metadata::stm::consumer_group::{
CompleteConsumerGroupRevocationRequest, JoinConsumerGroupRequest,
@@@ -3987,12 -4258,12 +4336,12 @@@
PollingConsumer::Consumer(consumer_id, usize::try_from(partition_id).unwrap()),
PollingArgs::new(PollingStrategy::offset(0), 1, true),
);
- match read_polled_messages(
+ read_polled_messages(
shard,
- transport_client_id,
+ ClientOrigin::capture(&shard.bus, transport_client_id),
&request,
resolved_poll,
- &attachment,
+ attachment,
transport_client_id,
)
.await
@@@ -4023,4 -4301,339 +4372,339 @@@
"the poll must reach the owner once routing can deliver it"
);
}
+
+ type BusShard =
+ IggyShard<Rc<IggyMessageBus>, PrepareJournal, IggySnapshot, TestMux, PapayaShardsTable>;
+
+ const RELAY_SESSION: u64 = 1;
+ const RELAY_REQUEST: u64 = 1;
+ const RELAY_POLL_INTERVAL: Duration = Duration::from_millis(1);
+ const RELAY_BUDGET: Duration = Duration::from_secs(5);
+
+ /// The owning shard commits a write and its reply returns to the shard
+ /// holding the socket. A full reply queue makes the reply wait for the
+ /// writer instead of closing the connection, and the write keeps its
+ /// place under the in-flight cap until its reply is queued.
+ #[compio::test]
+ async fn given_full_reply_queue_when_owner_shard_replies_should_wait_for_the_writer() {
+ const CLIENT: u128 = 0x41;
+ let (bus, shard, owner_inbox) = origin_shard_over_bus("relay-full-queue-test");
+ let (queue, conn_token) = install_stalled_client(&bus, CLIENT);
+ bus.send_to_client_instance(
+ CLIENT,
+ bus.client_instance(CLIENT),
+ committed_reply(CLIENT).into_frozen().into(),
+ )
+ .expect("the first reply fits the queue");
+
+ let drain = relay_write(&shard, ClientOrigin::capture(&shard.bus, CLIENT).queued()).await;
+ answer_submit(&owner_inbox);
+ // A reply that waits for room holds a sender of the queue.
+ let deadline = Instant::now() + RELAY_BUDGET;
+ while queue.sender_count() == 1 && !conn_token.is_triggered() {
+ assert!(
+ Instant::now() < deadline,
+ "the relay must take the committed reply"
+ );
+ compio::time::sleep(RELAY_POLL_INTERVAL).await;
+ }
+ assert!(
+ !conn_token.is_triggered(),
+ "a full queue must not close the connection a queued request came on"
+ );
+ assert_eq!(
+ drain.in_flight.get(),
+ 1,
+ "the write keeps its permit while its reply waits"
+ );
+
+ queue
+ .recv()
+ .await
+ .expect("the writer takes the first reply");
+ wait_for_relay(&drain).await;
+ assert_eq!(queue.len(), 1, "the waiting reply takes the freed slot");
+ assert!(!conn_token.is_triggered());
+ }
+
+ #[compio::test]
+ async fn given_waiting_reply_when_connection_disconnects_should_release_its_permit() {
+ const CLIENT: u128 = 0x46;
+ let (bus, shard, owner_inbox) = origin_shard_over_bus("waiting-reply-disconnect");
+ let (queue, conn_token) = install_stalled_client(&bus, CLIENT);
+ bus.send_to_client_instance(
+ CLIENT,
+ bus.client_instance(CLIENT),
+ committed_reply(CLIENT).into_frozen().into(),
+ )
+ .unwrap();
+ let drain = relay_write(&shard, ClientOrigin::capture(&shard.bus, CLIENT).queued()).await;
+ answer_submit(&owner_inbox);
+ let deadline = Instant::now() + RELAY_BUDGET;
+ while queue.sender_count() == 1 {
+ assert!(Instant::now() < deadline);
+ compio::time::sleep(RELAY_POLL_INTERVAL).await;
+ }
+ assert_eq!(drain.in_flight.get(), 1);
+ assert!(!conn_token.is_triggered());
+ bus.clients()
+ .close_peer_if_token_matches(
+ CLIENT,
+ bus.client_instance(CLIENT).unwrap(),
+ std::time::Duration::ZERO,
+ )
+ .await;
+ wait_for_relay(&drain).await;
+ assert_eq!(drain.in_flight.get(), 0);
+ assert!(queue.is_closed());
+ }
+
+ /// A reply that outlived its connection answers a request the connection
+ /// reusing the id never sent: it must neither reach that connection nor
+ /// close it when its queue is full.
+ #[compio::test]
+ async fn given_reused_connection_id_when_owner_shard_replies_should_not_reach_new_connection() {
+ const CLIENT: u128 = 0x43;
+ let (bus, shard, owner_inbox) = origin_shard_over_bus("relay-reused-id-test");
+ let _replaced = install_stalled_client(&bus, CLIENT);
+ let drain = relay_write(&shard, ClientOrigin::capture(&shard.bus, CLIENT).queued()).await;
+ assert!(bus.clients().remove(CLIENT));
+ let (reused_queue, reused_token) = install_stalled_client(&bus, CLIENT);
+ bus.send_to_client_instance(
+ CLIENT,
+ bus.client_instance(CLIENT),
+ committed_reply(CLIENT).into_frozen().into(),
+ )
+ .expect("the reused connection takes its own reply");
+
+ answer_submit(&owner_inbox);
+ wait_for_relay(&drain).await;
+
+ assert!(
+ !reused_token.is_triggered(),
+ "a stale reply must not close the connection that reuses the id"
+ );
+ assert_eq!(
+ reused_queue.len(),
+ 1,
+ "a stale reply must not reach the connection that reuses the id"
+ );
+ }
+
+ /// HTTP has no socket: it waits on a reply slot of its in-process
+ /// installation, so the relay must reach that slot through the
+ /// installation it captured, now that a reply by id reaches no socket.
+ #[compio::test]
+ async fn given_in_process_session_when_owner_shard_replies_should_fill_its_reply_slot() {
+ const CLIENT: u128 = 0x44;
+ let (bus, shard, owner_inbox) = origin_shard_over_bus("relay-in-process-test");
+ bus.clients()
+ .insert_in_process(CLIENT)
+ .expect("the client id is free");
+ let (_slot, mut reply) = bus
+ .clients()
+ .install_reply_slot(CLIENT, RELAY_REQUEST)
+ .expect("the reply slot installs");
+
+ let drain = relay_write(&shard, ClientOrigin::capture(&shard.bus, CLIENT)).await;
+ answer_submit(&owner_inbox);
+ wait_for_relay(&drain).await;
+
+ assert!(
+ matches!(reply.try_recv(), Ok(Some(_))),
+ "the committed reply must fill the session's reply slot"
+ );
+ }
+
+ /// A client can have more requests outstanding than its reply queue
+ /// holds, and their replies can land in one burst before its writer runs.
+ /// A client that takes each reply as it comes must keep its connection.
+ #[compio::test]
+ async fn given_more_requests_than_the_reply_queue_holds_when_the_client_reads_promptly_should_keep_the_connection()
+ {
+ const CLIENT: u128 = 0x45;
+ const REQUESTS: usize = 8;
+ let (bus, shard, _owner_inbox) = origin_shard_over_bus("prompt-reader-test");
+ // The test takes the replies off the queue in place of the writer.
+ let (queue, conn_token) = install_stalled_client(&bus, CLIENT);
+ let shard_handle: ShellShardHandle<Rc<IggyMessageBus>, PrepareJournal, IggySnapshot> =
+ Rc::new(RefCell::new(Some(Rc::downgrade(&shard))));
+ let sessions = Rc::new(RefCell::new(SessionManager::new()));
+ let server_config = Arc::new(ServerConfig::default());
+ let queues = ClientRequestQueues::default();
+ let active = ActiveClientRequests::default();
+ for _ in 0..REQUESTS {
+ enqueue_client_request(
+ &bus,
+ &shard_handle,
+ &sessions,
+ &server_config,
+ &queues,
+ &active,
+ CLIENT,
+ non_replicated_request(CLIENT, PING_CODE),
+ );
+ }
+
+ for _ in 0..REQUESTS {
+ compio::time::timeout(RELAY_BUDGET, queue.recv())
+ .await
+ .expect("every request must be answered")
+ .expect("the connection must stay open");
+ }
+ assert!(
+ !conn_token.is_triggered(),
+ "a client that reads its replies must keep its connection"
+ );
+ }
+
+ /// Shard 0 of a two-shard node over a real bus, so replies reach the
+ /// connection registry. Shard 1 owns the partition: the test holds its
+ /// inbox and answers its submits the way the owning plane does.
+ fn origin_shard_over_bus(
+ name: &str,
+ ) -> (Rc<IggyMessageBus>, Rc<BusShard>, Receiver<ShardFrame>) {
+ const POLL_COMPLETION_CAPACITY: usize = 1;
+ const INBOX_CAPACITY: usize = 4;
+ const OWNER_SHARD: u16 = 1;
+ let bus = Rc::new(IggyMessageBus::new(0));
+ let partitions = IggyPartitions::new(
+ ShardId::new(0),
+ PartitionsConfig {
+ messages_required_to_save: 1,
+ size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64),
+ validate_checksum: true,
+ segment_size: iggy_common::IggyByteSize::from(1_048_576_u64),
+ preallocate_segments: false,
- encryptor: None,
++ key_context: Arc::default(),
+ path_layout: PartitionPathLayout::default(),
+ },
+ );
+ let (origin_sender, origin_inbox, origin_replies) =
+ shard_channel(0, INBOX_CAPACITY, INBOX_CAPACITY);
+ let (owner_sender, owner_inbox, _owner_replies) =
+ shard_channel(OWNER_SHARD, INBOX_CAPACITY, INBOX_CAPACITY);
+ let shard = BusShard::new(
+ ShardIdentity::new(0, name.to_string()),
+ Rc::clone(&bus),
+ Rc::new(NoopHost),
+ IggyMetadata::new(None, None, None, None, TestMux::default(), None),
+ partitions,
+ vec![origin_sender, owner_sender],
+ origin_inbox,
+ origin_replies,
+ POLL_COMPLETION_CAPACITY,
+ None,
+ PapayaShardsTable::new(),
+ PartitionConsensusConfig::new(1, ReplicaTopology::new(0, 1), Rc::clone(&bus)),
+ None,
+ ShardMetrics::for_shard(),
+ )
+ .expect("the two-sender ring is canonically ordered");
+ shard.shards_table().insert(
+ relay_namespace(),
+ PartitionLocation::new(ShardId::new(OWNER_SHARD), 0),
+ );
+ (bus, Rc::new(shard), owner_inbox)
+ }
+
+ fn relay_namespace() -> IggyNamespace {
+ IggyNamespace::new(0, 0, 0)
+ }
+
+ /// A socket connection whose writer stopped draining, as when its client
+ /// stopped reading: its queue holds one reply. Returns the queue and the
+ /// connection's shutdown token.
+ fn install_stalled_client(
+ bus: &IggyMessageBus,
+ client_id: u128,
+ ) -> (BusReceiver, ShutdownToken) {
+ const STALLED_QUEUE_CAPACITY: usize = 1;
+ let (sender, queue) = async_channel::bounded(STALLED_QUEUE_CAPACITY);
+ let (conn_shutdown, conn_token) = Shutdown::new();
+ let writer_token = conn_token.clone();
+ let writer = compio::runtime::spawn(async move { writer_token.wait().await });
+ let reader_token = conn_token.clone();
+ let reader = compio::runtime::spawn(async move { reader_token.wait().await });
+ bus.clients()
+ .insert(client_id, sender, writer, reader, conn_shutdown)
+ .expect("the client id is free");
+ (queue, conn_token)
+ }
+
+ /// Relay a write from `origin` to the owning shard. The returned drain
+ /// counts the relay task until it has answered the client.
+ async fn relay_write(shard: &Rc<BusShard>, origin: ClientOrigin) -> Rc<ClientDrain> {
+ let namespace = relay_namespace();
+ let request = request_message(
+ Operation::SendMessages,
+ origin.id,
+ RELAY_SESSION,
+ RELAY_REQUEST,
+ &[],
+ )
+ .transmute_header(|header, routed: &mut RoutedRequestHeader| {
+ *routed = header;
+ routed.group = namespace.inner();
+ });
+ let header = *request.header();
+ let drain = Rc::new(ClientDrain::new(&ServerConfig::default()));
+ let in_flight = drain.admit(Admission::Cap).await;
+ let admission = relay_partition_reply(
+ shard,
+ namespace,
+ request,
+ origin,
+ &header,
+ None,
+ Some(in_flight),
+ )
+ .await;
+ assert_eq!(admission, PartitionAdmission::Admitted);
+ drain
+ }
+
+ /// Commit the write the owning shard received: its reply rides the
+ /// submit's own channel back to the shard holding the socket.
+ fn answer_submit(owner_inbox: &Receiver<ShardFrame>) {
+ let Ok(ShardFrame::Lifecycle(LifecycleFrame::PartitionSubmit { request, reply, .. })) =
+ owner_inbox.try_recv()
+ else {
+ panic!("the write must reach the owning shard as a submit");
+ };
+ let client = request.header().client;
+ assert!(
+ reply
+ .try_send(Some(committed_reply_to(request.header(), client)))
+ .is_ok(),
+ "the relay waits for the committed reply"
+ );
+ }
+
+ fn committed_reply(client: u128) -> Message<GenericHeader> {
+ let request = request_message(
+ Operation::SendMessages,
+ client,
+ RELAY_SESSION,
+ RELAY_REQUEST,
+ &[],
+ );
+ committed_reply_to(request.header(), client)
+ }
+
+ fn committed_reply_to(header: &RoutedRequestHeader, client: u128) -> Message<GenericHeader> {
+ const COMMITTED: u32 = 0;
+ build_deny_reply(header, client, 0, 0, COMMITTED).into_generic()
+ }
+
+ async fn wait_for_relay(drain: &ClientDrain) {
+ let deadline = Instant::now() + RELAY_BUDGET;
+ while drain.in_flight.get() > 0 {
+ assert!(
+ Instant::now() < deadline,
+ "the relay must answer once the owning shard replies"
+ );
+ compio::time::sleep(RELAY_POLL_INTERVAL).await;
+ }
+ }
}
diff --cc core/server/src/dispatch/test_support.rs
index 547496ae3,7155743e8..62299d653
--- a/core/server/src/dispatch/test_support.rs
+++ b/core/server/src/dispatch/test_support.rs
@@@ -60,9 -58,9 +62,10 @@@ use server_common::{MESSAGE_ALIGN, Mess
use shard::shards_table::PapayaShardsTable;
use shard::{IggyShard, PartitionConsensusConfig, ReplicaTopology, ShardIdentity};
use std::cell::{Cell, RefCell};
+ use std::future::Future;
use std::mem::size_of;
use std::rc::Rc;
+use std::sync::Arc;
pub type TestMux = MuxStateMachine<variadic!(Users, Streams)>;
pub type TestShard = IggyShard<SpyBus, PrepareJournal, IggySnapshot, TestMux, PapayaShardsTable>;
@@@ -238,16 -280,21 +285,51 @@@ pub const UNDECLARED_OPERATION: u8 = 13
/// forward-nonce counter from it, so passing a different value models a
/// restart.
pub fn test_shard(bus: &SpyBus, replica: u8, replica_count: u8, incarnation: u128) -> TestShard {
- test_shard_with_frontier(
+ test_shard_with_key_context(bus, replica, replica_count, incarnation, Arc::default())
+}
+
+/// [`test_shard`] holding `key_context` as its server keys.
+pub fn test_shard_with_key_context(
+ bus: &SpyBus,
+ replica: u8,
+ replica_count: u8,
+ incarnation: u128,
+ key_context: Arc<ServerKeyContext>,
++) -> TestShard {
++ build_test_shard(
+ bus,
+ replica,
+ replica_count,
+ incarnation,
- std::sync::Arc::default(),
++ key_context,
++ Arc::default(),
+ )
+ }
+
+ pub fn test_shard_with_frontier(
+ bus: &SpyBus,
+ replica: u8,
+ replica_count: u8,
+ incarnation: u128,
- frontier: std::sync::Arc<metadata::AppliedFrontier>,
++ frontier: Arc<metadata::AppliedFrontier>,
++) -> TestShard {
++ build_test_shard(
++ bus,
++ replica,
++ replica_count,
++ incarnation,
++ Arc::default(),
++ frontier,
++ )
++}
++
++fn build_test_shard(
++ bus: &SpyBus,
++ replica: u8,
++ replica_count: u8,
++ incarnation: u128,
++ key_context: Arc<ServerKeyContext>,
++ frontier: Arc<metadata::AppliedFrontier>,
) -> TestShard {
let consensus = VsrConsensus::new(
1,
diff --cc core/server/src/http/error.rs
index d86944095,1851322d2..dbf719ea4
--- a/core/server/src/http/error.rs
+++ b/core/server/src/http/error.rs
@@@ -76,16 -77,15 +77,17 @@@ impl IntoResponse for CustomError
| IggyError::InvalidPersonalAccessToken => StatusCode::UNAUTHORIZED,
IggyError::Unauthorized => StatusCode::FORBIDDEN,
IggyError::PartitionHistoryChanged => StatusCode::CONFLICT,
+ // Stored state this build cannot read, not a caller error.
+ IggyError::UnsupportedCriticalOption(_) => StatusCode::INTERNAL_SERVER_ERROR,
- // The pre-consensus retry frame: reaching this render
- // means the write path's replay budget is exhausted and
- // the op never committed - a transient server condition,
- // retryable like the other cannot-commit-right-now 503s
- // (see `service_unavailable`), never a caller error.
- IggyError::TransientNotCommitted
- | IggyError::TransientNotAccepted
- | IggyError::NotPrimary => StatusCode::SERVICE_UNAVAILABLE,
+ // The write path ran out of replays of a request id whose
+ // op may still commit (see `submit::submit_committed`).
+ // HTTP mints a new id per request, so a resend could apply
+ // the op twice: 504, the unknown outcome, never a
+ // retryable 503.
+ IggyError::TransientNotCommitted => StatusCode::GATEWAY_TIMEOUT,
+ IggyError::TransientNotAccepted | IggyError::NotPrimary => {
+ StatusCode::SERVICE_UNAVAILABLE
+ }
_ => StatusCode::BAD_REQUEST,
};
let response =
diff --cc core/server/src/http/extractor.rs
index b1e12df0f,576a3b602..130a63b15
--- a/core/server/src/http/extractor.rs
+++ b/core/server/src/http/extractor.rs
@@@ -40,9 -29,10 +40,10 @@@ use serde::de::{DeserializeOwned, Erro
use tracing::debug;
use super::HttpState;
-use super::error::AuthError;
+use super::error::{AuthError, WriteError};
use super::jwt::TokenSubject;
use super::session::HttpSession;
+ use super::state::HttpInner;
use crate::dispatch::session_ops::verify_pat_credentials_with_expiry;
use crate::http::ClientAddr;
@@@ -132,90 -91,59 +122,143 @@@ impl FromRequestParts<HttpState> for Au
}
}
+impl<T, S> FromRequest<S> for JsonMembers<T>
+where
+ T: DeserializeOwned,
+ S: Send + Sync,
+{
+ type Rejection = Response;
+
+ /// Runs axum's own `Json` extraction, so content type checks, body limits
+ /// and rejections stay what every other route answers.
+ async fn from_request(request: Request, state: &S) -> Result<Self, Self::Rejection> {
+ let (parts, body) = request.into_parts();
+ let bytes = Bytes::from_request(Request::from_parts(parts.clone(), body), state)
+ .await
+ .map_err(IntoResponse::into_response)?;
+ let value =
+ Json::<T>::from_request(Request::from_parts(parts, Body::from(bytes.clone())), state)
+ .await;
+ let members = Json::<BTreeMap<String, IgnoredAny>>::from_bytes(&bytes);
+ erase_buffer(bytes);
+ let Json(value) = value.map_err(IntoResponse::into_response)?;
+ let Json(members) = members.map_err(IntoResponse::into_response)?;
+ Ok(Self {
+ value,
+ members: members.into_keys().collect(),
+ })
+ }
+}
+
+impl<S> FromRequest<S> for JsonUpdate
+where
+ S: Send + Sync,
+{
+ type Rejection = WriteError;
+
+ async fn from_request(request: Request, state: &S) -> Result<Self, Self::Rejection> {
+ let (parts, body) = request.into_parts();
+ let bytes = Bytes::from_request(Request::from_parts(parts.clone(), body), state)
+ .await
+ .map_err(|_| WriteError::Rejected(IggyError::InvalidCommand))?;
+ let update = match Json::<OptionsUpdate>::from_request(
+ Request::from_parts(parts, Body::from(bytes.clone())),
+ state,
+ )
+ .await
+ {
+ Ok(Json(update)) => Ok(Self(update)),
+ Err(_) => Err(refused_member(&bytes).map_or(
+ WriteError::Rejected(IggyError::InvalidCommand),
+ WriteError::MalformedMember,
+ )),
+ };
+ erase_buffer(bytes);
+ update
+ }
+}
+
+impl<'de> Visitor<'de> for RefusedMemberVisitor<'_> {
+ type Value = ();
+
+ fn expecting(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str("an update body object")
+ }
+
+ fn visit_map<A: MapAccess<'de>>(self, mut access: A) -> Result<Self::Value, A::Error> {
+ let mut seen = BTreeSet::new();
+ while let Some(key) = access.next_key::<String>()? {
+ let fits = seen.insert(key.clone())
+ && if key == RESET_FIELD {
+ access.next_value::<Vec<String>>().is_ok_and(|names| {
+ let mut distinct = BTreeSet::new();
+ names.into_iter().all(|name| distinct.insert(name))
+ })
+ } else {
+ access.next_value::<Option<OptionInputValue>>().is_ok()
+ };
+ if !fits {
+ *self.refused = Some(key);
+ return Err(A::Error::custom("refused update member"));
+ }
+ }
+ Ok(())
+ }
+}
+
+ /// A verified bearer with what its VSR session is keyed and leased by, but no
+ /// session yet. A partition write resolves the session only where it is
+ /// served ([`Self::session`]): a relayed write runs under the serving node's
+ /// own session, and a `Register` on the relaying node would be a metadata
+ /// commit spent on nothing.
+ pub struct Credential {
+ key: String,
+ pub user_id: u32,
+ user_created_at: IggyTimestamp,
+ expiry: u64,
+ }
+
+ impl Credential {
+ /// Whether this node holds a live session for the credential, so serving
+ /// here Registers none.
+ pub(in crate::http) fn has_session(&self, state: &HttpInner) -> bool {
+ state.has_live_session(&self.key)
+ }
+
+ /// The credential's VSR session, minted and Registered on first use.
+ pub(in crate::http) async fn session(
+ &self,
+ state: &HttpInner,
+ ) -> Result<Rc<HttpSession>, AuthError> {
+ state
+ .resolve_session(&self.key, self.user_id, self.user_created_at, self.expiry)
+ .await
+ }
+ }
+
+ impl FromRequestParts<HttpState> for Credential {
+ type Rejection = AuthError;
+
+ async fn from_request_parts(
+ parts: &mut Parts,
+ state: &HttpState,
+ ) -> Result<Self, Self::Rejection> {
+ let bearer = bearer_token(&parts.headers)?;
+ let verified = verified_bearer(parts, state, bearer).await?;
+ Ok(Self {
+ key: credential_key(
+ bearer,
+ verified.jwt_id.as_deref(),
+ verified.user_id,
+ verified.user_created_at,
+ ),
+ user_id: verified.user_id,
+ user_created_at: verified.user_created_at,
+ expiry: verified.expiry,
+ })
+ }
+ }
+
/// Read-only caller identity: the account's id and creation timestamp, plus the
/// request path/query used for follower redirects. The timestamp prevents user
/// ID reuse from changing the caller across a wait.
@@@ -338,56 -283,97 +398,143 @@@ pub(in crate::http) async fn resolve_cr
let (user_id, user_created_at, expiry) =
verify_pat_credentials_with_expiry(&state.shard, bearer)
.map_err(|_| IggyError::Unauthenticated)?;
- Ok((None, user_id, user_created_at, expiry))
+ Ok(VerifiedBearer {
+ jwt_id: None,
+ user_id,
+ user_created_at,
+ expiry,
+ })
+ }
+
+ /// Table key of a credential [`resolve_credential`] resolved: the account and
+ /// `jti` of a JWT, or the PAT's hash, so the raw secret never becomes a key.
+ pub(in crate::http) fn credential_key(
+ bearer: &str,
+ jwt_id: Option<&str>,
+ user_id: u32,
+ user_created_at: IggyTimestamp,
+ ) -> String {
+ // External issuers choose their own jti, and their configured username
+ // can resolve to another account later. Scope the key to this account.
+ jwt_id.map_or_else(
+ || {
+ format!(
+ "{PAT_KEY_PREFIX}{}",
+ PersonalAccessToken::hash_token(bearer)
+ )
+ },
+ |jwt_id| {
+ format!(
+ "{JWT_KEY_PREFIX}{user_id}:{}:{jwt_id}",
+ user_created_at.as_micros()
+ )
+ },
+ )
}
+/// The member [`RefusedMemberVisitor`] finds in `bytes`, `None` for a body that
+/// is not a JSON object or has no member at fault.
+fn refused_member(bytes: &[u8]) -> Option<String> {
+ let mut refused = None;
+ let walked =
+ serde_json::Deserializer::from_slice(bytes).deserialize_map(RefusedMemberVisitor {
+ refused: &mut refused,
+ });
+ match walked {
+ Ok(()) => None,
+ Err(_) => refused,
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
+ use crate::http::reads::tests::read_state;
+ use axum::http::Request;
+
+ const LAYER_USER_ID: u32 = 7;
+
+ /// The partition route layer verifies the bearer before its handler runs,
+ /// so the extractor behind it reuses that proof. A bearer that would not
+ /// verify shows the extractor did not verify it again.
+ #[compio::test]
+ async fn given_a_bearer_the_route_layer_verified_when_extracting_should_not_verify_it_again() {
+ let state = read_state();
+ let verified = VerifiedBearer {
+ jwt_id: Some("layer-jti".to_owned()),
+ user_id: LAYER_USER_ID,
+ user_created_at: IggyTimestamp::from(1),
+ expiry: u64::MAX,
+ };
+
+ let mut parts = unverifiable_bearer_parts();
+ parts.extensions.insert(verified.clone());
+ let credential = Credential::from_request_parts(&mut parts, &state)
+ .await
+ .unwrap_or_else(|_| panic!("a verified bearer must not be verified again"));
+ assert_eq!(credential.user_id, LAYER_USER_ID);
+ assert_eq!(
+ credential.key,
+ credential_key(
+ "not-a-token",
+ verified.jwt_id.as_deref(),
+ LAYER_USER_ID,
+ verified.user_created_at
+ )
+ );
+ let identity = Identity::from_request_parts(&mut parts, &state)
+ .await
+ .unwrap_or_else(|_| panic!("a verified bearer must not be verified again"));
+ assert_eq!(identity.user_id, LAYER_USER_ID);
+
+ let mut unverified = unverifiable_bearer_parts();
+ assert!(
+ Credential::from_request_parts(&mut unverified, &state)
+ .await
+ .is_err(),
+ "without a layer the extractor still verifies the bearer"
+ );
+ }
+ #[test]
+ fn given_update_bodies_when_finding_refused_member_should_name_the_first_misfit() {
+ let cases = [
+ (r#"{"name":"orders","reset":["message_expiry"]}"#, None),
+ (r#"{"name":"orders","message_expiry":null}"#, None),
+ (r#"{"name":["orders"]}"#, Some("name")),
+ (
+ r#"{"name":"a","message_expiry":{"value":"1 day"}}"#,
+ Some("message_expiry"),
+ ),
+ (r#"{"reset":null}"#, Some("reset")),
+ (
+ r#"{"message_expiry":1.5,"name":"a","name":"b"}"#,
+ Some("message_expiry"),
+ ),
+ (r#"{"name":"a","name":"b"}"#, Some("name")),
+ (
+ r#"{"reset":["message_expiry","message_expiry"]}"#,
+ Some("reset"),
+ ),
+ (r#"["name"]"#, None),
+ (r#"{"name":"a""#, None),
+ ];
+ for (body, expected) in cases {
+ assert_eq!(
+ refused_member(body.as_bytes()).as_deref(),
+ expected,
+ "{body}"
+ );
+ }
+ }
++
+ fn unverifiable_bearer_parts() -> Parts {
+ Request::builder()
+ .uri("/streams/1/topics/1/messages")
+ .header(AUTHORIZATION, "Bearer not-a-token")
+ .body(())
+ .expect("request builds")
+ .into_parts()
+ .0
+ }
}
diff --cc core/server/src/http/forward.rs
index 9bf38c7b7,bbac57a38..1bc48c629
--- a/core/server/src/http/forward.rs
+++ b/core/server/src/http/forward.rs
@@@ -73,10 -97,12 +97,12 @@@ use axum::middleware::Next
use axum::response::{IntoResponse, Response};
use bytes::Bytes;
use configs::http::HttpTlsConfig;
- use consensus::MetadataHandle;
+ use consensus::{MetadataHandle, is_settled_primary};
+ use cyper::IntoUrl;
use futures::{Stream, StreamExt};
use iggy_binary_protocol::primitives::partition_history::PartitionContext;
- use iggy_common::{IggyError, erase_buffer};
-use iggy_common::{IggyError, IggyTimestamp};
++use iggy_common::{IggyError, IggyTimestamp, erase_buffer};
+ use message_bus::MessageBus;
use message_bus::transports::tls::load_pem;
use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
use rustls::crypto::{CryptoProvider, WebPkiSupportedAlgorithms};
@@@ -491,13 -894,35 +894,38 @@@ async fn forward_or_pass(state: HttpSta
// primary's applied op, and this node has to record it as this caller's
// read-your-writes floor (see `record_relayed_floor`).
let user_id = match resolve_credential(&state, bearer).await {
- Ok((_key, user_id, _user_created_at, _expiry)) => user_id,
+ Ok(verified) => verified.user_id,
Err(rejection) => return rejection.into_response(),
};
+ let deadline = Instant::now() + FORWARD_RETRY_DEADLINE;
+ let (parts, body) = request.into_parts();
+ // Buffered before a slot is taken: the slots also carry this node's
+ // partition relays, so a stalled upload must end with its request instead
+ // of keeping a slot for as long as it stalls. The router-wide
+ // `DefaultBodyLimit` only annotates the request, so the bound is passed
+ // here or the buffer would be unbounded.
+ let read = compio::time::timeout(
+ deadline.saturating_duration_since(Instant::now()),
+ to_bytes(body, state.forward.body_limit),
+ )
+ .await;
+ let Ok(read) = read else {
+ return close_connection(parts.version, not_admitted());
+ };
+ let Ok(body) = read else {
+ return error_response(
+ StatusCode::PAYLOAD_TOO_LARGE,
+ "payload_too_large",
+ "request body exceeds http.max_request_size",
+ );
+ };
let Some(_guard) = ForwardGuard::admit(&state.forward.in_flight) else {
++ erase_buffer(body);
return forward_busy();
};
- let response = forward(&state, request).await;
+ let response = forward(&state, &parts, &body, deadline).await;
++ // Kept for retries, so it may hold a create body's plaintext stream key.
++ erase_buffer(body);
record_relayed_floor(&state, user_id, &response);
response
}
@@@ -1172,10 -1925,470 +1928,471 @@@ mod tests
assert_eq!(parsed.context.metadata_op, 41);
headers.remove(PARTITION_ROUTE_HEADER);
- assert_eq!(partition_hop(&headers), Ok(PartitionHop::Unrouted));
+ assert_eq!(hop(&headers), Ok(PartitionHop::Unrouted));
+
+ let malformed = HeaderValue::from_static("3,7");
+ let headers = marked(&malformed, Some(&send_proof(&key, &malformed, now)));
+ assert_eq!(hop(&headers), Err(HopRefusal::MalformedRoute));
+ }
+
+ #[test]
+ fn given_relay_fields_without_a_valid_proof_when_reading_the_hop_should_refuse_them() {
+ let key = RelayKey::derive(KEY_MATERIAL);
+ let now = IggyTimestamp::now();
+ let later = now + IggyDuration::from(RELAY_PROOF_LIFETIME + Duration::from_secs(1));
+ let genuine = HeaderValue::from_static("3,7,0,41,9900");
+ let unreachable_frontier = HeaderValue::from_static("3,7,0,18446744073709551615,9900");
+ let proof = send_proof(&key, &genuine, now);
+ let early = send_proof(&key, &genuine, later);
+ for (case, method, headers, relay_key, at, refusal) in [
+ (
+ "no proof",
+ Method::POST,
+ marked(&genuine, None),
+ Some(&key),
+ now,
+ UnprovenRelay::Invalid,
+ ),
+ (
+ "forged frontier",
+ Method::POST,
+ marked(&unreachable_frontier, Some(&proof)),
+ Some(&key),
+ now,
+ UnprovenRelay::Invalid,
+ ),
+ (
+ "another method",
+ Method::GET,
+ marked(&genuine, Some(&proof)),
+ Some(&key),
+ now,
+ UnprovenRelay::Invalid,
+ ),
+ (
+ "expired proof",
+ Method::POST,
+ marked(&genuine, Some(&proof)),
+ Some(&key),
+ later,
+ UnprovenRelay::Expired(Duration::from_secs(1)),
+ ),
+ (
+ "proof made ahead of this node's clock",
+ Method::POST,
+ marked(&genuine, Some(&early)),
+ Some(&key),
+ now,
+ UnprovenRelay::Early(RELAY_PROOF_LIFETIME + Duration::from_secs(1)),
+ ),
+ (
+ "no cluster key material",
+ Method::POST,
+ marked(&genuine, Some(&proof)),
+ None,
+ now,
+ UnprovenRelay::Invalid,
+ ),
+ ] {
+ assert_eq!(
+ partition_hop(&received(method, SEND_PATH, headers), relay_key, at),
+ Err(HopRefusal::Unproven(refusal)),
+ "{case}"
+ );
+ }
+ }
+
+ #[test]
+ fn given_relay_headers_of_an_attempt_when_the_hop_target_reads_them_should_prove_the_route() {
+ let key = RelayKey::derive(KEY_MATERIAL);
+ let request = PartitionRequest {
+ method: Method::GET,
+ headers: forwarded_headers(&HeaderMap::new()),
+ path_and_query: format!("{SEND_PATH}?consumer_id=o'brien"),
+ body: Bytes::new(),
+ hop: PartitionHop::Contacted,
+ deadline: Instant::now(),
+ };
+ let route = PartitionRoute {
+ partition_id: 3,
+ context: PartitionContext {
+ incarnation: 7,
+ owner_generation: 0,
+ metadata_op: 41,
+ },
+ budget: Duration::from_millis(9_900),
+ };
+ let url = format!("http://127.0.0.1:3000{}", request.path_and_query);
+ let headers = relay_headers(&key, &request, route, &url).expect("relay headers");
+ // The URL percent-encodes the quote the caller sent raw, and the hop
+ // target receives the encoded form.
+ let target = received(
+ Method::GET,
+ &format!("{SEND_PATH}?consumer_id=o%27brien"),
+ headers.clone(),
+ );
+ assert_eq!(
+ partition_hop(&target, Some(&key), IggyTimestamp::now()),
+ Ok(PartitionHop::Target(route))
+ );
+ let moved = received(
+ Method::GET,
+ &format!("{SEND_PATH}?consumer_id=other"),
+ headers,
+ );
+ assert_eq!(
+ partition_hop(&moved, Some(&key), IggyTimestamp::now()),
+ Err(HopRefusal::Unproven(UnprovenRelay::Invalid))
+ );
+ }
+
+ #[compio::test]
+ async fn given_relay_fields_without_a_valid_proof_when_answering_should_name_the_refusal() {
+ let state = read_state_on(None, Some(KEY_MATERIAL));
+ let service = TowerToHyperService::new(router(
+ state.clone(),
+ state.forward.body_limit,
+ None,
+ None,
+ false,
+ ));
+ let route = HeaderValue::from_static("3,7,0,41,9900");
+ for request in [
+ received(Method::POST, SEND_PATH, marked(&route, None)),
+ received(Method::GET, METADATA_FLOOR_PATH, HeaderMap::new()),
+ ] {
+ let path = request.uri().path().to_owned();
+ let response = service.call(request).await.expect("the node answers");
+ assert_eq!(response.status(), StatusCode::FORBIDDEN, "{path}");
+ assert_eq!(
+ response.headers().get(RELAY_REFUSAL_HEADER),
+ Some(&HeaderValue::from_static(UnprovenRelay::Invalid.as_str())),
+ "{path}"
+ );
+ }
+ }
+
+ #[compio::test]
+ async fn given_expired_partition_admission_when_answering_the_metadata_floor_should_refuse() {
+ let epoch = Instant::now().checked_sub(Duration::from_secs(9)).unwrap();
+ for (credited_at, expected) in [
+ (Instant::now(), StatusCode::NO_CONTENT),
+ (epoch, StatusCode::SERVICE_UNAVAILABLE),
+ ] {
+ let frontier = Arc::new(
+ metadata::AppliedFrontier::default()
+ .with_partition_freshness(epoch, Duration::from_secs(5)),
+ );
+ frontier.credit_partition_freshness(credited_at, Duration::ZERO);
+ let state = read_state_with_frontier(None, Some(KEY_MATERIAL), frontier);
+ let proof = RelayKey::derive(KEY_MATERIAL)
+ .prove(
+ &Method::GET,
+ METADATA_FLOOR_PATH,
+ &NO_PARTITION_ROUTE,
+ IggyTimestamp::now(),
+ )
+ .expect("a proof is a valid header value");
+ let mut headers = HeaderMap::new();
+ headers.insert(RELAY_PROOF_HEADER, proof);
+
+ let response = answer_metadata_floor(
+ State(state),
+ received(Method::GET, METADATA_FLOOR_PATH, headers),
+ )
+ .await;
+
+ assert_eq!(response.status(), expected);
+ }
+ }
+
+ #[compio::test]
+ async fn given_a_refusal_of_this_nodes_relay_proof_when_classifying_should_answer_it_as_no_403()
+ {
+ let _ = rustls::crypto::ring::default_provider().install_default();
+ let forward =
+ build_forward_state(&HttpTlsConfig::default(), 1024, None).expect("forward client");
+ let (address, server) = answer_with(vec![
+ refused_proof_reply(UnprovenRelay::Expired(Duration::from_secs(1))),
+ plain_reply(StatusCode::FORBIDDEN),
+ ]);
+ let mut answers = Vec::new();
+ for _ in 0..2 {
+ let response = forward
+ .client
+ .get(format!("http://{address}/consumer-offsets"))
+ .expect("offset request")
+ .send()
+ .await
+ .expect("the target answers");
+ let AttemptOutcome::Relay(answer) = classify_forwarded_reply(
+ response,
+ &Method::GET,
+ RelayKind::Partition(PartitionEffect::Read),
+ )
+ .await
+ else {
+ panic!("a 403 is never retried")
+ };
+ let status = answer.status();
+ let body = to_bytes(answer.into_body(), usize::MAX)
+ .await
+ .expect("answer body");
+ let code = serde_json::from_slice::<serde_json::Value>(&body)
+ .ok()
+ .and_then(|error| error["code"].as_str().map(str::to_owned));
+ answers.push((status, code));
+ }
+ assert_eq!(
+ answers,
+ [
+ (
+ StatusCode::SERVICE_UNAVAILABLE,
+ Some("relay_refused".to_owned())
+ ),
+ (StatusCode::FORBIDDEN, None),
+ ]
+ );
+ server.join().expect("HTTP server finished");
+ }
+
+ #[compio::test]
+ async fn given_a_refusal_of_this_nodes_proof_when_reading_the_metadata_floor_should_stop() {
+ const FLOOR: u64 = 41;
+ let state = read_state_on(None, Some(KEY_MATERIAL));
+ let relay_key = state.forward.relay_key.as_ref().expect("a relay key");
+ let (address, server) = answer_with(vec![
+ refused_proof_reply(UnprovenRelay::Invalid),
+ plain_reply(StatusCode::SERVICE_UNAVAILABLE),
+ format!(
+ "HTTP/1.1 204 No Content\r\n{APPLIED_OP_HEADER}: {FLOOR}\r\nconnection: close\r\n\r\n"
+ ),
+ ]);
+ for read in [
+ ControlFlow::Break(None),
+ ControlFlow::Continue(()),
+ ControlFlow::Break(Some(FLOOR)),
+ ] {
+ assert_eq!(read_metadata_floor(&state, relay_key, address).await, read);
+ }
+ server.join().expect("HTTP server finished");
+ }
+
+ #[compio::test]
+ async fn given_a_rejected_proof_when_awaiting_the_metadata_floor_should_report_relay_refused() {
+ let (address, server) = answer_with(vec![refused_proof_reply(UnprovenRelay::Invalid)]);
+ let state = state_with_metadata_primary_at(address);
+ let refusal = crate::http::reads::await_metadata_floor(
+ &state,
+ Instant::now() + PARTITION_REQUEST_DEADLINE,
+ )
+ .await
+ .expect_err("a refused proof must stop the read");
+ assert!(matches!(refusal, ReadError::RelayRefused));
+ let response = refusal.into_response();
+ assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+ let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
+ let body: serde_json::Value = serde_json::from_slice(&body).unwrap();
+ assert_eq!(body["code"], "relay_refused");
+ assert_eq!(state.forward.in_flight.get(), 0);
+ server.join().unwrap();
+ }
+
+ #[compio::test]
+ async fn given_noncanonical_applied_op_when_reading_metadata_floor_should_retry() {
+ let state = read_state_on(None, Some(KEY_MATERIAL));
+ let relay_key = state.forward.relay_key.as_ref().unwrap();
+ let floors = ["+41", "041", "41"];
+ let (address, server) = answer_with(floors.iter().map(|floor| format!(
+ "HTTP/1.1 204 No Content\r\n{APPLIED_OP_HEADER}: {floor}\r\nconnection: close\r\n\r\n"
+ )).collect());
+ let mut outcomes = Vec::new();
+ for _ in floors {
+ outcomes.push(read_metadata_floor(&state, relay_key, address).await);
+ }
+ server.join().unwrap();
+ assert_eq!(
+ outcomes,
+ [
+ ControlFlow::Continue(()),
+ ControlFlow::Continue(()),
+ ControlFlow::Break(Some(41))
+ ]
+ );
+ }
+
+ #[test]
+ fn given_noncanonical_applied_op_when_recording_relay_should_preserve_floor() {
+ let state = read_state_on(None, Some(KEY_MATERIAL));
+ for value in ["+41", "041", "41"] {
+ let response = (StatusCode::NO_CONTENT, [(APPLIED_OP_HEADER, value)]).into_response();
+ record_relayed_floor(&state, READER_ID, &response);
+ assert_eq!(
+ state.metadata_watermarks.get(READER_ID),
+ if value == "41" { 41 } else { 0 }
+ );
+ }
+ }
+
+ /// Answer each of `replies` on a connection of its own once that
+ /// connection's request head arrived, as a node that answers exactly so.
+ fn answer_with(replies: Vec<String>) -> (SocketAddr, std::thread::JoinHandle<()>) {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("HTTP listener");
+ let address = listener.local_addr().expect("listener address");
+ let server = std::thread::spawn(move || {
+ for reply in replies {
+ let (mut socket, _) = listener.accept().expect("client connection");
+ read_request_head(&mut socket);
+ socket.write_all(reply.as_bytes()).expect("reply");
+ }
+ });
+ (address, server)
+ }
+
+ fn read_request_head(socket: &mut std::net::TcpStream) {
+ const READ_TIMEOUT: Duration = Duration::from_secs(10);
+ socket
+ .set_read_timeout(Some(READ_TIMEOUT))
+ .expect("read timeout");
+ let mut request = Vec::new();
+ let mut bytes = [0; 1024];
+ while !request.windows(4).any(|window| window == b"\r\n\r\n") {
+ let read = socket.read(&mut bytes).expect("request bytes");
+ assert_ne!(read, 0);
+ request.extend_from_slice(&bytes[..read]);
+ }
+ }
+
+ /// A state whose metadata primary, replica 0, serves HTTP at `address`.
+ fn state_with_metadata_primary_at(address: SocketAddr) -> HttpState {
+ let mut state = read_state_on(None, Some(KEY_MATERIAL));
+ let roster = Rc::get_mut(&mut Rc::get_mut(&mut state).expect("one state owner").roster)
+ .expect("one roster owner");
+ roster.enabled = true;
+ roster.nodes.push(
+ ResolvedClusterNode::try_from(ClusterNodeConfig {
+ name: "primary".to_owned(),
+ ip: address.ip().to_string(),
+ advertised_address: None,
+ advertised_addresses: Vec::new(),
+ replica_id: 0,
+ ports: TransportPorts {
+ http: Some(address.port()),
+ ..TransportPorts::default()
+ },
+ })
+ .expect("a valid roster node"),
+ );
+ state
+ }
+
+ fn floor_reply(floor: u64) -> String {
+ format!(
+ "HTTP/1.1 204 No Content\r\n{APPLIED_OP_HEADER}: {floor}\r\nconnection: close\r\n\r\n"
+ )
+ }
- headers.insert(PARTITION_ROUTE_HEADER, HeaderValue::from_static("3,7"));
- assert_eq!(partition_hop(&headers), Err(IggyError::InvalidCommand));
+ fn refused_proof_reply(unproven: UnprovenRelay) -> String {
+ format!(
+ "HTTP/1.1 403 Forbidden\r\n{RELAY_REFUSAL_HEADER}: {}\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
+ unproven.as_str()
+ )
+ }
+
+ fn plain_reply(status: StatusCode) -> String {
+ format!("HTTP/1.1 {status}\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
+ }
+
+ /// A contacted node relays with the metadata op it chose the partition
+ /// at, and the account of the caller can be as new as that op. The hop
+ /// target holds the relay until it applied the op and only then checks the
+ /// bearer and the grants, so it serves an account it did not know yet when
+ /// the relay arrived.
+ #[compio::test]
+ async fn given_a_relay_from_an_account_its_op_created_when_the_hop_target_lags_should_serve_it_once_applied()
+ {
+ const POLL_PATH: &str = "/streams/stream/topics/topic/messages?consumer_id=1&partition_id=0&kind=offset&value=0&count=1";
+ let (sender, shard_inbox, _shard_replies) = shard_channel(0, 1, 1);
+ let state = read_state_on(Some(sender), Some(KEY_MATERIAL));
+ let metadata = state.shard.plane.metadata();
+ let namespace = create_one_partition_in(&metadata.mux_stm);
+ state
+ .shard
+ .shards_table()
+ .insert(namespace, PartitionLocation::new(ShardId::new(0), 0));
+ let frontier = metadata.applied_frontier();
+ assert!(
+ frontier.get() < READER_CREATED_AT,
+ "the hop target has not applied the op that creates the reader"
+ );
+ let route = PartitionRoute {
+ partition_id: 0,
+ context: metadata
+ .mux_stm
+ .streams()
+ .poll_metadata(namespace, None, HTTP_UNREGISTERED_CLIENT_ID)
+ .expect("the partition resolves")
+ .context(READER_CREATED_AT),
+ budget: PARTITION_REQUEST_DEADLINE,
+ };
+ let route = HeaderValue::try_from(route.to_string()).expect("a route is a header value");
+ let proof = RelayKey::derive(KEY_MATERIAL)
+ .prove(&Method::GET, POLL_PATH, &route, IggyTimestamp::now())
+ .expect("a proof is a header value");
+ let token = state
+ .jwt
+ .generate(READER_ID, IggyTimestamp::from(READER_CREATED_AT))
+ .expect("a token for the reader");
+ let mut headers = marked(&route, Some(&proof));
+ headers.insert(
+ AUTHORIZATION,
+ HeaderValue::try_from(format!("Bearer {}", token.access_token))
+ .expect("a bearer is a header value"),
+ );
+ let service = TowerToHyperService::new(router(
+ state.clone(),
+ state.forward.body_limit,
+ None,
+ None,
+ false,
+ ));
+ let mut response = pin!(service.call(received(Method::GET, POLL_PATH, headers)));
+ assert!(
+ poll!(response.as_mut()).is_pending(),
+ "the hop target must hold the relay for its op"
+ );
+
+ commit_user_request(
+ &state,
+ READER_CREATED_AT,
+ Operation::CreateUser,
+ &create_reader_request().to_bytes(),
+ );
+ frontier.advance(READER_CREATED_AT);
+ assert!(
+ poll!(response.as_mut()).is_pending(),
+ "the poll waits for its partition"
+ );
+ let Ok(ShardFrame::Lifecycle(LifecycleFrame::PartitionRead {
+ read: PartitionRead::UnboundPollOnPrimary { metadata, .. },
+ reply,
+ ..
+ })) = shard_inbox.try_recv()
+ else {
+ panic!("the hop target must serve the relay once it applied its op");
+ };
+ reply
+ .try_send(PartitionReadReply::Poll {
+ context: metadata.context(READER_CREATED_AT),
+ fragments: Vec::new().into(),
+ current_offset: 0,
++ encryptor: None,
+ })
+ .expect("the poll waits for its reply");
+ assert_eq!(
+ response.await.expect("the hop target answers").status(),
+ StatusCode::OK
+ );
}
#[test]
diff --cc core/server/src/http/handlers.rs
index 8f51a5fc7,33c98d535..76f705930
--- a/core/server/src/http/handlers.rs
+++ b/core/server/src/http/handlers.rs
@@@ -124,11 -128,15 +124,15 @@@ use crate::http::error::
Consistency, ConsistencyQuery, CustomError, PartitionWriteError, ProduceAck, ProduceQuery,
ReadError, WriteError,
};
- use crate::http::extractor::{Authenticated, Identity, JsonMembers, JsonUpdate};
- use crate::http::forward::{PartitionEffect, PartitionHop, PartitionRequest, route_partition};
-use crate::http::extractor::{Authenticated, Credential, Identity};
++use crate::http::extractor::{Authenticated, Credential, Identity, JsonMembers, JsonUpdate};
+ use crate::http::forward::{
+ FirstAttempt, PartitionEffect, PartitionHop, PartitionRequest, require_local_primary,
+ serve_partition,
+ };
use crate::http::metrics::{TopicUsageSample, gauge_value};
use crate::http::reads::{
- authorize_data_plane, authorize_read, await_recovery_barrier, gate_local_read, read_local,
+ authorize_data_plane, authorize_read, await_metadata_floor, await_recovery_barrier,
+ check_partition_freshness, check_partition_incarnation, gate_local_read, read_local,
resolve_gate_stream, resolve_gate_topic, resolve_gate_topic_ids, resolve_gate_user,
topic_durability,
};
diff --cc core/server/src/http/reads.rs
index 5f098b77f,81eb6722c..9a2de98ff
--- a/core/server/src/http/reads.rs
+++ b/core/server/src/http/reads.rs
@@@ -759,11 -838,11 +838,11 @@@ pub(in crate::http) mod tests
(true, true, None),
(false, true, None),
(false, true, Some(Operation::DeleteStream)),
- (false, true, Some(Operation::UpdateStream)),
- (false, true, Some(Operation::UpdateTopic)),
+ (false, true, Some(Operation::UpdateStreamResolved)),
+ (false, true, Some(Operation::UpdateTopicResolved)),
] {
let (sender, shard_inbox, _shard_replies) = shard_channel(0, 1, 1);
- let state = read_state_on(Some(sender));
+ let state = read_state_on(Some(sender), None);
let namespace = create_one_partition_in(&state.shard.plane.metadata().mux_stm);
state
.shard
@@@ -878,9 -954,200 +957,201 @@@
}
}
+ #[compio::test]
+ #[allow(clippy::too_many_lines)]
+ async fn given_a_partition_read_when_freshness_expires_before_retry_should_not_read_again() {
+ for uri in [
+ "/streams/stream/topics/topic/messages?consumer_id=1&partition_id=0&kind=offset&value=0&count=1",
+ "/streams/stream/topics/topic/consumer-offsets?consumer_id=1&partition_id=0",
+ ] {
+ let (sender, shard_inbox, _shard_replies) = shard_channel(0, 1, 1);
+ let frontier = Arc::new(
+ AppliedFrontier::default().with_partition_freshness(Instant::now(), Duration::ZERO),
+ );
+ let state = read_state_with_frontier(Some(sender), None, Arc::clone(&frontier));
+ let namespace = create_one_partition_in(&state.shard.plane.metadata().mux_stm);
+ state
+ .shard
+ .shards_table()
+ .insert(namespace, PartitionLocation::new(ShardId::new(0), 0));
+ commit_user_request(
+ &state,
+ READER_CREATED_AT,
+ Operation::CreateUser,
+ &create_reader_request().to_bytes(),
+ );
+ let token = state
+ .jwt
+ .generate(READER_ID, IggyTimestamp::from(READER_CREATED_AT))
+ .unwrap();
+ let service = TowerToHyperService::new(router(
+ state.clone(),
+ state.forward.body_limit,
+ None,
+ None,
+ false,
+ ));
+ let request = Request::builder()
+ .uri(uri)
+ .header(AUTHORIZATION, format!("Bearer {}", token.access_token))
+ .body(Body::empty())
+ .unwrap();
+ frontier.credit_partition_freshness(Instant::now(), Duration::ZERO);
+ let mut response = pin!(service.call(request));
+ assert!(poll!(response.as_mut()).is_pending());
+ let Ok(ShardFrame::Lifecycle(LifecycleFrame::PartitionRead { reply, .. })) =
+ shard_inbox.try_recv()
+ else {
+ panic!("{uri} must start its first read while evidence is fresh");
+ };
+ reply
+ .try_send(PartitionReadReply::Rejected(IggyError::NotPrimary))
+ .unwrap();
+ assert!(poll!(response.as_mut()).is_pending());
+ let Ok(ShardFrame::Lifecycle(LifecycleFrame::PartitionRead {
+ read: PartitionRead::Primary,
+ reply,
+ ..
+ })) = shard_inbox.try_recv()
+ else {
+ panic!("{uri} must look up the primary after a refusal");
+ };
+ compio::time::sleep(AppliedFrontier::PARTITION_FRESHNESS_MARGIN).await;
+ assert!(!frontier.partition_requests_allowed(Instant::now()));
+ reply
+ .try_send(PartitionReadReply::Primary {
+ view: 0,
+ primary: 0,
+ })
+ .unwrap();
+ let receive = pin!(shard_inbox.recv());
+ let Either::Right((
+ Ok(ShardFrame::Lifecycle(LifecycleFrame::PartitionRead { read, reply, .. })),
+ _,
+ )) = select(response.as_mut(), receive).await
+ else {
+ panic!("{uri} must resolve its freshness refusal against the primary");
+ };
+ assert!(
+ matches!(read, PartitionRead::Primary),
+ "{uri} retried its data read after freshness expired"
+ );
+ reply
+ .try_send(PartitionReadReply::Primary {
+ view: 0,
+ primary: 0,
+ })
+ .unwrap();
+ assert_eq!(
+ response.await.unwrap().status(),
+ StatusCode::SERVICE_UNAVAILABLE
+ );
+ assert!(shard_inbox.try_recv().is_err());
+ }
+ }
+
+ #[compio::test]
+ async fn given_a_poll_waiting_on_its_owner_when_topic_recreated_should_refuse_changed_history()
+ {
+ assert_read_refused_after_topic_recreation(
+ "/streams/stream/topics/topic/messages?consumer_id=1&partition_id=0&kind=offset&value=0&count=1",
+ ).await;
+ }
+
+ #[compio::test]
+ async fn given_an_offset_read_waiting_on_its_owner_when_topic_recreated_should_refuse_changed_history()
+ {
+ assert_read_refused_after_topic_recreation(
+ "/streams/stream/topics/topic/consumer-offsets?consumer_id=1&partition_id=0",
+ )
+ .await;
+ }
+
+ async fn assert_read_refused_after_topic_recreation(uri: &str) {
+ let (sender, shard_inbox, _shard_replies) = shard_channel(0, 1, 1);
+ let state = read_state_on(Some(sender), None);
+ let metadata = state.shard.plane.metadata();
+ let namespace = create_one_partition_in(&metadata.mux_stm);
+ state
+ .shard
+ .shards_table()
+ .insert(namespace, PartitionLocation::new(ShardId::new(0), 0));
+ commit_user_request(
+ &state,
+ READER_CREATED_AT,
+ Operation::CreateUser,
+ &create_reader_request().to_bytes(),
+ );
+ let token = state
+ .jwt
+ .generate(READER_ID, IggyTimestamp::from(READER_CREATED_AT))
+ .unwrap();
+ let service = TowerToHyperService::new(router(
+ state.clone(),
+ state.forward.body_limit,
+ None,
+ None,
+ false,
+ ));
+ let request = Request::builder()
+ .uri(uri)
+ .header(AUTHORIZATION, format!("Bearer {}", token.access_token))
+ .body(Body::empty())
+ .unwrap();
+ let mut response = pin!(service.call(request));
+ assert!(poll!(response.as_mut()).is_pending());
+ let Ok(ShardFrame::Lifecycle(LifecycleFrame::PartitionRead { read, reply, .. })) =
+ shard_inbox.try_recv()
+ else {
+ panic!("{uri} must wait for its partition data");
+ };
+ let body = match read {
+ PartitionRead::UnboundPollOnPrimary { metadata, .. } => PartitionReadReply::Poll {
+ context: metadata.context(0),
+ fragments: Vec::new().into(),
+ current_offset: 0,
++ encryptor: None,
+ },
+ PartitionRead::ConsumerOffsetOnPrimary { .. } => PartitionReadReply::ConsumerOffset {
+ stored: Some(0),
+ current_offset: 0,
+ },
+ _ => panic!("{uri} sent an unexpected partition read"),
+ };
+ let incarnation = metadata
+ .mux_stm
+ .streams()
+ .created_revision_for_namespace(namespace)
+ .unwrap();
+ recreate_topic_in(&metadata.mux_stm, namespace);
+ assert_ne!(
+ metadata
+ .mux_stm
+ .streams()
+ .created_revision_for_namespace(namespace),
+ Some(incarnation)
+ );
+ reply.try_send(body).unwrap();
+ drop(shard_inbox);
+
+ let response = response.await.unwrap();
+ let expected = ReadError::Rejected(IggyError::HistoryUnavailable).into_response();
+ assert_eq!(response.status(), expected.status(), "{uri}");
+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let expected = axum::body::to_bytes(expected.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ assert_eq!(
+ body, expected,
+ "{uri} must refuse the old data without retrying an admitted read"
+ );
+ }
+
/// The account of [`create_reader_request`], the first one after root.
- const READER_ID: u32 = 1;
- const READER_CREATED_AT: u64 = 10;
+ pub(in crate::http) const READER_ID: u32 = 1;
+ pub(in crate::http) const READER_CREATED_AT: u64 = 10;
/// Replace the reader account under its ID, or revoke its grants, and
/// return the status that a read the reader started before must answer.
diff --cc core/server/src/partition_reconciler.rs
index 8e710253b,501fac5a1..d46083229
--- a/core/server/src/partition_reconciler.rs
+++ b/core/server/src/partition_reconciler.rs
@@@ -292,11 -295,9 +300,10 @@@ impl ReconcilerCtx
replica_count,
stats_registry,
failure_state: RefCell::new(AHashMap::new()),
+ unbuilt_owned: RefCell::new(AHashSet::new()),
last_revision: Cell::new(None),
last_pass_noop: Cell::new(false),
- group_offset_cleanup_inflight: Rc::new(RefCell::new(AHashSet::new())),
- group_offset_cleanup_completed: Rc::new(Cell::new(0)),
+ group_offset_cleanup: GroupOffsetCleanup::default(),
last_group_offset_reconcile_epoch: Cell::new(0),
lifecycle_reports: LifecycleReports::default(),
}
@@@ -563,8 -521,8 +570,8 @@@ async fn reconcile_once(ctx: &Reconcile
// namespace's legitimate convergence window.
if ctx.last_revision.get() == Some(revision)
&& ctx.last_pass_noop.get()
- && ctx.failure_state.borrow().is_empty()
+ && !ctx.has_retry_due(Instant::now())
- && ctx.group_offset_cleanup_completed.get() == 0
+ && ctx.group_offset_cleanup.completed.get() == 0
&& !ctx.shard.has_parked_partition_frames()
&& ctx.last_group_offset_reconcile_epoch.get() == group_offset_epoch
{
@@@ -586,9 -544,8 +593,9 @@@
reconcile_additions(ctx, target, &mut counters).await;
reconcile_removals(ctx, &target_set, &mut counters).await;
reconcile_parked_frames(ctx, &mut counters);
- reconcile_consumer_group_offsets(ctx, &mut counters);
+ reconcile_group_offsets(&ctx.shard, &ctx.group_offset_cleanup, &mut counters);
reconcile_segment_truncations(ctx, &mut counters);
+ reconcile_stream_keys(ctx);
report_lifecycles(ctx).await;
let local_set: AHashSet<IggyNamespace> =
@@@ -1979,13 -1888,13 +2019,14 @@@ pub fn install_tick_handler(shard: &Rc<
#[cfg(test)]
mod tests {
use super::{
- FailureCause, FailureRecord, LIFECYCLE_FENCE_RETRY_MICROS, PassCounters, ReconcilerCtx,
- TargetPartition, build_partition_fresh, current_revision, delete_partitions_from_disk,
- fetch_partition_build_inputs, group_offset_delete_request, reconcile_additions,
- reconcile_consumer_group_offsets, reconcile_once, reconcile_partition_lifecycles,
- reconcile_pending_revocations, reconcile_retired_lifecycles,
+ FailureCause, FailureRecord, GROUP_OFFSET_DELETES_PER_PASS, LIFECYCLE_FENCE_RETRY_MICROS,
+ PassCounters, ReconcilerCtx, TargetPartition, build_partition_fresh, current_revision,
+ delete_partitions_from_disk, fetch_partition_build_inputs, group_offset_delete_request,
+ reconcile_additions, reconcile_group_offsets, reconcile_once,
+ reconcile_partition_lifecycles, reconcile_pending_revocations,
+ reconcile_retired_lifecycles,
};
+ use bytes::Bytes;
use configs::server::ServerConfig;
use consensus::{MetadataHandle, PartitionsHandle, Sequencer, StateTransferStage};
use iggy_binary_protocol::codec::{WireDecode, WireEncode};
@@@ -2004,14 -1911,9 +2045,14 @@@
CreateTopicRequest, CreateTopicWithAssignmentsRequest, DeleteTopicRequest,
};
use iggy_binary_protocol::{
- AckLevel, Command, Operation, PrepareHeader, RepairRangeReplyHeader, ReplyHeader,
- RoutedRequestHeader, WireConsumer, WireIdentifier, WireOptions,
+ AckLevel, Command, Operation, PrepareHeader, RepairRangeReplyHeader, RoutedRequestHeader,
+ WireConsumer, WireIdentifier, WireOptions,
};
+ use iggy_common::wire_conversions::resource_options_to_wire;
+ use iggy_common::{
+ HeaderKey, HeaderValue, OptionKey, OptionValue, OptionsProvenance, ResourceOptions,
+ ServerKeyContext, StreamEncryptionKey, StreamKeyOrigin, WrapSet, critical_keys_marker,
+ };
use message_bus::IggyMessageBus;
use metadata::IggyMetadata;
use metadata::MuxStateMachine;
@@@ -2782,212 -2637,6 +2829,211 @@@
);
}
+ #[compio::test]
+ async fn given_topic_with_unknown_critical_option_when_reconciling_should_not_build_its_partition()
+ {
+ let tmp = TempDir::new().expect("tempdir for system path");
+ let config = test_config(&tmp);
+ let mux = TestMux::default();
+ seed_stream(&mux, 1, "stream-a");
+ let unreadable = ResourceOptions::from([
+ (
+ HeaderKey::from(OptionKey::CriticalKeys),
+ OptionValue::explicit(HeaderValue::try_from("a_future_key").expect("marker fits")),
+ ),
+ (
+ HeaderKey::try_from("a_future_key").expect("key fits"),
+ OptionValue::explicit(HeaderValue::from(7u64)),
+ ),
+ ]);
+ let request = CreateTopicWithAssignmentsRequest {
+ request: CreateTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ partitions_count: 1,
+ name: WireName::new("topic-a").expect("test topic name fits WireName"),
+ options: WireOptions::empty(),
+ },
+ derived_options: resource_options_to_wire(&unreadable, OptionsProvenance::All)
+ .expect("options encode"),
+ partitions: vec![assignment(0, 1)],
- created_view: 0,
+ };
+ mux.update(build_prepare(
+ 2,
+ Operation::CreateTopicWithAssignments,
+ &request,
+ ))
+ .expect("CreateTopicWithAssignments apply succeeds");
+ let shard = build_test_shard(0, &config, mux);
+ let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config));
+
+ reconcile_pass(&ctx).await;
+
+ assert!(
+ !shard
+ .plane
+ .partitions()
+ .contains(&IggyNamespace::new(0, 0, 0)),
+ "a topic this build cannot read must not be served"
+ );
+ }
+
+ #[compio::test]
+ async fn given_stream_key_that_does_not_open_when_reconciling_should_not_build_its_partition() {
+ let stream_context = key_context(CONFIG_KEY);
+ let ns = IggyNamespace::new(0, 0, 0);
+ for (node_context, opens) in [
+ (Arc::clone(&stream_context), true),
+ (key_context(OTHER_CONFIG_KEY), false),
+ ] {
+ let tmp = TempDir::new().expect("tempdir for system path");
+ let config = test_config(&tmp);
+ let partition_dir = config.get_partition_path(0, 0, 0);
+ let mux = TestMux::default();
+ seed_encrypted_stream(&mux, 1, "stream-a", &wrap_stream_key(&stream_context, 1));
+ seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]);
+ let shard = build_test_shard_with_keys(0, &config, mux, node_context);
+ let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config));
+
+ reconcile_pass(&ctx).await;
+
+ assert_eq!(
+ served_encrypted(&shard, ns),
+ opens.then_some(true),
+ "a partition is served only under its own stream's key"
+ );
+ assert_eq!(
+ ctx.failure_state
+ .borrow()
+ .contains_key(&(ns, FailureCause::Add)),
+ !opens
+ );
+ assert_eq!(Path::new(&partition_dir).exists(), opens);
+ }
+ }
+
+ #[compio::test]
+ async fn given_only_backed_off_add_when_reconciling_should_skip_until_retry_is_due() {
+ let tmp = TempDir::new().expect("tempdir for system path");
+ let config = test_config(&tmp);
+ let stream_context = key_context(CONFIG_KEY);
+ let node_context = key_context(OTHER_CONFIG_KEY);
+ let mux = TestMux::default();
+ seed_encrypted_stream(&mux, 1, "stream-a", &wrap_stream_key(&stream_context, 1));
+ seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]);
+ let shard = build_test_shard_with_keys(0, &config, mux, node_context);
+ let ctx = make_ctx(shard, 1, Rc::new(config));
+
+ assert!(reconcile_once(&ctx).await);
+ assert!(!ctx.failure_state.borrow().is_empty());
+ assert!(!reconcile_once(&ctx).await);
+
+ for record in ctx.failure_state.borrow_mut().values_mut() {
+ record.next_retry_at = Instant::now();
+ }
+ assert!(reconcile_once(&ctx).await);
+ }
+
+ #[compio::test]
+ async fn given_unbuilt_encrypted_partition_when_its_stream_is_deleted_should_remove_its_files()
+ {
+ for boot_deferred in [false, true] {
+ let tmp = TempDir::new().expect("tempdir for system path");
+ let config = test_config(&tmp);
+ let partition_dir = config.get_partition_path(0, 0, 0);
+ let stream_context = key_context(CONFIG_KEY);
+ let mux = TestMux::default();
+ seed_encrypted_stream(&mux, 1, "stream-a", &wrap_stream_key(&stream_context, 1));
+ seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]);
+ let shard = build_test_shard_with_keys(0, &config, mux, key_context(OTHER_CONFIG_KEY));
+ let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config));
+ let ns = IggyNamespace::new(0, 0, 0);
+ std::fs::create_dir_all(&partition_dir).expect("prior partition files");
+ if boot_deferred {
+ ctx.mark_unbuilt([ns]);
+ } else {
+ reconcile_pass(&ctx).await;
+ assert!(
+ ctx.failure_state
+ .borrow()
+ .contains_key(&(ns, FailureCause::Add))
+ );
+ }
+
+ seed_delete_stream(&shard.plane.metadata().mux_stm, 3, 0);
+ reconcile_pass(&ctx).await;
+
+ assert!(
+ !Path::new(&partition_dir).exists(),
+ "boot deferred {boot_deferred}: a deleted partition that never built keeps no files"
+ );
+ assert!(!shard.plane.partitions().is_tombstoned(&ns));
+ }
+ }
+
+ #[compio::test]
+ async fn given_rewrap_committed_after_a_failed_build_when_reconciling_should_build_under_the_new_wrap()
+ {
+ let tmp = TempDir::new().expect("tempdir for system path");
+ let config = test_config(&tmp);
+ let old_context = key_context(CONFIG_KEY);
+ let new_context = key_context(OTHER_CONFIG_KEY);
+ let stored = wrap_stream_key(&old_context, 1);
+ let mux = TestMux::default();
+ seed_encrypted_stream(&mux, 1, "stream-a", &stored);
+ seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]);
+ let shard = build_test_shard_with_keys(0, &config, mux, Arc::clone(&new_context));
+ let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config));
+ let ns = IggyNamespace::new(0, 0, 0);
+
+ reconcile_pass(&ctx).await;
+ assert_eq!(served_encrypted(&shard, ns), None);
+
+ let mux = &shard.plane.metadata().mux_stm;
+ let created_op = mux
+ .streams()
+ .read(|inner| inner.items.get(0).map(|stream| stream.created_op))
+ .expect("stream exists");
+ let rewrap = RewrapStreamKeysRequest {
+ entries: vec![RewrapStreamKeyEntry {
+ stream_id: 0,
+ created_op,
+ key_fingerprint: stored
+ .key_fingerprint_bytes()
+ .try_into()
+ .expect("8-byte fingerprint"),
+ wrap_set: Bytes::copy_from_slice(wrap_stream_key(&new_context, 1).as_ref()),
+ }],
+ };
+ mux.update(build_prepare(3, Operation::RewrapStreamKeys, &rewrap))
+ .expect("RewrapStreamKeys apply succeeds");
+ for record in ctx.failure_state.borrow_mut().values_mut() {
+ record.next_retry_at = Instant::now();
+ }
+ reconcile_pass(&ctx).await;
+
+ assert_eq!(served_encrypted(&shard, ns), Some(true));
+ }
+
+ #[compio::test]
+ async fn given_deleted_encrypted_stream_when_reconciling_should_forget_its_key() {
+ let tmp = TempDir::new().expect("tempdir for system path");
+ let config = test_config(&tmp);
+ let context = key_context(CONFIG_KEY);
+ let mux = TestMux::default();
+ seed_encrypted_stream(&mux, 1, "stream-a", &wrap_stream_key(&context, 1));
+ seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]);
+ let shard = build_test_shard_with_keys(0, &config, mux, context);
+ let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config));
+
+ reconcile_pass(&ctx).await;
+ assert!(shard.plane.partitions().has_stream_keys());
+
+ seed_delete_stream(&shard.plane.metadata().mux_stm, 3, 0);
+ reconcile_pass(&ctx).await;
+
+ assert!(!shard.plane.partitions().has_stream_keys());
+ }
+
/// A delete and a recreate of the same ids can commit while a pass runs,
/// so the pass holds an older epoch than the committed partition. The
/// directory of the committed incarnation must survive that deferral,
diff --cc core/server/src/rewrite.rs
index 05496a66b,4172861c1..ab5e6141f
--- a/core/server/src/rewrite.rs
+++ b/core/server/src/rewrite.rs
@@@ -666,535 -457,6 +666,535 @@@ fn admit_patch
#[cfg(test)]
mod tests {
use super::*;
+ use crate::dispatch::test_support::{
+ SpyBus, TestShard, create_one_partition, create_reader_request, prepare_message,
+ register_reply, request_message, test_shard, test_shard_with_key_context,
+ };
+ use bytes::Bytes;
+ use iggy_binary_protocol::requests::system::WireResetKeys;
+ use iggy_binary_protocol::requests::topics::CreateTopicWithAssignmentsRequest;
+ use iggy_binary_protocol::{MAX_OPTIONS, WireName};
+ use iggy_common::defaults::DEFAULT_ROOT_USER_ID;
+ use iggy_common::wire_conversions::resource_options_from_wire;
+ use iggy_common::{
+ HeaderKey, HeaderValue, IggyDuration, IggyExpiry, OptionKey, OptionValue, OptionsUpdate,
+ ServerKeyContext, StreamCreateOptions, StreamEncryptionKey, StreamKeyOrigin,
+ TopicUpdateOptions, UserUpdateOptions, stream_wrap_set, text,
+ };
+ use message_bus::installer::conn_info::ClientTransportKind;
+ use metadata::stm::StateMachine as _;
+ use std::collections::BTreeMap;
+ use std::str::FromStr;
+ use std::sync::Arc;
++ use std::time::Instant;
+
+ const CONFIG_KEY: &str = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=";
+
+ #[test]
+ #[allow(clippy::too_many_lines)]
+ fn given_patch_constraints_when_using_either_chain_should_return_same_error() {
+ let bus = SpyBus::default();
+ let shard = Rc::new(test_shard(&bus, 0, 1, 1));
+ create_one_partition(&shard);
+ let sessions = root_sessions(&shard, 1);
+ let topic = || ResourceTarget::Topic {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(0),
+ };
+ let stream = || ResourceTarget::Stream {
+ stream_id: WireIdentifier::numeric(0),
+ };
+ let raw_topic_patch = |key: OptionKey, value: &str| {
+ let patch = TopicUpdateOptions {
+ raw: BTreeMap::from([(key.to_string(), value.to_owned())]),
+ ..TopicUpdateOptions::default()
+ }
+ .to_patch()
+ .unwrap();
+ UpdateResourceRequest {
+ target: topic(),
+ reset: patch.reset,
+ options: patch.set,
+ }
+ .to_bytes()
+ };
+ let short_username = UserUpdateOptions {
+ raw: BTreeMap::from([(OptionKey::Username.to_string(), "ab".to_owned())]),
+ ..UserUpdateOptions::default()
+ }
+ .to_patch()
+ .unwrap();
+ let mut name_column = ResourceOptions::new();
+ name_column.insert(
+ HeaderKey::from(OptionKey::Name),
+ OptionValue::explicit(HeaderValue::try_from("shadow").unwrap()),
+ );
+ let json_patch = |target: ResourceTarget, body: &str| {
+ let patch = serde_json::from_str::<OptionsUpdate>(body)
+ .unwrap()
+ .to_patch()
+ .unwrap();
+ UpdateResourceRequest {
+ target,
+ reset: patch.reset,
+ options: patch.set,
+ }
+ .to_bytes()
+ };
+ let user = || ResourceTarget::User {
+ user_id: WireIdentifier::numeric(0),
+ };
+ let cases: Vec<(Operation, Bytes, IggyError)> = vec![
+ (
+ Operation::UpdateResource,
+ json_patch(topic(), r#"{"name": 42}"#),
+ IggyError::InvalidTopicName,
+ ),
+ (
+ Operation::UpdateResource,
+ json_patch(user(), r#"{"status": true}"#),
+ IggyError::InvalidUserStatus,
+ ),
+ (
+ Operation::UpdateResource,
+ raw_topic_patch(OptionKey::SegmentSize, "64 MiB"),
+ IggyError::OptionNotUpdatable(OptionKey::SegmentSize.to_string()),
+ ),
+ (
+ Operation::UpdateResource,
+ raw_topic_patch(OptionKey::MaxTopicSize, "1 MiB"),
+ IggyError::InvalidTopicSize(
+ MaxTopicSize::from(1024 * 1024u64),
+ IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE),
+ ),
+ ),
+ (
+ Operation::UpdateResource,
+ UpdateResourceRequest {
+ target: stream(),
+ reset: WireResetKeys::try_from(vec![WireName::new("name").unwrap()]).unwrap(),
+ options: WireOptions::empty(),
+ }
+ .to_bytes(),
+ IggyError::OptionNotResettable(OptionKey::Name.to_string()),
+ ),
+ (
+ Operation::UpdateResource,
+ UpdateResourceRequest {
+ target: stream(),
+ reset: WireResetKeys::default(),
+ options: resource_options_to_wire(
+ &ResourceOptions::from([(
+ HeaderKey::from(OptionKey::EncryptionKeySource),
+ OptionValue::explicit(HeaderValue::try_from("stream").unwrap()),
+ )]),
+ OptionsProvenance::All,
+ )
+ .unwrap(),
+ }
+ .to_bytes(),
+ IggyError::ReadOnlyOption(OptionKey::EncryptionKeySource.to_string()),
+ ),
+ (
+ Operation::UpdateResource,
+ UpdateResourceRequest {
+ target: ResourceTarget::User {
+ user_id: WireIdentifier::numeric(0),
+ },
+ reset: short_username.reset,
+ options: short_username.set,
+ }
+ .to_bytes(),
+ IggyError::InvalidUsername,
+ ),
+ (
+ Operation::CreateStream,
+ CreateStreamRequest {
+ name: WireName::new("orders").unwrap(),
+ options: resource_options_to_wire(&name_column, OptionsProvenance::All)
+ .unwrap(),
+ }
+ .to_bytes(),
+ IggyError::InvalidOptionValue(OptionKey::Name.to_string()),
+ ),
+ (
+ Operation::CreateStream,
+ CreateStreamRequest {
+ name: WireName::new("orders").unwrap(),
+ options: StreamCreateOptions {
+ encryption_key: Some(
+ StreamEncryptionKey::try_from([7u8; 32].as_slice()).unwrap(),
+ ),
+ ..StreamCreateOptions::default()
+ }
+ .to_wire()
+ .unwrap(),
+ }
+ .to_bytes(),
+ IggyError::StreamEncryptionUnavailable,
+ ),
+ ];
+
+ for (operation, body, expected) in cases {
+ let tcp = tcp_chain(
+ &shard,
+ &sessions,
+ 1,
+ request_message(operation, 1, 1, 1, &body),
+ )
+ .map(|_| ())
+ .map_err(|deny| deny.error);
+ let http = http_chain(
+ &shard,
+ DEFAULT_ROOT_USER_ID,
+ request_message(operation, 1, 1, 1, &body),
+ )
+ .map(|_| ());
+ assert_eq!(tcp, Err(expected.clone()), "binary chain, {operation:?}");
+ assert_eq!(http, Err(expected), "HTTP chain, {operation:?}");
+ }
+ }
+
+ #[test]
+ fn given_caller_without_update_permission_when_patching_topic_below_its_segment_should_leave_the_refusal_to_apply()
+ {
+ const READER: u32 = 1;
+ let bus = SpyBus::default();
+ let shard = Rc::new(test_shard(&bus, 0, 1, 1));
+ create_one_partition(&shard);
+ let created = shard
+ .plane
+ .metadata()
+ .mux_stm
+ .update(prepare_message(
+ Operation::CreateUser,
+ 1,
+ 3,
+ &create_reader_request().to_bytes(),
+ ))
+ .expect("the reader is created");
+ assert_eq!(created.code, 0);
+ let patch = TopicUpdateOptions {
+ raw: BTreeMap::from([(OptionKey::MaxTopicSize.to_string(), "1 MiB".to_owned())]),
+ ..TopicUpdateOptions::default()
+ }
+ .to_patch()
+ .expect("the patch encodes");
+ let body = UpdateResourceRequest {
+ target: ResourceTarget::Topic {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(0),
+ },
+ reset: patch.reset,
+ options: patch.set,
+ }
+ .to_bytes();
+ let chain = |user_id: u32| {
+ http_chain(
+ &shard,
+ user_id,
+ request_message(Operation::UpdateResource, 1, 1, 1, &body),
+ )
+ .map(|_| ())
+ };
+
+ assert_eq!(
+ chain(READER),
+ Ok(()),
+ "a caller apply refuses must not learn the topic's segment size here"
+ );
+ assert_eq!(
+ chain(DEFAULT_ROOT_USER_ID),
+ Err(IggyError::InvalidTopicSize(
+ MaxTopicSize::from(1024 * 1024u64),
+ IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE),
+ ))
+ );
+ }
+
+ #[test]
+ fn given_root_deactivation_when_admitting_patch_should_refuse_before_consensus() {
+ let bus = SpyBus::default();
+ let shard = Rc::new(test_shard(&bus, 0, 1, 1));
+ shard
+ .plane
+ .metadata()
+ .mux_stm
+ .users()
+ .ensure_root_user("iggy", "hash");
+ let patch = UserUpdateOptions {
+ status: Some(UserStatus::Inactive),
+ ..UserUpdateOptions::default()
+ }
+ .to_patch()
+ .expect("status patch encodes");
+ let body = UpdateResourceRequest {
+ target: ResourceTarget::User {
+ user_id: WireIdentifier::numeric(DEFAULT_ROOT_USER_ID),
+ },
+ reset: patch.reset,
+ options: patch.set,
+ }
+ .to_bytes();
+
+ let result = http_chain(
+ &shard,
+ DEFAULT_ROOT_USER_ID,
+ request_message(Operation::UpdateResource, 1, 1, 1, &body),
+ );
+ assert!(
+ matches!(result, Err(IggyError::CannotModifyRootUser)),
+ "actual: {:?}",
+ result.err()
+ );
+ assert!(bus.replica_sends.borrow().is_empty());
+ }
+
+ #[test]
+ fn given_manage_users_caller_when_patching_root_should_refuse_rename_and_deactivation() {
+ let bus = SpyBus::default();
+ let shard = Rc::new(test_shard(&bus, 0, 1, 1));
+ let metadata = shard.plane.metadata();
+ metadata.mux_stm.users().ensure_root_user("iggy", "hash");
+ let mut manager = create_reader_request();
+ manager.username = WireName::new("manager").expect("valid name");
+ manager
+ .permissions
+ .as_mut()
+ .expect("permissions")
+ .global
+ .manage_users = true;
+ let created = metadata
+ .mux_stm
+ .update(prepare_message(
+ Operation::CreateUser,
+ 1,
+ 3,
+ &manager.to_bytes(),
+ ))
+ .expect("create manager");
+ assert_eq!(created.code, 0);
+
+ for update in [
+ UserUpdateOptions {
+ username: Some("root-renamed".to_owned()),
+ ..UserUpdateOptions::default()
+ },
+ UserUpdateOptions {
+ status: Some(UserStatus::Inactive),
+ ..UserUpdateOptions::default()
+ },
+ ] {
+ let patch = update.to_patch().expect("patch encodes");
+ let body = UpdateResourceRequest {
+ target: ResourceTarget::User {
+ user_id: WireIdentifier::numeric(DEFAULT_ROOT_USER_ID),
+ },
+ reset: patch.reset,
+ options: patch.set,
+ }
+ .to_bytes();
+ let result = http_chain(
+ &shard,
+ 1,
+ request_message(Operation::UpdateResource, 1, 1, 1, &body),
+ );
+ assert!(matches!(result, Err(IggyError::CannotModifyRootUser)));
+ }
+ assert!(bus.replica_sends.borrow().is_empty());
+ }
+
+ #[test]
+ fn given_out_of_range_status_when_creating_user_with_either_chain_should_refuse_invalid_status()
+ {
+ let bus = SpyBus::default();
+ let shard = Rc::new(test_shard(&bus, 0, 1, 1));
+ let sessions = Rc::new(RefCell::new(SessionManager::new()));
+ let body = CreateUserRequest {
+ username: WireName::new("status-user").unwrap(),
+ password: "secret123".to_owned(),
+ status: 9,
+ permissions: None,
+ options: WireOptions::empty(),
+ }
+ .to_bytes();
+
+ let tcp = tcp_chain(
+ &shard,
+ &sessions,
+ 1,
+ request_message(Operation::CreateUser, 1, 1, 1, &body),
+ )
+ .map(|_| ())
+ .map_err(|deny| deny.error);
+ let http = http_chain(
+ &shard,
+ 0,
+ request_message(Operation::CreateUser, 1, 1, 1, &body),
+ )
+ .map(|_| ());
+
+ assert_eq!(tcp, Err(IggyError::InvalidUserStatus), "binary chain");
+ assert_eq!(http, Err(IggyError::InvalidUserStatus), "HTTP chain");
+ }
+
+ #[test]
+ fn given_text_stream_key_when_bounding_create_should_encode_no_request_with_the_key() {
+ let bus = SpyBus::default();
+ let shard = Rc::new(test_shard(&bus, 0, 1, 1));
+ let key = StreamEncryptionKey::try_from([7u8; 32].as_slice()).expect("32 bytes");
+ let body = CreateStreamRequest {
+ name: WireName::new("orders").expect("a name"),
+ options: text_key_block(&key),
+ }
+ .to_bytes();
+
+ let sent = request_message(Operation::CreateStream, 1, 1, 1, &body);
+
+ let bounded = static_bounds(&shard, Some(DEFAULT_ROOT_USER_ID), sent.clone())
+ .expect("the create admits");
+
+ let Bounded::StreamCreate {
+ request, options, ..
+ } = bounded
+ else {
+ panic!("a stream create goes on decoded");
+ };
+ assert_eq!(request.as_slice(), sent.as_slice(), "no request is rebuilt");
+ assert_eq!(
+ options.get(&HeaderKey::from(OptionKey::EncryptionKey)),
+ Some(&OptionValue::explicit(
+ HeaderValue::try_from(key.as_bytes().as_slice()).expect("a raw value")
+ )),
+ "the admitted key is canonical"
+ );
+ }
+
+ #[test]
+ fn given_stream_key_in_either_form_when_running_either_chain_should_forward_only_its_wrap_set()
+ {
+ let bus = SpyBus::default();
+ let context =
+ Arc::new(ServerKeyContext::from_config(CONFIG_KEY, &[], true).expect("a valid key"));
+ let shard = Rc::new(test_shard_with_key_context(
+ &bus,
+ 0,
+ 1,
+ 1,
+ Arc::clone(&context),
+ ));
+ let sessions = root_sessions(&shard, 1);
+ let key = StreamEncryptionKey::try_from([7u8; 32].as_slice()).expect("32 bytes");
+ let raw_block = StreamCreateOptions {
+ encryption_key: Some(key.clone()),
+ ..StreamCreateOptions::default()
+ }
+ .to_wire()
+ .expect("encodes");
+
+ for options in [raw_block, text_key_block(&key)] {
+ let body = CreateStreamRequest {
+ name: WireName::new("orders").expect("a name"),
+ options,
+ }
+ .to_bytes();
+ let (tcp, _) = tcp_chain(
+ &shard,
+ &sessions,
+ 1,
+ request_message(Operation::CreateStream, 1, 1, 1, &body),
+ )
+ .map_err(|deny| deny.error)
+ .expect("binary chain");
+ let (http, _) = http_chain(
+ &shard,
+ DEFAULT_ROOT_USER_ID,
+ request_message(Operation::CreateStream, 1, 1, 1, &body),
+ )
+ .expect("HTTP chain");
+
+ for forwarded in [tcp, http] {
+ let forwarded = request_body(&forwarded);
+ assert!(!contains(forwarded, key.as_bytes()), "raw key travels on");
+ assert!(
+ !contains(forwarded, text::as_base64(key.as_bytes()).as_bytes()),
+ "base64 key travels on"
+ );
+ let create = CreateStreamRequest::decode_from(forwarded).expect("decodes");
+ let stored = resource_options_from_wire(&create.options, true).expect("a block");
+ let set = stream_wrap_set(&stored)
+ .expect("readable")
+ .expect("a wrap set");
+ assert_eq!(set.origin(), StreamKeyOrigin::Client);
+ assert_eq!(context.open(&set).expect("opens"), key);
+ }
+ }
+ }
+
+ #[test]
+ fn given_stored_map_at_capacity_when_patching_a_new_key_should_refuse_at_the_edge() {
+ let bus = SpyBus::default();
+ let shard = Rc::new(test_shard(&bus, 0, 1, 1));
+ create_one_partition(&shard);
+ let full: ResourceOptions = (0..MAX_OPTIONS)
+ .map(|index| {
+ (
+ HeaderKey::from_str(&format!("future_{index:04}")).unwrap(),
+ OptionValue::explicit(HeaderValue::from(true)),
+ )
+ })
+ .collect();
+ shard
+ .plane
+ .metadata()
+ .mux_stm
+ .update(prepare_message(
+ Operation::CreateTopicWithAssignments,
+ 1,
+ 3,
+ &CreateTopicWithAssignmentsRequest {
+ request: CreateTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ partitions_count: 0,
+ name: WireName::new("full").unwrap(),
+ options: resource_options_to_wire(&full, OptionsProvenance::All).unwrap(),
+ },
+ derived_options: WireOptions::empty(),
+ partitions: Vec::new(),
- created_view: 0,
+ }
+ .to_bytes(),
+ ))
+ .unwrap();
+ let patch = TopicUpdateOptions {
+ message_expiry: Some(IggyExpiry::ExpireDuration(IggyDuration::from(
+ 60_000_000u64,
+ ))),
+ ..TopicUpdateOptions::default()
+ }
+ .to_patch()
+ .unwrap();
+ let body = UpdateResourceRequest {
+ target: ResourceTarget::Topic {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(1),
+ },
+ reset: patch.reset,
+ options: patch.set,
+ }
+ .to_bytes();
+
+ let bounded = static_bounds(
+ &shard,
+ Some(DEFAULT_ROOT_USER_ID),
+ request_message(Operation::UpdateResource, 1, 1, 1, &body),
+ )
+ .map(|_| ());
+
+ assert!(
+ matches!(bounded, Err(IggyError::OptionsBlockTooLarge(_))),
+ "{bounded:?}"
+ );
+ }
#[test]
fn create_topic_bounds_deny_pre_consensus() {
@@@ -1281,62 -543,4 +1281,64 @@@
"the cap still applies"
);
}
+
+ fn contains(haystack: &[u8], needle: &[u8]) -> bool {
+ haystack
+ .windows(needle.len())
+ .any(|window| window == needle)
+ }
+
+ fn text_key_block(key: &StreamEncryptionKey) -> WireOptions {
+ resource_options_to_wire(
+ &ResourceOptions::from([(
+ HeaderKey::from(OptionKey::EncryptionKey),
+ OptionValue::explicit(
+ HeaderValue::try_from(text::as_base64(key.as_bytes()).as_str())
+ .expect("a text value"),
+ ),
+ )]),
+ OptionsProvenance::All,
+ )
+ .expect("encodes")
+ }
+
+ /// A session table whose `transport` connection is bound to root.
+ fn root_sessions(shard: &TestShard, transport: u128) -> Rc<RefCell<SessionManager>> {
+ const CLIENT: u128 = 1;
+ const SESSION: u64 = 1;
+ let attachment = {
+ let mut client_table = shard.plane.metadata().client_table.borrow_mut();
+ client_table
+ .commit_register(
+ CLIENT,
+ DEFAULT_ROOT_USER_ID,
+ [0x5a; 32],
+ register_reply(CLIENT, SESSION),
+ )
+ .expect("root registers");
+ client_table
+ .attach_session(CLIENT, SESSION, DEFAULT_ROOT_USER_ID)
+ .expect("root attaches")
+ };
+ let sessions = Rc::new(RefCell::new(SessionManager::new()));
+ sessions.borrow_mut().ensure_connection(
+ transport,
+ "127.0.0.1:34567".parse().expect("a socket address"),
+ ClientTransportKind::Tcp,
++ Instant::now(),
+ );
+ sessions
+ .borrow_mut()
+ .bind_authenticated_connection(
+ transport,
+ CLIENT,
+ SESSION,
+ DEFAULT_ROOT_USER_ID,
+ attachment,
+ SESSION,
++ Instant::now(),
+ )
+ .expect("root binds");
+ sessions
+ }
}
diff --cc core/shard/src/lib.rs
index 60739f931,68c47fb72..7bae4597a
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@@ -57,7 -58,7 +58,7 @@@ use iggy_binary_protocol::
#[cfg(feature = "simulator")]
use iggy_common::PartitionStats;
use iggy_common::variadic;
- use iggy_common::{ConsumerKind, EncryptorKind, IggyError, IggyExpiry, IggyTimestamp, PollingKind};
-use iggy_common::{ConsumerKind, IggyError, IggyExpiry, IggyTimestamp};
++use iggy_common::{ConsumerKind, EncryptorKind, IggyError, IggyExpiry, IggyTimestamp};
use journal::superblock::{PingPongSuperblock, SuperblockStore};
use journal::{Journal, JournalHandle};
use message_bus::client_listener::RequestHandler;
@@@ -13761,6 -15364,566 +15369,566 @@@ mod metadata_repair_session_tests
}
}
+ #[cfg(test)]
+ mod partition_placement_tests {
+ use super::*;
+ use consensus::{CLIENTS_TABLE_MAX, LocalPipeline, build_reply_message_with};
+ use iggy_binary_protocol::requests::topics::{
+ CreateTopicRequest, CreateTopicWithAssignmentsRequest,
+ };
+ use iggy_binary_protocol::{
+ ReplyHeader, WireDecode, WireEncode, WireIdentifier, WireName, WireOptions,
+ };
+ use iggy_common::{IggyByteSize, PartitionStats};
+ use journal::prepare_journal::PrepareJournal;
+ use message_bus::IggyMessageBus;
+ use metadata::MuxStateMachine;
+ use metadata::impls::metadata::IggySnapshot;
+ use metadata::impls::recovery::{ReplicaIdentity, recover};
+ use metadata::stm::stream::Streams;
+ use metadata::stm::user::Users;
+ use partitions::{PartitionPathLayout, PartitionsConfig};
+ use server_common::sharding::METADATA_GROUP;
+ use shards_table::PapayaShardsTable;
+ use std::collections::BTreeSet;
+ use std::mem::size_of;
+ use std::sync::Arc;
+
+ type PlacementShard = IggyShard<
+ Rc<IggyMessageBus>,
+ PrepareJournal,
+ IggySnapshot,
+ MuxStateMachine<variadic!(Users, Streams)>,
+ PapayaShardsTable,
+ >;
+
+ pub fn owner(shard_id: u16) -> (PlacementShard, Receiver<ShardFrame>) {
+ let bus = Rc::new(IggyMessageBus::new(shard_id));
+ let segment_size = IggyByteSize::from(1_048_576_u64);
+ let partitions = IggyPartitions::new(
+ ShardId::new(shard_id),
+ PartitionsConfig {
+ messages_required_to_save: 100,
+ size_of_messages_required_to_save: segment_size,
+ validate_checksum: true,
+ segment_size,
+ preallocate_segments: false,
- encryptor: None,
++ key_context: Arc::default(),
+ path_layout: PartitionPathLayout::default(),
+ },
+ );
+ let metadata = IggyMetadata::new(
+ None,
+ None,
+ None,
+ None,
+ MuxStateMachine::<variadic!(Users, Streams)>::default(),
+ None,
+ );
+ let mut shard = PlacementShard::without_inbox(
+ ShardIdentity::new(shard_id, "partition-placement-test".to_string()),
+ bus.clone(),
+ metadata,
+ partitions,
+ PapayaShardsTable::new(),
+ PartitionConsensusConfig::new(1, ReplicaTopology::new(0, 3), bus),
+ );
+ let (senders, mut inboxes, _reply_inboxes) = shard_mesh_channels(shard_id + 1, 1, 1);
+ let target = inboxes[0].take().expect("shard zero inbox");
+ shard.attach_senders(senders);
+ (shard, target)
+ }
+
+ fn insert(shard: &PlacementShard, id: usize, revision: u64, view: u32) -> IggyNamespace {
+ let namespace = IggyNamespace::new(0, 0, id);
+ let mut consensus = VsrConsensus::new(
+ 1,
+ 0,
+ 3,
+ namespace.inner(),
+ shard.bus.clone(),
+ LocalPipeline::new(),
+ );
+ consensus.set_view(view);
+ consensus.init();
+ let mut partition = IggyPartition::with_in_memory_storage(
+ Arc::new(PartitionStats::default()),
+ consensus,
+ shard.plane.partitions().config().segment_size,
+ );
+ partition.set_created_revision(revision);
+ shard.plane.partitions().insert(namespace, partition);
+ namespace
+ }
+
+ fn publish(shard: &PlacementShard) {
+ let namespaces: Vec<_> = shard.plane.partitions().namespaces().copied().collect();
+ shard.publish_partition_views(&namespaces);
+ }
+
+ fn receive(inbox: &Receiver<ShardFrame>) -> Vec<PartitionPlacementView> {
+ let ShardFrame::Lifecycle(LifecycleFrame::PartitionPlacementViews(reports)) =
+ inbox.try_recv().expect("placement report queued")
+ else {
+ panic!("expected partition placement observations");
+ };
+ assert!(inbox.try_recv().is_err(), "at most one batch per sweep");
+ reports
+ }
+
+ #[compio::test]
+ async fn given_unchanged_placement_when_ticks_advance_should_report_at_probe_intervals() {
+ let (shard, inbox) = owner(1);
+ insert(&shard, 0, 7, 0);
+ shard.ticks.set(1);
+ publish(&shard);
+ assert_eq!(receive(&inbox).len(), 1);
+ let interval = u64::try_from(PARTITION_PLACEMENT_REPORT_INTERVAL_TICKS).unwrap();
+ for tick in 2..=interval {
+ shard.ticks.set(tick);
+ publish(&shard);
+ assert!(
+ inbox.try_recv().is_err(),
+ "placement report repeated at tick {tick}"
+ );
+ }
+ shard.ticks.set(interval + 1);
+ publish(&shard);
+ assert_eq!(receive(&inbox).len(), 1);
+ }
+
+ #[compio::test]
+ async fn given_more_groups_than_one_batch_when_an_interval_starts_should_report_each_group_within_it()
+ {
+ let (shard, inbox) = owner(1);
+ let count = PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN * 2 + 3;
+ for id in 0..count {
+ insert(&shard, id, 7, 0);
+ }
+ let batches =
+ u64::try_from(count.div_ceil(PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN)).unwrap();
+ let mut observed = BTreeSet::new();
+ for tick in 1..=batches {
+ shard.ticks.set(tick);
+ publish(&shard);
+ observed.extend(
+ receive(&inbox)
+ .into_iter()
+ .map(|report| report.namespace.partition_id()),
+ );
+ }
+ assert_eq!(observed, (0..count).collect());
+ shard.ticks.set(batches + 1);
+ publish(&shard);
+ assert!(
+ inbox.try_recv().is_err(),
+ "a finished pass waits for the next interval"
+ );
+ }
+
+ #[compio::test]
+ async fn given_more_groups_than_an_interval_of_batches_when_an_interval_runs_should_report_each_group_within_it()
+ {
+ let (shard, inbox) = owner(1);
+ let interval = u64::try_from(PARTITION_PLACEMENT_REPORT_INTERVAL_TICKS).unwrap();
+ let count = PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN * usize::try_from(interval).unwrap()
+ + PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN
+ + 3;
+ for id in 0..count {
+ insert(&shard, id, 7, 0);
+ }
+ let mut observed = BTreeSet::new();
+ for tick in 1..=interval {
+ shard.ticks.set(tick);
+ publish(&shard);
+ if !inbox.is_empty() {
+ observed.extend(
+ receive(&inbox)
+ .into_iter()
+ .map(|report| report.namespace.partition_id()),
+ );
+ }
+ }
+ assert_eq!(
+ observed.len(),
+ count,
+ "every group reports within one interval"
+ );
+
+ shard.ticks.set(interval + 1);
+ publish(&shard);
+ assert_eq!(
+ receive(&inbox)[0].namespace,
+ IggyNamespace::new(0, 0, 0),
+ "the next interval starts a new pass at the front"
+ );
+ }
+
+ #[compio::test]
+ async fn given_owned_groups_when_ticking_should_report_only_normal_serving_views_to_shard_zero()
+ {
+ for shard_id in [0, 1] {
+ let (shard, inbox) = owner(shard_id);
+ for id in 0..6 {
+ let view = u32::from(id == 1);
+ insert(&shard, id, 7, view);
+ }
+ let partitions = shard.plane.partitions();
+ partitions
+ .get_by_ns(&IggyNamespace::new(0, 0, 2))
+ .unwrap()
+ .consensus()
+ .begin_view_probe();
+ partitions
+ .get_by_ns(&IggyNamespace::new(0, 0, 3))
+ .unwrap()
+ .consensus()
+ .begin_state_transfer_await();
+ partitions
+ .get_by_ns(&IggyNamespace::new(0, 0, 4))
+ .unwrap()
+ .consensus()
+ .init_as_backup();
+ partitions.tombstone(IggyNamespace::new(0, 0, 5));
+
+ assert!(shard.tick_partitions(&mut Vec::new()).await.is_none());
+
+ let reports = receive(&inbox);
+ assert_eq!(reports.len(), 2);
+ assert_eq!(reports[0].namespace, IggyNamespace::new(0, 0, 0));
+ assert_eq!(reports[0].view, 0, "a normal primary contributes");
+ assert_eq!(reports[1].namespace, IggyNamespace::new(0, 0, 1));
+ assert_eq!(
+ reports[1].view, 1,
+ "a normal backup contributes its known view"
+ );
+ assert!(reports.iter().all(|report| report.created_revision == 7));
+ }
+ }
+
+ #[compio::test]
+ async fn given_more_groups_than_one_batch_when_publishing_should_bound_batches_and_cover_the_tail()
+ {
+ let (shard, inbox) = owner(1);
+ let count = PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN * 2 + 3;
+ for id in 0..count {
+ insert(&shard, id, 7, 0);
+ }
+ let mut observed = BTreeSet::new();
+ for _ in 0..count.div_ceil(PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN) {
+ publish(&shard);
+ let reports = receive(&inbox);
+ assert_eq!(reports.len(), PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN);
+ observed.extend(
+ reports
+ .into_iter()
+ .map(|report| report.namespace.partition_id()),
+ );
+ }
+ assert_eq!(observed, (0..count).collect());
+
+ let removed = shard.partition_placement_cursor.get().unwrap();
+ shard.plane.partitions().remove(&removed);
+ publish(&shard);
+ let reports = receive(&inbox);
+ assert_eq!(
+ reports[0].namespace.partition_id(),
+ removed.partition_id() + 1
+ );
+ assert!(reports.iter().all(|report| report.namespace != removed));
+ }
+
+ #[compio::test]
+ async fn given_a_full_inbox_when_retried_should_publish_current_incarnation_without_skipping_groups()
+ {
+ let (shard, inbox) = owner(1);
+ let namespace = insert(&shard, 0, 7, 0);
+ insert(&shard, 1, 7, 0);
+ assert!(
+ shard.senders[0]
+ .try_send(ShardFrame::lifecycle(LifecycleFrame::MetadataCommitTick))
+ .is_ok()
+ );
+
+ publish(&shard);
+ assert_eq!(shard.partition_placement_cursor.get(), None);
+ assert_eq!(
+ inbox.len(),
+ 1,
+ "no deferred report queue grows behind the full inbox"
+ );
+ assert!(matches!(
+ inbox.try_recv(),
+ Ok(ShardFrame::Lifecycle(LifecycleFrame::MetadataCommitTick))
+ ));
+ shard.plane.partitions().remove(&namespace);
+ insert(&shard, 0, 11, 9);
+
+ publish(&shard);
+ let reports = receive(&inbox);
+ assert_eq!(reports.len(), 2);
+ assert_eq!(reports[0].namespace, namespace);
+ assert_eq!(reports[0].created_revision, 11);
+ assert_eq!(
+ reports[0].view, 9,
+ "retry rereads rather than retaining the old observation"
+ );
+ assert_eq!(reports[1].namespace, IggyNamespace::new(0, 0, 1));
+ }
+
+ #[compio::test]
+ async fn given_an_ineligible_prefix_when_publishing_should_bound_visits_and_reach_live_groups()
+ {
+ let (shard, inbox) = owner(1);
+ for id in 0..=PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN {
+ let namespace = insert(&shard, id, 7, 0);
+ if id < PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN {
+ shard
+ .plane
+ .partitions()
+ .get_by_ns(&namespace)
+ .unwrap()
+ .consensus()
+ .begin_view_probe();
+ }
+ }
+ publish(&shard);
+ assert!(
+ inbox.try_recv().is_err(),
+ "one sweep visits at most its bounded range"
+ );
+
+ publish(&shard);
+ let reports = receive(&inbox);
+ assert_eq!(reports.len(), 1);
+ assert_eq!(
+ reports[0].namespace.partition_id(),
+ PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN
+ );
+ }
+
+ #[compio::test]
+ #[allow(clippy::too_many_lines)]
+ async fn given_reopened_pending_create_when_rebuilding_pipeline_should_restore_placement_reservations()
+ {
+ type PlacementMetadata = IggyMetadata<
+ VsrConsensus<Rc<IggyMessageBus>>,
+ PrepareJournal,
+ IggySnapshot,
+ MuxStateMachine<variadic!(Users, Streams)>,
+ >;
+ const JOURNAL_SLOTS: usize = 128;
+ const FIRST_CLIENT: u128 = 7;
+ const NEXT_CLIENT: u128 = 8;
+ const ADMISSION_BUDGET: Duration = Duration::from_secs(5);
+ let register = |metadata: &PlacementMetadata, client| {
+ let reply = build_reply_message_with(
+ &PrepareHeader {
+ client,
+ op: 1,
+ operation: Operation::Register,
+ ..Default::default()
+ },
+ 0,
+ |_| {},
+ );
+ metadata
+ .client_table
+ .borrow_mut()
+ .commit_register(client, 0, [0x5a; 32], reply)
+ .unwrap();
+ };
+ let request = |client, name: &str| {
+ let body = CreateTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ partitions_count: 2,
+ name: WireName::new(name).unwrap(),
+ options: WireOptions::empty(),
+ }
+ .to_bytes();
+ let header_size = size_of::<RoutedRequestHeader>();
+ let mut message = Message::<RoutedRequestHeader>::new(header_size + body.len());
+ message.as_mut_slice()[header_size..].copy_from_slice(&body);
+ message.transmute_header(|_, header| {
+ *header = RoutedRequestHeader {
+ command: Command::Request,
+ operation: Operation::CreateTopic,
+ size: u32::try_from(header_size + body.len()).unwrap(),
+ client,
+ session: 1,
+ request: 1,
+ group: METADATA_GROUP,
+ ..Default::default()
+ };
+ })
+ };
+ let capture = async |metadata: &PlacementMetadata, request, op| {
+ let mut submit = Box::pin(metadata.submit_request_in_process(request));
+ compio::time::timeout(ADMISSION_BUDGET, async {
+ loop {
+ assert!(
+ submit.as_mut().now_or_never().is_none(),
+ "three replicas need another acknowledgment before commit"
+ );
+ let journal = metadata.journal.as_ref().unwrap();
+ let header = journal.handle().header(op).map(|header| *header);
+ if let Some(header) = header {
+ break journal.handle().entry(&header).await.unwrap();
+ }
+ compio::time::sleep(Duration::from_millis(1)).await;
+ }
+ })
+ .await
+ .expect("admission must append within its budget")
+ };
+
+ let directory = tempfile::tempdir().unwrap();
+ let metadata_directory = directory.path().join(metadata::impls::METADATA_DIR);
+ std::fs::create_dir(&metadata_directory).unwrap();
+ let inherited = {
+ let journal = PrepareJournal::open_with_slots(
+ &metadata_directory.join("journal.wal"),
+ 0,
+ JOURNAL_SLOTS,
+ )
+ .await
+ .unwrap();
+ let consensus = VsrConsensus::new(
+ 1,
+ 0,
+ 3,
+ METADATA_GROUP,
+ Rc::new(IggyMessageBus::new(0)),
+ LocalPipeline::new(),
+ );
+ consensus.init();
+ let metadata = PlacementMetadata::new(
+ Some(consensus),
+ Some(journal),
+ None,
+ None,
+ MuxStateMachine::default(),
+ None,
+ );
+ register(&metadata, FIRST_CLIENT);
+ let prepare = capture(&metadata, request(FIRST_CLIENT, "inherited"), 1).await;
+ assert_eq!(metadata.consensus.as_ref().unwrap().commit_min(), 0);
+ prepare
+ };
+ let original = CreateTopicWithAssignmentsRequest::decode_from(inherited.body())
+ .unwrap()
+ .partitions;
+ assert_eq!(
+ original
+ .iter()
+ .map(|partition| (partition.consensus_group_id, partition.initial_view))
+ .collect::<Vec<_>>(),
+ [(1, 0), (2, 1)]
+ );
+
+ let recovered = recover::<MuxStateMachine<variadic!(Users, Streams)>>(
+ directory.path(),
+ ReplicaIdentity {
+ cluster: 1,
+ replica_id: 0,
+ replica_count: 3,
+ },
+ JOURNAL_SLOTS,
+ CLIENTS_TABLE_MAX,
+ |_| {},
+ |_, _| {},
+ )
+ .await
+ .unwrap();
+ assert_eq!(recovered.last_applied_op, None);
+ assert_eq!(recovered.last_journaled_op, Some(inherited.header().op));
+ assert_eq!(recovered.chain_break_op, None);
+ let reopened = recovered
+ .journal
+ .handle()
+ .entry(inherited.header())
+ .await
+ .unwrap();
+ assert_eq!(reopened.as_slice(), inherited.as_slice());
+
+ // Enter the recovered-primary seam; the network election itself is covered separately.
+ let consensus = VsrConsensus::new(
+ 1,
+ 0,
+ 3,
+ METADATA_GROUP,
+ Rc::new(IggyMessageBus::new(0)),
+ LocalPipeline::new(),
+ );
+ consensus.init();
+ consensus
+ .sequencer()
+ .set_sequence(recovered.last_journaled_op.unwrap());
+ consensus.set_last_prepare_checksum(inherited.header().checksum);
+ let metadata = PlacementMetadata::new(
+ Some(consensus),
+ Some(recovered.journal),
+ recovered.snapshot,
+ Some(Rc::new(recovered.superblock)),
+ recovered.mux_stm,
+ None,
+ );
+ register(&metadata, NEXT_CLIENT);
+ let consensus = metadata.consensus.as_ref().unwrap();
+ assert!(consensus.pipeline_is_empty());
+ dispatch_vsr_actions(
+ consensus,
+ metadata.journal.as_ref(),
+ &[VsrAction::RebuildPipeline {
+ from_op: recovered.last_applied_op.unwrap_or(0) + 1,
+ to_op: recovered.last_journaled_op.unwrap(),
+ }],
+ )
+ .await;
+ assert!(consensus.pipeline_holds_entry(inherited.header().op, inherited.header().checksum));
+ let refused = metadata
+ .submit_request_in_process(request(NEXT_CLIENT, "next"))
+ .await
+ .unwrap()
+ .try_into_typed::<ReplyHeader>()
+ .unwrap();
+ assert_eq!(
+ iggy_binary_protocol::result_code(refused.body()),
+ Some(IggyError::TransientNotAccepted.as_code())
+ );
+ assert!(!refused.header().has_committed_result());
+ assert_eq!(metadata.allocator.highest(), 0);
+ metadata.refresh_placement().await;
+ assert_eq!(metadata.allocator.highest(), 2);
+ let next = capture(&metadata, request(NEXT_CLIENT, "next"), 2).await;
+ let assignments = CreateTopicWithAssignmentsRequest::decode_from(next.body())
+ .unwrap()
+ .partitions;
+ assert_eq!(
+ assignments
+ .iter()
+ .map(|partition| (partition.consensus_group_id, partition.initial_view))
+ .collect::<Vec<_>>(),
+ [(3, 2), (4, 0)]
+ );
+ assert_eq!(next.header().parent, inherited.header().checksum);
+ assert!(consensus.pipeline_holds_entry(inherited.header().op, inherited.header().checksum));
+ let retained = metadata
+ .journal
+ .as_ref()
+ .unwrap()
+ .handle()
+ .entry(inherited.header())
+ .await
+ .unwrap();
+ assert_eq!(
+ retained.as_slice(),
+ inherited.as_slice(),
+ "reconstruction must preserve the original assignment bytes"
+ );
+ }
+ }
+
#[cfg(test)]
mod partition_ack_durability_tests {
use super::*;
diff --cc core/shard/src/primary_rebalance/tests.rs
index 000000000,f30157277..2d78f4e25
mode 000000,100644..100644
--- a/core/shard/src/primary_rebalance/tests.rs
+++ b/core/shard/src/primary_rebalance/tests.rs
@@@ -1,0 -1,1191 +1,1191 @@@
+ // Licensed to the Apache Software Foundation (ASF) under one
+ // or more contributor license agreements. See the NOTICE file
+ // distributed with this work for additional information
+ // regarding copyright ownership. The ASF licenses this file
+ // to you under the Apache License, Version 2.0 (the
+ // "License"); you may not use this file except in compliance
+ // with the License. You may obtain a copy of the License at
+ //
+ // http://www.apache.org/licenses/LICENSE-2.0
+ //
+ // Unless required by applicable law or agreed to in writing,
+ // software distributed under the License is distributed on an
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ // KIND, either express or implied. See the License for the
+ // specific language governing permissions and limitations
+ // under the License.
+
+ use super::*;
+ use crate::shards_table::PapayaShardsTable;
+ use crate::{
+ PartitionConsensusConfig, Receiver, ReplicaTopology, ShardIdentity, shard_mesh_channels,
+ };
+ use consensus::{LocalPipeline, PlaneKind, StateTransferStage};
+ use iggy_binary_protocol::{
+ CommitHeader, DoViewChangeHeader, StartViewChangeHeader, StartViewHeader,
+ };
+ use iggy_common::{IggyByteSize, PartitionStats, variadic};
+ use journal::prepare_journal::PrepareJournal;
+ use journal::superblock::{PingPongSuperblock, SuperblockContents};
+ use message_bus::{BusMessage, SendError};
+ use metadata::impls::metadata::{IggySnapshot, PartitionPlacementView, StreamsFrontend};
+ use metadata::stm::stream::Streams;
+ use metadata::stm::user::Users;
+ use metadata::{IggyMetadata, MuxStateMachine};
+ use partitions::{IggyPartitions, PartitionPathLayout, PartitionsConfig};
+ use server_common::MESSAGE_ALIGN;
+ use server_common::iobuf::Frozen;
+ use server_common::sharding::{METADATA_GROUP, ShardId};
+ use std::cell::{Cell, RefCell};
+ use std::pin::pin;
+ use std::rc::Rc;
+ use std::sync::Arc;
+ use std::task::Poll;
+
+ const CLUSTER: u128 = 7;
+ const REPLICAS: u8 = 3;
+ const METADATA_VIEW: u32 = 3;
+ const PARTITION_VIEW: u32 = 4;
+ const CREATED_REVISION: u64 = 11;
+ const COMMITTED: u64 = 13;
+ /// The primary of `PARTITION_VIEW`.
+ const SOURCE: u8 = 1;
+ const TARGET: u8 = 2;
+ const NONCE: u128 = 17;
+
+ #[derive(Debug, Default)]
+ struct RecordingBus {
+ sent: RefCell<Vec<(u8, PrimaryRebalanceHeader)>>,
+ now: Cell<Option<Instant>>,
+ disconnected: Cell<bool>,
+ }
+
+ #[allow(clippy::future_not_send, clippy::unused_async_trait_impl)]
+ impl MessageBus for RecordingBus {
+ fn monotonic_now(&self) -> Instant {
+ self.now.get().unwrap_or_else(Instant::now)
+ }
+
+ fn track_background(&self, _handle: message_bus::JoinHandle<()>) {}
+
+ async fn send_to_client(
+ &self,
+ _client_id: u128,
+ _data: impl Into<BusMessage>,
+ ) -> Result<(), SendError> {
+ panic!("readiness exchange must not send client traffic");
+ }
+
+ async fn send_to_replica(
+ &self,
+ replica: u8,
+ frame: Frozen<MESSAGE_ALIGN>,
+ ) -> Result<(), SendError> {
+ if self.disconnected.get() {
+ return Err(SendError::ReplicaNotConnected(replica));
+ }
+ assert_eq!(frame.len(), HEADER_SIZE);
+ let mut message = Message::<PrimaryRebalanceHeader>::new(HEADER_SIZE);
+ message.as_mut_slice().copy_from_slice(frame.as_slice());
+ message.header().validate().unwrap();
+ message.header().verify_frame().unwrap();
+ self.sent.borrow_mut().push((replica, *message.header()));
+ Ok(())
+ }
+
+ fn set_connection_lost_fn(&self, _callback: message_bus::ConnectionLostFn) {}
+ fn set_replica_forward_fn(&self, _callback: message_bus::ReplicaForwardFn) {}
+ }
+
+ type TestShard<SB = PingPongSuperblock> = IggyShard<
+ Rc<RecordingBus>,
+ PrepareJournal,
+ IggySnapshot,
+ MuxStateMachine<variadic!(Users, Streams)>,
+ PapayaShardsTable,
+ SB,
+ >;
+
+ fn owner() -> (TestShard, Receiver<ShardFrame>) {
+ owner_with_superblock(None)
+ }
+
+ fn owner_with_superblock<SB: SuperblockStore>(
+ superblock: Option<Rc<SB>>,
+ ) -> (TestShard<SB>, Receiver<ShardFrame>) {
+ owner_at(superblock, 0, REPLICAS, METADATA_VIEW)
+ }
+
+ fn owner_at<SB: SuperblockStore>(
+ superblock: Option<Rc<SB>>,
+ replica: u8,
+ replica_count: u8,
+ view: u32,
+ ) -> (TestShard<SB>, Receiver<ShardFrame>) {
+ let bus = Rc::new(RecordingBus::default());
+ bus.now.set(Some(Instant::now()));
+ let mut consensus = VsrConsensus::new(
+ CLUSTER,
+ replica,
+ replica_count,
+ METADATA_GROUP,
+ Rc::clone(&bus),
+ LocalPipeline::new(),
+ );
+ consensus.set_view(view);
+ consensus.init();
+ let segment_size = IggyByteSize::from(1_048_576_u64);
+ let partitions = IggyPartitions::new(
+ ShardId::new(0),
+ PartitionsConfig {
+ messages_required_to_save: 100,
+ size_of_messages_required_to_save: segment_size,
+ validate_checksum: true,
+ segment_size,
+ preallocate_segments: false,
- encryptor: None,
++ key_context: Arc::default(),
+ path_layout: PartitionPathLayout::default(),
+ },
+ );
+ let metadata = IggyMetadata::new(
+ Some(consensus),
+ None,
+ None,
+ superblock,
+ MuxStateMachine::<variadic!(Users, Streams)>::default(),
+ None,
+ )
+ .with_applied_frontier(Arc::new(
+ metadata::AppliedFrontier::default()
+ .with_partition_freshness(bus.monotonic_now(), Duration::from_secs(5)),
+ ));
+ let mut shard = TestShard::<SB>::without_inbox(
+ ShardIdentity::new(0, "primary-rebalance-test".to_owned()),
+ Rc::clone(&bus),
+ metadata,
+ partitions,
+ PapayaShardsTable::new(),
+ PartitionConsensusConfig::new(CLUSTER, ReplicaTopology::new(replica, replica_count), bus),
+ );
+ let (senders, mut inboxes, _reply_inboxes) = shard_mesh_channels(1, 1, 1);
+ shard.attach_senders(senders);
+ (shard, inboxes[0].take().unwrap())
+ }
+
+ fn arm<SB: SuperblockStore>(shard: &TestShard<SB>, phase: MovePhase) -> PrimaryRebalanceHeader {
+ let namespace = IggyNamespace::new(1, 1, 0);
+ let consensus = shard.plane.metadata().consensus.as_ref().unwrap();
+ let mut probe = probe_header(consensus, NONCE);
+ probe.namespace = namespace.inner();
+ probe.created_revision = CREATED_REVISION;
+ probe.partition_view = PARTITION_VIEW;
+ probe.commit = COMMITTED;
+ probe.kind = if phase == MovePhase::Capture {
+ PrimaryRebalanceKind::SourceProbe
+ } else {
+ PrimaryRebalanceKind::PartitionProbe
+ };
+ *shard.primary_rebalance.borrow_mut() = PrimaryRebalance {
+ view: Some(METADATA_VIEW),
+ peers: vec![Peer::default(); usize::from(REPLICAS)],
+ eligible: vec![true; usize::from(REPLICAS)],
+ pending: Some(PendingMove {
+ plan: PartitionPrimaryTransfer {
+ namespace,
+ created_revision: CREATED_REVISION,
+ view: PARTITION_VIEW,
+ target: TARGET,
+ },
+ probe,
+ phase,
+ since: Instant::now(),
+ }),
+ ..PrimaryRebalance::default()
+ };
+ probe.kind = if phase == MovePhase::Capture {
+ PrimaryRebalanceKind::SourceReply
+ } else {
+ PrimaryRebalanceKind::PartitionReply
+ };
+ probe.replica = if phase == MovePhase::Capture {
+ consensus.primary_index(PARTITION_VIEW)
+ } else {
+ TARGET
+ };
+ probe.ready = 1;
+ probe
+ }
+
+ #[compio::test]
+ async fn given_pending_target_probe_when_reply_is_stale_or_mismatched_should_not_grant() {
+ for changed in [
+ "cluster",
+ "nonce",
+ "metadata view",
+ "namespace",
+ "incarnation",
+ "partition view",
+ "commit",
+ "target",
+ "expired",
+ ] {
+ let (shard, inbox) = owner();
+ let mut reply = arm(&shard, MovePhase::Probe);
+ match changed {
+ "cluster" => reply.cluster += 1,
+ "nonce" => reply.nonce += 1,
+ "metadata view" => reply.view += 1,
+ "namespace" => reply.namespace += 1,
+ "incarnation" => reply.created_revision += 1,
+ "partition view" => reply.partition_view += 1,
+ "commit" => reply.commit += 1,
+ "target" => reply.replica = 0,
+ "expired" => {
+ shard
+ .primary_rebalance
+ .borrow_mut()
+ .pending
+ .as_mut()
+ .unwrap()
+ .since = Instant::now()
+ .checked_sub(PROBE_TIMEOUT + PROBE_INTERVAL)
+ .unwrap();
+ }
+ _ => unreachable!(),
+ }
+ shard.on_primary_rebalance(reply).await;
+ assert!(
+ inbox.try_recv().is_err(),
+ "{changed} reply granted a transfer"
+ );
+ assert!(shard.bus.sent.borrow().is_empty());
+ assert!(
+ shard
+ .primary_rebalance
+ .borrow()
+ .pending
+ .as_ref()
+ .is_some_and(|pending| pending.phase == MovePhase::Probe)
+ );
+ }
+ }
+
+ #[compio::test]
+ async fn given_primary_frontier_when_target_confirms_it_should_grant_the_source_primary_once() {
+ let (shard, inbox) = owner();
+ let mut source = arm(&shard, MovePhase::Capture);
+ source.commit = COMMITTED + 1;
+ let mut wrong_source = source;
+ wrong_source.replica = TARGET;
+ shard.on_primary_rebalance(wrong_source).await;
+ assert!(shard.bus.sent.borrow().is_empty());
+ assert!(inbox.try_recv().is_err());
+
+ shard.on_primary_rebalance(source).await;
+ let (target, probe) = shard.bus.sent.borrow()[0];
+ assert_eq!(target, TARGET);
+ assert_eq!(probe.kind, PrimaryRebalanceKind::PartitionProbe);
+ assert_eq!(probe.commit, source.commit);
+ assert_eq!(probe.nonce, source.nonce);
+ assert_eq!(probe.replica, 0);
+ assert!(inbox.try_recv().is_err());
+ shard.on_primary_rebalance(source).await;
+ assert_eq!(shard.bus.sent.borrow().len(), 1);
+
+ let mut reply = probe;
+ reply.kind = PrimaryRebalanceKind::PartitionReply;
+ reply.replica = TARGET;
+ reply.ready = 1;
+ shard.on_primary_rebalance(reply).await;
+ assert!(
+ inbox.try_recv().is_err(),
+ "the coordinator must not grant its own replica of the partition"
+ );
+ let (source_replica, grant) = shard.bus.sent.borrow()[1];
+ assert_eq!(source_replica, source.replica);
+ assert_eq!(grant.kind, PrimaryRebalanceKind::PrimaryTransfer);
+ assert_eq!(grant.replica, 0);
+ assert_eq!(grant.view, METADATA_VIEW);
+ assert_eq!(grant.nonce, NONCE);
+ assert_eq!(grant.namespace, probe.namespace);
+ assert_eq!(grant.created_revision, CREATED_REVISION);
+ assert_eq!(grant.partition_view, PARTITION_VIEW);
+ assert_eq!(grant.target, TARGET);
+ shard.on_primary_rebalance(reply).await;
+ assert_eq!(
+ shard.bus.sent.borrow().len(),
+ 2,
+ "a replay must not issue a second grant"
+ );
+
+ let (source_node, source_inbox) =
+ owner_at::<PingPongSuperblock>(None, source_replica, REPLICAS, METADATA_VIEW);
+ let received_at = source_node.bus.monotonic_now();
+ source_node.on_primary_rebalance(grant).await;
+ let ShardFrame::Lifecycle(LifecycleFrame::PrimaryRebalance(frame)) =
+ source_inbox.try_recv().unwrap()
+ else {
+ panic!("the source must route the grant to the partition's shard");
+ };
+ let LocalRebalance::Transfer { namespace, request } = *frame else {
+ panic!("a grant must yield a transfer");
+ };
+ assert_eq!(namespace.inner(), probe.namespace);
+ assert_eq!(request.created_revision, CREATED_REVISION);
+ assert_eq!(request.expected_view, PARTITION_VIEW);
+ assert_eq!(request.target, TARGET);
+ assert_eq!(
+ request.expires_at,
+ received_at + GRANT_TIMEOUT,
+ "the source must time the grant on its own clock"
+ );
+ }
+
+ #[compio::test]
+ async fn given_a_grant_frame_when_its_sender_is_not_the_ready_metadata_primary_should_drop_it() {
+ for changed in ["none", "cluster", "metadata view", "sender", "not ready"] {
+ let (source_node, inbox) =
+ owner_at::<PingPongSuperblock>(None, SOURCE, REPLICAS, METADATA_VIEW);
+ let consensus = source_node.plane.metadata().consensus.as_ref().unwrap();
+ let mut grant = probe_header(consensus, NONCE);
+ grant.replica = consensus.primary_index(METADATA_VIEW);
+ grant.kind = PrimaryRebalanceKind::PrimaryTransfer;
+ grant.namespace = IggyNamespace::new(1, 1, 0).inner();
+ grant.created_revision = CREATED_REVISION;
+ grant.partition_view = PARTITION_VIEW;
+ grant.target = TARGET;
+ match changed {
+ "none" => {}
+ "cluster" => grant.cluster += 1,
+ "metadata view" => grant.view += 1,
+ "sender" => grant.replica = TARGET,
+ "not ready" => consensus.begin_view_probe(),
+ _ => unreachable!(),
+ }
+ source_node.on_primary_rebalance(grant).await;
+ assert_eq!(inbox.try_recv().is_ok(), changed == "none", "{changed}");
+ }
+ }
+
+ #[compio::test]
+ async fn given_the_coordinator_leads_the_partition_when_target_confirms_it_should_grant_through_its_own_ingress()
+ {
+ let (shard, inbox) = owner();
+ let mut reply = arm(&shard, MovePhase::Probe);
+ let consensus = shard.plane.metadata().consensus.as_ref().unwrap();
+ // The coordinator leads the metadata view, so it leads this partition view too.
+ let led_view = METADATA_VIEW;
+ assert_eq!(consensus.primary_index(led_view), consensus.replica());
+ {
+ let mut state = shard.primary_rebalance.borrow_mut();
+ let pending = state.pending.as_mut().unwrap();
+ pending.plan.view = led_view;
+ pending.probe.partition_view = led_view;
+ }
+ reply.partition_view = led_view;
+ shard.on_primary_rebalance(reply).await;
+ assert!(
+ shard.bus.sent.borrow().is_empty(),
+ "a local grant must not leave the node"
+ );
+ let ShardFrame::Lifecycle(LifecycleFrame::PrimaryRebalance(frame)) = inbox.try_recv().unwrap()
+ else {
+ panic!("the grant must enter the metadata owner's inbox");
+ };
+ let frame = *frame;
+ let LocalRebalance::Wire(grant) = &frame else {
+ panic!("a local grant must take the wire path");
+ };
+ assert_eq!(grant.kind, PrimaryRebalanceKind::PrimaryTransfer);
+ shard.on_local_rebalance(frame).await;
+ let ShardFrame::Lifecycle(LifecycleFrame::PrimaryRebalance(frame)) = inbox.try_recv().unwrap()
+ else {
+ panic!("the checked grant must reach the partition's shard");
+ };
+ let LocalRebalance::Transfer { request, .. } = *frame else {
+ panic!("a grant must yield a transfer");
+ };
+ assert_eq!(request.expected_view, led_view);
+ assert_eq!(request.target, TARGET);
+ }
+
+ #[compio::test]
+ async fn given_target_rejection_when_reply_is_correlated_should_drop_the_move_and_ignore_late_success()
+ {
+ let (shard, inbox) = owner();
+ let mut reply = arm(&shard, MovePhase::Probe);
+ reply.ready = 0;
+ shard.on_primary_rebalance(reply).await;
+ assert!(shard.primary_rebalance.borrow().pending.is_none());
+ reply.ready = 1;
+ shard.on_primary_rebalance(reply).await;
+ assert!(inbox.try_recv().is_err());
+ assert!(shard.bus.sent.borrow().is_empty());
+ }
+
+ /// Ticks the coordinator at `at` with every peer freshly ready, so only the
+ /// pending move can change.
+ #[allow(clippy::future_not_send)]
+ async fn tick_with_ready_peers(shard: &TestShard, at: Instant) {
+ for peer in &mut shard.primary_rebalance.borrow_mut().peers {
+ peer.ready_at = Some(at);
+ }
+ shard.bus.now.set(Some(at));
+ shard.tick_primary_rebalance().await;
+ }
+
+ #[compio::test]
+ async fn given_a_sent_grant_when_the_new_view_is_unseen_should_hold_the_move_until_it_shows_or_times_out()
+ {
+ for shows in [true, false] {
+ let (shard, _inbox) = owner();
+ arm(&shard, MovePhase::Transfer);
+ let metadata = shard.plane.metadata();
+ let namespace = IggyNamespace::new(1, 1, 0);
+ let streams = metadata.mux_stm.streams();
+ streams.seed_namespace(namespace, namespace.inner(), PARTITION_VIEW);
+ let created_revision = streams.created_revision_for_namespace(namespace).unwrap();
+ let observe = |view| {
+ metadata.observe_partition_views(&[PartitionPlacementView {
+ namespace,
+ created_revision,
+ view,
+ }]);
+ };
+ observe(PARTITION_VIEW);
+ let sent_at = shard.bus.monotonic_now();
+ {
+ let mut state = shard.primary_rebalance.borrow_mut();
+ let pending = state.pending.as_mut().unwrap();
+ pending.plan.created_revision = created_revision;
+ pending.since = sent_at;
+ }
+ let pending = || shard.primary_rebalance.borrow().pending.is_some();
+
+ tick_with_ready_peers(&shard, sent_at + GRANT_TIMEOUT * 2).await;
+ assert!(
+ pending(),
+ "an expired grant may have started a view change that is not visible yet"
+ );
+ if shows {
+ observe(PARTITION_VIEW + 1);
+ tick_with_ready_peers(&shard, sent_at + GRANT_TIMEOUT * 2).await;
+ } else {
+ tick_with_ready_peers(&shard, sent_at + TRANSFER_TIMEOUT).await;
+ assert!(pending(), "the move lasts the whole transfer timeout");
+ tick_with_ready_peers(&shard, sent_at + TRANSFER_TIMEOUT + Duration::from_nanos(1))
+ .await;
+ }
+ assert!(!pending(), "shows={shows}");
+ }
+ }
+
+ #[compio::test]
+ async fn given_an_unsent_grant_when_the_coordinator_ticks_should_plan_again_before_the_transfer_timeout()
+ {
+ let (shard, _inbox) = owner();
+ let reply = arm(&shard, MovePhase::Probe);
+ let metadata = shard.plane.metadata();
+ let streams = metadata.mux_stm.streams();
+ // Both partitions of the seeded topic start on the source, so a move
+ // stays planned after the refused one.
+ let last = IggyNamespace::new(1, 1, 1);
+ streams.seed_namespace(last, last.inner(), PARTITION_VIEW);
+ let views: Vec<_> = (0..=last.partition_id())
+ .map(|partition| {
+ let namespace = IggyNamespace::new(1, 1, partition);
+ PartitionPlacementView {
+ namespace,
+ created_revision: streams.created_revision_for_namespace(namespace).unwrap(),
+ view: PARTITION_VIEW,
+ }
+ })
+ .collect();
+ metadata.observe_partition_views(&views);
+ shard
+ .primary_rebalance
+ .borrow_mut()
+ .pending
+ .as_mut()
+ .unwrap()
+ .plan
+ .created_revision = views[0].created_revision;
+
+ shard.bus.disconnected.set(true);
+ shard.on_primary_rebalance(reply).await;
+ shard.bus.disconnected.set(false);
+ assert!(shard.bus.sent.borrow().is_empty());
+
+ let replan_after = STABLE_INTERVAL * 2;
+ assert!(replan_after < TRANSFER_TIMEOUT);
+ tick_with_ready_peers(&shard, shard.bus.monotonic_now() + replan_after).await;
+ assert!(
+ shard
+ .bus
+ .sent
+ .borrow()
+ .iter()
+ .any(|(replica, probe)| *replica == SOURCE
+ && probe.kind == PrimaryRebalanceKind::SourceProbe),
+ "the coordinator must plan again before the transfer timeout"
+ );
+ }
+
+ fn suspect_primary(group: u64, view: u32) -> MessageBag {
+ MessageBag::StartViewChange(
+ Message::<StartViewChangeHeader>::new(HEADER_SIZE).transmute_header(
+ |_, header: &mut StartViewChangeHeader| {
+ header.cluster = CLUSTER;
+ header.group = group;
+ header.view = view;
+ header.replica = 1;
+ header.command = Command::StartViewChange;
+ header.size = u32::try_from(HEADER_SIZE).unwrap();
+ },
+ ),
+ )
+ }
+
+ #[compio::test]
+ async fn given_pending_target_probe_when_metadata_authority_changes_should_not_grant_while_handler_suspends()
+ {
+ let superblock = Rc::new(HeldSuperblock::default());
+ let (shard, inbox) = owner_with_superblock(Some(Rc::clone(&superblock)));
+ let reply = arm(&shard, MovePhase::Probe);
+ let mut dispatch =
+ pin!(shard.dispatch_message(suspect_primary(METADATA_GROUP, METADATA_VIEW + 1), None));
+ assert_eq!(futures::poll!(&mut dispatch), Poll::Pending);
+ assert!(superblock.entered.get());
+ shard.on_primary_rebalance(reply).await;
+ assert!(
+ shard.bus.sent.borrow().is_empty(),
+ "a reply handled during the suspension granted a move"
+ );
+ assert!(inbox.try_recv().is_err());
+ assert!(shard.primary_rebalance.borrow().pending.is_none());
+ }
+
+ #[derive(Default)]
+ struct HeldSuperblock {
+ entered: Cell<bool>,
+ }
+
+ #[allow(clippy::future_not_send)]
+ impl SuperblockStore for HeldSuperblock {
+ async fn write(&self, _payload: &[u8]) -> std::io::Result<()> {
+ self.entered.set(true);
+ std::future::pending().await
+ }
+
+ fn read_latest(&self) -> impl Future<Output = std::io::Result<SuperblockContents>> {
+ std::future::ready(Ok(SuperblockContents::Empty))
+ }
+ }
+
+ #[test]
+ fn given_pending_move_when_unrelated_or_current_traffic_arrives_should_preserve_authority() {
+ let partition_group = IggyNamespace::new(1, 1, 0).inner();
+ let cases = [
+ (partition_group, METADATA_VIEW + 1, false),
+ (METADATA_GROUP, METADATA_VIEW - 1, false),
+ // Includes a late vote for the view this replica already leads.
+ (METADATA_GROUP, METADATA_VIEW, false),
+ (METADATA_GROUP, METADATA_VIEW + 1, true),
+ ];
+ for command in [
+ Command::StartViewChange,
+ Command::DoViewChange,
+ Command::Commit,
+ ] {
+ for (group, view, should_revoke) in cases {
+ let (shard, _inbox) = owner();
+ arm(&shard, MovePhase::Transfer);
+ let message = match command {
+ Command::StartViewChange => suspect_primary(group, view),
+ Command::DoViewChange => MessageBag::DoViewChange(
+ Message::<DoViewChangeHeader>::new(HEADER_SIZE).transmute_header(
+ |_, header: &mut DoViewChangeHeader| {
+ header.cluster = CLUSTER;
+ header.group = group;
+ header.view = view;
+ header.replica = 1;
+ header.command = command;
+ header.size = u32::try_from(HEADER_SIZE).unwrap();
+ },
+ ),
+ ),
+ _ => {
+ MessageBag::Commit(Message::<CommitHeader>::new(HEADER_SIZE).transmute_header(
+ |_, header: &mut CommitHeader| {
+ header.cluster = CLUSTER;
+ header.group = group;
+ header.view = view;
+ header.replica = 1;
+ header.command = command;
+ header.size = u32::try_from(HEADER_SIZE).unwrap();
+ },
+ ))
+ }
+ };
+ shard.fence_rebalance_authority(&message);
+ assert_eq!(
+ shard.primary_rebalance.borrow().pending.is_none(),
+ should_revoke,
+ "group={group}, view={view}, command={command:?}"
+ );
+ }
+ }
+ }
+
+ #[compio::test]
+ async fn given_a_new_authority_when_peers_have_not_answered_should_keep_them_eligible_until_their_first_probe_times_out()
+ {
+ let (shard, _inbox) = owner_at::<PingPongSuperblock>(None, 0, REPLICAS, 0);
+ let epoch = shard.bus.monotonic_now();
+ let eligible = || shard.primary_rebalance.borrow().eligible.clone();
+ shard.tick_primary_rebalance().await;
+ assert_eq!(
+ eligible(),
+ [true; 3],
+ "placement must not narrow to the coordinator before any peer answers"
+ );
+
+ let (replica, mut not_ready) = shard.bus.sent.borrow()[0];
+ not_ready.replica = replica;
+ not_ready.kind = PrimaryRebalanceKind::MetadataReply;
+ shard.on_primary_rebalance(not_ready).await;
+ shard.tick_primary_rebalance().await;
+ assert_eq!(eligible(), [true, false, true]);
+
+ shard.bus.now.set(Some(epoch + PROBE_TIMEOUT));
+ shard.tick_primary_rebalance().await;
+ assert_eq!(eligible(), [true, false, true]);
+ shard
+ .bus
+ .now
+ .set(Some(epoch + PROBE_TIMEOUT + Duration::from_nanos(1)));
+ shard.tick_primary_rebalance().await;
+ assert_eq!(
+ eligible(),
+ [true, false, false],
+ "an unanswered first probe ends the presumption at its timeout"
+ );
+ }
+
+ #[test]
+ fn given_health_challenge_when_reply_is_overdue_or_replayed_should_not_extend_readiness() {
+ let (shard, _inbox) = owner();
+ let consensus = shard.plane.metadata().consensus.as_ref().unwrap();
+ let mut reply = probe_header(consensus, NONCE);
+ reply.kind = PrimaryRebalanceKind::MetadataReply;
+ reply.replica = TARGET;
+ reply.ready = 1;
+ for changed in ["nonce", "view", "replica", "expired", "unsent"] {
+ let mut state = PrimaryRebalance {
+ view: Some(METADATA_VIEW),
+ peers: vec![Peer::default(); usize::from(REPLICAS)],
+ ..PrimaryRebalance::default()
+ };
+ let previous = Instant::now()
+ .checked_sub(PROBE_TIMEOUT + PROBE_INTERVAL)
+ .unwrap();
+ state.peers[usize::from(TARGET)] = Peer {
+ nonce: Some(NONCE),
+ sent_at: Some(Instant::now()),
+ ready_at: Some(previous),
+ answered_at: None,
+ replied: false,
+ };
+ let mut invalid = reply;
+ match changed {
+ "nonce" => invalid.nonce += 1,
+ "view" => invalid.view += 1,
+ "replica" => invalid.replica = REPLICAS,
+ "expired" => state.peers[usize::from(TARGET)].sent_at = Some(previous),
+ "unsent" => state.peers[usize::from(TARGET)].sent_at = None,
+ _ => unreachable!(),
+ }
+ state.observe_peer(&invalid, Instant::now());
+ assert_eq!(
+ state.peers[usize::from(TARGET)].ready_at,
+ Some(previous),
+ "{changed}"
+ );
+ assert_eq!(
+ state.peers[usize::from(TARGET)].nonce,
+ Some(NONCE),
+ "{changed}"
+ );
+ state.peers[usize::from(TARGET)].sent_at = Some(Instant::now());
+ state.observe_peer(&reply, Instant::now());
+ let accepted_at = state.peers[usize::from(TARGET)].ready_at;
+ assert!(accepted_at.is_some_and(|ready| ready > previous));
+ assert_eq!(state.peers[usize::from(TARGET)].nonce, None);
+ reply.ready = 0;
+ state.observe_peer(&reply, Instant::now());
+ assert_eq!(state.peers[usize::from(TARGET)].ready_at, accepted_at);
+ reply.ready = 1;
+ }
+ }
+
+ fn ready_partition(replica: u8) -> (IggyPartition<Rc<RecordingBus>>, PrimaryRebalanceHeader) {
+ let (shard, _inbox) = owner();
+ let probe = arm(&shard, MovePhase::Probe);
+ let mut consensus = VsrConsensus::new(
+ CLUSTER,
+ replica,
+ REPLICAS,
+ probe.namespace,
+ Rc::clone(&shard.bus),
+ LocalPipeline::new(),
+ );
+ consensus.set_view(PARTITION_VIEW);
+ consensus.init();
+ consensus.restore_commit_state(COMMITTED, COMMITTED);
+ let mut partition = IggyPartition::with_in_memory_storage(
+ Arc::new(PartitionStats::default()),
+ consensus,
+ shard.plane.partitions().config().segment_size,
+ );
+ partition.set_created_revision(CREATED_REVISION);
+ (partition, probe)
+ }
+
+ #[test]
+ fn given_partition_probe_when_local_history_is_not_ready_should_refuse() {
+ for changed in [
+ "incarnation",
+ "view",
+ "observed newer view",
+ "prefix",
+ "unapplied",
+ "recovering",
+ "unproved",
+ "transfer",
+ "ceded",
+ "recovery barrier",
+ "transition",
+ ] {
+ let (mut partition, mut probe) = ready_partition(TARGET);
+ assert!(
+ partition_ready(&partition, &probe),
+ "ready fixture: {changed}"
+ );
+ match changed {
+ "incarnation" => probe.created_revision += 1,
+ "view" => probe.partition_view += 1,
+ "observed newer view" => partition.consensus().observe_newer_view(PARTITION_VIEW + 1),
+ "prefix" => probe.commit += 1,
+ "unapplied" => partition.consensus().advance_commit_max(COMMITTED + 1),
+ "recovering" => partition.consensus().begin_view_probe(),
+ "unproved" => partition.consensus().fence_view_log(COMMITTED, COMMITTED),
+ "transfer" => partition
+ .consensus()
+ .set_state_transfer_stage(StateTransferStage::AwaitingTarget),
+ "ceded" => partition.consensus().init_as_backup(),
+ "recovery barrier" => partition.consensus().set_recovery_barrier(COMMITTED + 1),
+ "transition" => {
+ partition.defer_view_truncation((COMMITTED + 1)..=u64::MAX, Vec::new(), None);
+ }
+ _ => unreachable!(),
+ }
+ assert!(!partition_ready(&partition, &probe), "{changed}");
+ }
+ }
+
+ #[test]
+ fn given_busy_source_history_when_capturing_frontier_should_allow_drain_but_not_target_readiness() {
+ let (mut partition, probe) = ready_partition(SOURCE);
+ assert!(partition.consensus().is_primary());
+ partition.consensus().advance_commit_max(COMMITTED + 1);
+ assert!(partition_history_ready(&partition, &probe));
+ assert!(!partition_ready(&partition, &probe));
+ let request = PreparedPrimaryTransfer {
+ created_revision: CREATED_REVISION,
+ expected_view: PARTITION_VIEW,
+ target: TARGET,
+ expires_at: Instant::now() + GRANT_TIMEOUT,
+ };
+ assert!(partition.defer_primary_transfer(request));
+ assert!(partition.history_is_busy());
+ assert!(partition_history_ready(&partition, &probe));
+ assert!(!partition_ready(&partition, &probe));
+ }
+
+ #[compio::test]
+ async fn given_three_or_five_replicas_when_a_quorum_answers_not_ready_should_credit_probe_send_times()
+ {
+ for replicas in [3, 5] {
+ let (shard, _inbox) = owner_at::<PingPongSuperblock>(None, 0, replicas, 0);
+ let consensus = shard.plane.metadata().consensus.as_ref().unwrap();
+ let frontier = shard.plane.metadata().applied_frontier();
+ let epoch = shard.bus.monotonic_now();
+ shard.tick_primary_rebalance().await;
+ assert_eq!(shard.bus.sent.borrow().len(), usize::from(replicas - 1));
+ assert!(
+ shard
+ .bus
+ .sent
+ .borrow()
+ .iter()
+ .all(|(_, header)| header.quorum_age_ns
+ == PrimaryRebalanceHeader::NO_QUORUM_EVIDENCE)
+ );
+ assert!(!frontier.partition_requests_allowed(epoch));
+ shard.bus.now.set(Some(epoch + Duration::from_secs(3)));
+ for peer in 1..=2 {
+ let mut reply = shard.bus.sent.borrow()[usize::from(peer - 1)].1;
+ reply.replica = peer;
+ reply.kind = PrimaryRebalanceKind::MetadataReply;
+ reply.ready = 0;
+ // The second challenge was sent later; receipt time is identical.
+ if peer == 2 {
+ shard.primary_rebalance.borrow_mut().peers[usize::from(peer)].sent_at =
+ Some(epoch + Duration::from_secs(2));
+ }
+ shard.on_primary_rebalance(reply).await;
+ assert_eq!(
+ frontier.partition_requests_allowed(shard.bus.monotonic_now()),
+ replicas == 3 || peer == 2
+ );
+ }
+ let deadline = epoch + Duration::from_secs(if replicas == 3 { 10 } else { 8 });
+ assert!(
+ frontier
+ .partition_requests_allowed(deadline.checked_sub(Duration::from_nanos(1)).unwrap())
+ );
+ assert!(!frontier.partition_requests_allowed(deadline));
+ assert_eq!(
+ shard
+ .primary_rebalance
+ .borrow()
+ .quorum_evidence(consensus.quorum_replication()),
+ Some(epoch + Duration::from_secs(if replicas == 3 { 2 } else { 0 }))
+ );
+ consensus.begin_view_probe();
+ shard.tick_primary_rebalance().await;
+ assert!(
+ frontier
+ .partition_requests_allowed(deadline.checked_sub(Duration::from_nanos(1)).unwrap()),
+ "authority loss must not shorten credit"
+ );
+ assert!(!frontier.partition_requests_allowed(deadline));
+ }
+ }
+
+ #[compio::test]
+ async fn given_a_refused_probe_when_the_interval_elapses_should_probe_again_without_waiting_for_the_timeout()
+ {
+ let (shard, _inbox) = owner_at::<PingPongSuperblock>(None, 0, REPLICAS, 0);
+ let epoch = shard.bus.monotonic_now();
+ shard.bus.disconnected.set(true);
+ shard.tick_primary_rebalance().await;
+ shard.bus.disconnected.set(false);
+ shard.bus.now.set(Some(
+ (epoch + PROBE_INTERVAL)
+ .checked_sub(Duration::from_nanos(1))
+ .unwrap(),
+ ));
+ shard.tick_primary_rebalance().await;
+ assert!(
+ shard.bus.sent.borrow().is_empty(),
+ "a refused probe must still pace the peer"
+ );
+ shard.bus.now.set(Some(epoch + PROBE_INTERVAL));
+ shard.tick_primary_rebalance().await;
+ assert_eq!(shard.bus.sent.borrow().len(), usize::from(REPLICAS - 1));
+ }
+
+ #[compio::test]
+ async fn given_admission_opening_when_peers_answered_earlier_probes_should_reprobe_each_once_with_evidence()
+ {
+ let (primary, _inbox) = owner_at::<PingPongSuperblock>(None, 0, REPLICAS, 0);
+ let (backup, _backup_inbox) = owner_at::<PingPongSuperblock>(None, 1, REPLICAS, 0);
+ let epoch = primary.bus.monotonic_now();
+ let advance = |millis| {
+ primary
+ .bus
+ .now
+ .set(Some(epoch + Duration::from_millis(millis)));
+ };
+ let sent = |index: usize| primary.bus.sent.borrow()[index];
+ let answer = |index: usize| {
+ let (replica, mut reply) = sent(index);
+ reply.replica = replica;
+ reply.kind = PrimaryRebalanceKind::MetadataReply;
+ reply
+ };
+
+ primary.tick_primary_rebalance().await;
+ advance(10);
+ primary.on_primary_rebalance(answer(0)).await;
+ assert!(
+ primary
+ .plane
+ .metadata()
+ .applied_frontier()
+ .partition_requests_allowed(primary.bus.monotonic_now())
+ );
+ advance(20);
+ primary.tick_primary_rebalance().await;
+ assert_eq!(
+ primary.bus.sent.borrow().len(),
+ 3,
+ "only the peer that already answered is probed again"
+ );
+ let (replica, reprobe) = sent(2);
+ assert_eq!(replica, 1);
+ assert_ne!(
+ reprobe.quorum_age_ns,
+ PrimaryRebalanceHeader::NO_QUORUM_EVIDENCE
+ );
+
+ let backup_frontier = backup.plane.metadata().applied_frontier();
+ backup.on_primary_rebalance(sent(0).1).await;
+ assert!(!backup_frontier.partition_requests_allowed(backup.bus.monotonic_now()));
+ backup.on_primary_rebalance(reprobe).await;
+ assert!(backup_frontier.partition_requests_allowed(backup.bus.monotonic_now()));
+
+ advance(30);
+ primary.on_primary_rebalance(answer(1)).await;
+ advance(40);
+ primary.tick_primary_rebalance().await;
+ assert_eq!(primary.bus.sent.borrow().len(), 4);
+ let (replica, reprobe) = sent(3);
+ assert_eq!(replica, 2);
+ assert_ne!(
+ reprobe.quorum_age_ns,
+ PrimaryRebalanceHeader::NO_QUORUM_EVIDENCE
+ );
+
+ advance(50);
+ primary.on_primary_rebalance(answer(2)).await;
+ primary.on_primary_rebalance(answer(3)).await;
+ advance(60);
+ primary.tick_primary_rebalance().await;
+ assert_eq!(
+ primary.bus.sent.borrow().len(),
+ 4,
+ "one opening re-probes each peer once"
+ );
+ }
+
+ #[compio::test]
+ async fn given_backup_credit_when_probes_are_stale_replayed_or_lack_evidence_should_not_extend_it()
+ {
+ let (shard, _inbox) = owner_at::<PingPongSuperblock>(None, 1, 3, 0);
+ let consensus = shard.plane.metadata().consensus.as_ref().unwrap();
+ let frontier = shard.plane.metadata().applied_frontier();
+ let epoch = shard.bus.monotonic_now();
+ let mut probe = probe_header(consensus, 100);
+ probe.replica = 0;
+ probe.quorum_age_ns = 0;
+ let mut previous = probe;
+ previous.nonce -= 1;
+ shard.on_primary_rebalance(previous).await;
+ shard.on_primary_rebalance(probe).await;
+ assert!(frontier.partition_requests_allowed(epoch));
+ shard.bus.now.set(Some(epoch + Duration::from_secs(1)));
+ for altered in ["replay", "reordered", "view", "cluster", "primary"] {
+ let mut invalid = probe;
+ match altered {
+ "replay" => {}
+ "reordered" => invalid.nonce -= 1,
+ "view" => invalid.view += 1,
+ "cluster" => invalid.cluster += 1,
+ "primary" => invalid.replica = 2,
+ _ => unreachable!(),
+ }
+ shard.on_primary_rebalance(invalid).await;
+ assert!(
+ !frontier.partition_requests_allowed(epoch + Duration::from_secs(8)),
+ "{altered}"
+ );
+ }
+ shard.primary_rebalance.borrow_mut().reset_authority();
+ shard.on_primary_rebalance(probe).await;
+ assert!(!frontier.partition_requests_allowed(epoch + Duration::from_secs(8)));
+ probe.nonce += 1;
+ probe.quorum_age_ns = PrimaryRebalanceHeader::NO_QUORUM_EVIDENCE;
+ shard.on_primary_rebalance(probe).await;
+ probe.quorum_age_ns = 0;
+ shard.on_primary_rebalance(probe).await;
+ assert!(
+ !frontier.partition_requests_allowed(epoch + Duration::from_secs(8)),
+ "an absent-evidence probe is consumed too"
+ );
+ probe.nonce += 1;
+ probe.quorum_age_ns = u64::try_from(Duration::from_secs(8).as_nanos()).unwrap();
+ shard.on_primary_rebalance(probe).await;
+ assert!(!frontier.partition_requests_allowed(epoch + Duration::from_secs(8)));
+ probe.nonce += 1;
+ probe.quorum_age_ns = u64::try_from(Duration::from_secs(1).as_nanos()).unwrap();
+ shard.on_primary_rebalance(probe).await;
+ assert!(
+ !frontier.partition_requests_allowed(epoch + Duration::from_secs(8)),
+ "receipt must not hide the advertised evidence age"
+ );
+ probe.nonce += 1;
+ probe.quorum_age_ns = 0;
+ shard.on_primary_rebalance(probe).await;
+ assert!(frontier.partition_requests_allowed(epoch + Duration::from_secs(8)));
+ assert!(!frontier.partition_requests_allowed(epoch + Duration::from_secs(9)));
+ }
+
+ #[compio::test]
+ async fn given_an_adopted_view_when_its_head_is_not_applied_and_advertised_should_not_credit_a_probe()
+ {
+ let (shard, _inbox) = owner_at::<PingPongSuperblock>(None, 1, 3, 0);
+ let consensus = shard.plane.metadata().consensus.as_ref().unwrap();
+ let frontier = shard.plane.metadata().applied_frontier();
+ let epoch = shard.bus.monotonic_now();
+ let start = Message::<StartViewHeader>::new(HEADER_SIZE).transmute_header(
+ |_, header: &mut StartViewHeader| {
+ header.cluster = CLUSTER;
+ header.command = Command::StartView;
+ header.size = u32::try_from(HEADER_SIZE).unwrap();
+ header.group = METADATA_GROUP;
+ header.view = 3;
+ header.replica = 0;
+ header.op = COMMITTED;
+ header.commit = COMMITTED;
+ },
+ );
+ let mut header = *start.header();
+ consensus.handle_start_view(PlaneKind::Metadata, &header, &[]);
+ assert_eq!(consensus.adopted_view_head(), Some((3, COMMITTED)));
+ header.replica = 2;
+ header.op += 10;
+ consensus.handle_start_view(PlaneKind::Metadata, &header, &[]);
+ assert_eq!(
+ consensus.adopted_view_head(),
+ Some((3, COMMITTED)),
+ "rejected offers cannot raise the floor"
+ );
+ let mut probe = probe_header(consensus, NONCE);
+ probe.replica = 0;
+ probe.quorum_age_ns = 0;
+ probe.commit = COMMITTED - 1;
+ frontier.advance(COMMITTED - 1);
+ shard.on_primary_rebalance(probe).await;
+ assert!(!frontier.partition_requests_allowed(epoch));
+ probe.nonce += 1;
+ probe.commit = COMMITTED;
+ shard.on_primary_rebalance(probe).await;
+ assert!(!frontier.partition_requests_allowed(epoch));
+ frontier.advance(COMMITTED);
+ shard.on_primary_rebalance(probe).await;
+ assert!(
+ !frontier.partition_requests_allowed(epoch),
+ "an earlier discarded packet cannot become fresh by replay"
+ );
+ probe.nonce += 1;
+ shard.on_primary_rebalance(probe).await;
+ assert!(frontier.partition_requests_allowed(epoch));
+ }
+
+ #[test]
+ fn given_empty_primary_restart_with_lower_nonce_when_metadata_advances_should_resume_without_accepting_old_probes()
+ {
+ let (shard, _inbox) = owner();
+ let consensus = shard.plane.metadata().consensus.as_ref().unwrap();
+ let mut state = PrimaryRebalance::default();
+ let epoch = Instant::now();
+ let at = |seconds| epoch + Duration::from_secs(seconds);
+ let mut probe = probe_header(consensus, 100);
+ assert_eq!(state.accept_probe(&probe, at(0)), None);
+ probe.nonce = 1;
+ assert_eq!(state.accept_probe(&probe, at(1)), None);
+ probe.commit = 1;
+ assert_eq!(
+ state.accept_probe(&probe, at(2)),
+ Some(at(0)),
+ "the lower nonce must not have been accepted"
+ );
+ let mut old = probe;
+ old.commit = 0;
+ old.nonce = 101;
+ assert_eq!(state.accept_probe(&old, at(3)), None);
+ state.reset_authority();
+ assert_eq!(state.accept_probe(&probe, at(4)), None);
+ probe.nonce += 1;
+ assert_eq!(
+ state.accept_probe(&probe, at(5)),
+ Some(at(2)),
+ "replay protection must survive the authority reset"
+ );
+ }
+
+ #[compio::test]
+ async fn given_a_delayed_probe_when_crediting_a_backup_should_age_its_evidence_from_the_previous_probe()
+ {
+ let (shard, _inbox) = owner_at::<PingPongSuperblock>(None, 1, 3, 0);
+ let consensus = shard.plane.metadata().consensus.as_ref().unwrap();
+ let frontier = shard.plane.metadata().applied_frontier();
+ let epoch = shard.bus.monotonic_now();
+ let window = Duration::from_secs(5) + metadata::AppliedFrontier::PARTITION_FRESHNESS_MARGIN;
+ let mut probe = probe_header(consensus, NONCE);
+ probe.replica = 0;
+ probe.quorum_age_ns = 0;
+ shard.on_primary_rebalance(probe).await;
+ assert!(
+ !frontier.partition_requests_allowed(epoch),
+ "nothing bounds when the first probe of a view left the primary"
+ );
+ let transit = Duration::from_secs(3);
+ shard.bus.now.set(Some(epoch + transit));
+ probe.nonce += 1;
+ shard.on_primary_rebalance(probe).await;
+ assert!(
+ frontier.partition_requests_allowed(
+ (epoch + window)
+ .checked_sub(Duration::from_nanos(1))
+ .unwrap()
+ )
+ );
+ assert!(
+ !frontier.partition_requests_allowed(epoch + window),
+ "the transit of the probe must not extend the credit"
+ );
+ }
+
+ #[compio::test]
+ async fn given_an_overdue_probe_when_probing_again_should_withhold_evidence_until_the_peer_replies()
+ {
+ let (shard, _inbox) = owner_at::<PingPongSuperblock>(None, 0, REPLICAS, 0);
+ let epoch = shard.bus.monotonic_now();
+ let advance = |millis| {
+ shard
+ .bus
+ .now
+ .set(Some(epoch + Duration::from_millis(millis)));
+ };
+ let sent = |index: usize| shard.bus.sent.borrow()[index];
+ let answer = |index: usize| {
+ let (replica, mut reply) = sent(index);
+ reply.replica = replica;
+ reply.kind = PrimaryRebalanceKind::MetadataReply;
+ reply
+ };
+ let carries_evidence =
+ |index: usize| sent(index).1.quorum_age_ns != PrimaryRebalanceHeader::NO_QUORUM_EVIDENCE;
+
+ shard.tick_primary_rebalance().await;
+ advance(10);
+ shard.on_primary_rebalance(answer(0)).await;
+ advance(3_100);
+ shard.tick_primary_rebalance().await;
+ assert_eq!(sent(2).0, 1);
+ assert!(carries_evidence(2), "a peer that answered gets evidence");
+ assert_eq!(sent(3).0, 2);
+ assert!(
+ !carries_evidence(3),
+ "the overdue probe can still reach the peer after this one"
+ );
+ advance(3_200);
+ shard.on_primary_rebalance(answer(3)).await;
+ advance(4_200);
+ shard.tick_primary_rebalance().await;
+ assert_eq!(sent(4).0, 2);
+ assert!(carries_evidence(4), "the reply restores the evidence");
+ }
diff --cc core/simulator/src/lib.rs
index e7b8cf532,80c6351e0..3c02208e5
--- a/core/simulator/src/lib.rs
+++ b/core/simulator/src/lib.rs
@@@ -1971,24 -2152,28 +2169,29 @@@ fn stamp_metadata_watermark
#[cfg(test)]
mod tests {
- use super::partition_repair_driver_tests::{cluster, is_commit_for, sharded_cluster};
+ use super::partition_repair_driver_tests::{
+ cluster, is_commit_for, prepare_for, sharded_cluster,
+ };
use super::*;
- use crate::client::SimClient;
+ use crate::client::{SIM_ROTATED_WRAP_SET, SIM_STREAM_WRAP_SET, SimClient};
use crate::workload::apply_sim_commands;
use bytes::Bytes;
- use consensus::{Consensus, Project, Sequencer, Status, client_table::COMMITTED_WINDOW_BITS};
+ use consensus::{
+ Consensus, PlaneKind, Project, Sequencer, Status, client_table::COMMITTED_WINDOW_BITS,
+ };
use futures::FutureExt;
use iggy_binary_protocol::requests::partitions::InstallConsumerGroupOwnerRequest;
+ use iggy_binary_protocol::requests::streams::{RewrapStreamKeyEntry, RewrapStreamKeysRequest};
use iggy_binary_protocol::requests::users::CreateUserRequest;
use iggy_binary_protocol::{
- AckLevel, ConsensusHeader, Operation, PrepareOkHeader, RoutedRequestHeader, WireEncode,
- WireGlobalPermissions, WireIdentifier, WireName, WireOptions, WirePermissions,
+ AckLevel, ConsensusHeader, Operation, PrepareOkHeader, RoutedRequestHeader,
+ StartViewHeader, WireDecode, WireEncode, WireGlobalPermissions, WireIdentifier, WireName,
+ WireOptions, WirePermissions,
};
- use iggy_common::{ConsumerKind, IggyError};
+ use iggy_common::{ConsumerKind, HeaderKey, IggyError, OptionKey, WrapSet};
use journal::{Journal, JournalHandle};
- use message_bus::AUTO_COMMIT_CLIENT_ID;
use message_bus::lifecycle::Shutdown;
+ use message_bus::{AUTO_COMMIT_CLIENT_ID, ConnectionInstaller, MessageBus};
use metadata::stm::stream::Streams;
use server_common::MessageBag;
use server_common::sharding::{IggyNamespace, LIST_CLIENTS_GATHER_TIMEOUT, METADATA_GROUP};
diff --cc core/simulator/src/replica.rs
index bd2a94715,438ca2ae9..f9ae0b470
--- a/core/simulator/src/replica.rs
+++ b/core/simulator/src/replica.rs
@@@ -21,10 -20,14 +21,14 @@@ use crate::deps::SimSuperblock
use crate::deps::{MemStorage, SimJournal, SimMuxStateMachine, SimSnapshot};
use configs::server::PersonalAccessTokenConfig;
use configs::server::ServerConfig;
- use consensus::{ClientTable, ConsensusClock, LocalPipeline, Sequencer, VsrConsensus, VsrState};
+ use consensus::{
+ ClientTable, ConsensusClock, HistoryRecovery, LocalPipeline, Sequencer, VsrConsensus, VsrState,
+ };
+ use futures::FutureExt;
-use iggy_common::IggyByteSize;
use iggy_common::variadic;
+use iggy_common::{IggyByteSize, ServerKeyContext};
use journal::Journal;
+ use message_bus::MessageBus;
use metadata::impls::metadata::IggySnapshot;
use metadata::impls::metadata::StreamsFrontend;
use metadata::stm::mux::WithFactory;
diff --cc core/simulator/src/workload/actions.rs
index c4dffe319,64c6bd07f..696b660a5
--- a/core/simulator/src/workload/actions.rs
+++ b/core/simulator/src/workload/actions.rs
@@@ -52,7 -52,7 +52,8 @@@ pub enum Action
JoinFixtureGroup,
LeaveFixtureGroup,
StoreStaleOwnerOffset,
+ PollMessages,
+ CreateKeyedStream,
}
// Lock the discriminants of the first three variants. Inserting a new variant before
diff --cc core/simulator/src/workload/ops/mod.rs
index 07cb8f22b,c6639d8e5..a5eb37d9b
--- a/core/simulator/src/workload/ops/mod.rs
+++ b/core/simulator/src/workload/ops/mod.rs
@@@ -216,5 -216,5 +217,6 @@@ op_dispatch!
JoinFixtureGroup => join_fixture_group,
LeaveFixtureGroup => leave_fixture_group,
StoreStaleOwnerOffset => store_stale_owner_offset,
+ PollMessages => poll_messages,
+ CreateKeyedStream => create_keyed_stream,
}
diff --cc core/simulator/src/workload/oracle.rs
index ab53d7151,f302e964e..1ef4242fa
--- a/core/simulator/src/workload/oracle.rs
+++ b/core/simulator/src/workload/oracle.rs
@@@ -48,7 -50,9 +50,9 @@@ use crate::workload::
use crate::{Simulator, rebalancing_timeout_micros};
use consensus::{Consensus, MetadataHandle, PartitionsHandle, Status};
use iggy_binary_protocol::primitives::partition_history::ConsumerGroupOwner;
+ use iggy_binary_protocol::requests::partitions::RetireConsumerGroupOwnersRequest;
+ use iggy_binary_protocol::{Operation, WireDecode};
-use iggy_common::ConsumerKind;
+use iggy_common::{ConsumerKind, ResourceOptions, UserStatus};
use metadata::impls::metadata::StreamsFrontend;
use server_common::sharding::IggyNamespace;
use std::collections::{BTreeMap, BTreeSet};
@@@ -1051,16 -1122,25 +1179,28 @@@ fn shadow_metadata(shadow: &Shadow) ->
mod tests {
use super::*;
use crate::packet::PacketSimulatorOptions;
- use crate::tests::commit_sends_everywhere;
+ use crate::partition_repair_driver_tests::cluster;
- use crate::tests::complete_metadata_op;
++ use crate::tests::{commit_sends_everywhere, complete_metadata_op};
+ use crate::workload::actions::Action;
use crate::workload::effect::Effect;
- use crate::workload::options::WorkloadOptions;
+ use crate::workload::options::{ActionWeights, WorkloadOptions};
+ use crate::workload::{create_fixture_groups, run_group_lifecycle};
+ use iggy_binary_protocol::batch::{BATCH_HEADER_SIZE, BATCH_MESSAGE_HEADER_SIZE};
+ use iggy_binary_protocol::{HEADER_SIZE, Operation};
+ use iggy_common::{IggyDuration, IggyExpiry, TopicUpdateOptions};
+ use journal::Journal;
+ use server_common::iobuf::Owned;
+ use std::time::Duration;
const SEED: u64 = 0x000F_F5E7;
+ const CLIENT: u128 = 1;
const CONSUMER: u32 = 7;
const OFFSET: u64 = 9;
+ /// Batches the data fixture commits, one message each.
+ const SENDS: usize = 3;
+ /// First payload byte of a `SendMessages` prepare: past the prepare header,
+ /// the batch header and the frame header of its first message.
+ const FIRST_PAYLOAD_BYTE: usize = HEADER_SIZE + BATCH_HEADER_SIZE + BATCH_MESSAGE_HEADER_SIZE;
fn namespace() -> IggyNamespace {
IggyNamespace::new(1, 1, 0)
diff --cc foreign/cpp/BUILD.bazel
index a7cd98926,417d7d711..b1ca2a1c8
--- a/foreign/cpp/BUILD.bazel
+++ b/foreign/cpp/BUILD.bazel
@@@ -196,7 -196,7 +196,8 @@@ cc_binary
"tests/bdd/background_steps.cpp",
"tests/bdd/messaging_steps.cpp",
"tests/bdd/raw_command_steps.cpp",
+ "tests/bdd/resource_options_steps.cpp",
+ "tests/bdd/session_contract_steps.cpp",
"tests/bdd/world.hpp",
],
copts = [
diff --cc foreign/cpp/README.md
index 129eb0fd6,e98848e33..5c89d98fb
--- a/foreign/cpp/README.md
+++ b/foreign/cpp/README.md
@@@ -25,31 -25,17 +25,42 @@@ bazel test //:uni
bazel test //:e2e
```
+The E2E suite expects a local server on the default ports with encryption enabled, because the resource options cases create streams with their own keys and expect a server-generated key for keyless streams. Start it with `IGGY_ENCRYPTION_ENABLED=true` and `IGGY_ENCRYPTION_KEY` set to a base64 encoded 32-byte key, for example `IGGY_ENCRYPTION_ENABLED=true IGGY_ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64) cargo run --bin iggy-server -- --with-default-root-credentials`.
+
+ Receipt access follows the server's [session and revocation policy](../../core/server/README.md#sessions-and-retry-receipts).
+ An unresolved operation cannot recover its receipt after its original session
+ ends, even after a new login or account reactivation. A fresh send can duplicate
+ that operation.
+
+ Each node counts the messages in its stream, topic and system stats. After a
+ successful send, a node that is not the partition primary shows the new count
+ only after the next prepare or commit broadcast. When no other write follows,
+ this delay is at most `cluster.commit_broadcast_interval` (500 ms by default).
+ See [metadata freshness and counters](../../core/server/README.md#metadata-freshness-and-counters).
+
+## Resource options
+
+Streams, topics and users take typed create and update options. Unset settings use the server default on create and keep their stored value on update. `SetRawEntries()` and `RawReset()` name keys newer than this SDK build and leave the decision to the server. `DescribeOptions()` serves each scope's catalog.
+
+```cpp
+const std::string key_base64 = LoadStreamKey();
+const auto stream = client.CreateStream("payments", iggy::StreamCreateOptions().SetEncryptionKeyBase64(key_base64));
+const bool same_key = stream.EncryptionKeyFingerprint() == iggy::EncryptionKeyFingerprint(key_base64);
+
+const auto stream_id = iggy::Identifier::Numeric(stream.Id());
+const auto topic = client.CreateTopic(stream_id, "orders",
+ iggy::TopicCreateOptions()
+ .SetPartitionsCount(3)
+ .SetDurability(iggy::Durability::Persisted)
+ .SetPartitionResizePolicy(iggy::PartitionResizePolicy::Fixed));
+client.UpdateTopic(stream_id, iggy::Identifier::Numeric(topic.Id()),
+ iggy::TopicUpdateOptions()
+ .SetMessageExpiry(iggy::Expiry::Duration(3600000000))
+ .Reset(iggy::TopicOptionKey::MaxTopicSize));
+```
+
+A stream key must be 32 random bytes, never derived from a password, because reads expose its fingerprint. It travels inside the create request, so protecting it in transit with TLS is the caller's job. The server stores it wrapped and never returns it. Reads report only its source and fingerprints. Without a key, a server with encryption enabled generates one for the stream. An invalid key is refused before anything is sent.
+
Binary mutations retain an uncertain operation until it is resolved or abandoned.
`IggyException::OperationId()` carries an exact `std::uint64_t` for
`operation_uncertain`, `lane_blocked`, and `operation_not_retained`.
diff --cc foreign/cpp/include/iggy.hpp
index 013ae67b9,4c00beb92..1e6006752
--- a/foreign/cpp/include/iggy.hpp
+++ b/foreign/cpp/include/iggy.hpp
@@@ -29,8 -29,8 +29,9 @@@
#include <cstdint>
#include <limits>
#include <map>
+ #include <memory>
#include <optional>
+#include <set>
#include <stdexcept>
#include <string>
#include <string_view>
diff --cc foreign/cpp/src/client.rs
index 610bb5206,21aceed60..b9a62e9fe
--- a/foreign/cpp/src/client.rs
+++ b/foreign/cpp/src/client.rs
@@@ -93,20 -82,7 +93,15 @@@ impl ffi::ErrorInfo
fn record(&mut self, error: &IggyError) {
self.code = error.as_code();
self.kind = error.as_string().to_owned();
+ // HTTP reports an option refusal only in its body, binary transports by the error itself.
+ if let IggyError::HttpResponseError(_, body) = error
+ && let Ok(body) = serde_json::from_str::<HttpErrorBody>(body)
+ && is_option_error(&IggyError::from_code(body.id))
+ {
+ self.code = body.id;
+ self.kind = IggyError::from_code_as_string(body.id).to_owned();
+ }
- let operation = match error {
- IggyError::OperationUncertain(op)
- | IggyError::LaneBlocked(op)
- | IggyError::OperationNotRetained(op) => Some(*op),
- _ => None,
- };
+ let operation = error.operation_id();
self.has_operation_id = operation.is_some();
self.operation_id = operation.unwrap_or_default();
}
@@@ -675,8 -642,9 +669,9 @@@ impl Client
RUNTIME.block_on(async {
self.inner
- .update_topic(&rust_stream_id, &rust_topic_id, &topic_name, &update_options)
+ .update_topic(&rust_stream_id, &rust_topic_id, &update_options)
- .await.inspect_err(|error| error_info.record(error))
+ .await
+ .inspect_err(|error| error_info.record(error))
.map_err(|error| {
format!(
"Could not update topic '{rust_topic_id}' on stream '{rust_stream_id}': {error}"
diff --cc foreign/cpp/tests/unit/unit_tests.cpp
index 911256047,60e21f448..c97b19354
--- a/foreign/cpp/tests/unit/unit_tests.cpp
+++ b/foreign/cpp/tests/unit/unit_tests.cpp
@@@ -22,8 -21,8 +22,9 @@@
#include <limits>
#include <map>
#include <optional>
+#include <set>
#include <string>
+ #include <type_traits>
#include <utility>
#include <vector>
diff --cc foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj
index aba8afc71,e52a301a0..aa5a38bf3
--- a/foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj
+++ b/foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj
@@@ -67,12 -63,9 +67,15 @@@ under the License
<ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\raw_command.feature">
<Link>Features\raw_command.feature</Link>
</ReqnrollFeatureFile>
+ <ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\stream_options.feature">
+ <Link>Features\stream_options.feature</Link>
+ </ReqnrollFeatureFile>
+ <ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\resource_options_patch.feature">
+ <Link>Features\resource_options_patch.feature</Link>
+ </ReqnrollFeatureFile>
+ <ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\session_contract.feature">
+ <Link>Features\session_contract.feature</Link>
+ </ReqnrollFeatureFile>
</ItemGroup>
<ItemGroup>
diff --cc foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs
index f1c110f71,6b1afdbc8..f5b60e16c
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs
@@@ -72,12 -72,22 +72,28 @@@ public class IggyServerFixture : IAsync
/// </summary>
public static readonly TimeSpan SessionReportInterval = TimeSpan.FromMilliseconds(500);
+ private static readonly TimeSpan StatsConvergenceTimeout = TimeSpan.FromSeconds(10);
+ private static readonly TimeSpan StatsPollInterval = TimeSpan.FromMilliseconds(100);
+
+ /// <summary>Node-local counters catch up after partition commit broadcasts.</summary>
+ public static async Task<T> ReadConvergedStatsAsync<T>(Func<Task<T>> read, Func<T, bool> converged)
+ {
+ var deadline = Environment.TickCount64 + (long)StatsConvergenceTimeout.TotalMilliseconds;
+ var result = await read();
+ while (!converged(result) && Environment.TickCount64 < deadline)
+ {
+ await Task.Delay(StatsPollInterval);
+ result = await read();
+ }
+ return result;
+ }
+
+ /// <summary>
+ /// Server encryption key every node shares. With encryption enabled a stream created without its own key
+ /// gets one the server generates, so the stream options tests read a server sourced key.
+ /// </summary>
+ public static readonly string ServerEncryptionKey = "/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=";
+
/// <summary>
/// Extra environment variables for every node, layered over the cluster's base configuration.
/// Heartbeat verification is on for every fixture so the eviction tests share the general server
diff --cc foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs
index a52251258,66cc7d814..daba7e9d5
--- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs
@@@ -787,59 -912,30 +913,59 @@@ public sealed partial class TcpMessageS
}
// Typed, raw and replayed calls share this success path, and only the request body carries the new credentials.
- private void RefreshCommittedCredentials(int code, ReadOnlySpan<byte> body)
+ private void RefreshCommittedCredentials(int code, ReadOnlySpan<byte> body, ulong generation)
{
- if (code is not (CommandCodes.CHANGE_PASSWORD_CODE or CommandCodes.UPDATE_USER_CODE))
- {
- return;
- }
-
- var position = 0;
- var user = ReadIdentifier(body, ref position);
if (code == CommandCodes.CHANGE_PASSWORD_CODE)
{
+ var position = 0;
+ var user = ReadIdentifier(body, ref position);
var currentPasswordLength = body[position];
- RefreshRememberedCredentials(user, null, ReadName(body, position + 1 + currentPasswordLength));
+ RefreshRememberedCredentials(user, null, ReadName(body, position + 1 + currentPasswordLength), generation);
}
- else if (body[position] == 1)
+ else if (code == CommandCodes.UPDATE_RESOURCE_CODE && body.Length > 0 && body[0] == (byte)OptionsScope.User)
{
- RefreshRememberedCredentials(user, ReadName(body, position + 1), null, generation);
+ var position = 1;
+ var user = ReadIdentifier(body, ref position);
+ var resetCount = body[position++];
+ for (var reset = 0; reset < resetCount; reset++)
+ {
+ position += 1 + body[position];
+ }
+
+ if (PatchedUsername(body[position..]) is { } username)
+ {
- RefreshRememberedCredentials(user, username, null);
++ RefreshRememberedCredentials(user, username, null, generation);
+ }
}
static string ReadName(ReadOnlySpan<byte> body, int position) =>
Encoding.UTF8.GetString(body.Slice(position + 1, body[position]));
}
+ /// <summary>The username a user update sets, read from the options block that ends its body.</summary>
+ private static string? PatchedUsername(ReadOnlySpan<byte> options)
+ {
+ var position = 0;
+ while (position < options.Length)
+ {
+ var keyKind = options[position];
+ var keyLength = BinaryPrimitives.ReadInt32LittleEndian(options[(position + 1)..]);
+ var key = options.Slice(position + 5, keyLength);
+ position += 5 + keyLength;
+ var valueKind = options[position];
+ var valueLength = BinaryPrimitives.ReadInt32LittleEndian(options[(position + 1)..]);
+ var value = options.Slice(position + 5, valueLength);
+ position += 5 + valueLength;
+ if (keyKind == StringKind && valueKind == StringKind && key.SequenceEqual("username"u8))
+ {
+ return Encoding.UTF8.GetString(value);
+ }
+ }
+
+ return null;
+ }
+
- private void RefreshRememberedCredentials(Identifier user, string? username, string? password)
+ private void RefreshRememberedCredentials(Identifier user, string? username, string? password, ulong generation)
{
lock (_pollRoutingGate)
{
diff --cc foreign/csharp/Iggy_SDK/Vsr/VsrError.cs
index 3389571ca,ea9fa6ee5..b2c88eb35
--- a/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs
+++ b/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs
@@@ -36,11 -36,12 +36,13 @@@ internal static class VsrErro
internal const int INVALID_CREDENTIALS = 42;
internal const int INVALID_USERNAME = 43;
internal const int INVALID_PASSWORD = 44;
+ internal const int USER_INACTIVE = 47;
internal const int PERSONAL_ACCESS_TOKEN_ALREADY_EXISTS = 51;
internal const int INVALID_PERSONAL_ACCESS_TOKEN = 53;
+ internal const int PERSONAL_ACCESS_TOKEN_EXPIRED = 54;
internal const int TRANSIENT_NOT_COMMITTED = 57;
internal const int TRANSIENT_NOT_ACCEPTED = 58;
+ internal const int INVALID_ENCRYPTION_KEY = 70;
internal const int INVALID_SIZE_BYTES = 80;
internal const int REQUEST_TOO_OLD = 85;
internal const int HISTORY_UNAVAILABLE = 87;
diff --cc foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.SignIn.cs
index 000000000,7f040159e..0563212ba
mode 000000,100644..100644
--- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.SignIn.cs
+++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/NativeRecoveryTests.SignIn.cs
@@@ -1,0 -1,710 +1,710 @@@
+ // Licensed to the Apache Software Foundation (ASF) under one
+ // or more contributor license agreements. See the NOTICE file
+ // distributed with this work for additional information
+ // regarding copyright ownership. The ASF licenses this file
... 3365 lines suppressed ...