You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by pi...@apache.org on 2026/08/15 07:13:34 UTC
(iggy) branch master updated: feat(common)!: extensible key-value options for topics, streams, users (#3880)
This is an automated email from the ASF dual-hosted git repository.
spetz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 7c0fd6842 feat(common)!: extensible key-value options for topics, streams, users (#3880)
7c0fd6842 is described below
commit 7c0fd68421aa610897fdbc7331f1cb22848fb386
Author: Grzegorz Koszyk <11...@users.noreply.github.com>
AuthorDate: Sat Aug 15 09:13:29 2026 +0200
feat(common)!: extensible key-value options for topics, streams, users (#3880)
This PR reworks current Streams/Topics/Users metadata storage by
replacing the hard coded values with an extensible key value pair
storage. Additionally it moves some of the partition related settings
such as `messages_required_to_save` or `segment_size` from the static
configuration (config.toml) to topic property.
---------
Co-authored-by: Hubert Gruszecki <h....@gmail.com>
---
Cargo.lock | 3 -
Cargo.toml | 1 -
.../features/step_definitions/messaging_steps.cpp | 10 +-
bdd/go/tests/basic_messaging.go | 1 -
bdd/go/tests/tcp_test/stream_feature_get_by_id.go | 6 +-
bdd/go/tests/tcp_test/topic_feature_create.go | 20 +-
bdd/go/tests/tcp_test/topic_feature_update.go | 20 +-
bdd/go/tests/tcp_test/topic_steps.go | 4 +-
.../org/apache/iggy/bdd/BasicMessagingSteps.java | 1 -
bdd/rust/tests/steps/topics.rs | 12 +-
core/ai/mcp/src/service/mod.rs | 72 +-
core/ai/mcp/src/service/requests.rs | 23 +-
.../bench/src/actors/producer/client/high_level.rs | 1 -
core/bench/src/args/common.rs | 10 +
core/bench/src/benchmarks/benchmark.rs | 19 +-
core/binary_protocol/src/codec.rs | 43 +-
core/binary_protocol/src/codes.rs | 1 +
core/binary_protocol/src/dispatch.rs | 4 +
core/binary_protocol/src/lib.rs | 1 +
core/binary_protocol/src/primitives/mod.rs | 1 +
core/binary_protocol/src/primitives/options.rs | 499 +++++++
.../src/primitives/partition_assignment.rs | 11 +-
.../src/requests/streams/create_stream.rs | 44 +-
.../src/requests/streams/update_stream.rs | 54 +-
.../src/requests/system/describe_options.rs | 88 ++
core/binary_protocol/src/requests/system/mod.rs | 4 +
.../src/requests/topics/create_topic.rs | 108 +-
.../topics/create_topic_with_assignments.rs | 79 +-
.../src/requests/topics/update_topic.rs | 88 +-
.../src/requests/users/create_user.rs | 65 +-
.../src/requests/users/update_user.rs | 48 +-
.../src/responses/clients/get_client.rs | 2 +-
.../consumer_groups/get_consumer_group.rs | 2 +-
.../consumer_groups/sync_consumer_group.rs | 4 +-
.../src/responses/messages/send_messages.rs | 4 +-
.../src/responses/streams/get_stream.rs | 77 +-
.../src/responses/streams/get_streams.rs | 4 +-
.../src/responses/streams/stream_response.rs | 21 +-
.../src/responses/system/describe_options.rs | 220 +++
.../src/responses/system/get_cluster_metadata.rs | 2 +-
.../src/responses/system/get_stats.rs | 2 +-
core/binary_protocol/src/responses/system/mod.rs | 2 +
.../src/responses/topics/get_topic.rs | 24 +-
.../src/responses/topics/get_topics.rs | 48 +-
.../src/responses/users/get_user.rs | 3 +-
.../src/responses/users/get_users.rs | 4 +-
.../src/responses/users/user_response.rs | 21 +-
core/cli/src/args/common.rs | 30 +
core/cli/src/args/mod.rs | 9 +-
core/cli/src/args/system.rs | 9 +-
core/cli/src/args/topic.rs | 16 +-
.../src/commands/binary_streams/update_stream.rs | 16 +-
core/cli/src/commands/binary_system/mod.rs | 1 +
core/cli/src/commands/binary_system/options.rs | 86 ++
core/cli/src/commands/binary_system/session.rs | 6 +-
.../cli/src/commands/binary_topics/create_topic.rs | 38 +-
.../cli/src/commands/binary_topics/update_topic.rs | 37 +-
core/cli/src/commands/binary_users/update_user.rs | 6 +
core/cli/src/commands/cli_command.rs | 6 +
core/cli/src/logging.rs | 17 +-
core/cli/src/main.rs | 10 +-
core/common/src/error/iggy_error.rs | 10 +
core/common/src/http/streams/update_stream.rs | 6 +
core/common/src/http/topics/create_topic.rs | 20 +-
core/common/src/http/topics/update_topic.rs | 47 +-
core/common/src/http/users/update_user.rs | 5 +
core/common/src/lib.rs | 1 +
core/common/src/traits/binary_impls/streams.rs | 22 +-
core/common/src/traits/binary_impls/system.rs | 28 +-
core/common/src/traits/binary_impls/topics.rs | 27 +-
core/common/src/traits/binary_impls/users.rs | 7 +-
core/common/src/traits/stream_client.rs | 9 +-
core/common/src/traits/system_client.rs | 11 +-
core/common/src/traits/topic_client.rs | 21 +-
core/common/src/traits/user_client.rs | 2 +
core/common/src/types/message/user_headers.rs | 22 +
core/common/src/types/mod.rs | 1 +
core/common/src/types/options/mod.rs | 1531 ++++++++++++++++++++
core/common/src/types/stream/mod.rs | 8 +-
core/common/src/types/topic/mod.rs | 15 +-
core/common/src/types/user/user_info.rs | 7 +
core/common/src/wire_conversions.rs | 254 +++-
core/configs/src/common/defaults.rs | 33 +-
core/configs/src/common/displays.rs | 29 +-
core/configs/src/common/system.rs | 65 +-
core/configs/src/common/validators.rs | 27 +-
core/configs/src/configs_impl/file_provider.rs | 215 +++
core/configs/src/configs_impl/mod.rs | 2 +-
core/configs/src/lib.rs | 2 +-
core/configs/src/server_config/server.rs | 48 +-
core/configs/src/server_config/validators.rs | 101 +-
core/harness_derive/src/attrs.rs | 44 +-
core/harness_derive/src/codegen.rs | 26 +-
core/harness_derive/src/lib.rs | 6 +-
core/integration/src/harness/config/mod.rs | 2 +-
core/integration/src/harness/config/resolve.rs | 130 +-
core/integration/src/harness/handle/server.rs | 9 +-
.../src/harness/orchestrator/builder.rs | 9 +-
core/integration/src/harness/seeds.rs | 53 +-
.../test_consumer_group_create_command.rs | 12 +-
.../test_consumer_group_delete_command.rs | 12 +-
.../test_consumer_group_get_command.rs | 12 +-
.../test_consumer_group_list_command.rs | 12 +-
.../test_consumer_offset_get_command.rs | 10 +-
.../test_consumer_offset_set_command.rs | 10 +-
.../tests/cli/general/test_help_command.rs | 1 +
.../tests/cli/general/test_overview_command.rs | 1 +
.../cli/message/test_message_flush_command.rs | 12 +-
.../tests/cli/message/test_message_poll_command.rs | 10 +-
.../message/test_message_poll_to_file_command.rs | 10 +-
.../tests/cli/message/test_message_send_command.rs | 10 +-
.../message/test_message_send_from_file_command.rs | 10 +-
.../cli/partition/test_partition_create_command.rs | 15 +-
.../cli/partition/test_partition_delete_command.rs | 12 +-
.../tests/cli/stream/test_stream_purge_command.rs | 10 +-
.../tests/cli/system/test_stats_command.rs | 22 +-
.../tests/cli/topic/test_topic_create_command.rs | 32 +-
.../tests/cli/topic/test_topic_delete_command.rs | 12 +-
.../tests/cli/topic/test_topic_get_command.rs | 12 +-
.../tests/cli/topic/test_topic_list_command.rs | 12 +-
.../tests/cli/topic/test_topic_purge_command.rs | 10 +-
.../tests/cli/topic/test_topic_update_command.rs | 53 +-
.../tests/cluster/client_table_restart.rs | 5 +-
.../cluster/multi_shard_partition_convergence.rs | 10 +-
.../tests/cluster/partition_state_transfer.rs | 41 +-
.../tests/cluster/register_forwarding.rs | 10 +-
core/integration/tests/config_provider/mod.rs | 13 +-
.../tests/connectors/fixtures/doris/container.rs | 116 +-
.../connectors/fixtures/elasticsearch/container.rs | 62 +-
.../tests/connectors/fixtures/s3/fixture.rs | 58 +-
.../data_integrity/verify_after_server_restart.rs | 50 +-
.../verify_auto_commit_offset_replicates.rs | 10 +-
.../verify_cluster_replica_data_identical.rs | 20 +-
.../verify_consumer_group_partition_assignment.rs | 126 +-
core/integration/tests/sdk/consumer_group.rs | 10 +-
.../tests/sdk/consumer_group_membership.rs | 20 +-
core/integration/tests/sdk/consumer_offset.rs | 10 +-
core/integration/tests/sdk/mod.rs | 1 +
core/integration/tests/sdk/options.rs | 521 +++++++
core/integration/tests/sdk/producer/mod.rs | 10 +-
core/integration/tests/sdk/send_confirmation.rs | 10 +-
core/integration/tests/server/flush_vsr.rs | 10 +-
core/integration/tests/server/general.rs | 1 -
core/integration/tests/server/http_rbac.rs | 2 +-
core/integration/tests/server/http_vsr.rs | 61 +-
core/integration/tests/server/message_cleanup.rs | 11 +-
core/integration/tests/server/message_retrieval.rs | 102 +-
.../tests/server/partition_view_durability_vsr.rs | 10 +-
.../integration/tests/server/poll_semantics_vsr.rs | 30 +-
core/integration/tests/server/purge_delete.rs | 17 +-
core/integration/tests/server/purge_vsr.rs | 25 +-
.../server/scenarios/authentication_scenario.rs | 41 +-
.../concurrent_produce_consume_scenario.rs | 22 +-
.../tests/server/scenarios/concurrent_scenario.rs | 30 +-
...umer_group_auto_commit_reconnection_scenario.rs | 10 +-
...onsumer_group_duplicate_name_create_scenario.rs | 14 +-
.../scenarios/consumer_group_join_scenario.rs | 16 +-
...er_group_new_messages_after_restart_scenario.rs | 10 +-
.../consumer_group_offset_cleanup_scenario.rs | 10 +-
...h_multiple_clients_polling_messages_scenario.rs | 10 +-
...with_single_client_polling_messages_scenario.rs | 10 +-
.../consumer_timestamp_polling_scenario.rs | 10 +-
.../tests/server/scenarios/encryption_scenario.rs | 42 +-
.../scenarios/invalid_consumer_offset_scenario.rs | 11 +-
.../server/scenarios/log_rotation_scenario.rs | 15 +-
.../server/scenarios/message_cleanup_scenario.rs | 149 +-
.../server/scenarios/message_headers_scenario.rs | 10 +-
.../server/scenarios/message_size_scenario.rs | 10 +-
.../tests/server/scenarios/offset_scenario.rs | 27 +-
.../tests/server/scenarios/permissions_scenario.rs | 116 +-
.../server/scenarios/purge_delete_scenario.rs | 195 ++-
.../scenarios/read_during_persistence_scenario.rs | 21 +-
.../scenarios/reconnect_after_restart_scenario.rs | 58 +-
.../scenarios/restart_offset_skip_scenario.rs | 15 +-
.../scenarios/segment_rotation_race_scenario.rs | 31 +-
.../scenarios/single_message_per_batch_scenario.rs | 14 +-
.../stale_client_consumer_group_scenario.rs | 48 +-
.../scenarios/stream_size_validation_scenario.rs | 10 +-
.../tests/server/scenarios/system_scenario.rs | 69 +-
.../tests/server/scenarios/tcp_tls_scenario.rs | 10 +-
.../tests/server/scenarios/timestamp_scenario.rs | 27 +-
.../tests/server/scenarios/user_scenario.rs | 2 +
.../server/scenarios/websocket_tls_scenario.rs | 10 +-
core/integration/tests/server/specific.rs | 34 +-
.../tests/server/topic_admission_vsr.rs | 148 +-
core/metadata/src/impls/metadata.rs | 150 +-
core/metadata/src/stm/authz.rs | 10 +-
core/metadata/src/stm/consumer_group.rs | 13 +-
core/metadata/src/stm/result.rs | 10 +-
core/metadata/src/stm/snapshot.rs | 168 ++-
core/metadata/src/stm/stream.rs | 359 ++++-
core/metadata/src/stm/user.rs | 33 +-
core/partitions/src/iggy_partition.rs | 97 +-
core/partitions/src/messages_writer.rs | 46 +-
core/partitions/src/state_transfer.rs | 16 +-
.../src/client_wrappers/binary_stream_client.rs | 20 +-
.../src/client_wrappers/binary_system_client.rs | 14 +-
.../sdk/src/client_wrappers/binary_topic_client.rs | 133 +-
core/sdk/src/client_wrappers/binary_user_client.rs | 22 +-
core/sdk/src/clients/binary_streams.rs | 10 +-
core/sdk/src/clients/binary_system.rs | 8 +-
core/sdk/src/clients/binary_topics.rs | 33 +-
core/sdk/src/clients/binary_users.rs | 4 +-
core/sdk/src/clients/producer.rs | 22 +-
core/sdk/src/clients/producer_builder.rs | 5 -
core/sdk/src/http/streams.rs | 9 +-
core/sdk/src/http/system.rs | 9 +
core/sdk/src/http/topics.rs | 41 +-
core/sdk/src/http/users.rs | 3 +
core/sdk/src/prelude.rs | 20 +-
.../stream_builder/build/build_iggy_producer.rs | 2 -
.../src/stream_builder/build/build_stream_topic.rs | 13 +-
.../stream_builder/config/config_iggy_consumer.rs | 14 -
.../stream_builder/config/config_iggy_producer.rs | 15 -
core/sdk/src/vsr.rs | 3 +-
core/server/config.toml | 94 +-
core/server/src/bootstrap.rs | 188 ++-
core/server/src/dispatch.rs | 146 +-
core/server/src/dispatch/authz.rs | 9 +-
core/server/src/http.rs | 7 +-
core/server/src/http/handlers.rs | 179 ++-
core/server/src/partition_helpers.rs | 66 +-
core/server/src/partition_reconciler.rs | 40 +-
core/server/src/responses.rs | 232 ++-
core/server/src/segment_cleaner.rs | 8 +-
core/server/src/segment_recovery.rs | 5 +-
core/server_common/Cargo.toml | 3 -
.../src/deduplication/message_deduplicator.rs | 125 --
core/server_common/src/deduplication/mod.rs | 20 -
core/server_common/src/lib.rs | 2 -
core/server_common/src/messages_batch_mut.rs | 89 +-
core/simulator/src/client.rs | 16 +-
core/simulator/src/workload/ops/create_stream.rs | 5 +-
core/simulator/src/workload/ops/create_topic.rs | 3 +
core/simulator/src/workload/ops/create_user.rs | 11 +-
core/simulator/src/workload/ops/update_stream.rs | 3 +
core/simulator/src/workload/ops/update_topic.rs | 3 +
core/simulator/src/workload/ops/update_user.rs | 10 +-
core/tools/src/data-seeder/seeder.rs | 50 +-
examples/go/getting-started/producer/main.go | 3 +-
.../apache/iggy/examples/async/AsyncProducer.java | 2 -
.../producer/GettingStartedProducer.java | 11 +-
.../producer/MessageEnvelopeProducer.java | 8 +-
.../producer/MessageHeadersProducer.java | 8 +-
.../multitenant/producer/MultiTenantProducer.java | 1 -
.../sinkdataproducer/SinkDataProducer.java | 1 -
.../iggy/examples/streambuilder/StreamBasic.java | 8 +-
.../examples/tcptls/producer/TcpTlsProducer.java | 11 +-
examples/node/src/multi-tenant/producer.ts | 1 -
examples/node/src/sink-data-producer/producer.ts | 2 -
examples/node/src/stream-builder/example.ts | 1 -
examples/node/src/utils/index.ts | 1 -
examples/python/basic/producer.py | 1 -
examples/python/getting-started/producer.py | 1 -
examples/python/message-headers/common.py | 1 -
examples/rust/src/getting-started/producer/main.rs | 10 +-
.../message-compression/producer/main.rs | 11 +-
examples/rust/src/multi-tenant/producer/main.rs | 1 -
examples/rust/src/new-sdk/producer/main.rs | 1 -
examples/rust/src/shared/system.rs | 13 +-
.../stream-builder/stream-producer-config/main.rs | 3 -
examples/rust/src/tcp-tls/producer/main.rs | 10 +-
foreign/cpp/include/iggy.hpp | 118 ++
foreign/cpp/src/client.rs | 103 +-
foreign/cpp/src/lib.rs | 43 +-
foreign/cpp/src/type_conversion.rs | 70 +-
foreign/cpp/tests/e2e/client.cpp | 72 +-
foreign/cpp/tests/e2e/consumer_group.cpp | 156 +-
foreign/cpp/tests/e2e/message.cpp | 152 +-
foreign/cpp/tests/e2e/partition.cpp | 52 +-
foreign/cpp/tests/e2e/stream.cpp | 44 +-
foreign/cpp/tests/e2e/topic.cpp | 485 ++++---
foreign/cpp/tests/unit/unit_tests.cpp | 83 ++
foreign/csharp/Benchmarks/Program.cs | 1 -
.../Fixtures/VsrCluster.cs | 1 -
.../Iggy_SDK.Tests.Integration/TopicsTests.cs | 16 +-
.../csharp/Iggy_SDK/Contracts/Auth/UserResponse.cs | 8 +
.../Iggy_SDK/Contracts/Http/CreateTopicRequest.cs | 14 +-
.../Iggy_SDK/Contracts/Http/UpdateTopicRequest.cs | 6 +-
foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs | 71 +
.../csharp/Iggy_SDK/Contracts/StreamResponse.cs | 7 +
.../csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs | 240 ++-
foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs | 106 ++
foreign/csharp/Iggy_SDK/Contracts/TopicResponse.cs | 39 +-
foreign/csharp/Iggy_SDK/Headers/HeaderValue.cs | 14 +
foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs | 18 +
foreign/csharp/Iggy_SDK/IggyClient/IIggyTopic.cs | 26 +-
.../Implementations/HttpMessageStream.cs | 87 +-
.../IggyClient/Implementations/TcpMessageStream.cs | 31 +-
.../JsonConverters/ResourceOptionsConverter.cs | 140 ++
.../JsonConverters/UserHeadersConverter.cs | 2 +-
foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 212 ++-
.../csharp/Iggy_SDK/Publishers/IggyPublisher.cs | 8 +-
.../Iggy_SDK/Publishers/IggyPublisherBuilder.cs | 4 +-
.../Iggy_SDK/Publishers/IggyPublisherConfig.cs | 7 -
foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs | 1 +
.../ClientTests/HttpTopicOptionsTests.cs | 179 +++
.../ContractsTests/TopicContractsTests.cs | 103 ++
.../Iggy_SDK_Tests/MapperTests/BinaryMapper.cs | 105 +-
.../MapperTests/OptionsBlockGoldenVectorTests.cs | 109 ++
.../MapperTests/ResourceOptionsConverterTests.cs | 187 +++
.../PublisherTests/IggyTypedPublisherTests.cs | 3 +-
.../UtilityTests/TopicOptionsTests.cs | 70 +
.../csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs | 44 +-
.../Iggy_SDK_Tests/Utils/Topics/TopicFactory.cs | 5 +-
foreign/csharp/README.md | 1 -
.../go/benchmarks/send_messages_benchmark_test.go | 1 -
.../binary_response_deserializer.go | 201 ++-
.../binary_response_deserializer_test.go | 96 +-
.../vsr_response_deserializer_test.go | 80 +-
foreign/go/client/tcp/tcp_core_test.go | 12 +-
.../go/client/tcp/tcp_stream_management_test.go | 6 +-
foreign/go/client/tcp/tcp_topic_management.go | 15 +-
foreign/go/client/tcp/tcp_topic_management_test.go | 29 +-
foreign/go/client/tcp/tcp_utilities.go | 14 +
foreign/go/contracts/client.go | 16 +-
foreign/go/contracts/compression_algorithm.go | 39 +-
foreign/go/contracts/compression_algorithm_test.go | 90 ++
foreign/go/contracts/options.go | 70 +
foreign/go/contracts/stream.go | 13 +-
foreign/go/contracts/topic_options.go | 98 ++
foreign/go/contracts/topic_options_test.go | 135 ++
foreign/go/contracts/topics.go | 23 +-
foreign/go/contracts/users.go | 9 +-
foreign/go/internal/command/code.go | 1 +
foreign/go/internal/command/stream.go | 11 +-
foreign/go/internal/command/system.go | 22 +-
foreign/go/internal/command/topic.go | 187 ++-
foreign/go/internal/command/topic_test.go | 229 ++-
foreign/go/internal/command/update_user.go | 29 +-
foreign/go/internal/command/update_user_test.go | 76 +
foreign/go/tests/e2e_helpers_test.go | 2 +-
foreign/go/tests/e2e_test.go | 142 ++
.../common/provision/ResourceProvisioner.java | 2 -
.../org/apache/iggy/client/async/SystemClient.java | 14 +
.../org/apache/iggy/client/async/TopicsClient.java | 55 +-
.../iggy/client/async/tcp/StreamsTcpClient.java | 6 +
.../iggy/client/async/tcp/SystemTcpClient.java | 19 +
.../iggy/client/async/tcp/TopicsTcpClient.java | 91 +-
.../iggy/client/async/tcp/UsersTcpClient.java | 6 +
.../apache/iggy/client/blocking/SystemClient.java | 12 +
.../apache/iggy/client/blocking/TopicsClient.java | 62 +-
.../client/blocking/http/StreamsHttpClient.java | 37 +-
.../client/blocking/http/SystemHttpClient.java | 24 +
.../client/blocking/http/TopicsHttpClient.java | 139 +-
.../iggy/client/blocking/tcp/SystemTcpClient.java | 7 +
.../iggy/client/blocking/tcp/TopicsTcpClient.java | 14 +-
.../java/org/apache/iggy/message/HeaderValue.java | 37 +-
.../org/apache/iggy/serde/BytesDeserializer.java | 151 +-
.../org/apache/iggy/serde/BytesSerializer.java | 24 +
.../java/org/apache/iggy/serde/CommandCode.java | 1 +
.../java/org/apache/iggy/stream/StreamBase.java | 18 +-
.../java/org/apache/iggy/system/OptionSpec.java | 36 +
.../StreamBase.java => system/OptionsScope.java} | 22 +-
.../src/main/java/org/apache/iggy/topic/Topic.java | 26 +-
.../java/org/apache/iggy/topic/TopicDetails.java | 22 +-
.../java/org/apache/iggy/topic/TopicOptions.java | 109 ++
.../client/async/AsyncClientIntegrationTest.java | 24 +-
.../client/async/AsyncConnectionPoolAuthTest.java | 3 -
.../iggy/client/async/AsyncConsumerGroupsTest.java | 18 +-
.../async/tcp/TopicsTcpClientPayloadTest.java | 116 ++
.../iggy/client/blocking/IntegrationTest.java | 2 -
.../iggy/client/blocking/SystemClientBaseTest.java | 27 +
.../iggy/client/blocking/TopicsClientBaseTest.java | 46 +-
.../client/blocking/tcp/TlsConnectionTest.java | 10 +-
.../client/blocking/tcp/TopicsTcpClientTest.java | 43 +
.../org/apache/iggy/message/HeaderValueTest.java | 25 +
.../apache/iggy/serde/BytesDeserializerTest.java | 65 +-
.../iggy/serde/OptionsBlockGoldenVectorTest.java | 65 +
.../org/apache/iggy/stream/StreamDetailsTest.java | 8 +-
.../org/apache/iggy/topic/TopicDetailsTest.java | 7 +-
.../org/apache/iggy/topic/TopicOptionsTest.java | 115 ++
foreign/node/src/e2e/tcp.topic.e2e.ts | 3 +-
foreign/node/src/wire/command-set.ts | 4 +-
foreign/node/src/wire/command.code.ts | 1 +
foreign/node/src/wire/options.utils.test.ts | 139 ++
foreign/node/src/wire/options.utils.ts | 192 +++
foreign/node/src/wire/stream/stream.utils.ts | 11 +-
.../src/wire/system/describe-options.command.ts | 103 ++
foreign/node/src/wire/system/index.ts | 1 +
.../src/wire/topic/create-topic.command.test.ts | 94 +-
.../node/src/wire/topic/create-topic.command.ts | 97 +-
foreign/node/src/wire/topic/topic.utils.test.ts | 67 +-
foreign/node/src/wire/topic/topic.utils.ts | 59 +-
.../node/src/wire/topic/update-topic.command.ts | 55 +-
foreign/node/src/wire/user/user.utils.ts | 10 +-
foreign/php/iggy-php.stubs.php | 10 +-
foreign/php/src/client.rs | 35 +-
foreign/python/apache_iggy.pyi | 118 +-
foreign/python/src/client.rs | 139 +-
foreign/python/src/lib.rs | 3 +
foreign/python/src/options.rs | 109 ++
foreign/python/src/topic.rs | 63 +-
foreign/python/src/user_headers.rs | 12 +
foreign/python/tests/test_topic.py | 235 ++-
package-lock.json | 110 ++
package.json | 5 +
.../performance/run-standard-performance-suite.sh | 6 +-
scripts/performance/utils.sh | 11 +-
399 files changed, 14653 insertions(+), 4024 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 9a69b24d1..53a0f0e82 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11943,8 +11943,6 @@ dependencies = [
"human-repr",
"iggy_binary_protocol",
"iggy_common",
- "lending-iterator",
- "moka",
"nix",
"opentelemetry",
"opentelemetry-appender-tracing",
@@ -11961,7 +11959,6 @@ dependencies = [
"smallvec",
"tempfile",
"thiserror 2.0.19",
- "tokio",
"tracing",
"tracing-appender",
"tracing-opentelemetry",
diff --git a/Cargo.toml b/Cargo.toml
index 7820ec9cb..18a95963f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -227,7 +227,6 @@ metadata = { path = "core/metadata" }
mimalloc = "0.1"
mime_guess = "2.0"
mockall = "0.15.0"
-moka = { version = "0.12.15", features = ["future"] }
mongodb = { version = "3.8.0", features = ["rustls-tls"] }
nix = { version = "0.31.3", features = ["feature", "fs", "resource", "sched"] }
nonzero_lit = "0.1.2"
diff --git a/bdd/cpp/features/step_definitions/messaging_steps.cpp b/bdd/cpp/features/step_definitions/messaging_steps.cpp
index df4afc7db..6be384e28 100644
--- a/bdd/cpp/features/step_definitions/messaging_steps.cpp
+++ b/bdd/cpp/features/step_definitions/messaging_steps.cpp
@@ -78,9 +78,9 @@ WHEN("^I create a topic with name \"([^\"]{1,255})\" in stream ([0-9]+) with ([0
context->client->create_topic(bdd::make_numeric_identifier(static_cast<std::uint32_t>(stream_id)), topic_name,
static_cast<std::uint32_t>(partitions_count),
- std::string(compression.compression_algorithm_value()), 0,
+ std::string(compression.compression_algorithm_value()),
std::string(message_expiry.expiry_kind()), message_expiry.expiry_value(),
- std::string(max_topic_size.max_topic_size()));
+ std::string(max_topic_size.max_topic_size()), {});
}
THEN("^the topic should be created successfully$") {
@@ -117,9 +117,9 @@ WHEN("^I send ([0-9]+) messages to stream ([0-9]+), topic ([0-9]+), partition ([
rust::Vec<iggy::ffi::IggyMessageToSend> messages;
for (int index = 0; index < message_count; ++index) {
- iggy::ffi::IggyMessageToSend message = iggy::ffi::make_message(
- bdd::to_payload(bdd::expected_payload(static_cast<std::uint32_t>(index))),
- rust::Vec<iggy::ffi::HeaderEntry>());
+ iggy::ffi::IggyMessageToSend message =
+ iggy::ffi::make_message(bdd::to_payload(bdd::expected_payload(static_cast<std::uint32_t>(index))),
+ rust::Vec<iggy::ffi::HeaderEntry>());
// Assign an explicit, 1-based id so the last-sent/last-polled comparison is meaningful.
message.id_lo = static_cast<std::uint64_t>(index + 1);
messages.push_back(std::move(message));
diff --git a/bdd/go/tests/basic_messaging.go b/bdd/go/tests/basic_messaging.go
index 960543c9f..e5d7089f6 100644
--- a/bdd/go/tests/basic_messaging.go
+++ b/bdd/go/tests/basic_messaging.go
@@ -269,7 +269,6 @@ func (s basicMessagingSteps) whenCreateTopic(ctx context.Context,
iggcon.CompressionAlgorithmNone,
iggcon.IggyExpiryNeverExpire,
0,
- nil,
)
if err != nil {
return fmt.Errorf("failed to create topic: %w", err)
diff --git a/bdd/go/tests/tcp_test/stream_feature_get_by_id.go b/bdd/go/tests/tcp_test/stream_feature_get_by_id.go
index 41e7e62e6..71d4cc497 100644
--- a/bdd/go/tests/tcp_test/stream_feature_get_by_id.go
+++ b/bdd/go/tests/tcp_test/stream_feature_get_by_id.go
@@ -63,8 +63,7 @@ var _ = ginkgo.Describe("GET STREAM BY ID:", func() {
2,
iggcon.CompressionAlgorithmNone,
iggcon.Millisecond,
- math.MaxUint64,
- nil)
+ math.MaxUint64)
itShouldNotReturnError(err)
t2, err := client.CreateTopic(
context.Background(),
@@ -73,8 +72,7 @@ var _ = ginkgo.Describe("GET STREAM BY ID:", func() {
2,
iggcon.CompressionAlgorithmNone,
iggcon.Millisecond,
- math.MaxUint64,
- nil)
+ math.MaxUint64)
itShouldNotReturnError(err)
itShouldSuccessfullyCreateTopic(streamId, t1.Id, t1Name, client)
itShouldSuccessfullyCreateTopic(streamId, t2.Id, t2Name, client)
diff --git a/bdd/go/tests/tcp_test/topic_feature_create.go b/bdd/go/tests/tcp_test/topic_feature_create.go
index d5777601b..902abcd2b 100644
--- a/bdd/go/tests/tcp_test/topic_feature_create.go
+++ b/bdd/go/tests/tcp_test/topic_feature_create.go
@@ -32,7 +32,6 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() {
ginkgo.Context("and tries to create topic unique name and id", func() {
client := createAuthorizedConnection()
streamId, _ := successfullyCreateStream(prefix, client)
- replicationFactor := uint8(1)
name := createRandomString(32)
defer deleteStreamAfterTests(streamId, client)
streamIdentifier, _ := iggcon.NewIdentifier(streamId)
@@ -43,8 +42,7 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() {
2,
iggcon.CompressionAlgorithmNone,
iggcon.Millisecond,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldNotReturnError(err)
})
@@ -52,7 +50,6 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() {
ginkgo.Context("and tries to create topic for a non existing stream", func() {
client := createAuthorizedConnection()
streamId := createRandomUInt32()
- replicationFactor := uint8(1)
name := createRandomString(32)
streamIdentifier, _ := iggcon.NewIdentifier(streamId)
_, err := client.CreateTopic(
@@ -62,8 +59,7 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() {
2,
iggcon.CompressionAlgorithmNone,
iggcon.Millisecond,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound)
})
@@ -74,7 +70,6 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() {
defer deleteStreamAfterTests(streamId, client)
_, name := successfullyCreateTopic(streamId, client)
- replicationFactor := uint8(1)
streamIdentifier, _ := iggcon.NewIdentifier(streamId)
_, err := client.CreateTopic(
context.Background(),
@@ -83,8 +78,7 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() {
2,
iggcon.CompressionAlgorithmNone,
iggcon.IggyExpiryServerDefault,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldReturnSpecificError(err, ierror.ErrTopicNameAlreadyExists)
})
@@ -94,7 +88,6 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() {
defer deleteStreamAfterTests(streamId, createAuthorizedConnection())
streamIdentifier, _ := iggcon.NewIdentifier(streamId)
- replicationFactor := uint8(1)
_, err := client.CreateTopic(
context.Background(),
streamIdentifier,
@@ -102,8 +95,7 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() {
2,
iggcon.CompressionAlgorithmNone,
iggcon.IggyExpiryServerDefault,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldReturnSpecificError(err, ierror.ErrInvalidTopicName)
})
@@ -112,7 +104,6 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() {
ginkgo.When("User is not logged in", func() {
ginkgo.Context("and tries to create topic", func() {
client := createClient()
- replicationFactor := uint8(1)
streamIdentifier, _ := iggcon.NewIdentifier[uint32](10)
_, err := client.CreateTopic(
context.Background(),
@@ -121,8 +112,7 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() {
2,
iggcon.CompressionAlgorithmNone,
iggcon.IggyExpiryServerDefault,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldReturnUnauthenticatedError(err)
})
diff --git a/bdd/go/tests/tcp_test/topic_feature_update.go b/bdd/go/tests/tcp_test/topic_feature_update.go
index e8ce2e527..292883021 100644
--- a/bdd/go/tests/tcp_test/topic_feature_update.go
+++ b/bdd/go/tests/tcp_test/topic_feature_update.go
@@ -35,7 +35,6 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() {
defer deleteStreamAfterTests(streamId, client)
topicId, _ := successfullyCreateTopic(streamId, client)
newName := createRandomString(128)
- replicationFactor := uint8(1)
streamIdentifier, _ := iggcon.NewIdentifier(streamId)
topicIdentifier, _ := iggcon.NewIdentifier(topicId)
err := client.UpdateTopic(
@@ -45,8 +44,7 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() {
newName,
iggcon.CompressionAlgorithmNone,
iggcon.Microsecond,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldNotReturnError(err)
itShouldSuccessfullyUpdateTopic(streamId, topicId, newName, client)
})
@@ -57,7 +55,6 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() {
defer deleteStreamAfterTests(streamId, client)
_, topic1Name := successfullyCreateTopic(streamId, client)
topic2Id, _ := successfullyCreateTopic(streamId, client)
- replicationFactor := uint8(1)
streamIdentifier, _ := iggcon.NewIdentifier(streamId)
topic2Identifier, _ := iggcon.NewIdentifier(topic2Id)
err := client.UpdateTopic(
@@ -67,15 +64,13 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() {
topic1Name,
iggcon.CompressionAlgorithmNone,
iggcon.IggyExpiryServerDefault,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldReturnSpecificError(err, ierror.ErrTopicNameAlreadyExists)
})
ginkgo.Context("and tries to update non-existing topic", func() {
client := createAuthorizedConnection()
- replicationFactor := uint8(1)
err := client.UpdateTopic(
context.Background(),
randomU32Identifier(),
@@ -83,8 +78,7 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() {
createRandomString(128),
1,
0,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound)
})
@@ -93,7 +87,6 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() {
client := createAuthorizedConnection()
streamId, _ := successfullyCreateStream(prefix, client)
defer deleteStreamAfterTests(streamId, createAuthorizedConnection())
- replicationFactor := uint8(1)
streamIdentifier, _ := iggcon.NewIdentifier(streamId)
err := client.UpdateTopic(
context.Background(),
@@ -102,8 +95,7 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() {
createRandomString(128),
1,
0,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldReturnSpecificError(err, ierror.ErrTopicIdNotFound)
})
@@ -113,7 +105,6 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() {
streamId, _ := successfullyCreateStream(prefix, client)
defer deleteStreamAfterTests(streamId, createAuthorizedConnection())
topicId, _ := successfullyCreateTopic(streamId, client)
- replicationFactor := uint8(1)
streamIdentifier, _ := iggcon.NewIdentifier(streamId)
topicIdentifier, _ := iggcon.NewIdentifier(topicId)
err := client.UpdateTopic(
@@ -123,8 +114,7 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() {
createRandomString(256),
1,
0,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldReturnSpecificError(err, ierror.ErrInvalidTopicName)
})
diff --git a/bdd/go/tests/tcp_test/topic_steps.go b/bdd/go/tests/tcp_test/topic_steps.go
index fae8d6a40..a359f688d 100644
--- a/bdd/go/tests/tcp_test/topic_steps.go
+++ b/bdd/go/tests/tcp_test/topic_steps.go
@@ -31,7 +31,6 @@ import (
//operations
func successfullyCreateTopic(streamId uint32, client iggcon.Client) (uint32, string) {
- replicationFactor := uint8(1)
name := createRandomString(128)
streamIdentifier, _ := iggcon.NewIdentifier(streamId)
topic, err := client.CreateTopic(
@@ -41,8 +40,7 @@ func successfullyCreateTopic(streamId uint32, client iggcon.Client) (uint32, str
2,
1,
0,
- math.MaxUint64,
- &replicationFactor)
+ math.MaxUint64)
itShouldSuccessfullyCreateTopic(streamId, topic.Id, name, client)
itShouldNotReturnError(err)
diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java b/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java
index c14615b28..6b644b441 100644
--- a/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java
+++ b/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java
@@ -108,7 +108,6 @@ public class BasicMessagingSteps {
CompressionAlgorithm.None,
BigInteger.ZERO,
BigInteger.ZERO,
- Optional.empty(),
topicName);
context.lastTopicId = topic.id();
diff --git a/bdd/rust/tests/steps/topics.rs b/bdd/rust/tests/steps/topics.rs
index 4515bba9d..2506908b8 100644
--- a/bdd/rust/tests/steps/topics.rs
+++ b/bdd/rust/tests/steps/topics.rs
@@ -17,7 +17,7 @@
use crate::common::global_context::GlobalContext;
use cucumber::{then, when};
-use iggy::prelude::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, TopicClient};
+use iggy::prelude::{Identifier, IggyExpiry, TopicClient, TopicCreateOptions};
#[when(regex = r"^I create a topic with name (.+) in stream (\d+) with (\d+) partitions$")]
pub async fn when_create_topic(
@@ -31,11 +31,11 @@ pub async fn when_create_topic(
.create_topic(
&Identifier::numeric(stream_id).unwrap(),
&topic_name,
- partitions_count,
- CompressionAlgorithm::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(partitions_count),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await
.expect("Should be able to create topic");
diff --git a/core/ai/mcp/src/service/mod.rs b/core/ai/mcp/src/service/mod.rs
index 1653b02fb..7ba7b027d 100644
--- a/core/ai/mcp/src/service/mod.rs
+++ b/core/ai/mcp/src/service/mod.rs
@@ -16,10 +16,12 @@
// under the License.
use iggy::prelude::{
- ClusterClient, Consumer, ConsumerGroupClient, ConsumerOffsetClient, Identifier, IggyClient,
- IggyError, IggyMessage, IggyTimestamp, MessageClient, PartitionClient, Partitioning,
- PersonalAccessTokenClient, PollingKind, PollingStrategy, SegmentClient, StreamClient,
- SystemClient, SystemSnapshotType, TopicClient, UserClient, UserStatus,
+ ClusterClient, CompressionAlgorithm, Consumer, ConsumerGroupClient, ConsumerOffsetClient,
+ Identifier, IggyClient, IggyError, IggyExpiry, IggyMessage, IggyTimestamp, MaxTopicSize,
+ MessageClient, PartitionClient, Partitioning, PersonalAccessTokenClient, PollingKind,
+ PollingStrategy, SegmentClient, StreamClient, StreamUpdateOptions, SystemClient,
+ SystemSnapshotType, TopicClient, TopicCreateOptions, TopicUpdateOptions, UserClient,
+ UserStatus, UserUpdateOptions,
};
use requests::*;
use rmcp::{
@@ -94,7 +96,11 @@ impl IggyService {
Parameters(UpdateStream { stream_id, name }): Parameters<UpdateStream>,
) -> Result<CallToolResult, ErrorData> {
self.permissions.ensure_update()?;
- request(self.client.update_stream(&id(&stream_id)?, &name).await)
+ request(
+ self.client
+ .update_stream(&id(&stream_id)?, &name, &StreamUpdateOptions::default())
+ .await,
+ )
}
#[tool(description = "Delete stream")]
@@ -148,9 +154,9 @@ impl IggyService {
name,
partitions_count,
compression_algorithm,
- replication_factor,
message_expiry,
max_size,
+ options,
}): Parameters<CreateTopic>,
) -> Result<CallToolResult, ErrorData> {
self.permissions.ensure_create()?;
@@ -166,11 +172,21 @@ impl IggyService {
.create_topic(
&id(&stream_id)?,
&name,
- partitions_count,
- compression_algorithm,
- replication_factor,
- message_expiry,
- max_size,
+ &TopicCreateOptions {
+ partitions_count: Some(partitions_count),
+ compression_algorithm: (compression_algorithm
+ != CompressionAlgorithm::default())
+ .then_some(compression_algorithm),
+ message_expiry: (message_expiry != IggyExpiry::ServerDefault)
+ .then_some(message_expiry),
+ max_topic_size: (max_size != MaxTopicSize::ServerDefault)
+ .then_some(max_size),
+ // Raw keys ride as strings and are parsed server-side by
+ // the same rules a config value goes through, so a key
+ // added after this build shipped is still reachable.
+ raw: options,
+ ..TopicCreateOptions::default()
+ },
)
.await,
)
@@ -184,30 +200,23 @@ impl IggyService {
topic_id,
name,
compression_algorithm,
- replication_factor,
message_expiry,
max_size,
+ options,
}): Parameters<UpdateTopic>,
) -> Result<CallToolResult, ErrorData> {
self.permissions.ensure_update()?;
- let compression_algorithm = compression_algorithm
- .and_then(|ca| ca.parse().ok())
- .unwrap_or_default();
- let message_expiry = message_expiry
- .and_then(|me| me.parse().ok())
- .unwrap_or_default();
- let max_size = max_size.and_then(|ms| ms.parse().ok()).unwrap_or_default();
+ // Absent means "leave alone" now, so an unparsable value must not
+ // silently become a reset to the server default.
+ let update_options = TopicUpdateOptions {
+ compression_algorithm: compression_algorithm.and_then(|ca| ca.parse().ok()),
+ message_expiry: message_expiry.and_then(|me| me.parse().ok()),
+ max_topic_size: max_size.and_then(|ms| ms.parse().ok()),
+ raw: options,
+ };
request(
self.client
- .update_topic(
- &id(&stream_id)?,
- &id(&topic_id)?,
- &name,
- compression_algorithm,
- replication_factor,
- message_expiry,
- max_size,
- )
+ .update_topic(&id(&stream_id)?, &id(&topic_id)?, &name, &update_options)
.await,
)
}
@@ -691,7 +700,12 @@ impl IggyService {
});
request(
self.client
- .update_user(&id(&user_id)?, username.as_deref(), status)
+ .update_user(
+ &id(&user_id)?,
+ username.as_deref(),
+ status,
+ &UserUpdateOptions::default(),
+ )
.await,
)
}
diff --git a/core/ai/mcp/src/service/requests.rs b/core/ai/mcp/src/service/requests.rs
index a0833eb1e..0a498c65a 100644
--- a/core/ai/mcp/src/service/requests.rs
+++ b/core/ai/mcp/src/service/requests.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
use iggy::prelude;
use rmcp::schemars::{self, JsonSchema};
@@ -82,14 +82,18 @@ pub struct CreateTopic {
#[schemars(description = "compression algorithm (optional, can be one of 'none', 'gzip')")]
pub compression_algorithm: Option<String>,
- #[schemars(description = "replication factor (optional, must be greater than 0)")]
- pub replication_factor: Option<u8>,
-
#[schemars(description = "message expiry (optional)")]
pub message_expiry: Option<String>,
#[schemars(description = "maximum size (optional)")]
pub max_size: Option<String>,
+
+ #[schemars(
+ description = "additional options as string key-values, e.g. {\"segment_size\": \"128 MiB\"}; \
+ call describe_options for the keys the server accepts (optional)"
+ )]
+ #[serde(default)]
+ pub options: BTreeMap<String, String>,
}
#[derive(Debug, Deserialize, JsonSchema)]
@@ -106,14 +110,19 @@ pub struct UpdateTopic {
#[schemars(description = "compression algorithm (optional, can be one of 'none', 'gzip')")]
pub compression_algorithm: Option<String>,
- #[schemars(description = "replication factor (optional, must be greater than 0)")]
- pub replication_factor: Option<u8>,
-
#[schemars(description = "message expiry (optional)")]
pub message_expiry: Option<String>,
#[schemars(description = "maximum size (optional)")]
pub max_size: Option<String>,
+
+ #[schemars(
+ description = "options to change, as string key-values; only compression_algorithm, \
+ message_expiry and max_topic_size may be updated, and a key left out \
+ keeps its current value (optional)"
+ )]
+ #[serde(default)]
+ pub options: BTreeMap<String, String>,
}
#[derive(Debug, Deserialize, JsonSchema)]
diff --git a/core/bench/src/actors/producer/client/high_level.rs b/core/bench/src/actors/producer/client/high_level.rs
index 840b076f7..3db74f869 100644
--- a/core/bench/src/actors/producer/client/high_level.rs
+++ b/core/bench/src/actors/producer/client/high_level.rs
@@ -105,7 +105,6 @@ impl BenchmarkInit for HighLevelProducerClient {
.create_stream_if_not_exists()
.create_topic_if_not_exists(
self.config.partitions,
- Some(1),
IggyExpiry::NeverExpire,
MaxTopicSize::ServerDefault,
)
diff --git a/core/bench/src/args/common.rs b/core/bench/src/args/common.rs
index 1a6a3a46b..a7c5f9f7d 100644
--- a/core/bench/src/args/common.rs
+++ b/core/bench/src/args/common.rs
@@ -100,6 +100,12 @@ pub struct IggyBenchArgs {
/// consumers start with fresh data and accurate latency measurements.
#[arg(long, default_value_t = false)]
pub reuse_streams: bool,
+
+ /// Fsync every write on the benchmark topic instead of leaving it to the
+ /// page cache. Set as a topic option at creation, so it has no effect with
+ /// `--reuse-streams` against an already-created topic.
+ #[arg(long, default_value_t = false)]
+ pub enforce_fsync: bool,
}
impl IggyBenchArgs {
@@ -327,6 +333,10 @@ impl IggyBenchArgs {
self.reuse_streams
}
+ pub const fn enforce_fsync(&self) -> bool {
+ self.enforce_fsync
+ }
+
pub fn username(&self) -> &str {
&self.username
}
diff --git a/core/bench/src/benchmarks/benchmark.rs b/core/bench/src/benchmarks/benchmark.rs
index d1724e5a1..3d61bc331 100644
--- a/core/bench/src/benchmarks/benchmark.rs
+++ b/core/bench/src/benchmarks/benchmark.rs
@@ -130,21 +130,26 @@ pub trait Benchmarkable: Send {
.max_topic_size()
.map_or(MaxTopicSize::Unlimited, MaxTopicSize::Custom);
let message_expiry = self.args().message_expiry();
+ let enforce_fsync = self.args().enforce_fsync();
info!(
- "Creating the test topic '{}' for stream '{}' with max topic size: {:?}, message expiry: {}",
- topic_name, stream_name, max_topic_size, message_expiry
+ "Creating the test topic '{}' for stream '{}' with max topic size: {:?}, message expiry: {}, enforce fsync: {}",
+ topic_name, stream_name, max_topic_size, message_expiry, enforce_fsync
);
client
.create_topic(
&stream_id,
&topic_name,
- partitions_count,
- CompressionAlgorithm::default(),
- None,
- message_expiry,
- max_topic_size,
+ &TopicCreateOptions {
+ partitions_count: Some(partitions_count),
+ message_expiry: (message_expiry != IggyExpiry::ServerDefault)
+ .then_some(message_expiry),
+ max_topic_size: (max_topic_size != MaxTopicSize::ServerDefault)
+ .then_some(max_topic_size),
+ enforce_fsync: enforce_fsync.then_some(true),
+ ..TopicCreateOptions::default()
+ },
)
.await?;
}
diff --git a/core/binary_protocol/src/codec.rs b/core/binary_protocol/src/codec.rs
index 801ba787c..4edb5304b 100644
--- a/core/binary_protocol/src/codec.rs
+++ b/core/binary_protocol/src/codec.rs
@@ -251,16 +251,23 @@ pub fn read_bytes(buf: &[u8], offset: usize, len: usize) -> Result<&[u8], WireEr
})
}
-/// Cap a pre-allocation hint so a bogus wire count cannot cause OOM.
-/// The actual count is validated by the decode loop - this only limits
-/// the upfront allocation.
-#[inline]
+/// Capacity to preallocate for `count` elements that still have to be decoded
+/// out of `remaining` bytes, each at least `min_element_size` bytes on the
+/// wire.
+///
+/// A count read off the wire is chosen by the peer, and `Vec::with_capacity`
+/// reserves for it before a single element has been bounds-checked. A hostile
+/// or corrupt frame claiming `u32::MAX` elements therefore aborts the process
+/// on allocation failure instead of returning a decode error. The bytes left
+/// to decode are the honest ceiling on how many elements can exist.
+///
+/// A zero `min_element_size` is treated as one byte rather than passing the
+/// wire count through: an element that occupies no bytes cannot exist, and
+/// trusting the count is the exact hole this function was written to close.
#[must_use]
-pub fn capped_capacity(count: usize, remaining: usize, min_item_size: usize) -> usize {
- if min_item_size == 0 {
- return count;
- }
- count.min(remaining / min_item_size)
+#[inline]
+pub fn bounded_capacity(count: usize, remaining: usize, min_element_size: usize) -> usize {
+ count.min(remaining / min_element_size.max(1))
}
#[cfg(test)]
@@ -268,17 +275,17 @@ mod tests {
use super::*;
#[test]
- fn capped_capacity_limits_allocation() {
- assert_eq!(capped_capacity(1_000_000, 100, 10), 10);
- assert_eq!(capped_capacity(5, 100, 10), 5);
- assert_eq!(capped_capacity(10, 100, 10), 10);
- assert_eq!(capped_capacity(11, 100, 10), 10);
- assert_eq!(capped_capacity(0, 100, 10), 0);
- assert_eq!(capped_capacity(100, 0, 10), 0);
+ fn bounded_capacity_limits_allocation() {
+ assert_eq!(bounded_capacity(1_000_000, 100, 10), 10);
+ assert_eq!(bounded_capacity(5, 100, 10), 5);
+ assert_eq!(bounded_capacity(10, 100, 10), 10);
+ assert_eq!(bounded_capacity(11, 100, 10), 10);
+ assert_eq!(bounded_capacity(0, 100, 10), 0);
+ assert_eq!(bounded_capacity(100, 0, 10), 0);
}
#[test]
- fn capped_capacity_zero_item_size_returns_count() {
- assert_eq!(capped_capacity(1_000_000, 100, 0), 1_000_000);
+ fn bounded_capacity_zero_element_size_still_bounds_by_the_buffer() {
+ assert_eq!(bounded_capacity(1_000_000, 100, 0), 100);
}
}
diff --git a/core/binary_protocol/src/codes.rs b/core/binary_protocol/src/codes.rs
index 3a538130c..90420293d 100644
--- a/core/binary_protocol/src/codes.rs
+++ b/core/binary_protocol/src/codes.rs
@@ -25,6 +25,7 @@ pub const PING_CODE: u32 = 1;
pub const GET_STATS_CODE: u32 = 10;
pub const GET_SNAPSHOT_FILE_CODE: u32 = 11;
pub const GET_CLUSTER_METADATA_CODE: u32 = 12;
+pub const DESCRIBE_OPTIONS_CODE: u32 = 13;
pub const GET_ME_CODE: u32 = 20;
pub const GET_CLIENT_CODE: u32 = 21;
pub const GET_CLIENTS_CODE: u32 = 22;
diff --git a/core/binary_protocol/src/dispatch.rs b/core/binary_protocol/src/dispatch.rs
index 92f020303..576fadb0f 100644
--- a/core/binary_protocol/src/dispatch.rs
+++ b/core/binary_protocol/src/dispatch.rs
@@ -193,6 +193,8 @@ pub const COMMAND_TABLE: &[CommandMeta] = &[
CommandMeta::non_replicated(SYNC_CONSUMER_GROUP_CODE, "consumer_group.sync"),
// Login + Register (PAT - Personal Access Token variant)
CommandMeta::non_replicated(LOGIN_REGISTER_WITH_PAT_CODE, "user.login_register_with_pat"),
+ // Options catalog discovery
+ CommandMeta::non_replicated(DESCRIBE_OPTIONS_CODE, "options.describe"),
];
/// Lookup command metadata by command code.
@@ -256,6 +258,7 @@ pub const fn lookup_command(code: u32) -> Option<&'static CommandMeta> {
LEAVE_CONSUMER_GROUP_CODE => 49,
SYNC_CONSUMER_GROUP_CODE => 50,
LOGIN_REGISTER_WITH_PAT_CODE => 51,
+ DESCRIBE_OPTIONS_CODE => 52,
_ => return None,
};
Some(&COMMAND_TABLE[idx])
@@ -367,6 +370,7 @@ mod tests {
DELETE_CONSUMER_GROUP_CODE,
JOIN_CONSUMER_GROUP_CODE,
LEAVE_CONSUMER_GROUP_CODE,
+ DESCRIBE_OPTIONS_CODE,
];
for code in all_codes {
assert!(
diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs
index 0825d0f44..b308e5f73 100644
--- a/core/binary_protocol/src/lib.rs
+++ b/core/binary_protocol/src/lib.rs
@@ -90,6 +90,7 @@ pub use message_view::{
pub use primitives::ack_level::AckLevel;
pub use primitives::consumer::{KIND_CONSUMER_GROUP, WireConsumer};
pub use primitives::identifier::{MAX_WIRE_NAME_LENGTH, WireIdentifier, WireName};
+pub use primitives::options::{MAX_OPTIONS, MAX_OPTIONS_BYTES, WireOptions, validate_options};
pub use primitives::partition_assignment::CreatedPartitionAssignment;
pub use primitives::partitioning::{MAX_MESSAGES_KEY_LENGTH, WirePartitioning};
pub use primitives::permissions::{
diff --git a/core/binary_protocol/src/primitives/mod.rs b/core/binary_protocol/src/primitives/mod.rs
index f3c7ee4e3..c72528925 100644
--- a/core/binary_protocol/src/primitives/mod.rs
+++ b/core/binary_protocol/src/primitives/mod.rs
@@ -20,6 +20,7 @@
pub mod ack_level;
pub mod consumer;
pub mod identifier;
+pub mod options;
pub mod partition_assignment;
pub mod partitioning;
pub mod permissions;
diff --git a/core/binary_protocol/src/primitives/options.rs b/core/binary_protocol/src/primitives/options.rs
new file mode 100644
index 000000000..737b6f328
--- /dev/null
+++ b/core/binary_protocol/src/primitives/options.rs
@@ -0,0 +1,499 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Key-value options block for `Create*` requests.
+//!
+//! Options reuse the user-headers TLV encoding and its structural validator
+//! wholesale, so unknown value kind codes remain forwardable across
+//! mixed-version clusters. On top of the structural walk this layer enforces
+//! the stricter contract options need and messages do not: string keys only,
+//! no duplicate keys, and hard bounds on entry count, key length, and total
+//! block size, all rejected at the wire edge before a request reaches
+//! admission.
+
+use std::borrow::Cow;
+
+use bytes::{BufMut, Bytes, BytesMut};
+
+use crate::WireError;
+use crate::codec::WireEncode;
+use crate::primitives::user_headers::{
+ WireHeaderKind, WireUserHeaderEntry, WireUserHeaderIterator, validate_user_headers,
+};
+
+/// Maximum number of key-value entries in one options block.
+pub const MAX_OPTIONS: u32 = 1024;
+
+/// Maximum total byte length of an encoded options block.
+///
+/// Mirrors `iggy_common::MAX_USER_HEADERS_SIZE`: options ride the user-headers
+/// codec, so they inherit its budget rather than inventing a second one. The
+/// constant is duplicated because `iggy_common` depends on this crate, not the
+/// other way round; `options_block_budget_matches_user_headers` pins the two
+/// together from the side that can see both.
+///
+/// Sized so [`MAX_OPTIONS`] is actually reachable: the cheapest entry is 12
+/// bytes (kind + length + one byte, for key and value each), so 1024 entries
+/// need at least 12 KiB. A smaller budget would silently cap the entry count
+/// below `MAX_OPTIONS` and make that limit a lie.
+pub const MAX_OPTIONS_BYTES: usize = 100 * 1000;
+
+/// Wire kind code for UTF-8 string fields, matching `HeaderKind::String`
+/// in `iggy_common`.
+const STRING_KIND: WireHeaderKind = WireHeaderKind(2);
+
+/// Validate the structural and semantic integrity of an options byte buffer.
+///
+/// Runs [`validate_user_headers`] first, then enforces the options contract:
+///
+/// - Total block size within [`MAX_OPTIONS_BYTES`]
+/// - At most [`MAX_OPTIONS`] entries
+/// - Every key is a UTF-8 string (length already bounded by the codec)
+/// - No duplicate keys
+///
+/// Value kind codes are deliberately NOT restricted to the currently defined
+/// range, mirroring the user-headers forward-compatibility contract.
+///
+/// Returns the number of key-value pairs on success.
+///
+/// # Errors
+///
+/// Returns `WireError::UnexpectedEof` if the buffer is truncated mid-entry,
+/// or `WireError::Validation` if any constraint above is violated.
+pub fn validate_options(buf: &[u8]) -> Result<u32, WireError> {
+ if buf.len() > MAX_OPTIONS_BYTES {
+ return Err(WireError::Validation(Cow::Owned(format!(
+ "options block is {} bytes, exceeds maximum {MAX_OPTIONS_BYTES}",
+ buf.len()
+ ))));
+ }
+
+ let pairs = validate_user_headers(buf)?;
+ if pairs == 0 {
+ return Ok(0);
+ }
+ if pairs > MAX_OPTIONS {
+ return Err(WireError::Validation(Cow::Owned(format!(
+ "options block has {pairs} entries, exceeds maximum {MAX_OPTIONS}"
+ ))));
+ }
+
+ let mut keys: Vec<&[u8]> = Vec::with_capacity(pairs as usize);
+ for entry in WireUserHeaderIterator::new(buf) {
+ if entry.key_kind != STRING_KIND {
+ return Err(WireError::Validation(Cow::Owned(format!(
+ "option key kind {} is not a string",
+ entry.key_kind.0
+ ))));
+ }
+ if std::str::from_utf8(entry.key).is_err() {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "option key is not valid UTF-8",
+ )));
+ }
+ keys.push(entry.key);
+ }
+
+ keys.sort_unstable();
+ for pair in keys.windows(2) {
+ if pair[0] == pair[1] {
+ return Err(WireError::Validation(Cow::Owned(format!(
+ "duplicate option key: {}",
+ String::from_utf8_lossy(pair[0])
+ ))));
+ }
+ }
+
+ Ok(pairs)
+}
+
+/// Encoded size of a `u32`-length-prefixed options block.
+///
+/// Response elements (topic/stream/user entries in a list) cannot run their
+/// options to end-of-payload the way requests do, so they carry the block
+/// behind a length prefix.
+#[must_use]
+pub fn options_prefixed_size(options: &WireOptions) -> usize {
+ 4 + options.encoded_size()
+}
+
+/// Encode a `u32`-length-prefixed options block.
+pub fn encode_options_prefixed(options: &WireOptions, buf: &mut BytesMut) {
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(options.encoded_size() as u32);
+ options.encode(buf);
+}
+
+/// Decode a `u32`-length-prefixed options block at `pos`.
+///
+/// Returns the block and the total bytes consumed (prefix included).
+///
+/// # Errors
+///
+/// Returns `WireError::UnexpectedEof` on truncation, or
+/// `WireError::Validation` when the block violates [`validate_options`].
+pub fn decode_options_prefixed(buf: &[u8], pos: usize) -> Result<(WireOptions, usize), WireError> {
+ let length = crate::codec::read_u32_le(buf, pos)? as usize;
+ let start = pos + 4;
+ // `checked_add`: on a 32-bit target a wire-supplied length wraps, the
+ // truncation guard below then passes, and the slice panics instead.
+ let end = start
+ .checked_add(length)
+ .ok_or_else(|| WireError::UnexpectedEof {
+ offset: start,
+ need: length,
+ have: buf.len().saturating_sub(start),
+ })?;
+ if buf.len() < end {
+ return Err(WireError::UnexpectedEof {
+ offset: start,
+ need: length,
+ have: buf.len().saturating_sub(start),
+ });
+ }
+ let options = WireOptions::from_slice(&buf[start..end])?;
+ Ok((options, 4 + length))
+}
+
+/// Pre-validated options as a contiguous TLV byte buffer.
+///
+/// Construction validates both structural TLV integrity and the options
+/// contract via [`validate_options`]. The inner bytes are immutable.
+///
+/// Like `WireUserHeaders`, this type is opaque at the wire layer. Domain
+/// interpretation of value kind codes happens in `iggy_common`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireOptions(Bytes);
+
+impl WireOptions {
+ /// Validate and wrap a borrowed byte slice (copies into new `Bytes`).
+ ///
+ /// # Errors
+ /// Returns `WireError` if the buffer violates [`validate_options`].
+ pub fn from_slice(buf: &[u8]) -> Result<Self, WireError> {
+ validate_options(buf)?;
+ Ok(Self(Bytes::copy_from_slice(buf)))
+ }
+
+ /// Validate and wrap an owned `Bytes` buffer (zero-copy).
+ ///
+ /// # Errors
+ /// Returns `WireError` if the buffer violates [`validate_options`].
+ pub fn from_bytes(buf: Bytes) -> Result<Self, WireError> {
+ validate_options(&buf)?;
+ Ok(Self(buf))
+ }
+
+ /// Wrap pre-validated bytes without re-checking.
+ ///
+ /// # Safety contract (not `unsafe`, but caller must ensure):
+ /// The bytes must satisfy [`validate_options`]. Iterating invalid bytes
+ /// will panic.
+ pub const fn from_validated(buf: Bytes) -> Self {
+ Self(buf)
+ }
+
+ /// Empty options (zero-length buffer).
+ #[must_use]
+ pub const fn empty() -> Self {
+ Self(Bytes::new())
+ }
+
+ #[must_use]
+ pub const fn is_empty(&self) -> bool {
+ self.0.is_empty()
+ }
+
+ /// The raw validated bytes, suitable for wire transmission.
+ #[must_use]
+ pub fn as_bytes(&self) -> &[u8] {
+ &self.0
+ }
+
+ /// Consume into the inner `Bytes` handle (zero-copy).
+ #[must_use]
+ pub fn into_bytes(self) -> Bytes {
+ self.0
+ }
+
+ /// Zero-copy iteration over the pre-validated entries.
+ #[must_use]
+ pub fn iter(&self) -> WireUserHeaderIterator<'_> {
+ WireUserHeaderIterator::new(&self.0)
+ }
+}
+
+impl<'a> IntoIterator for &'a WireOptions {
+ type Item = WireUserHeaderEntry<'a>;
+ type IntoIter = WireUserHeaderIterator<'a>;
+
+ fn into_iter(self) -> Self::IntoIter {
+ self.iter()
+ }
+}
+
+impl WireEncode for WireOptions {
+ fn encoded_size(&self) -> usize {
+ self.0.len()
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ buf.put_slice(&self.0);
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::primitives::user_headers::encode_user_headers;
+
+ const STRING: u8 = 2;
+ const UINT64: u8 = 12;
+
+ fn encode(entries: &[(u8, &[u8], u8, &[u8])]) -> BytesMut {
+ let mut buf = BytesMut::new();
+ encode_user_headers(entries, &mut buf);
+ buf
+ }
+
+ #[test]
+ fn empty_buffer_is_zero_options() {
+ assert_eq!(validate_options(&[]).unwrap(), 0);
+ assert!(WireOptions::empty().is_empty());
+ }
+
+ #[test]
+ fn valid_block_roundtrips() {
+ let value = 1_073_741_824u64.to_le_bytes();
+ let buf = encode(&[
+ (STRING, b"segment_size", UINT64, &value),
+ (STRING, b"message_expiry", STRING, b"7 days"),
+ ]);
+ assert_eq!(validate_options(&buf).unwrap(), 2);
+
+ let options = WireOptions::from_slice(&buf).unwrap();
+ let entries: Vec<_> = options.iter().collect();
+ assert_eq!(entries.len(), 2);
+ assert_eq!(entries[0].key, b"segment_size");
+ assert_eq!(entries[0].value, value);
+ assert_eq!(entries[1].key, b"message_expiry");
+ assert_eq!(entries[1].value, b"7 days");
+ }
+
+ #[test]
+ fn unknown_value_kind_is_forwardable() {
+ let buf = encode(&[(STRING, b"future_option", 200, b"opaque")]);
+ assert_eq!(validate_options(&buf).unwrap(), 1);
+ }
+
+ #[test]
+ fn duplicate_key_is_rejected() {
+ let buf = encode(&[
+ (STRING, b"segment_size", STRING, b"1 GiB"),
+ (STRING, b"segment_size", STRING, b"2 GiB"),
+ ]);
+ assert!(matches!(
+ validate_options(&buf),
+ Err(WireError::Validation(_))
+ ));
+ }
+
+ #[test]
+ fn non_string_key_kind_is_rejected() {
+ let buf = encode(&[(UINT64, &1u64.to_le_bytes(), STRING, b"value")]);
+ assert!(matches!(
+ validate_options(&buf),
+ Err(WireError::Validation(_))
+ ));
+ }
+
+ #[test]
+ fn non_utf8_key_is_rejected() {
+ let buf = encode(&[(STRING, &[0xFF, 0xFE], STRING, b"value")]);
+ assert!(matches!(
+ validate_options(&buf),
+ Err(WireError::Validation(_))
+ ));
+ }
+
+ #[test]
+ fn key_length_is_bounded_by_the_user_headers_codec() {
+ // Options add no key-length rule of their own: every field is already
+ // bounded to 1..=255 by `validate_user_headers`, so the codec's limit
+ // is the option key's limit.
+ let over = [b'k'; 256];
+ let buf = encode(&[(STRING, &over, STRING, b"value")]);
+ assert!(matches!(
+ validate_options(&buf),
+ Err(WireError::Validation(_))
+ ));
+
+ let at_limit = [b'k'; 255];
+ let buf = encode(&[(STRING, &at_limit, STRING, b"value")]);
+ assert_eq!(validate_options(&buf).unwrap(), 1);
+ }
+
+ #[test]
+ fn entry_count_over_limit_is_rejected() {
+ let keys: Vec<String> = (0..=MAX_OPTIONS).map(|i| format!("key_{i}")).collect();
+ let entries: Vec<(u8, &[u8], u8, &[u8])> = keys
+ .iter()
+ .map(|key| (STRING, key.as_bytes(), STRING, b"v".as_slice()))
+ .collect();
+ let buf = encode(&entries);
+ assert!(matches!(
+ validate_options(&buf),
+ Err(WireError::Validation(_))
+ ));
+
+ let buf = encode(&entries[..MAX_OPTIONS as usize]);
+ assert_eq!(validate_options(&buf).unwrap(), MAX_OPTIONS);
+ }
+
+ #[test]
+ fn block_over_byte_limit_is_rejected() {
+ // 200 maximum-size entries are ~104 KB, over the byte budget while
+ // still well under `MAX_OPTIONS`, so this proves the byte cap trips
+ // independently of the entry count.
+ let value = [b'v'; 255];
+ let keys: Vec<String> = (0..200).map(|i| format!("{i:0>255}")).collect();
+ let entries: Vec<(u8, &[u8], u8, &[u8])> = keys
+ .iter()
+ .map(|key| (STRING, key.as_bytes(), STRING, value.as_slice()))
+ .collect();
+ let buf = encode(&entries);
+ assert!(buf.len() > MAX_OPTIONS_BYTES);
+ assert!(matches!(
+ validate_options(&buf),
+ Err(WireError::Validation(_))
+ ));
+ }
+
+ #[test]
+ fn truncated_block_errors() {
+ let buf = encode(&[(STRING, b"segment_size", STRING, b"1 GiB")]);
+ for i in 1..buf.len() {
+ assert!(
+ validate_options(&buf[..i]).is_err(),
+ "expected error for truncation at byte {i}"
+ );
+ }
+ }
+
+ #[test]
+ fn wire_options_constructors_validate() {
+ let buf = encode(&[
+ (STRING, b"enforce_fsync", STRING, b"true"),
+ (STRING, b"enforce_fsync", STRING, b"false"),
+ ]);
+ assert!(WireOptions::from_slice(&buf).is_err());
+ assert!(WireOptions::from_bytes(buf.freeze()).is_err());
+ }
+
+ #[test]
+ fn encoded_size_matches_bytes() {
+ let buf = encode(&[(STRING, b"key", STRING, b"value")]);
+ let options = WireOptions::from_slice(&buf).unwrap();
+ assert_eq!(options.encoded_size(), buf.len());
+ assert_eq!(options.as_bytes(), &buf[..]);
+ }
+
+ #[test]
+ fn prefixed_roundtrip() {
+ let buf = encode(&[(STRING, b"key", STRING, b"value")]);
+ let options = WireOptions::from_slice(&buf).unwrap();
+ let mut encoded = BytesMut::new();
+ encode_options_prefixed(&options, &mut encoded);
+ assert_eq!(encoded.len(), options_prefixed_size(&options));
+ let (decoded, consumed) = decode_options_prefixed(&encoded, 0).unwrap();
+ assert_eq!(consumed, encoded.len());
+ assert_eq!(decoded, options);
+ }
+
+ /// The cross-SDK golden vector for an options block.
+ ///
+ /// Every SDK writes this block from its own encoder, so "it round-trips
+ /// through my own decoder" proves nothing about interoperability. The bytes
+ /// here are the contract: `foreign/{node,go,java}` pin the identical vector
+ /// in their own unit tests, and a change to the TLV layout has to break all
+ /// of them together instead of leaving one SDK talking to itself.
+ ///
+ /// `enforce_fsync=true` (a `Bool`) and `segment_size=1 GiB` (a `Uint64`)
+ /// cover both a one-byte and an eight-byte value. What the vector pins is
+ /// the per-entry byte layout, not a key order: these two land sorted only
+ /// because `iggy_common` holds options in a `BTreeMap`, and
+ /// `unsorted_keys_are_accepted` covers the SDKs that emit insertion order.
+ const GOLDEN_OPTIONS_BLOCK: &[u8] = &[
+ 2, 13, 0, 0, 0, // key kind String, length 13
+ b'e', b'n', b'f', b'o', b'r', b'c', b'e', b'_', b'f', b's', b'y', b'n', b'c', 3, 1, 0, 0,
+ 0, 1, // value kind Bool, length 1, true
+ 2, 12, 0, 0, 0, // key kind String, length 12
+ b's', b'e', b'g', b'm', b'e', b'n', b't', b'_', b's', b'i', b'z', b'e', 12, 8, 0, 0,
+ 0, // value kind Uint64, length 8
+ 0, 0, 0, 64, 0, 0, 0, 0, // 1 GiB little-endian
+ ];
+
+ #[test]
+ fn golden_options_block_is_byte_stable() {
+ let encoded = encode(&[
+ (STRING, b"enforce_fsync", 3, &[1]),
+ (
+ STRING,
+ b"segment_size",
+ UINT64,
+ &1_073_741_824u64.to_le_bytes(),
+ ),
+ ]);
+
+ assert_eq!(
+ &encoded[..],
+ GOLDEN_OPTIONS_BLOCK,
+ "the options TLV layout changed; update every SDK's copy of this vector"
+ );
+ assert_eq!(validate_options(GOLDEN_OPTIONS_BLOCK).unwrap(), 2);
+ }
+
+ #[test]
+ fn unsorted_keys_are_accepted() {
+ let unsorted = encode(&[
+ (
+ STRING,
+ b"segment_size",
+ UINT64,
+ &1_073_741_824u64.to_le_bytes(),
+ ),
+ (STRING, b"enforce_fsync", 3, &[1]),
+ ]);
+
+ assert_ne!(&unsorted[..], GOLDEN_OPTIONS_BLOCK);
+ assert_eq!(validate_options(&unsorted).unwrap(), 2);
+ }
+
+ #[test]
+ fn prefixed_truncation_errors() {
+ let buf = encode(&[(STRING, b"key", STRING, b"value")]);
+ let options = WireOptions::from_slice(&buf).unwrap();
+ let mut encoded = BytesMut::new();
+ encode_options_prefixed(&options, &mut encoded);
+ for i in 0..encoded.len() {
+ assert!(
+ decode_options_prefixed(&encoded[..i], 0).is_err(),
+ "expected error for truncation at byte {i}"
+ );
+ }
+ }
+}
diff --git a/core/binary_protocol/src/primitives/partition_assignment.rs b/core/binary_protocol/src/primitives/partition_assignment.rs
index 2c36732bd..27e8f9e72 100644
--- a/core/binary_protocol/src/primitives/partition_assignment.rs
+++ b/core/binary_protocol/src/primitives/partition_assignment.rs
@@ -25,9 +25,14 @@ pub struct CreatedPartitionAssignment {
pub consensus_group_id: u64,
}
+impl CreatedPartitionAssignment {
+ /// Fixed wire size: a `u32` partition id and a `u64` consensus group id.
+ pub const ENCODED_SIZE: usize = 12;
+}
+
impl WireEncode for CreatedPartitionAssignment {
fn encoded_size(&self) -> usize {
- 12
+ Self::ENCODED_SIZE
}
fn encode(&self, buf: &mut BytesMut) {
@@ -38,10 +43,10 @@ impl WireEncode for CreatedPartitionAssignment {
impl WireDecode for CreatedPartitionAssignment {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
- if buf.len() < 12 {
+ if buf.len() < Self::ENCODED_SIZE {
return Err(WireError::UnexpectedEof {
offset: 0,
- need: 12,
+ need: Self::ENCODED_SIZE,
have: buf.len(),
});
}
diff --git a/core/binary_protocol/src/requests/streams/create_stream.rs b/core/binary_protocol/src/requests/streams/create_stream.rs
index 93af61681..ebcb91525 100644
--- a/core/binary_protocol/src/requests/streams/create_stream.rs
+++ b/core/binary_protocol/src/requests/streams/create_stream.rs
@@ -18,39 +18,56 @@
use crate::WireError;
use crate::codec::{WireDecode, WireEncode};
use crate::primitives::identifier::WireName;
+use crate::primitives::options::WireOptions;
use bytes::BytesMut;
-/// `CreateStream` request. Wire format: `[name_len:1][name:N]`
+/// `CreateStream` request.
+///
+/// Wire format: `[name_len:1][name:N][options TLV to end]`
+///
+/// The options block runs to the end of the payload; its validator requires
+/// exact consumption, so no length prefix is needed.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CreateStreamRequest {
pub name: WireName,
+ pub options: WireOptions,
}
impl WireEncode for CreateStreamRequest {
fn encoded_size(&self) -> usize {
- self.name.encoded_size()
+ self.name.encoded_size() + self.options.encoded_size()
}
fn encode(&self, buf: &mut BytesMut) {
self.name.encode(buf);
+ self.options.encode(buf);
}
}
impl WireDecode for CreateStreamRequest {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
let (name, consumed) = WireName::decode(buf)?;
- Ok((Self { name }, consumed))
+ let options = WireOptions::from_slice(&buf[consumed..])?;
+ Ok((Self { name, options }, buf.len()))
}
}
#[cfg(test)]
mod tests {
use super::*;
+ use crate::primitives::user_headers::encode_user_headers;
+
+ fn sample_options() -> WireOptions {
+ let mut buf = BytesMut::new();
+ encode_user_headers(&[(2, b"future_key", 2, b"value")], &mut buf);
+ WireOptions::from_bytes(buf.freeze()).unwrap()
+ }
#[test]
fn roundtrip() {
let req = CreateStreamRequest {
name: WireName::new("test-stream").unwrap(),
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
assert_eq!(bytes.len(), 1 + 11);
@@ -59,6 +76,18 @@ mod tests {
assert_eq!(decoded, req);
}
+ #[test]
+ fn roundtrip_with_options() {
+ let req = CreateStreamRequest {
+ name: WireName::new("test-stream").unwrap(),
+ options: sample_options(),
+ };
+ let bytes = req.to_bytes();
+ let (decoded, consumed) = CreateStreamRequest::decode(&bytes).unwrap();
+ assert_eq!(consumed, bytes.len());
+ assert_eq!(decoded, req);
+ }
+
#[test]
fn empty_name_rejected() {
let buf = [0u8];
@@ -66,23 +95,28 @@ mod tests {
}
#[test]
- fn truncated_returns_error() {
+ fn truncated_options_return_error() {
let req = CreateStreamRequest {
name: WireName::new("test").unwrap(),
+ options: sample_options(),
};
let bytes = req.to_bytes();
- for i in 0..bytes.len() {
+ let name_end = 1 + 4;
+ for i in name_end + 1..bytes.len() {
assert!(
CreateStreamRequest::decode(&bytes[..i]).is_err(),
"expected error for truncation at byte {i}"
);
}
+ let (decoded, _) = CreateStreamRequest::decode(&bytes[..name_end]).unwrap();
+ assert!(decoded.options.is_empty());
}
#[test]
fn wire_compat_byte_layout() {
let req = CreateStreamRequest {
name: WireName::new("test").unwrap(),
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
assert_eq!(&bytes[..], &[4, b't', b'e', b's', b't']);
diff --git a/core/binary_protocol/src/requests/streams/update_stream.rs b/core/binary_protocol/src/requests/streams/update_stream.rs
index 5fa962981..d36fadd16 100644
--- a/core/binary_protocol/src/requests/streams/update_stream.rs
+++ b/core/binary_protocol/src/requests/streams/update_stream.rs
@@ -19,23 +19,33 @@ use crate::WireError;
use crate::WireIdentifier;
use crate::codec::{WireDecode, WireEncode};
use crate::primitives::identifier::WireName;
+use crate::primitives::options::WireOptions;
use bytes::BytesMut;
-/// `UpdateStream` request. Wire format: `[identifier][name_len:1][name:N]`
+/// `UpdateStream` request.
+///
+/// Wire format: `[identifier][name_len:1][name:N][options TLV to end]`
+///
+/// The options block mirrors `CreateStream`'s, so a stream setting added to
+/// the catalog is updatable without another layout change. Keys absent from
+/// the block are left alone rather than reset: an update patches the stored
+/// map, so a client built before a key existed cannot erase it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UpdateStreamRequest {
pub stream_id: WireIdentifier,
pub name: WireName,
+ pub options: WireOptions,
}
impl WireEncode for UpdateStreamRequest {
fn encoded_size(&self) -> usize {
- self.stream_id.encoded_size() + self.name.encoded_size()
+ self.stream_id.encoded_size() + self.name.encoded_size() + self.options.encoded_size()
}
fn encode(&self, buf: &mut BytesMut) {
self.stream_id.encode(buf);
self.name.encode(buf);
+ self.options.encode(buf);
}
}
@@ -44,19 +54,35 @@ impl WireDecode for UpdateStreamRequest {
let (stream_id, mut pos) = WireIdentifier::decode(buf)?;
let (name, consumed) = WireName::decode(&buf[pos..])?;
pos += consumed;
- Ok((Self { stream_id, name }, pos))
+ let options = WireOptions::from_slice(&buf[pos..])?;
+ Ok((
+ Self {
+ stream_id,
+ name,
+ options,
+ },
+ buf.len(),
+ ))
}
}
#[cfg(test)]
mod tests {
use super::*;
+ use crate::primitives::user_headers::encode_user_headers;
+
+ fn sample_options() -> WireOptions {
+ let mut buf = BytesMut::new();
+ encode_user_headers(&[(2, b"future_key", 2, b"value")], &mut buf);
+ WireOptions::from_bytes(buf.freeze()).unwrap()
+ }
#[test]
fn roundtrip() {
let req = UpdateStreamRequest {
stream_id: WireIdentifier::named("old-name").unwrap(),
name: WireName::new("new-name").unwrap(),
+ options: sample_options(),
};
let bytes = req.to_bytes();
let (decoded, consumed) = UpdateStreamRequest::decode(&bytes).unwrap();
@@ -65,10 +91,11 @@ mod tests {
}
#[test]
- fn truncated_returns_error() {
+ fn truncated_fixed_fields_return_error() {
let req = UpdateStreamRequest {
stream_id: WireIdentifier::numeric(1),
name: WireName::new("test").unwrap(),
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
for i in 0..bytes.len() {
@@ -78,4 +105,23 @@ mod tests {
);
}
}
+
+ #[test]
+ fn truncated_options_return_error() {
+ // One pair, so any strict-interior truncation is invalid. Cutting at
+ // the block boundary is a legitimate update carrying no options.
+ let req = UpdateStreamRequest {
+ stream_id: WireIdentifier::numeric(1),
+ name: WireName::new("test").unwrap(),
+ options: sample_options(),
+ };
+ let bytes = req.to_bytes();
+ let fixed_end = bytes.len() - req.options.encoded_size();
+ for i in fixed_end + 1..bytes.len() {
+ assert!(
+ UpdateStreamRequest::decode(&bytes[..i]).is_err(),
+ "expected error for truncation at byte {i}"
+ );
+ }
+ }
}
diff --git a/core/binary_protocol/src/requests/system/describe_options.rs b/core/binary_protocol/src/requests/system/describe_options.rs
new file mode 100644
index 000000000..a01b7147f
--- /dev/null
+++ b/core/binary_protocol/src/requests/system/describe_options.rs
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::WireError;
+use crate::codec::{WireDecode, WireEncode, read_u8};
+use bytes::{BufMut, BytesMut};
+use std::borrow::Cow;
+
+/// Resource whose option catalog is being described.
+pub const OPTIONS_SCOPE_TOPIC: u8 = 1;
+pub const OPTIONS_SCOPE_STREAM: u8 = 2;
+pub const OPTIONS_SCOPE_USER: u8 = 3;
+
+/// `DescribeOptions` request. Wire format: `[scope:u8]`
+///
+/// Because unknown option keys are rejected at create, a client cannot probe
+/// support by trying; this read-only command serves the catalog instead.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DescribeOptionsRequest {
+ pub scope: u8,
+}
+
+impl WireEncode for DescribeOptionsRequest {
+ fn encoded_size(&self) -> usize {
+ 1
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ buf.put_u8(self.scope);
+ }
+}
+
+impl WireDecode for DescribeOptionsRequest {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let scope = read_u8(buf, 0)?;
+ if !(OPTIONS_SCOPE_TOPIC..=OPTIONS_SCOPE_USER).contains(&scope) {
+ return Err(WireError::Validation(Cow::Owned(format!(
+ "unknown options scope: {scope}"
+ ))));
+ }
+ Ok((Self { scope }, 1))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn roundtrip() {
+ for scope in [
+ OPTIONS_SCOPE_TOPIC,
+ OPTIONS_SCOPE_STREAM,
+ OPTIONS_SCOPE_USER,
+ ] {
+ let req = DescribeOptionsRequest { scope };
+ let bytes = req.to_bytes();
+ let (decoded, consumed) = DescribeOptionsRequest::decode(&bytes).unwrap();
+ assert_eq!(consumed, bytes.len());
+ assert_eq!(decoded, req);
+ }
+ }
+
+ #[test]
+ fn unknown_scope_rejected() {
+ assert!(DescribeOptionsRequest::decode(&[0]).is_err());
+ assert!(DescribeOptionsRequest::decode(&[4]).is_err());
+ }
+
+ #[test]
+ fn empty_buffer_rejected() {
+ assert!(DescribeOptionsRequest::decode(&[]).is_err());
+ }
+}
diff --git a/core/binary_protocol/src/requests/system/mod.rs b/core/binary_protocol/src/requests/system/mod.rs
index b490d39c9..c39bb2fb0 100644
--- a/core/binary_protocol/src/requests/system/mod.rs
+++ b/core/binary_protocol/src/requests/system/mod.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+pub mod describe_options;
pub mod get_client;
pub mod get_clients;
pub mod get_cluster_metadata;
@@ -23,6 +24,9 @@ pub mod get_snapshot;
pub mod get_stats;
pub mod ping;
+pub use describe_options::{
+ DescribeOptionsRequest, OPTIONS_SCOPE_STREAM, OPTIONS_SCOPE_TOPIC, OPTIONS_SCOPE_USER,
+};
pub use get_client::GetClientRequest;
pub use get_clients::GetClientsRequest;
pub use get_cluster_metadata::GetClusterMetadataRequest;
diff --git a/core/binary_protocol/src/requests/topics/create_topic.rs b/core/binary_protocol/src/requests/topics/create_topic.rs
index 67cc5de17..dadbc3011 100644
--- a/core/binary_protocol/src/requests/topics/create_topic.rs
+++ b/core/binary_protocol/src/requests/topics/create_topic.rs
@@ -17,41 +17,44 @@
use crate::WireError;
use crate::WireIdentifier;
-use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le};
+use crate::codec::{WireDecode, WireEncode, read_u32_le};
use crate::primitives::identifier::WireName;
+use crate::primitives::options::WireOptions;
use bytes::{BufMut, BytesMut};
/// `CreateTopic` request.
///
/// Wire format:
-/// `[stream_id:WireIdentifier][partitions_count:u32_le][compression_algorithm:u8]
-/// [message_expiry:u64_le][max_topic_size:u64_le][replication_factor:u8][name_len:u8][name:N]`
+/// `[stream_id:WireIdentifier][partitions_count:u32_le][name_len:u8][name:N][options TLV to end]`
+///
+/// The fixed fields are the shape of the operation itself: which stream, how
+/// many partitions to allocate, and what to call the topic. `partitions_count`
+/// is an argument, not a setting -- admission consumes it to compute the
+/// partition assignments and it is deliberately never persisted as an option
+/// (`CreatePartitions` would make a stored count stale).
+///
+/// Every actual topic SETTING -- expiry, size caps, segment sizing, durability
+/// and any future knob -- rides the options block, so adding a knob never
+/// changes this layout. The block runs to the end of the payload; its
+/// validator requires exact consumption, so no length prefix is needed.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CreateTopicRequest {
pub stream_id: WireIdentifier,
pub partitions_count: u32,
- pub compression_algorithm: u8,
- pub message_expiry: u64,
- pub max_topic_size: u64,
- pub replication_factor: u8,
pub name: WireName,
+ pub options: WireOptions,
}
-const FIXED_FIELDS_SIZE: usize = 4 + 1 + 8 + 8 + 1; // 22 bytes
-
impl WireEncode for CreateTopicRequest {
fn encoded_size(&self) -> usize {
- self.stream_id.encoded_size() + FIXED_FIELDS_SIZE + self.name.encoded_size()
+ self.stream_id.encoded_size() + 4 + self.name.encoded_size() + self.options.encoded_size()
}
fn encode(&self, buf: &mut BytesMut) {
self.stream_id.encode(buf);
buf.put_u32_le(self.partitions_count);
- buf.put_u8(self.compression_algorithm);
- buf.put_u64_le(self.message_expiry);
- buf.put_u64_le(self.max_topic_size);
- buf.put_u8(self.replication_factor);
self.name.encode(buf);
+ self.options.encode(buf);
}
}
@@ -60,27 +63,17 @@ impl WireDecode for CreateTopicRequest {
let (stream_id, mut pos) = WireIdentifier::decode(buf)?;
let partitions_count = read_u32_le(buf, pos)?;
pos += 4;
- let compression_algorithm = read_u8(buf, pos)?;
- pos += 1;
- let message_expiry = read_u64_le(buf, pos)?;
- pos += 8;
- let max_topic_size = read_u64_le(buf, pos)?;
- pos += 8;
- let replication_factor = read_u8(buf, pos)?;
- pos += 1;
let (name, consumed) = WireName::decode(&buf[pos..])?;
pos += consumed;
+ let options = WireOptions::from_slice(&buf[pos..])?;
Ok((
Self {
stream_id,
partitions_count,
- compression_algorithm,
- message_expiry,
- max_topic_size,
- replication_factor,
name,
+ options,
},
- pos,
+ buf.len(),
))
}
}
@@ -88,16 +81,26 @@ impl WireDecode for CreateTopicRequest {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::primitives::user_headers::encode_user_headers;
+
+ fn sample_options() -> WireOptions {
+ let mut buf = BytesMut::new();
+ encode_user_headers(
+ &[
+ (2, b"message_expiry", 2, b"7 days"),
+ (2, b"max_topic_size", 12, &1024u64.to_le_bytes()),
+ ],
+ &mut buf,
+ );
+ WireOptions::from_bytes(buf.freeze()).unwrap()
+ }
fn sample_request() -> CreateTopicRequest {
CreateTopicRequest {
stream_id: WireIdentifier::numeric(1),
partitions_count: 3,
- compression_algorithm: 1,
- message_expiry: 3600,
- max_topic_size: 1_000_000,
- replication_factor: 1,
name: WireName::new("orders").unwrap(),
+ options: sample_options(),
}
}
@@ -111,15 +114,12 @@ mod tests {
}
#[test]
- fn roundtrip_named_stream() {
+ fn roundtrip_without_options() {
let req = CreateTopicRequest {
stream_id: WireIdentifier::named("my-stream").unwrap(),
- partitions_count: 10,
- compression_algorithm: 2,
- message_expiry: 0,
- max_topic_size: u64::MAX,
- replication_factor: 3,
+ partitions_count: 1,
name: WireName::new("events").unwrap(),
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
let (decoded, consumed) = CreateTopicRequest::decode(&bytes).unwrap();
@@ -128,8 +128,38 @@ mod tests {
}
#[test]
- fn truncated_returns_error() {
- let req = sample_request();
+ fn truncated_options_return_error() {
+ // A single key-value pair: any strict-interior truncation of the
+ // block is structurally invalid (a multi-pair block truncates
+ // cleanly at pair boundaries).
+ let mut buf = BytesMut::new();
+ encode_user_headers(&[(2, b"message_expiry", 2, b"7 days")], &mut buf);
+ let req = CreateTopicRequest {
+ stream_id: WireIdentifier::numeric(1),
+ partitions_count: 2,
+ name: WireName::new("orders").unwrap(),
+ options: WireOptions::from_bytes(buf.freeze()).unwrap(),
+ };
+ let bytes = req.to_bytes();
+ let fixed_end = bytes.len() - req.options.encoded_size();
+ for i in fixed_end + 1..bytes.len() {
+ assert!(
+ CreateTopicRequest::decode(&bytes[..i]).is_err(),
+ "expected error for truncation at byte {i}"
+ );
+ }
+ let (decoded, _) = CreateTopicRequest::decode(&bytes[..fixed_end]).unwrap();
+ assert!(decoded.options.is_empty());
+ }
+
+ #[test]
+ fn truncated_fixed_fields_return_error() {
+ let req = CreateTopicRequest {
+ stream_id: WireIdentifier::numeric(1),
+ partitions_count: 1,
+ name: WireName::new("orders").unwrap(),
+ options: WireOptions::empty(),
+ };
let bytes = req.to_bytes();
for i in 0..bytes.len() {
assert!(
diff --git a/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs b/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs
index b5199abb3..b6665b618 100644
--- a/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs
+++ b/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs
@@ -17,6 +17,7 @@
use crate::WireError;
use crate::codec::{WireDecode, WireEncode, read_u32_le};
+use crate::primitives::options::{WireOptions, decode_options_prefixed};
use crate::primitives::partition_assignment::CreatedPartitionAssignment;
use crate::requests::topics::CreateTopicRequest;
use bytes::{BufMut, BytesMut};
@@ -25,15 +26,25 @@ fn usize_to_u32(value: usize, context: &str) -> u32 {
u32::try_from(value).unwrap_or_else(|_| panic!("{context} exceeds u32"))
}
+/// Internal create-topic form the admitting primary replicates.
+///
+/// `request` carries the client's explicit options verbatim; `derived_options`
+/// carries the values the primary resolved from server defaults for keys the
+/// client did not send. Splitting the two preserves per-key provenance across
+/// replication, so `GetTopic` can distinguish client-pinned settings from
+/// defaults filled at creation.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CreateTopicWithAssignmentsRequest {
pub request: CreateTopicRequest,
+ pub derived_options: WireOptions,
pub partitions: Vec<CreatedPartitionAssignment>,
}
impl WireEncode for CreateTopicWithAssignmentsRequest {
fn encoded_size(&self) -> usize {
4 + self.request.encoded_size()
+ + 4
+ + self.derived_options.encoded_size()
+ 4
+ self
.partitions
@@ -48,6 +59,11 @@ impl WireEncode for CreateTopicWithAssignmentsRequest {
"create topic request size",
));
self.request.encode(buf);
+ buf.put_u32_le(usize_to_u32(
+ self.derived_options.encoded_size(),
+ "create topic derived options size",
+ ));
+ self.derived_options.encode(buf);
buf.put_u32_le(usize_to_u32(
self.partitions.len(),
"create topic partition count",
@@ -61,8 +77,19 @@ impl WireEncode for CreateTopicWithAssignmentsRequest {
impl WireDecode for CreateTopicWithAssignmentsRequest {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
let request_size = read_u32_le(buf, 0)? as usize;
- let request_start = 4;
- let request_end = request_start + request_size;
+ let request_start: usize = 4;
+ // `checked_add`: on a 32-bit target a wire-supplied length wraps, the
+ // truncation guard below then passes, and the slice panics instead.
+ // This decode runs on journal replay, so that panic would take a
+ // replica down rather than fail one client request.
+ let request_end =
+ request_start
+ .checked_add(request_size)
+ .ok_or_else(|| WireError::UnexpectedEof {
+ offset: request_start,
+ need: request_size,
+ have: buf.len().saturating_sub(request_start),
+ })?;
if buf.len() < request_end {
return Err(WireError::UnexpectedEof {
offset: request_start,
@@ -72,9 +99,17 @@ impl WireDecode for CreateTopicWithAssignmentsRequest {
}
let request = CreateTopicRequest::decode_from(&buf[request_start..request_end])?;
- let partitions_count = read_u32_le(buf, request_end)? as usize;
- let mut offset = request_end + 4;
- let mut partitions = Vec::with_capacity(partitions_count);
+
+ let (derived_options, derived_consumed) = decode_options_prefixed(buf, request_end)?;
+ let derived_end = request_end + derived_consumed;
+
+ let partitions_count = read_u32_le(buf, derived_end)? as usize;
+ let mut offset = derived_end + 4;
+ let mut partitions = Vec::with_capacity(crate::codec::bounded_capacity(
+ partitions_count,
+ buf.len().saturating_sub(offset),
+ CreatedPartitionAssignment::ENCODED_SIZE,
+ ));
for _ in 0..partitions_count {
let (partition, consumed) = CreatedPartitionAssignment::decode(&buf[offset..])?;
offset += consumed;
@@ -84,6 +119,7 @@ impl WireDecode for CreateTopicWithAssignmentsRequest {
Ok((
Self {
request,
+ derived_options,
partitions,
},
offset,
@@ -97,8 +133,17 @@ mod tests {
use crate::WireIdentifier;
use crate::codec::{WireDecode, WireEncode};
use crate::primitives::identifier::WireName;
+ use crate::primitives::options::WireOptions;
use crate::primitives::partition_assignment::CreatedPartitionAssignment;
+ use crate::primitives::user_headers::encode_user_headers;
use crate::requests::topics::CreateTopicRequest;
+ use bytes::BytesMut;
+
+ fn options_block(entries: &[(u8, &[u8], u8, &[u8])]) -> WireOptions {
+ let mut buf = BytesMut::new();
+ encode_user_headers(entries, &mut buf);
+ WireOptions::from_bytes(buf.freeze()).unwrap()
+ }
#[test]
fn roundtrip() {
@@ -106,12 +151,10 @@ mod tests {
request: CreateTopicRequest {
stream_id: WireIdentifier::numeric(1),
partitions_count: 2,
- compression_algorithm: 1,
- message_expiry: 3600,
- max_topic_size: 1024,
- replication_factor: 1,
name: WireName::new("events").unwrap(),
+ options: options_block(&[(2, b"message_expiry", 2, b"7 days")]),
},
+ derived_options: options_block(&[(2, b"max_topic_size", 12, &1024u64.to_le_bytes())]),
partitions: vec![
CreatedPartitionAssignment {
partition_id: 0,
@@ -128,4 +171,22 @@ mod tests {
assert_eq!(consumed, bytes.len());
assert_eq!(decoded, request);
}
+
+ #[test]
+ fn roundtrip_with_empty_blocks() {
+ let request = CreateTopicWithAssignmentsRequest {
+ request: CreateTopicRequest {
+ stream_id: WireIdentifier::numeric(7),
+ partitions_count: 0,
+ name: WireName::new("bare").unwrap(),
+ options: WireOptions::empty(),
+ },
+ derived_options: WireOptions::empty(),
+ partitions: vec![],
+ };
+ let bytes = request.to_bytes();
+ let (decoded, consumed) = CreateTopicWithAssignmentsRequest::decode(&bytes).unwrap();
+ assert_eq!(consumed, bytes.len());
+ assert_eq!(decoded, request);
+ }
}
diff --git a/core/binary_protocol/src/requests/topics/update_topic.rs b/core/binary_protocol/src/requests/topics/update_topic.rs
index f7151242b..902971b37 100644
--- a/core/binary_protocol/src/requests/topics/update_topic.rs
+++ b/core/binary_protocol/src/requests/topics/update_topic.rs
@@ -17,44 +17,48 @@
use crate::WireError;
use crate::WireIdentifier;
-use crate::codec::{WireDecode, WireEncode, read_u8, read_u64_le};
+use crate::codec::{WireDecode, WireEncode};
use crate::primitives::identifier::WireName;
-use bytes::{BufMut, BytesMut};
+use crate::primitives::options::WireOptions;
+use bytes::BytesMut;
/// `UpdateTopic` request.
///
/// Wire format:
-/// `[stream_id:WireIdentifier][topic_id:WireIdentifier][compression_algorithm:u8]
-/// [message_expiry:u64_le][max_topic_size:u64_le][replication_factor:u8][name_len:u8][name:N]`
+/// `[stream_id:WireIdentifier][topic_id:WireIdentifier][name_len:u8][name:N]
+/// [options TLV to end]`
+///
+/// Identity and the new name are the only fixed fields; every SETTING rides the
+/// options block, which mirrors `CreateTopic`'s and carries the same catalog.
+/// A knob added there is updatable here without another layout change, and no
+/// setting has two homes to disagree between.
+///
+/// Keys absent from the block are LEFT ALONE rather than reset to their
+/// defaults. A client built before a key existed cannot send it, so treating
+/// the block as the topic's complete option set would let an old client wipe a
+/// newer knob just by updating the name -- the same forward-compatibility
+/// argument that makes unknown keys survive a round trip.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UpdateTopicRequest {
pub stream_id: WireIdentifier,
pub topic_id: WireIdentifier,
- pub compression_algorithm: u8,
- pub message_expiry: u64,
- pub max_topic_size: u64,
- pub replication_factor: u8,
pub name: WireName,
+ pub options: WireOptions,
}
-const FIXED_FIELDS_SIZE: usize = 1 + 8 + 8 + 1; // 18 bytes
-
impl WireEncode for UpdateTopicRequest {
fn encoded_size(&self) -> usize {
self.stream_id.encoded_size()
+ self.topic_id.encoded_size()
- + FIXED_FIELDS_SIZE
+ self.name.encoded_size()
+ + self.options.encoded_size()
}
fn encode(&self, buf: &mut BytesMut) {
self.stream_id.encode(buf);
self.topic_id.encode(buf);
- buf.put_u8(self.compression_algorithm);
- buf.put_u64_le(self.message_expiry);
- buf.put_u64_le(self.max_topic_size);
- buf.put_u8(self.replication_factor);
self.name.encode(buf);
+ self.options.encode(buf);
}
}
@@ -63,27 +67,17 @@ impl WireDecode for UpdateTopicRequest {
let (stream_id, mut pos) = WireIdentifier::decode(buf)?;
let (topic_id, consumed) = WireIdentifier::decode(&buf[pos..])?;
pos += consumed;
- let compression_algorithm = read_u8(buf, pos)?;
- pos += 1;
- let message_expiry = read_u64_le(buf, pos)?;
- pos += 8;
- let max_topic_size = read_u64_le(buf, pos)?;
- pos += 8;
- let replication_factor = read_u8(buf, pos)?;
- pos += 1;
let (name, name_consumed) = WireName::decode(&buf[pos..])?;
pos += name_consumed;
+ let options = WireOptions::from_slice(&buf[pos..])?;
Ok((
Self {
stream_id,
topic_id,
- compression_algorithm,
- message_expiry,
- max_topic_size,
- replication_factor,
name,
+ options,
},
- pos,
+ buf.len(),
))
}
}
@@ -91,16 +85,20 @@ impl WireDecode for UpdateTopicRequest {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::primitives::user_headers::encode_user_headers;
+
+ fn sample_options() -> WireOptions {
+ let mut buf = BytesMut::new();
+ encode_user_headers(&[(2, b"future_option", 9, &[3u8])], &mut buf);
+ WireOptions::from_bytes(buf.freeze()).unwrap()
+ }
fn sample_request() -> UpdateTopicRequest {
UpdateTopicRequest {
stream_id: WireIdentifier::numeric(1),
topic_id: WireIdentifier::numeric(2),
- compression_algorithm: 1,
- message_expiry: 7200,
- max_topic_size: 500_000,
- replication_factor: 2,
name: WireName::new("updated-topic").unwrap(),
+ options: sample_options(),
}
}
@@ -118,11 +116,8 @@ mod tests {
let req = UpdateTopicRequest {
stream_id: WireIdentifier::named("stream-a").unwrap(),
topic_id: WireIdentifier::named("topic-b").unwrap(),
- compression_algorithm: 0,
- message_expiry: 0,
- max_topic_size: u64::MAX,
- replication_factor: 1,
name: WireName::new("new-name").unwrap(),
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
let (decoded, consumed) = UpdateTopicRequest::decode(&bytes).unwrap();
@@ -131,9 +126,28 @@ mod tests {
}
#[test]
- fn truncated_returns_error() {
+ fn truncated_options_return_error() {
+ // One key-value pair, so any strict-interior truncation of the block
+ // is structurally invalid. Cutting at the block boundary is NOT an
+ // error: that is a legitimate update carrying no options.
let req = sample_request();
let bytes = req.to_bytes();
+ let fixed_end = bytes.len() - req.options.encoded_size();
+ for i in fixed_end + 1..bytes.len() {
+ assert!(
+ UpdateTopicRequest::decode(&bytes[..i]).is_err(),
+ "expected error for truncation at byte {i}"
+ );
+ }
+ }
+
+ #[test]
+ fn truncated_fixed_fields_return_error() {
+ let req = UpdateTopicRequest {
+ options: WireOptions::empty(),
+ ..sample_request()
+ };
+ let bytes = req.to_bytes();
for i in 0..bytes.len() {
assert!(
UpdateTopicRequest::decode(&bytes[..i]).is_err(),
diff --git a/core/binary_protocol/src/requests/users/create_user.rs b/core/binary_protocol/src/requests/users/create_user.rs
index 2cb6634ed..d58ee4985 100644
--- a/core/binary_protocol/src/requests/users/create_user.rs
+++ b/core/binary_protocol/src/requests/users/create_user.rs
@@ -18,6 +18,7 @@
use crate::WireError;
use crate::codec::{WireDecode, WireEncode, read_str, read_u8, read_u32_le};
use crate::primitives::identifier::WireName;
+use crate::primitives::options::WireOptions;
use crate::primitives::permissions::WirePermissions;
use bytes::{BufMut, BytesMut};
use std::borrow::Cow;
@@ -26,13 +27,18 @@ use std::borrow::Cow;
///
/// Wire format:
/// `[username_len:u8][username:N][password_len:u8][password:N][status:u8]
-/// [has_permissions:u8][permissions_len:u32_le?][permissions:M?]`
+/// [has_permissions:u8][permissions_len:u32_le?][permissions:M?]
+/// [options TLV to end]`
+///
+/// The options block runs to the end of the payload; its validator requires
+/// exact consumption, so no length prefix is needed.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CreateUserRequest {
pub username: WireName,
pub password: String,
pub status: u8,
pub permissions: Option<WirePermissions>,
+ pub options: WireOptions,
}
impl WireEncode for CreateUserRequest {
@@ -46,6 +52,7 @@ impl WireEncode for CreateUserRequest {
.permissions
.as_ref()
.map_or(0, |p| 4 + p.encoded_size())
+ + self.options.encoded_size()
}
fn encode(&self, buf: &mut BytesMut) {
@@ -67,6 +74,7 @@ impl WireEncode for CreateUserRequest {
} else {
buf.put_u8(0);
}
+ self.options.encode(buf);
}
}
@@ -100,14 +108,17 @@ impl WireDecode for CreateUserRequest {
None
};
+ let options = WireOptions::from_slice(&buf[pos..])?;
+
Ok((
Self {
username,
password,
status,
permissions,
+ options,
},
- pos,
+ buf.len(),
))
}
}
@@ -135,6 +146,13 @@ mod tests {
}
}
+ fn sample_options() -> WireOptions {
+ use crate::primitives::user_headers::encode_user_headers;
+ let mut buf = BytesMut::new();
+ encode_user_headers(&[(2, b"future_key", 2, b"value")], &mut buf);
+ WireOptions::from_bytes(buf.freeze()).unwrap()
+ }
+
#[test]
fn roundtrip_without_permissions() {
let req = CreateUserRequest {
@@ -142,6 +160,7 @@ mod tests {
password: "secret123".to_string(),
status: 1,
permissions: None,
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
let (decoded, consumed) = CreateUserRequest::decode(&bytes).unwrap();
@@ -156,6 +175,22 @@ mod tests {
password: "p@ssw0rd".to_string(),
status: 2,
permissions: Some(sample_permissions()),
+ options: WireOptions::empty(),
+ };
+ let bytes = req.to_bytes();
+ let (decoded, consumed) = CreateUserRequest::decode(&bytes).unwrap();
+ assert_eq!(consumed, bytes.len());
+ assert_eq!(decoded, req);
+ }
+
+ #[test]
+ fn roundtrip_with_permissions_and_options() {
+ let req = CreateUserRequest {
+ username: WireName::new("admin").unwrap(),
+ password: "p@ssw0rd".to_string(),
+ status: 2,
+ permissions: Some(sample_permissions()),
+ options: sample_options(),
};
let bytes = req.to_bytes();
let (decoded, consumed) = CreateUserRequest::decode(&bytes).unwrap();
@@ -170,6 +205,7 @@ mod tests {
password: "pw".to_string(),
status: 0,
permissions: Some(sample_permissions()),
+ options: sample_options(),
};
assert_eq!(req.encoded_size(), req.to_bytes().len());
}
@@ -181,6 +217,7 @@ mod tests {
password: "pass".to_string(),
status: 1,
permissions: None,
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
for i in 0..bytes.len() {
@@ -191,6 +228,27 @@ mod tests {
}
}
+ #[test]
+ fn truncated_options_return_error() {
+ let req = CreateUserRequest {
+ username: WireName::new("user").unwrap(),
+ password: "pass".to_string(),
+ status: 1,
+ permissions: None,
+ options: sample_options(),
+ };
+ let bytes = req.to_bytes();
+ let fixed_end = bytes.len() - req.options.encoded_size();
+ for i in fixed_end + 1..bytes.len() {
+ assert!(
+ CreateUserRequest::decode(&bytes[..i]).is_err(),
+ "expected error for truncation at byte {i}"
+ );
+ }
+ let (decoded, _) = CreateUserRequest::decode(&bytes[..fixed_end]).unwrap();
+ assert!(decoded.options.is_empty());
+ }
+
#[test]
fn none_permissions_wire_layout() {
let req = CreateUserRequest {
@@ -198,10 +256,11 @@ mod tests {
password: "p".to_string(),
status: 0,
permissions: None,
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
// username: [1, b'u'] + password: [1, b'p'] + status: [0]
- // + has_perm: [0]
+ // + has_perm: [0]; empty options add zero bytes
let expected: &[u8] = &[1, b'u', 1, b'p', 0, 0];
assert_eq!(&bytes[..], expected);
}
diff --git a/core/binary_protocol/src/requests/users/update_user.rs b/core/binary_protocol/src/requests/users/update_user.rs
index 4e2d70aa1..125e1df02 100644
--- a/core/binary_protocol/src/requests/users/update_user.rs
+++ b/core/binary_protocol/src/requests/users/update_user.rs
@@ -19,18 +19,25 @@ use crate::WireError;
use crate::WireIdentifier;
use crate::codec::{WireDecode, WireEncode, read_u8};
use crate::primitives::identifier::WireName;
+use crate::primitives::options::WireOptions;
use bytes::{BufMut, BytesMut};
/// `UpdateUser` request.
///
/// Wire format:
/// `[user_id:WireIdentifier][has_username:u8][username_len:u8?][username:N?]
-/// [has_status:u8][status:u8?]`
+/// [has_status:u8][status:u8?][options TLV to end]`
+///
+/// The options block mirrors `CreateUser`'s, so a user setting added to the
+/// catalog is updatable without another layout change. Keys absent from the
+/// block are left alone rather than reset: an update patches the stored map,
+/// so a client built before a key existed cannot erase it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UpdateUserRequest {
pub user_id: WireIdentifier,
pub username: Option<WireName>,
pub status: Option<u8>,
+ pub options: WireOptions,
}
impl WireEncode for UpdateUserRequest {
@@ -40,6 +47,7 @@ impl WireEncode for UpdateUserRequest {
+ self.username.as_ref().map_or(0, WireEncode::encoded_size)
+ 1 // has_status
+ self.status.map_or(0, |_| 1)
+ + self.options.encoded_size()
}
fn encode(&self, buf: &mut BytesMut) {
@@ -62,6 +70,7 @@ impl WireEncode for UpdateUserRequest {
buf.put_u8(0);
}
}
+ self.options.encode(buf);
}
}
@@ -89,13 +98,15 @@ impl WireDecode for UpdateUserRequest {
None
};
+ let options = WireOptions::from_slice(&buf[pos..])?;
Ok((
Self {
user_id,
username,
status,
+ options,
},
- pos,
+ buf.len(),
))
}
}
@@ -103,6 +114,13 @@ impl WireDecode for UpdateUserRequest {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::primitives::user_headers::encode_user_headers;
+
+ fn sample_options() -> WireOptions {
+ let mut buf = BytesMut::new();
+ encode_user_headers(&[(2, b"future_key", 2, b"value")], &mut buf);
+ WireOptions::from_bytes(buf.freeze()).unwrap()
+ }
#[test]
fn roundtrip_both_present() {
@@ -110,6 +128,7 @@ mod tests {
user_id: WireIdentifier::numeric(1),
username: Some(WireName::new("new-name").unwrap()),
status: Some(2),
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap();
@@ -123,6 +142,7 @@ mod tests {
user_id: WireIdentifier::numeric(5),
username: None,
status: None,
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap();
@@ -136,6 +156,7 @@ mod tests {
user_id: WireIdentifier::named("admin").unwrap(),
username: Some(WireName::new("super-admin").unwrap()),
status: None,
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap();
@@ -149,6 +170,7 @@ mod tests {
user_id: WireIdentifier::numeric(10),
username: None,
status: Some(0),
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap();
@@ -162,16 +184,38 @@ mod tests {
user_id: WireIdentifier::numeric(1),
username: Some(WireName::new("test").unwrap()),
status: Some(1),
+ options: WireOptions::empty(),
};
assert_eq!(req.encoded_size(), req.to_bytes().len());
}
+ #[test]
+ fn truncated_options_return_error() {
+ // One pair, so any strict-interior truncation is invalid. Cutting at
+ // the block boundary is a legitimate update carrying no options.
+ let req = UpdateUserRequest {
+ user_id: WireIdentifier::numeric(1),
+ username: Some(WireName::new("name").unwrap()),
+ status: Some(1),
+ options: sample_options(),
+ };
+ let bytes = req.to_bytes();
+ let fixed_end = bytes.len() - req.options.encoded_size();
+ for i in fixed_end + 1..bytes.len() {
+ assert!(
+ UpdateUserRequest::decode(&bytes[..i]).is_err(),
+ "expected error for truncation at byte {i}"
+ );
+ }
+ }
+
#[test]
fn truncated_returns_error() {
let req = UpdateUserRequest {
user_id: WireIdentifier::numeric(1),
username: Some(WireName::new("name").unwrap()),
status: Some(1),
+ options: WireOptions::empty(),
};
let bytes = req.to_bytes();
for i in 0..bytes.len() {
diff --git a/core/binary_protocol/src/responses/clients/get_client.rs b/core/binary_protocol/src/responses/clients/get_client.rs
index 3fa4ffedc..aa721c0c3 100644
--- a/core/binary_protocol/src/responses/clients/get_client.rs
+++ b/core/binary_protocol/src/responses/clients/get_client.rs
@@ -57,7 +57,7 @@ impl WireDecode for ClientDetailsResponse {
let (client, mut pos) = ClientResponse::decode(buf)?;
let count = client.consumer_groups_count as usize;
let remaining = buf.len().saturating_sub(pos);
- let mut consumer_groups = Vec::with_capacity(crate::codec::capped_capacity(
+ let mut consumer_groups = Vec::with_capacity(crate::codec::bounded_capacity(
count,
remaining,
ConsumerGroupInfoResponse::SIZE,
diff --git a/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs b/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs
index 551cf0da1..cc4f175f3 100644
--- a/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs
+++ b/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs
@@ -52,7 +52,7 @@ impl WireDecode for ConsumerGroupMemberResponse {
let id = read_u32_le(buf, 0)?;
let partitions_count = read_u32_le(buf, 4)?;
let remaining = buf.len().saturating_sub(8);
- let mut partitions = Vec::with_capacity(crate::codec::capped_capacity(
+ let mut partitions = Vec::with_capacity(crate::codec::bounded_capacity(
partitions_count as usize,
remaining,
4,
diff --git a/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs b/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs
index 9f9c3283a..3fe5d6551 100644
--- a/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs
+++ b/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs
@@ -16,7 +16,7 @@
// under the License.
use crate::WireError;
-use crate::codec::{WireDecode, WireEncode, capped_capacity, read_u32_le, read_u64_le};
+use crate::codec::{WireDecode, WireEncode, bounded_capacity, read_u32_le, read_u64_le};
use bytes::{BufMut, BytesMut};
/// `SyncConsumerGroup` response.
@@ -53,7 +53,7 @@ impl WireDecode for SyncConsumerGroupResponse {
let partitions_count = read_u32_le(buf, 8)?;
let remaining = buf.len().saturating_sub(12);
let mut partitions =
- Vec::with_capacity(capped_capacity(partitions_count as usize, remaining, 4));
+ Vec::with_capacity(bounded_capacity(partitions_count as usize, remaining, 4));
let mut offset = 12;
for _ in 0..partitions_count {
partitions.push(read_u32_le(buf, offset)?);
diff --git a/core/binary_protocol/src/responses/messages/send_messages.rs b/core/binary_protocol/src/responses/messages/send_messages.rs
index 22a7a4624..32785b7be 100644
--- a/core/binary_protocol/src/responses/messages/send_messages.rs
+++ b/core/binary_protocol/src/responses/messages/send_messages.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::codec::{WireDecode, WireEncode, capped_capacity, read_u32_le, read_u64_le};
+use crate::codec::{WireDecode, WireEncode, bounded_capacity, read_u32_le, read_u64_le};
use crate::error::WireError;
use bytes::{BufMut, BytesMut};
use std::borrow::Cow;
@@ -122,7 +122,7 @@ impl WireDecode for SendMessagesResponse {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
let confirmations_count = read_u32_le(buf, 0)?;
let remaining = buf.len().saturating_sub(4);
- let mut confirmations = Vec::with_capacity(capped_capacity(
+ let mut confirmations = Vec::with_capacity(bounded_capacity(
confirmations_count as usize,
remaining,
CONFIRMATION_SIZE,
diff --git a/core/binary_protocol/src/responses/streams/get_stream.rs b/core/binary_protocol/src/responses/streams/get_stream.rs
index 25aae5177..fd5b2acdc 100644
--- a/core/binary_protocol/src/responses/streams/get_stream.rs
+++ b/core/binary_protocol/src/responses/streams/get_stream.rs
@@ -18,18 +18,26 @@
use crate::WireError;
use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le};
use crate::primitives::identifier::WireName;
+use crate::primitives::options::{
+ WireOptions, decode_options_prefixed, encode_options_prefixed, options_prefixed_size,
+};
use crate::responses::streams::StreamResponse;
use bytes::{BufMut, BytesMut};
-use std::borrow::Cow;
/// Topic header within a `GetStream` response.
///
-/// Wire format (51 + `name_len` bytes):
+/// Wire format (50 + `name_len` + options bytes):
/// ```text
/// [id:4][created_at:8][partitions_count:4][message_expiry:8]
-/// [compression_algorithm:1][max_topic_size:8][replication_factor:1]
+/// [compression_algorithm:1][max_topic_size:8]
/// [size_bytes:8][messages_count:8][name_len:1][name:N]
+/// [explicit_options_len:4][explicit options TLV]
+/// [derived_options_len:4][derived options TLV]
/// ```
+///
+/// `options` are the keys the client pinned at create; `derived_options` are
+/// the defaults the admitting primary resolved. A round-tripping client
+/// re-sends only `options`; the effective configuration is the union.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TopicHeader {
pub id: u32,
@@ -38,19 +46,23 @@ pub struct TopicHeader {
pub message_expiry: u64,
pub compression_algorithm: u8,
pub max_topic_size: u64,
- pub replication_factor: u8,
pub size_bytes: u64,
pub messages_count: u64,
pub name: WireName,
+ pub options: WireOptions,
+ pub derived_options: WireOptions,
}
impl TopicHeader {
- const FIXED_SIZE: usize = 4 + 8 + 4 + 8 + 1 + 8 + 1 + 8 + 8 + 1; // 51
+ pub(crate) const FIXED_SIZE: usize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1; // 50
}
impl WireEncode for TopicHeader {
fn encoded_size(&self) -> usize {
- Self::FIXED_SIZE + self.name.len()
+ Self::FIXED_SIZE
+ + self.name.len()
+ + options_prefixed_size(&self.options)
+ + options_prefixed_size(&self.derived_options)
}
fn encode(&self, buf: &mut BytesMut) {
@@ -60,10 +72,11 @@ impl WireEncode for TopicHeader {
buf.put_u64_le(self.message_expiry);
buf.put_u8(self.compression_algorithm);
buf.put_u64_le(self.max_topic_size);
- buf.put_u8(self.replication_factor);
buf.put_u64_le(self.size_bytes);
buf.put_u64_le(self.messages_count);
self.name.encode(buf);
+ encode_options_prefixed(&self.options, buf);
+ encode_options_prefixed(&self.derived_options, buf);
}
}
@@ -75,11 +88,14 @@ impl WireDecode for TopicHeader {
let message_expiry = read_u64_le(buf, 16)?;
let compression_algorithm = read_u8(buf, 24)?;
let max_topic_size = read_u64_le(buf, 25)?;
- let replication_factor = read_u8(buf, 33)?;
- let size_bytes = read_u64_le(buf, 34)?;
- let messages_count = read_u64_le(buf, 42)?;
- let (name, name_consumed) = WireName::decode(&buf[50..])?;
- let consumed = 50 + name_consumed;
+ let size_bytes = read_u64_le(buf, 33)?;
+ let messages_count = read_u64_le(buf, 41)?;
+ let (name, name_consumed) = WireName::decode(&buf[49..])?;
+ let mut consumed = 49 + name_consumed;
+ let (options, options_consumed) = decode_options_prefixed(buf, consumed)?;
+ consumed += options_consumed;
+ let (derived_options, derived_consumed) = decode_options_prefixed(buf, consumed)?;
+ consumed += derived_consumed;
Ok((
Self {
@@ -89,10 +105,11 @@ impl WireDecode for TopicHeader {
message_expiry,
compression_algorithm,
max_topic_size,
- replication_factor,
size_bytes,
messages_count,
name,
+ options,
+ derived_options,
},
consumed,
))
@@ -132,22 +149,26 @@ impl WireEncode for GetStreamResponse {
}
}
+/// Smallest a topic header can encode as: the fixed ids, timestamps, sizes,
+/// counts and name length, plus the shortest name `WireName` accepts (one byte,
+/// since it rejects an empty one) and two empty length-prefixed option blocks.
+const MIN_TOPIC_HEADER_SIZE: usize = TopicHeader::FIXED_SIZE + 1 + 4 + 4;
+
impl WireDecode for GetStreamResponse {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
let (stream, mut pos) = StreamResponse::decode(buf)?;
- let mut topics = Vec::new();
- while pos < buf.len() {
+ // Count-driven: a topic element carries variable-length options
+ // blocks, so "consume until the buffer ends" no longer delimits it.
+ let mut topics = Vec::with_capacity(crate::codec::bounded_capacity(
+ stream.topics_count as usize,
+ buf.len().saturating_sub(pos),
+ MIN_TOPIC_HEADER_SIZE,
+ ));
+ for _ in 0..stream.topics_count {
let (topic, consumed) = TopicHeader::decode(&buf[pos..])?;
pos += consumed;
topics.push(topic);
}
- if topics.len() != stream.topics_count as usize {
- return Err(WireError::Validation(Cow::Owned(format!(
- "stream.topics_count={} but decoded {} topics",
- stream.topics_count,
- topics.len()
- ))));
- }
Ok((Self { stream, topics }, pos))
}
}
@@ -164,6 +185,7 @@ mod tests {
size_bytes: 2048,
messages_count: 200,
name: WireName::new("my-stream").unwrap(),
+ options: WireOptions::empty(),
}
}
@@ -175,10 +197,11 @@ mod tests {
message_expiry: 0,
compression_algorithm: 1,
max_topic_size: 0,
- replication_factor: 1,
size_bytes: 1024,
messages_count: 100,
name: WireName::new(name).unwrap(),
+ options: WireOptions::empty(),
+ derived_options: WireOptions::empty(),
}
}
@@ -213,12 +236,18 @@ mod tests {
fn topic_header_roundtrip() {
let topic = sample_topic(5, "events");
let bytes = topic.to_bytes();
- assert_eq!(bytes.len(), TopicHeader::FIXED_SIZE + 6);
+ assert_eq!(bytes.len(), TopicHeader::FIXED_SIZE + 6 + 4 + 4);
let (decoded, consumed) = TopicHeader::decode(&bytes).unwrap();
assert_eq!(consumed, bytes.len());
assert_eq!(decoded, topic);
}
+ #[test]
+ fn min_topic_header_size_matches_the_shortest_encoding() {
+ let bytes = sample_topic(1, "t").to_bytes();
+ assert_eq!(bytes.len(), MIN_TOPIC_HEADER_SIZE);
+ }
+
#[test]
fn topic_header_truncated_returns_error() {
let topic = sample_topic(1, "t");
diff --git a/core/binary_protocol/src/responses/streams/get_streams.rs b/core/binary_protocol/src/responses/streams/get_streams.rs
index 3221c45e4..11e30ab55 100644
--- a/core/binary_protocol/src/responses/streams/get_streams.rs
+++ b/core/binary_protocol/src/responses/streams/get_streams.rs
@@ -61,7 +61,7 @@ impl WireDecode for GetStreamsResponse {
#[cfg(test)]
mod tests {
use super::*;
- use crate::WireName;
+ use crate::{WireName, WireOptions};
#[test]
fn roundtrip_empty() {
@@ -84,6 +84,7 @@ mod tests {
size_bytes: 512,
messages_count: 50,
name: WireName::new("s1").unwrap(),
+ options: WireOptions::empty(),
},
StreamResponse {
id: 2,
@@ -92,6 +93,7 @@ mod tests {
size_bytes: 0,
messages_count: 0,
name: WireName::new("stream-two").unwrap(),
+ options: WireOptions::empty(),
},
],
};
diff --git a/core/binary_protocol/src/responses/streams/stream_response.rs b/core/binary_protocol/src/responses/streams/stream_response.rs
index aeeef4f58..8beb76f6d 100644
--- a/core/binary_protocol/src/responses/streams/stream_response.rs
+++ b/core/binary_protocol/src/responses/streams/stream_response.rs
@@ -18,14 +18,21 @@
use crate::WireError;
use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le};
use crate::primitives::identifier::WireName;
+use crate::primitives::options::{
+ WireOptions, decode_options_prefixed, encode_options_prefixed, options_prefixed_size,
+};
use bytes::{BufMut, BytesMut};
/// Stream header on the wire. Used in both single-stream and multi-stream responses.
///
-/// Wire format (33 + `name_len` bytes):
+/// Wire format (33 + `name_len` + options bytes):
/// ```text
/// [id:4][created_at:8][topics_count:4][size_bytes:8][messages_count:8][name_len:1][name:N]
+/// [options_len:4][options TLV]
/// ```
+///
+/// Streams have no server-derived options (no catalog keys resolve against
+/// config yet), so a single client-explicit block suffices.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StreamResponse {
pub id: u32,
@@ -34,6 +41,7 @@ pub struct StreamResponse {
pub size_bytes: u64,
pub messages_count: u64,
pub name: WireName,
+ pub options: WireOptions,
}
impl StreamResponse {
@@ -42,7 +50,7 @@ impl StreamResponse {
impl WireEncode for StreamResponse {
fn encoded_size(&self) -> usize {
- Self::FIXED_SIZE + self.name.len()
+ Self::FIXED_SIZE + self.name.len() + options_prefixed_size(&self.options)
}
fn encode(&self, buf: &mut BytesMut) {
@@ -52,6 +60,7 @@ impl WireEncode for StreamResponse {
buf.put_u64_le(self.size_bytes);
buf.put_u64_le(self.messages_count);
self.name.encode(buf);
+ encode_options_prefixed(&self.options, buf);
}
}
@@ -63,7 +72,9 @@ impl WireDecode for StreamResponse {
let size_bytes = read_u64_le(buf, 16)?;
let messages_count = read_u64_le(buf, 24)?;
let (name, name_consumed) = WireName::decode(&buf[32..])?;
- let consumed = 32 + name_consumed;
+ let mut consumed = 32 + name_consumed;
+ let (options, options_consumed) = decode_options_prefixed(buf, consumed)?;
+ consumed += options_consumed;
Ok((
Self {
@@ -73,6 +84,7 @@ impl WireDecode for StreamResponse {
size_bytes,
messages_count,
name,
+ options,
},
consumed,
))
@@ -91,6 +103,7 @@ mod tests {
size_bytes: 1024,
messages_count: 100,
name: WireName::new("test-stream").unwrap(),
+ options: WireOptions::empty(),
}
}
@@ -98,7 +111,7 @@ mod tests {
fn roundtrip() {
let resp = sample();
let bytes = resp.to_bytes();
- assert_eq!(bytes.len(), StreamResponse::FIXED_SIZE + 11);
+ assert_eq!(bytes.len(), StreamResponse::FIXED_SIZE + 11 + 4);
let (decoded, consumed) = StreamResponse::decode(&bytes).unwrap();
assert_eq!(consumed, bytes.len());
assert_eq!(decoded, resp);
diff --git a/core/binary_protocol/src/responses/system/describe_options.rs b/core/binary_protocol/src/responses/system/describe_options.rs
new file mode 100644
index 000000000..154196ef7
--- /dev/null
+++ b/core/binary_protocol/src/responses/system/describe_options.rs
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::WireError;
+use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le};
+use crate::primitives::identifier::WireName;
+use bytes::{BufMut, Bytes, BytesMut};
+
+/// One catalog entry in a `DescribeOptions` response.
+///
+/// Wire format:
+/// ```text
+/// [key_len:u8][key:N][kind:u8]
+/// [default_len:u32][default bytes][description_len:u32][description]
+/// ```
+///
+/// `kind` is this key's canonical header kind code: what the server encodes its
+/// default under, and what a value set at create is stored as, since create
+/// admission re-encodes the block from its own parse (a `String` value parsed by
+/// the same rules as a config file value is accepted and canonicalized). An
+/// update stores the client's bytes verbatim and is the exception. `default` is
+/// that default in `kind`'s encoding, empty when the key has none, and is a
+/// build constant rather than a per-node value.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct OptionDescriptor {
+ pub key: WireName,
+ pub kind: u8,
+ pub default_value: Bytes,
+ pub description: String,
+}
+
+impl WireEncode for OptionDescriptor {
+ fn encoded_size(&self) -> usize {
+ self.key.encoded_size() + 1 + 4 + self.default_value.len() + 4 + self.description.len()
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.key.encode(buf);
+ buf.put_u8(self.kind);
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(self.default_value.len() as u32);
+ buf.put_slice(&self.default_value);
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(self.description.len() as u32);
+ buf.put_slice(self.description.as_bytes());
+ }
+}
+
+impl WireDecode for OptionDescriptor {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let (key, mut pos) = WireName::decode(buf)?;
+ let kind = read_u8(buf, pos)?;
+ pos += 1;
+ let default_len = read_u32_le(buf, pos)? as usize;
+ pos += 4;
+ // `checked_add`: on a 32-bit target a wire-supplied length wraps, the
+ // truncation guard below then passes, and the slice panics instead.
+ let default_end = pos
+ .checked_add(default_len)
+ .ok_or_else(|| WireError::UnexpectedEof {
+ offset: pos,
+ need: default_len,
+ have: buf.len().saturating_sub(pos),
+ })?;
+ if buf.len() < default_end {
+ return Err(WireError::UnexpectedEof {
+ offset: pos,
+ need: default_len,
+ have: buf.len().saturating_sub(pos),
+ });
+ }
+ let default_value = Bytes::copy_from_slice(&buf[pos..default_end]);
+ pos = default_end;
+ let description_len = read_u32_le(buf, pos)? as usize;
+ pos += 4;
+ let description_end =
+ pos.checked_add(description_len)
+ .ok_or_else(|| WireError::UnexpectedEof {
+ offset: pos,
+ need: description_len,
+ have: buf.len().saturating_sub(pos),
+ })?;
+ if buf.len() < description_end {
+ return Err(WireError::UnexpectedEof {
+ offset: pos,
+ need: description_len,
+ have: buf.len().saturating_sub(pos),
+ });
+ }
+ let description = String::from_utf8_lossy(&buf[pos..description_end]).into_owned();
+ pos = description_end;
+
+ Ok((
+ Self {
+ key,
+ kind,
+ default_value,
+ description,
+ },
+ pos,
+ ))
+ }
+}
+
+/// `DescribeOptions` response: count-prefixed catalog entries.
+///
+/// Wire format: `[count:u32][OptionDescriptor]*`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DescribeOptionsResponse {
+ pub entries: Vec<OptionDescriptor>,
+}
+
+impl WireEncode for DescribeOptionsResponse {
+ fn encoded_size(&self) -> usize {
+ 4 + self
+ .entries
+ .iter()
+ .map(WireEncode::encoded_size)
+ .sum::<usize>()
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(self.entries.len() as u32);
+ for entry in &self.entries {
+ entry.encode(buf);
+ }
+ }
+}
+
+/// Smallest a descriptor can encode as: a one-character `WireName` (a length
+/// byte plus at least one byte of name), a kind byte, and empty
+/// length-prefixed default and description.
+const MIN_DESCRIPTOR_SIZE: usize = 2 + 1 + 4 + 4;
+
+impl WireDecode for DescribeOptionsResponse {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let count = read_u32_le(buf, 0)? as usize;
+ let mut pos = 4;
+ let mut entries = Vec::with_capacity(crate::codec::bounded_capacity(
+ count,
+ buf.len().saturating_sub(pos),
+ MIN_DESCRIPTOR_SIZE,
+ ));
+ for _ in 0..count {
+ let (entry, consumed) = OptionDescriptor::decode(&buf[pos..])?;
+ pos += consumed;
+ entries.push(entry);
+ }
+ Ok((Self { entries }, pos))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn sample() -> DescribeOptionsResponse {
+ DescribeOptionsResponse {
+ entries: vec![
+ OptionDescriptor {
+ key: WireName::new("partitions_count").unwrap(),
+ kind: 11,
+ default_value: Bytes::copy_from_slice(&1u32.to_le_bytes()),
+ description: "Number of partitions to create".to_string(),
+ },
+ OptionDescriptor {
+ key: WireName::new("max_topic_size").unwrap(),
+ kind: 12,
+ default_value: Bytes::new(),
+ description: String::new(),
+ },
+ ],
+ }
+ }
+
+ #[test]
+ fn roundtrip() {
+ let resp = sample();
+ let bytes = resp.to_bytes();
+ let (decoded, consumed) = DescribeOptionsResponse::decode(&bytes).unwrap();
+ assert_eq!(consumed, bytes.len());
+ assert_eq!(decoded, resp);
+ }
+
+ #[test]
+ fn roundtrip_empty() {
+ let resp = DescribeOptionsResponse { entries: vec![] };
+ let bytes = resp.to_bytes();
+ assert_eq!(bytes.len(), 4);
+ let (decoded, consumed) = DescribeOptionsResponse::decode(&bytes).unwrap();
+ assert_eq!(consumed, bytes.len());
+ assert_eq!(decoded, resp);
+ }
+
+ #[test]
+ fn truncated_returns_error() {
+ let bytes = sample().to_bytes();
+ for i in 5..bytes.len() {
+ assert!(
+ DescribeOptionsResponse::decode(&bytes[..i]).is_err(),
+ "expected error for truncation at byte {i}"
+ );
+ }
+ }
+}
diff --git a/core/binary_protocol/src/responses/system/get_cluster_metadata.rs b/core/binary_protocol/src/responses/system/get_cluster_metadata.rs
index 0b343e644..22748b4e7 100644
--- a/core/binary_protocol/src/responses/system/get_cluster_metadata.rs
+++ b/core/binary_protocol/src/responses/system/get_cluster_metadata.rs
@@ -138,7 +138,7 @@ impl WireDecode for ClusterMetadataResponse {
pos += 4;
let remaining = buf.len().saturating_sub(pos);
- let mut nodes = Vec::with_capacity(crate::codec::capped_capacity(
+ let mut nodes = Vec::with_capacity(crate::codec::bounded_capacity(
nodes_count,
remaining,
NODE_FIXED_SIZE,
diff --git a/core/binary_protocol/src/responses/system/get_stats.rs b/core/binary_protocol/src/responses/system/get_stats.rs
index 3e84bd439..7921876eb 100644
--- a/core/binary_protocol/src/responses/system/get_stats.rs
+++ b/core/binary_protocol/src/responses/system/get_stats.rs
@@ -207,7 +207,7 @@ impl WireDecode for StatsResponse {
pos += 4;
let remaining = buf.len().saturating_sub(pos);
- let mut cache_metrics = Vec::with_capacity(crate::codec::capped_capacity(
+ let mut cache_metrics = Vec::with_capacity(crate::codec::bounded_capacity(
cache_count,
remaining,
CacheMetricEntry::SIZE,
diff --git a/core/binary_protocol/src/responses/system/mod.rs b/core/binary_protocol/src/responses/system/mod.rs
index 8b3cafc70..113eee391 100644
--- a/core/binary_protocol/src/responses/system/mod.rs
+++ b/core/binary_protocol/src/responses/system/mod.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+pub mod describe_options;
pub mod get_cluster_metadata;
pub mod get_me;
pub mod get_snapshot;
@@ -22,6 +23,7 @@ pub mod get_stats;
mod ping;
pub use super::EmptyResponse;
+pub use describe_options::{DescribeOptionsResponse, OptionDescriptor};
pub use get_cluster_metadata::{ClusterMetadataResponse, ClusterNodeResponse};
pub use get_me::GetMeResponse;
pub use get_snapshot::GetSnapshotResponse;
diff --git a/core/binary_protocol/src/responses/topics/get_topic.rs b/core/binary_protocol/src/responses/topics/get_topic.rs
index b90e52156..793800a3c 100644
--- a/core/binary_protocol/src/responses/topics/get_topic.rs
+++ b/core/binary_protocol/src/responses/topics/get_topic.rs
@@ -19,7 +19,6 @@ use crate::WireError;
use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le};
use crate::responses::streams::get_stream::TopicHeader;
use bytes::{BufMut, BytesMut};
-use std::borrow::Cow;
/// Partition details within a `GetTopic` response.
///
@@ -114,19 +113,19 @@ impl WireEncode for GetTopicResponse {
impl WireDecode for GetTopicResponse {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
let (topic, mut pos) = TopicHeader::decode(buf)?;
- let mut partitions = Vec::new();
- while pos < buf.len() {
+ // Count-driven so the element stays delimited even when embedded in a
+ // larger payload; the header's variable-length options blocks removed
+ // the old "everything after the header is partitions" property.
+ let mut partitions = Vec::with_capacity(crate::codec::bounded_capacity(
+ topic.partitions_count as usize,
+ buf.len().saturating_sub(pos),
+ PartitionResponse::FIXED_SIZE,
+ ));
+ for _ in 0..topic.partitions_count {
let (partition, consumed) = PartitionResponse::decode(&buf[pos..])?;
pos += consumed;
partitions.push(partition);
}
- if partitions.len() != topic.partitions_count as usize {
- return Err(WireError::Validation(Cow::Owned(format!(
- "topic.partitions_count={} but decoded {} partitions",
- topic.partitions_count,
- partitions.len()
- ))));
- }
Ok((Self { topic, partitions }, pos))
}
}
@@ -134,7 +133,7 @@ impl WireDecode for GetTopicResponse {
#[cfg(test)]
mod tests {
use super::*;
- use crate::WireName;
+ use crate::{WireName, WireOptions};
fn sample_topic(partitions_count: u32) -> TopicHeader {
TopicHeader {
@@ -144,10 +143,11 @@ mod tests {
message_expiry: 0,
compression_algorithm: 1,
max_topic_size: 0,
- replication_factor: 1,
size_bytes: 2048,
messages_count: 200,
name: WireName::new("my-topic").unwrap(),
+ options: WireOptions::empty(),
+ derived_options: WireOptions::empty(),
}
}
diff --git a/core/binary_protocol/src/responses/topics/get_topics.rs b/core/binary_protocol/src/responses/topics/get_topics.rs
index b36d4ee12..6852866e3 100644
--- a/core/binary_protocol/src/responses/topics/get_topics.rs
+++ b/core/binary_protocol/src/responses/topics/get_topics.rs
@@ -16,18 +16,23 @@
// under the License.
use crate::WireError;
-use crate::codec::{WireDecode, WireEncode};
+use crate::codec::{WireDecode, WireEncode, read_u32_le};
use crate::responses::streams::get_stream::TopicHeader;
-use bytes::BytesMut;
+use bytes::{BufMut, BytesMut};
-/// `GetTopics` response: sequential topic headers.
+/// `GetTopics` response: count-prefixed topic headers.
///
/// Wire format:
/// ```text
-/// [TopicHeader]*
+/// [topics_count:u32_le][TopicHeader]*
/// ```
///
-/// Empty payload means zero topics.
+/// The count prefix is what lets a decoder pre-size its `Vec`: an element
+/// carries two variable-length option blocks, so the payload length alone says
+/// nothing useful about how many elements are in it. Detecting a short read
+/// does not need it - the element decoder errors on one, which is how
+/// `GetStreams` and `GetUsers` still run their loops to end-of-payload. This
+/// response is the odd one out of the three, and every SDK special-cases it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GetTopicsResponse {
pub topics: Vec<TopicHeader>,
@@ -35,21 +40,37 @@ pub struct GetTopicsResponse {
impl WireEncode for GetTopicsResponse {
fn encoded_size(&self) -> usize {
- self.topics.iter().map(WireEncode::encoded_size).sum()
+ 4 + self
+ .topics
+ .iter()
+ .map(WireEncode::encoded_size)
+ .sum::<usize>()
}
fn encode(&self, buf: &mut BytesMut) {
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(self.topics.len() as u32);
for topic in &self.topics {
topic.encode(buf);
}
}
}
+/// Smallest a topic header can encode as: the fixed ids, timestamps, sizes,
+/// counts and name length, plus the shortest name `WireName` accepts (one byte,
+/// since it rejects an empty one) and two empty length-prefixed option blocks.
+const MIN_TOPIC_HEADER_SIZE: usize = TopicHeader::FIXED_SIZE + 1 + 4 + 4;
+
impl WireDecode for GetTopicsResponse {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
- let mut topics = Vec::new();
- let mut pos = 0;
- while pos < buf.len() {
+ let topics_count = read_u32_le(buf, 0)? as usize;
+ let mut pos = 4;
+ let mut topics = Vec::with_capacity(crate::codec::bounded_capacity(
+ topics_count,
+ buf.len().saturating_sub(pos),
+ MIN_TOPIC_HEADER_SIZE,
+ ));
+ for _ in 0..topics_count {
let (topic, consumed) = TopicHeader::decode(&buf[pos..])?;
pos += consumed;
topics.push(topic);
@@ -61,7 +82,7 @@ impl WireDecode for GetTopicsResponse {
#[cfg(test)]
mod tests {
use super::*;
- use crate::WireName;
+ use crate::{WireName, WireOptions};
fn sample_topic(id: u32, name: &str) -> TopicHeader {
TopicHeader {
@@ -71,10 +92,11 @@ mod tests {
message_expiry: 0,
compression_algorithm: 1,
max_topic_size: 0,
- replication_factor: 1,
size_bytes: 1024,
messages_count: 100,
name: WireName::new(name).unwrap(),
+ options: WireOptions::empty(),
+ derived_options: WireOptions::empty(),
}
}
@@ -82,9 +104,9 @@ mod tests {
fn roundtrip_empty() {
let resp = GetTopicsResponse { topics: vec![] };
let bytes = resp.to_bytes();
- assert!(bytes.is_empty());
+ assert_eq!(bytes.len(), 4);
let (decoded, consumed) = GetTopicsResponse::decode(&bytes).unwrap();
- assert_eq!(consumed, 0);
+ assert_eq!(consumed, bytes.len());
assert_eq!(decoded, resp);
}
diff --git a/core/binary_protocol/src/responses/users/get_user.rs b/core/binary_protocol/src/responses/users/get_user.rs
index cae86cf65..6392e37ce 100644
--- a/core/binary_protocol/src/responses/users/get_user.rs
+++ b/core/binary_protocol/src/responses/users/get_user.rs
@@ -89,10 +89,10 @@ impl WireDecode for UserDetailsResponse {
#[cfg(test)]
mod tests {
use super::*;
- use crate::WireName;
use crate::primitives::permissions::{
WireGlobalPermissions, WireStreamPermissions, WireTopicPermissions,
};
+ use crate::{WireName, WireOptions};
fn sample_user() -> UserResponse {
UserResponse {
@@ -100,6 +100,7 @@ mod tests {
created_at: 1_710_000_000_000,
status: 1,
username: WireName::new("admin").unwrap(),
+ options: WireOptions::empty(),
}
}
diff --git a/core/binary_protocol/src/responses/users/get_users.rs b/core/binary_protocol/src/responses/users/get_users.rs
index 5e522aea1..87811f6f4 100644
--- a/core/binary_protocol/src/responses/users/get_users.rs
+++ b/core/binary_protocol/src/responses/users/get_users.rs
@@ -59,7 +59,7 @@ impl WireDecode for GetUsersResponse {
#[cfg(test)]
mod tests {
use super::*;
- use crate::WireName;
+ use crate::{WireName, WireOptions};
#[test]
fn roundtrip_empty() {
@@ -80,12 +80,14 @@ mod tests {
created_at: 100,
status: 1,
username: WireName::new("admin").unwrap(),
+ options: WireOptions::empty(),
},
UserResponse {
id: 2,
created_at: 200,
status: 2,
username: WireName::new("alice").unwrap(),
+ options: WireOptions::empty(),
},
],
};
diff --git a/core/binary_protocol/src/responses/users/user_response.rs b/core/binary_protocol/src/responses/users/user_response.rs
index 6eb03f0b0..ef495a9eb 100644
--- a/core/binary_protocol/src/responses/users/user_response.rs
+++ b/core/binary_protocol/src/responses/users/user_response.rs
@@ -18,20 +18,28 @@
use crate::WireError;
use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le};
use crate::primitives::identifier::WireName;
+use crate::primitives::options::{
+ WireOptions, decode_options_prefixed, encode_options_prefixed, options_prefixed_size,
+};
use bytes::{BufMut, BytesMut};
/// User header on the wire. Used in both single-user and multi-user responses.
///
-/// Wire format (13 + `username_len` bytes):
+/// Wire format (13 + `username_len` + options bytes):
/// ```text
/// [id:4][created_at:8][status:1][username_len:1][username:N]
+/// [options_len:4][options TLV]
/// ```
+///
+/// Users have no server-derived options (no catalog keys resolve against
+/// config yet), so a single client-explicit block suffices.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UserResponse {
pub id: u32,
pub created_at: u64,
pub status: u8,
pub username: WireName,
+ pub options: WireOptions,
}
impl UserResponse {
@@ -40,7 +48,7 @@ impl UserResponse {
impl WireEncode for UserResponse {
fn encoded_size(&self) -> usize {
- Self::FIXED_SIZE + self.username.encoded_size()
+ Self::FIXED_SIZE + self.username.encoded_size() + options_prefixed_size(&self.options)
}
fn encode(&self, buf: &mut BytesMut) {
@@ -48,6 +56,7 @@ impl WireEncode for UserResponse {
buf.put_u64_le(self.created_at);
buf.put_u8(self.status);
self.username.encode(buf);
+ encode_options_prefixed(&self.options, buf);
}
}
@@ -57,7 +66,9 @@ impl WireDecode for UserResponse {
let created_at = read_u64_le(buf, 4)?;
let status = read_u8(buf, 12)?;
let (username, name_consumed) = WireName::decode(&buf[13..])?;
- let consumed = 13 + name_consumed;
+ let mut consumed = 13 + name_consumed;
+ let (options, options_consumed) = decode_options_prefixed(buf, consumed)?;
+ consumed += options_consumed;
Ok((
Self {
@@ -65,6 +76,7 @@ impl WireDecode for UserResponse {
created_at,
status,
username,
+ options,
},
consumed,
))
@@ -81,6 +93,7 @@ mod tests {
created_at: 1_710_000_000_000,
status: 1,
username: WireName::new("admin").unwrap(),
+ options: WireOptions::empty(),
}
}
@@ -88,7 +101,7 @@ mod tests {
fn roundtrip() {
let resp = sample();
let bytes = resp.to_bytes();
- assert_eq!(bytes.len(), UserResponse::FIXED_SIZE + 1 + 5);
+ assert_eq!(bytes.len(), UserResponse::FIXED_SIZE + 1 + 5 + 4);
let (decoded, consumed) = UserResponse::decode(&bytes).unwrap();
assert_eq!(consumed, bytes.len());
assert_eq!(decoded, resp);
diff --git a/core/cli/src/args/common.rs b/core/cli/src/args/common.rs
index bb186bf11..2accf0117 100644
--- a/core/cli/src/args/common.rs
+++ b/core/cli/src/args/common.rs
@@ -122,3 +122,33 @@ impl From<ListModeExt> for GetStatsOutput {
}
}
}
+
+/// Longest an option key or value may be, matching the header-field codec the
+/// options block rides. Enforced here so `--set` reports the offending pair
+/// with clap's own error rather than failing at encode time.
+const MAX_OPTION_FIELD_LEN: usize = 255;
+
+/// Parse a repeatable `--set KEY=VALUE` pair.
+///
+/// Shared by every resource's option args: divergent copies disagreed on
+/// whether an empty value was legal, which decided whether a pair reached the
+/// encoder at all.
+pub(crate) fn parse_key_value(raw: &str) -> Result<(String, String), String> {
+ let (key, value) = raw
+ .split_once('=')
+ .ok_or_else(|| format!("expected KEY=VALUE, got: {raw}"))?;
+ for (label, field) in [("key", key), ("value", value)] {
+ if field.is_empty() {
+ return Err(format!(
+ "expected non-empty {label} in KEY=VALUE, got: {raw}"
+ ));
+ }
+ if field.len() > MAX_OPTION_FIELD_LEN {
+ return Err(format!(
+ "option {label} is {} bytes, maximum {MAX_OPTION_FIELD_LEN}",
+ field.len()
+ ));
+ }
+ }
+ Ok((key.to_string(), value.to_string()))
+}
diff --git a/core/cli/src/args/mod.rs b/core/cli/src/args/mod.rs
index a7159addb..42d517810 100644
--- a/core/cli/src/args/mod.rs
+++ b/core/cli/src/args/mod.rs
@@ -37,7 +37,7 @@ use crate::args::{
partition::PartitionAction,
personal_access_token::PersonalAccessTokenAction,
stream::StreamAction,
- system::{PingArgs, StatsArgs},
+ system::{OptionsArgs, PingArgs, StatsArgs},
topic::TopicAction,
};
@@ -162,6 +162,13 @@ pub(crate) enum Command {
/// server address and protocol type.
#[clap(verbatim_doc_comment)]
Me,
+ /// list the options a resource's create command accepts
+ ///
+ /// Prints the server's option catalog for one scope (topic, stream, user):
+ /// every key, the kind its default is reported in, that default, and the
+ /// bounds each value is checked against. These are the keys `--set` takes.
+ #[clap(verbatim_doc_comment)]
+ Options(OptionsArgs),
/// get iggy server statistics
///
/// Collect basic Iggy server statistics like number of streams, topics, partitions, etc.
diff --git a/core/cli/src/args/system.rs b/core/cli/src/args/system.rs
index 39f58fc37..a7195a2f1 100644
--- a/core/cli/src/args/system.rs
+++ b/core/cli/src/args/system.rs
@@ -17,7 +17,7 @@
use crate::args::common::ListModeExt;
use clap::Args;
-use iggy::prelude::{SnapshotCompression, SystemSnapshotType};
+use iggy::prelude::{OptionsScope, SnapshotCompression, SystemSnapshotType};
use iggy_cli::commands::utils::login_session_expiry::LoginSessionExpiry;
#[derive(Debug, Clone, Args)]
@@ -38,6 +38,13 @@ pub(crate) struct LoginArgs {
pub(crate) expiry: Option<Vec<LoginSessionExpiry>>,
}
+#[derive(Debug, Clone, Args)]
+pub(crate) struct OptionsArgs {
+ /// Resource whose option catalog to print (topic, stream, user)
+ #[arg(value_parser = clap::value_parser!(OptionsScope))]
+ pub(crate) scope: OptionsScope,
+}
+
#[derive(Debug, Clone, Args)]
pub(crate) struct StatsArgs {
/// List mode (table, list, JSON, TOML)
diff --git a/core/cli/src/args/topic.rs b/core/cli/src/args/topic.rs
index 694d763e0..875b3705f 100644
--- a/core/cli/src/args/topic.rs
+++ b/core/cli/src/args/topic.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::args::common::ListMode;
+use crate::args::common::{ListMode, parse_key_value};
use clap::{Args, Subcommand};
use iggy::prelude::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize};
@@ -117,16 +117,21 @@ pub(crate) struct TopicCreateArgs {
/// Can't be lower than segment size in the config.
#[arg(short, long, default_value = "server_default", verbatim_doc_comment)]
pub(crate) max_topic_size: MaxTopicSize,
- /// Replication factor for the topic
- #[arg(short, long, default_value = "1")]
- pub(crate) replication_factor: u8,
/// Message expiry time in human-readable format like "unlimited" or "15days 2min 2s"
///
/// "server_default" or skipping parameter makes CLI to use server default (from current server config) expiry time
#[arg(default_value = "server_default", value_parser = clap::value_parser!(IggyExpiry), verbatim_doc_comment)]
pub(crate) message_expiry: Vec<IggyExpiry>,
+ /// Additional topic option as key=value, repeatable
+ ///
+ /// Values are sent as strings and parsed server-side through each option's
+ /// own FromStr (e.g. --set segment_size=128MiB). The server rejects keys it
+ /// does not support; run "iggy options topic" to list the ones it accepts.
+ #[arg(long = "set", value_name = "KEY=VALUE", value_parser = parse_key_value, verbatim_doc_comment)]
+ pub(crate) set: Vec<(String, String)>,
}
+/// Parse one `--set key=value` occurrence.
#[derive(Debug, Clone, Args)]
pub(crate) struct TopicDeleteArgs {
/// Stream ID to delete topic
@@ -164,9 +169,6 @@ pub(crate) struct TopicUpdateArgs {
/// Can't be lower than segment size in the config.
#[arg(short, long, default_value = "server_default", verbatim_doc_comment)]
pub(crate) max_topic_size: MaxTopicSize,
- #[arg(short, long, default_value = "1")]
- /// New replication factor for the topic
- pub(crate) replication_factor: u8,
/// New message expiry time in human-readable format like "unlimited" or "15days 2min 2s"
///
/// "server_default" or skipping parameter makes CLI to use server default (from current server config) expiry time
diff --git a/core/cli/src/commands/binary_streams/update_stream.rs b/core/cli/src/commands/binary_streams/update_stream.rs
index 0f612d5b1..209c7af59 100644
--- a/core/cli/src/commands/binary_streams/update_stream.rs
+++ b/core/cli/src/commands/binary_streams/update_stream.rs
@@ -20,17 +20,25 @@ use anyhow::Context;
use async_trait::async_trait;
use iggy_common::Client;
use iggy_common::Identifier;
+use iggy_common::StreamUpdateOptions;
use iggy_common::update_stream::UpdateStream;
+use std::collections::BTreeMap;
use tracing::{Level, event};
pub struct UpdateStreamCmd {
update_stream: UpdateStream,
+ options: StreamUpdateOptions,
}
impl UpdateStreamCmd {
pub fn new(stream_id: Identifier, name: String) -> Self {
UpdateStreamCmd {
- update_stream: UpdateStream { stream_id, name },
+ update_stream: UpdateStream {
+ stream_id,
+ name,
+ options: BTreeMap::new(),
+ },
+ options: StreamUpdateOptions::default(),
}
}
}
@@ -46,7 +54,11 @@ impl CliCommand for UpdateStreamCmd {
async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> {
client
- .update_stream(&self.update_stream.stream_id, &self.update_stream.name)
+ .update_stream(
+ &self.update_stream.stream_id,
+ &self.update_stream.name,
+ &self.options,
+ )
.await
.with_context(|| {
format!(
diff --git a/core/cli/src/commands/binary_system/mod.rs b/core/cli/src/commands/binary_system/mod.rs
index 979ebeabe..2bbc45a3b 100644
--- a/core/cli/src/commands/binary_system/mod.rs
+++ b/core/cli/src/commands/binary_system/mod.rs
@@ -18,6 +18,7 @@
pub mod login;
pub mod logout;
pub mod me;
+pub mod options;
pub mod ping;
pub mod session;
pub mod session_status;
diff --git a/core/cli/src/commands/binary_system/options.rs b/core/cli/src/commands/binary_system/options.rs
new file mode 100644
index 000000000..5fc7f6beb
--- /dev/null
+++ b/core/cli/src/commands/binary_system/options.rs
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::commands::cli_command::{CliCommand, PRINT_TARGET};
+use anyhow::Context;
+use async_trait::async_trait;
+use comfy_table::Table;
+use iggy_common::{Client, HeaderValue, OptionSpec, OptionsScope};
+use tracing::{Level, event};
+
+/// `iggy options <scope>`: print the server's option catalog.
+///
+/// This is the discovery surface for `--set`. A key outside the catalog is
+/// refused at create, and the binary transports carry only the error code back,
+/// so without this command a CLI user has no way to see which keys their server
+/// knows or what each one is bounded by.
+pub struct DescribeOptionsCmd {
+ scope: OptionsScope,
+}
+
+impl DescribeOptionsCmd {
+ pub fn new(scope: OptionsScope) -> Self {
+ Self { scope }
+ }
+}
+
+#[async_trait]
+impl CliCommand for DescribeOptionsCmd {
+ fn explain(&self) -> String {
+ format!("describe {} options", self.scope)
+ }
+
+ async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> {
+ let specs = client
+ .describe_options(self.scope)
+ .await
+ .with_context(|| "Problem sending describe_options command".to_owned())?;
+
+ if specs.is_empty() {
+ event!(target: PRINT_TARGET, Level::INFO,
+ "{} resources accept no options yet", self.scope);
+ return Ok(());
+ }
+
+ let mut table = Table::new();
+ table.set_header(vec!["Key", "Kind", "Default", "Description"]);
+ for spec in &specs {
+ table.add_row(vec![
+ spec.key.as_str(),
+ spec.kind.to_string().as_str(),
+ render_default(spec).as_str(),
+ spec.description.as_str(),
+ ]);
+ }
+ event!(target: PRINT_TARGET, Level::INFO, "{table}");
+
+ Ok(())
+ }
+}
+
+/// The catalog's default in the string form `--set` takes, so the printed value
+/// can be handed straight back to a create.
+fn render_default(spec: &OptionSpec) -> String {
+ if spec.default_value.is_empty() {
+ return String::new();
+ }
+ match HeaderValue::from_raw(spec.kind, &spec.default_value) {
+ Ok(value) => value.to_string_value(),
+ // A kind this build cannot name still has a readable byte count.
+ Err(_) => format!("<{} bytes>", spec.default_value.len()),
+ }
+}
diff --git a/core/cli/src/commands/binary_system/session.rs b/core/cli/src/commands/binary_system/session.rs
index 5133de4d9..737241a52 100644
--- a/core/cli/src/commands/binary_system/session.rs
+++ b/core/cli/src/commands/binary_system/session.rs
@@ -27,7 +27,7 @@ use windows_native_keyring_store::store::Store;
#[cfg(secret_service_keyring)]
use zbus_secret_service_keyring_store::store::Store;
-use crate::commands::cli_command::PRINT_TARGET;
+use crate::commands::cli_command::DIAGNOSTIC_TARGET;
const SESSION_TOKEN_NAME: &str = "iggy-cli-session";
const SESSION_KEYRING_SERVICE_NAME: &str = "iggy-cli-session";
@@ -79,7 +79,7 @@ impl ServerSession {
pub fn is_active(&self) -> bool {
if let Err(e) = ensure_default_store() {
- warn!(target: PRINT_TARGET, "keyring backend unavailable, treating session as inactive: {e}");
+ warn!(target: DIAGNOSTIC_TARGET, "keyring backend unavailable, treating session as inactive: {e}");
return false;
}
if let Ok(entry) = Entry::new(&self.get_service_name(), &self.get_token_name()) {
@@ -98,7 +98,7 @@ impl ServerSession {
pub fn get_token(&self) -> Option<String> {
if let Err(e) = ensure_default_store() {
- warn!(target: PRINT_TARGET, "keyring backend unavailable, cannot read session token: {e}");
+ warn!(target: DIAGNOSTIC_TARGET, "keyring backend unavailable, cannot read session token: {e}");
return None;
}
if let Ok(entry) = Entry::new(&self.get_service_name(), &self.get_token_name())
diff --git a/core/cli/src/commands/binary_topics/create_topic.rs b/core/cli/src/commands/binary_topics/create_topic.rs
index a6e0d662a..ec9209ddb 100644
--- a/core/cli/src/commands/binary_topics/create_topic.rs
+++ b/core/cli/src/commands/binary_topics/create_topic.rs
@@ -21,14 +21,15 @@ use async_trait::async_trait;
use core::fmt;
use iggy_common::Client;
use iggy_common::create_topic::CreateTopic;
-use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize};
+use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, TopicCreateOptions};
+use std::collections::BTreeMap;
use tracing::{Level, event};
pub struct CreateTopicCmd {
create_topic: CreateTopic,
message_expiry: IggyExpiry,
max_topic_size: MaxTopicSize,
- replication_factor: u8,
+ raw_options: BTreeMap<String, String>,
}
impl CreateTopicCmd {
@@ -40,7 +41,7 @@ impl CreateTopicCmd {
name: String,
message_expiry: IggyExpiry,
max_topic_size: MaxTopicSize,
- replication_factor: u8,
+ raw_options: BTreeMap<String, String>,
) -> Self {
Self {
create_topic: CreateTopic {
@@ -50,11 +51,11 @@ impl CreateTopicCmd {
name,
message_expiry,
max_topic_size,
- replication_factor: Some(replication_factor),
+ options: raw_options.clone(),
},
message_expiry,
max_topic_size,
- replication_factor,
+ raw_options,
}
}
}
@@ -70,11 +71,19 @@ impl CliCommand for CreateTopicCmd {
.create_topic(
&self.create_topic.stream_id,
&self.create_topic.name,
- self.create_topic.partitions_count,
- self.create_topic.compression_algorithm,
- self.create_topic.replication_factor,
- self.create_topic.message_expiry,
- self.create_topic.max_topic_size,
+ &TopicCreateOptions {
+ partitions_count: Some(self.create_topic.partitions_count),
+ compression_algorithm: (self.create_topic.compression_algorithm
+ != CompressionAlgorithm::default())
+ .then_some(self.create_topic.compression_algorithm),
+ message_expiry: (self.create_topic.message_expiry != IggyExpiry::ServerDefault)
+ .then_some(self.create_topic.message_expiry),
+ max_topic_size: (self.create_topic.max_topic_size
+ != MaxTopicSize::ServerDefault)
+ .then_some(self.create_topic.max_topic_size),
+ raw: self.raw_options.clone(),
+ ..TopicCreateOptions::default()
+ },
)
.await
.with_context(|| {
@@ -87,13 +96,12 @@ impl CliCommand for CreateTopicCmd {
})?;
event!(target: PRINT_TARGET, Level::INFO,
- "Topic with name: {}, partitions count: {}, compression algorithm: {}, message expiry: {}, max topic size: {}, replication factor: {} created in stream with ID: {}",
+ "Topic with name: {}, partitions count: {}, compression algorithm: {}, message expiry: {}, max topic size: {} created in stream with ID: {}",
self.create_topic.name,
self.create_topic.partitions_count,
self.create_topic.compression_algorithm,
self.message_expiry,
self.max_topic_size,
- self.replication_factor,
self.create_topic.stream_id,
);
@@ -107,13 +115,13 @@ impl fmt::Display for CreateTopicCmd {
let compression_algorithm = &self.create_topic.compression_algorithm;
let message_expiry = &self.message_expiry;
let max_topic_size = &self.max_topic_size;
- let replication_factor = self.replication_factor;
let stream_id = &self.create_topic.stream_id;
write!(
f,
- "create topic with name: {topic_name}, message expiry: {message_expiry}, compression algorithm: {compression_algorithm}, \
- max topic size: {max_topic_size}, replication factor: {replication_factor} in stream with ID: {stream_id}",
+ "create topic with name: {topic_name}, message expiry: {message_expiry}, \
+ compression algorithm: {compression_algorithm}, max topic size: {max_topic_size} \
+ in stream with ID: {stream_id}",
)
}
}
diff --git a/core/cli/src/commands/binary_topics/update_topic.rs b/core/cli/src/commands/binary_topics/update_topic.rs
index 2615a975f..04f3b827a 100644
--- a/core/cli/src/commands/binary_topics/update_topic.rs
+++ b/core/cli/src/commands/binary_topics/update_topic.rs
@@ -21,14 +21,16 @@ use async_trait::async_trait;
use core::fmt;
use iggy_common::Client;
use iggy_common::update_topic::UpdateTopic;
-use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize};
+use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, TopicUpdateOptions};
+use std::collections::BTreeMap;
use tracing::{Level, event};
pub struct UpdateTopicCmd {
update_topic: UpdateTopic,
+ compression_algorithm: CompressionAlgorithm,
message_expiry: IggyExpiry,
max_topic_size: MaxTopicSize,
- replication_factor: u8,
+ options: TopicUpdateOptions,
}
impl UpdateTopicCmd {
@@ -39,21 +41,26 @@ impl UpdateTopicCmd {
name: String,
message_expiry: IggyExpiry,
max_topic_size: MaxTopicSize,
- replication_factor: u8,
) -> Self {
Self {
update_topic: UpdateTopic {
stream_id,
topic_id,
name,
- compression_algorithm,
- message_expiry,
- max_topic_size,
- replication_factor: Some(replication_factor),
+ compression_algorithm: Some(compression_algorithm),
+ message_expiry: Some(message_expiry),
+ max_topic_size: Some(max_topic_size),
+ options: BTreeMap::new(),
},
+ compression_algorithm,
message_expiry,
max_topic_size,
- replication_factor,
+ options: TopicUpdateOptions {
+ compression_algorithm: Some(compression_algorithm),
+ message_expiry: Some(message_expiry),
+ max_topic_size: Some(max_topic_size),
+ ..TopicUpdateOptions::default()
+ },
}
}
}
@@ -66,7 +73,7 @@ impl CliCommand for UpdateTopicCmd {
async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> {
client
- .update_topic(&self.update_topic.stream_id, &self.update_topic.topic_id, &self.update_topic.name, self.update_topic.compression_algorithm, self.replication_factor.into(), self.message_expiry, self.max_topic_size)
+ .update_topic(&self.update_topic.stream_id, &self.update_topic.topic_id, &self.update_topic.name, &self.options)
.await
.with_context(|| {
format!(
@@ -79,13 +86,12 @@ impl CliCommand for UpdateTopicCmd {
})?;
event!(target: PRINT_TARGET, Level::INFO,
- "Topic with ID: {} updated name: {}, updated message expiry: {}, updated compression algorithm: {}, updated max topic size: {}, updated replication factor: {} in stream with ID: {}",
+ "Topic with ID: {} updated name: {}, updated message expiry: {}, updated compression algorithm: {}, updated max topic size: {} in stream with ID: {}",
self.update_topic.topic_id,
self.update_topic.name,
self.message_expiry,
- self.update_topic.compression_algorithm,
+ self.compression_algorithm,
self.max_topic_size,
- self.replication_factor,
self.update_topic.stream_id,
);
@@ -97,17 +103,16 @@ impl fmt::Display for UpdateTopicCmd {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let topic_id = &self.update_topic.topic_id;
let topic_name = &self.update_topic.name;
- let compression_algorithm = &self.update_topic.compression_algorithm;
+ let compression_algorithm = &self.compression_algorithm;
let message_expiry = &self.message_expiry;
let max_topic_size = &self.max_topic_size;
- let replication_factor = self.replication_factor;
let stream_id = &self.update_topic.stream_id;
write!(
f,
"update topic with ID: {topic_id}, name: {topic_name}, message expiry: \
- {message_expiry}, compression algorithm: {compression_algorithm}, max topic size: {max_topic_size}, replication \
- factor: {replication_factor}, in stream with ID: {stream_id}",
+ {message_expiry}, compression algorithm: {compression_algorithm}, max topic size: \
+ {max_topic_size}, in stream with ID: {stream_id}",
)
}
}
diff --git a/core/cli/src/commands/binary_users/update_user.rs b/core/cli/src/commands/binary_users/update_user.rs
index e397e61c8..7366c6c6f 100644
--- a/core/cli/src/commands/binary_users/update_user.rs
+++ b/core/cli/src/commands/binary_users/update_user.rs
@@ -21,7 +21,9 @@ use async_trait::async_trait;
use iggy_common::Client;
use iggy_common::Identifier;
use iggy_common::UserStatus;
+use iggy_common::UserUpdateOptions;
use iggy_common::update_user::UpdateUser;
+use std::collections::BTreeMap;
use tracing::{Level, event};
#[derive(Debug, Clone)]
@@ -48,6 +50,7 @@ impl UpdateUserCmd {
user_id,
username,
status,
+ options: BTreeMap::new(),
},
}
}
@@ -76,6 +79,9 @@ impl CliCommand for UpdateUserCmd {
&self.update_user.user_id,
self.update_user.username.as_deref(),
self.update_user.status,
+ // `iggy user name` / `iggy user status` are single-purpose
+ // commands; users have no option keys to set yet.
+ &UserUpdateOptions::default(),
)
.await
.with_context(|| {
diff --git a/core/cli/src/commands/cli_command.rs b/core/cli/src/commands/cli_command.rs
index 9ecdf2d77..eb2520a02 100644
--- a/core/cli/src/commands/cli_command.rs
+++ b/core/cli/src/commands/cli_command.rs
@@ -21,6 +21,12 @@ use iggy_common::Client;
pub static PRINT_TARGET: &str = "iggy::cli::output";
+/// Diagnostics that must not land on [`PRINT_TARGET`]: that channel is the
+/// command's result, and callers pipe it. Routed to stderr instead, so a
+/// degraded-but-working run (no keyring, for one) stays visible without
+/// corrupting parseable output.
+pub static DIAGNOSTIC_TARGET: &str = "iggy::cli::diagnostic";
+
#[async_trait]
pub trait CliCommand {
fn explain(&self) -> String;
diff --git a/core/cli/src/logging.rs b/core/cli/src/logging.rs
index a2363511d..9dec12246 100644
--- a/core/cli/src/logging.rs
+++ b/core/cli/src/logging.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use iggy_cli::commands::cli_command::PRINT_TARGET;
+use iggy_cli::commands::cli_command::{DIAGNOSTIC_TARGET, PRINT_TARGET};
use std::path::PathBuf;
use tracing_appender::non_blocking::WorkerGuard;
use tracing_subscriber::{
@@ -59,6 +59,21 @@ impl Logging {
layers.push(stdout_layer.with_filter(stdout_filter).boxed());
+ // Diagnostics go to stderr even under `--quiet`: quiet suppresses the
+ // command's result, not a warning that the run was degraded. Scoped to
+ // DIAGNOSTIC_TARGET rather than "every non-print event" so dependency
+ // logs stay suppressed as before.
+ let stderr_filter =
+ filter::filter_fn(|metadata| metadata.target().contains(DIAGNOSTIC_TARGET));
+ let stderr_layer = fmt::Layer::default()
+ .without_time()
+ .with_target(false)
+ .with_writer(std::io::stderr)
+ .with_filter(LevelFilter::WARN)
+ .boxed();
+
+ layers.push(stderr_layer.with_filter(stderr_filter).boxed());
+
if let Some(file_path) = debug {
let _ = std::fs::remove_file(file_path); // Remove file if it exists
let file_appender = tracing_appender::rolling::never("", file_path);
diff --git a/core/cli/src/main.rs b/core/cli/src/main.rs
index 3187811d6..93a73618f 100644
--- a/core/cli/src/main.rs
+++ b/core/cli/src/main.rs
@@ -46,6 +46,8 @@ use iggy_cli::commands::binary_context::show_context::ShowContextCmd;
use iggy_cli::commands::binary_context::use_context::UseContextCmd;
use iggy_cli::commands::binary_segments::delete_segments::DeleteSegmentsCmd;
use iggy_cli::commands::binary_system::snapshot::GetSnapshotCmd;
+#[cfg(feature = "login-session")]
+use iggy_cli::commands::cli_command::DIAGNOSTIC_TARGET;
use iggy_cli::commands::cli_command::{CliCommand, PRINT_TARGET};
use iggy_cli::commands::{
binary_client::{get_client::GetClientCmd, get_clients::GetClientsCmd},
@@ -75,7 +77,7 @@ use iggy_cli::commands::{
create_stream::CreateStreamCmd, delete_stream::DeleteStreamCmd, get_stream::GetStreamCmd,
get_streams::GetStreamsCmd, purge_stream::PurgeStreamCmd, update_stream::UpdateStreamCmd,
},
- binary_system::{me::GetMeCmd, ping::PingCmd, stats::GetStatsCmd},
+ binary_system::{me::GetMeCmd, options::DescribeOptionsCmd, ping::PingCmd, stats::GetStatsCmd},
binary_topics::{
create_topic::CreateTopicCmd, delete_topic::DeleteTopicCmd, get_topic::GetTopicCmd,
get_topics::GetTopicsCmd, purge_topic::PurgeTopicCmd, update_topic::UpdateTopicCmd,
@@ -131,7 +133,7 @@ fn get_command(
args.name.clone(),
args.message_expiry.clone().into(),
args.max_topic_size,
- args.replication_factor,
+ args.set.iter().cloned().collect(),
)),
TopicAction::Delete(args) => Box::new(DeleteTopicCmd::new(
args.stream_id.clone(),
@@ -144,7 +146,6 @@ fn get_command(
args.name.clone(),
args.message_expiry.clone().into(),
args.max_topic_size,
- args.replication_factor,
)),
TopicAction::Get(args) => Box::new(GetTopicCmd::new(
args.stream_id.clone(),
@@ -181,6 +182,7 @@ fn get_command(
},
Command::Ping(args) => Box::new(PingCmd::new(args.count)),
Command::Me => Box::new(GetMeCmd::new()),
+ Command::Options(args) => Box::new(DescribeOptionsCmd::new(args.scope)),
Command::Stats(args) => Box::new(GetStatsCmd::new(cli_options.quiet, args.output.into())),
Command::Snapshot(args) => Box::new(GetSnapshotCmd::new(
args.compression,
@@ -382,7 +384,7 @@ async fn main() -> Result<(), IggyCmdError> {
// token-name lookup) sees the backend regardless of code-path ordering.
#[cfg(feature = "login-session")]
if let Err(e) = ensure_default_store() {
- tracing::warn!(target: PRINT_TARGET, "keyring backend unavailable: {e}");
+ tracing::warn!(target: DIAGNOSTIC_TARGET, "keyring backend unavailable: {e}");
}
let command = args.command.clone().unwrap();
diff --git a/core/common/src/error/iggy_error.rs b/core/common/src/error/iggy_error.rs
index fead3058d..d80006ade 100644
--- a/core/common/src/error/iggy_error.rs
+++ b/core/common/src/error/iggy_error.rs
@@ -400,6 +400,16 @@ pub enum IggyError {
InvalidBatchChecksum(u64, u64, u64) = 4039,
#[error("Invalid header kind code: {0}")]
InvalidHeaderKind(u8) = 4040,
+ /// The key is only populated where the error text itself travels, which
+ /// today means HTTP. A binary transport sends the code alone, so a client
+ /// rebuilding this variant from it renders an empty key; `DescribeOptions`
+ /// is what tells that client which keys exist.
+ #[error("Unsupported option key: {0}")]
+ UnsupportedOptionKey(String) = 4041,
+ #[error("Invalid option value for key: {0}")]
+ InvalidOptionValue(String) = 4042,
+ #[error("Options block exceeds its limits: {0}")]
+ OptionsBlockTooLarge(String) = 4043,
#[error("Cannot sed messages due to client disconnection")]
CannotSendMessagesDueToClientDisconnection = 4050,
#[error("Background send error")]
diff --git a/core/common/src/http/streams/update_stream.rs b/core/common/src/http/streams/update_stream.rs
index dc64962b3..1c92741ac 100644
--- a/core/common/src/http/streams/update_stream.rs
+++ b/core/common/src/http/streams/update_stream.rs
@@ -20,6 +20,7 @@ use crate::Identifier;
use crate::Validatable;
use crate::error::IggyError;
use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
/// `UpdateStream` command is used to update an existing stream.
/// It has additional payload:
@@ -32,6 +33,10 @@ pub struct UpdateStream {
pub stream_id: Identifier,
/// Unique stream name (string), max length is 255 characters.
pub name: String,
+ /// Additional stream options as string key-values. Restricted to the keys
+ /// an update may change; anything else is rejected.
+ #[serde(default)]
+ pub options: BTreeMap<String, String>,
}
impl Default for UpdateStream {
@@ -39,6 +44,7 @@ impl Default for UpdateStream {
UpdateStream {
stream_id: Identifier::default(),
name: "stream".to_string(),
+ options: BTreeMap::new(),
}
}
}
diff --git a/core/common/src/http/topics/create_topic.rs b/core/common/src/http/topics/create_topic.rs
index e476f123a..99a18f706 100644
--- a/core/common/src/http/topics/create_topic.rs
+++ b/core/common/src/http/topics/create_topic.rs
@@ -23,6 +23,7 @@ use crate::error::IggyError;
use crate::utils::expiry::IggyExpiry;
use crate::utils::topic_size::MaxTopicSize;
use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
/// `CreateTopic` command is used to create a new topic in a stream.
/// It has additional payload:
@@ -31,7 +32,6 @@ use serde::{Deserialize, Serialize};
/// - `message_expiry` - message expiry, if `NeverExpire` then messages will never expire.
/// - `max_topic_size` - maximum size of the topic, if `Unlimited` then topic size is unlimited.
/// Can't be lower than segment size in the config.
-/// - `replication_factor` - replication factor for the topic.
/// - `name` - unique topic name, max length is 255 characters.
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct CreateTopic {
@@ -47,10 +47,16 @@ pub struct CreateTopic {
/// Max topic size, if `Unlimited` then topic size is unlimited.
/// Can't be lower than segment size in the config.
pub max_topic_size: MaxTopicSize,
- /// Replication factor for the topic.
- pub replication_factor: Option<u8>,
/// Unique topic name, max length is 255 characters.
pub name: String,
+ /// Additional topic options as string key-values, parsed by the server
+ /// with the same rules as its config file. Unknown keys are rejected.
+ ///
+ /// A response renders each option as `{"value": "<string>", "explicit":
+ /// bool}`, in this same string form, so the values a `GET` reports can be
+ /// sent straight back here.
+ #[serde(default)]
+ pub options: BTreeMap<String, String>,
}
impl Default for CreateTopic {
@@ -61,8 +67,8 @@ impl Default for CreateTopic {
compression_algorithm: CompressionAlgorithm::None,
message_expiry: IggyExpiry::NeverExpire,
max_topic_size: MaxTopicSize::ServerDefault,
- replication_factor: None,
name: "topic".to_string(),
+ options: BTreeMap::new(),
}
}
}
@@ -77,12 +83,6 @@ impl Validatable<IggyError> for CreateTopic {
return Err(IggyError::TooManyPartitions);
}
- if let Some(replication_factor) = self.replication_factor
- && replication_factor == 0
- {
- return Err(IggyError::InvalidReplicationFactor);
- }
-
Ok(())
}
}
diff --git a/core/common/src/http/topics/update_topic.rs b/core/common/src/http/topics/update_topic.rs
index ac2303f9a..7208c88b9 100644
--- a/core/common/src/http/topics/update_topic.rs
+++ b/core/common/src/http/topics/update_topic.rs
@@ -23,16 +23,19 @@ use crate::error::IggyError;
use crate::utils::expiry::IggyExpiry;
use crate::utils::topic_size::MaxTopicSize;
use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
/// `UpdateTopic` command is used to update a topic in a stream.
/// It has additional payload:
/// - `stream_id` - unique stream ID (numeric or name).
/// - `topic_id` - unique topic ID (numeric or name).
-/// - `message_expiry` - message expiry, if `NeverExpire` then messages will never expire.
-/// - `max_topic_size` - maximum size of the topic in bytes, if `Unlimited` then topic size is unlimited.
-/// Can't be lower than segment size in the config.
-/// - `replication_factor` - replication factor for the topic.
/// - `name` - unique topic name, max length is 255 characters.
+/// - `compression_algorithm`, `message_expiry`, `max_topic_size` - omit a field
+/// to leave the topic's current value alone. Named here for REST ergonomics;
+/// the server folds them into the same option keys the binary protocol uses,
+/// so there is still one source per setting.
+/// - `options` - additional option keys as strings; only keys the update path
+/// accepts are allowed.
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct UpdateTopic {
/// Unique stream ID (numeric or name).
@@ -41,17 +44,21 @@ pub struct UpdateTopic {
/// Unique topic ID (numeric or name).
#[serde(skip)]
pub topic_id: Identifier,
- /// Compression algorithm for the topic.
- pub compression_algorithm: CompressionAlgorithm,
- /// Message expiry, if `NeverExpire` then messages will never expire.
- pub message_expiry: IggyExpiry,
- /// Max topic size, if `Unlimited` then topic size is unlimited.
- /// Can't be lower than segment size in the config.
- pub max_topic_size: MaxTopicSize,
- /// Replication factor for the topic.
- pub replication_factor: Option<u8>,
+ /// Compression algorithm; omit to leave the current one alone.
+ #[serde(default)]
+ pub compression_algorithm: Option<CompressionAlgorithm>,
+ /// Message expiry; omit to leave the current one alone.
+ #[serde(default)]
+ pub message_expiry: Option<IggyExpiry>,
+ /// Max topic size; omit to leave the current one alone.
+ #[serde(default)]
+ pub max_topic_size: Option<MaxTopicSize>,
/// Unique topic name, max length is 255 characters.
pub name: String,
+ /// Additional topic options as string key-values. Restricted to the keys
+ /// an update may change; anything else is rejected.
+ #[serde(default)]
+ pub options: BTreeMap<String, String>,
}
impl Default for UpdateTopic {
@@ -59,11 +66,11 @@ impl Default for UpdateTopic {
UpdateTopic {
stream_id: Identifier::default(),
topic_id: Identifier::default(),
- compression_algorithm: Default::default(),
- message_expiry: IggyExpiry::NeverExpire,
- max_topic_size: MaxTopicSize::ServerDefault,
- replication_factor: None,
+ compression_algorithm: None,
+ message_expiry: None,
+ max_topic_size: None,
name: "topic".to_string(),
+ options: BTreeMap::new(),
}
}
}
@@ -74,12 +81,6 @@ impl Validatable<IggyError> for UpdateTopic {
return Err(IggyError::InvalidTopicName);
}
- if let Some(replication_factor) = self.replication_factor
- && replication_factor == 0
- {
- return Err(IggyError::InvalidReplicationFactor);
- }
-
Ok(())
}
}
diff --git a/core/common/src/http/users/update_user.rs b/core/common/src/http/users/update_user.rs
index 0bce1d423..f29711cb3 100644
--- a/core/common/src/http/users/update_user.rs
+++ b/core/common/src/http/users/update_user.rs
@@ -21,6 +21,7 @@ use crate::UserStatus;
use crate::Validatable;
use crate::error::IggyError;
use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
/// `UpdateUser` command is used to update a user's username and status.
/// It has additional payload:
@@ -33,6 +34,10 @@ pub struct UpdateUser {
pub user_id: Identifier,
pub username: Option<String>,
pub status: Option<UserStatus>,
+ /// Additional user options as string key-values. Restricted to the keys
+ /// an update may change; anything else is rejected.
+ #[serde(default)]
+ pub options: BTreeMap<String, String>,
}
impl Validatable<IggyError> for UpdateUser {
diff --git a/core/common/src/lib.rs b/core/common/src/lib.rs
index b3bfc1c8f..777b5ef27 100644
--- a/core/common/src/lib.rs
+++ b/core/common/src/lib.rs
@@ -109,6 +109,7 @@ pub use types::either::Either;
pub use types::http::HttpMethod;
pub use types::identifier::*;
pub use types::message::*;
+pub use types::options::*;
pub use types::partition::*;
pub use types::permissions::permissions_global::*;
pub use types::permissions::personal_access_token::*;
diff --git a/core/common/src/traits/binary_impls/streams.rs b/core/common/src/traits/binary_impls/streams.rs
index c78339787..81375d97d 100644
--- a/core/common/src/traits/binary_impls/streams.rs
+++ b/core/common/src/traits/binary_impls/streams.rs
@@ -17,8 +17,9 @@
use crate::traits::binary_auth::fail_if_not_authenticated;
use crate::wire_conversions::{identifier_to_wire, streams_from_wire};
-use crate::{BinaryClient, Identifier, IggyError, Stream, StreamClient, StreamDetails};
-use iggy_binary_protocol::WireName;
+use crate::{
+ BinaryClient, Identifier, IggyError, Stream, StreamClient, StreamDetails, StreamUpdateOptions,
+};
use iggy_binary_protocol::codec::WireEncode;
use iggy_binary_protocol::codes::{
CREATE_STREAM_CODE, DELETE_STREAM_CODE, GET_STREAM_CODE, GET_STREAMS_CODE, PURGE_STREAM_CODE,
@@ -30,6 +31,7 @@ use iggy_binary_protocol::requests::streams::{
};
use iggy_binary_protocol::responses::streams::get_stream::GetStreamResponse;
use iggy_binary_protocol::responses::streams::get_streams::GetStreamsResponse;
+use iggy_binary_protocol::{WireName, WireOptions};
#[async_trait::async_trait]
impl<B: BinaryClient> StreamClient for B {
@@ -58,7 +60,7 @@ impl<B: BinaryClient> StreamClient for B {
return Ok(Vec::new());
}
let wire_resp = super::decode_response::<GetStreamsResponse>(&response)?;
- Ok(streams_from_wire(wire_resp))
+ Ok(streams_from_wire(wire_resp)?)
}
async fn create_stream(&self, name: &str) -> Result<StreamDetails, IggyError> {
@@ -67,14 +69,23 @@ impl<B: BinaryClient> StreamClient for B {
let response = self
.send_raw_with_response(
CREATE_STREAM_CODE,
- CreateStreamRequest { name: wire_name }.to_bytes(),
+ CreateStreamRequest {
+ name: wire_name,
+ options: WireOptions::empty(),
+ }
+ .to_bytes(),
)
.await?;
let wire_resp = super::decode_response::<GetStreamResponse>(&response)?;
Ok(StreamDetails::try_from(wire_resp)?)
}
- async fn update_stream(&self, stream_id: &Identifier, name: &str) -> Result<(), IggyError> {
+ async fn update_stream(
+ &self,
+ stream_id: &Identifier,
+ name: &str,
+ options: &StreamUpdateOptions,
+ ) -> Result<(), IggyError> {
fail_if_not_authenticated(self).await?;
let wire_id = identifier_to_wire(stream_id)?;
let wire_name = WireName::new(name).map_err(|_| IggyError::InvalidFormat)?;
@@ -83,6 +94,7 @@ impl<B: BinaryClient> StreamClient for B {
UpdateStreamRequest {
stream_id: wire_id,
name: wire_name,
+ options: options.to_wire()?,
}
.to_bytes(),
)
diff --git a/core/common/src/traits/binary_impls/system.rs b/core/common/src/traits/binary_impls/system.rs
index 09f82a997..b9a526c09 100644
--- a/core/common/src/traits/binary_impls/system.rs
+++ b/core/common/src/traits/binary_impls/system.rs
@@ -18,20 +18,21 @@
use crate::traits::binary_auth::fail_if_not_authenticated;
use crate::wire_conversions::clients_from_wire;
use crate::{
- BinaryClient, ClientInfo, ClientInfoDetails, IggyDuration, IggyError, Snapshot,
- SnapshotCompression, Stats, SystemClient, SystemSnapshotType,
+ BinaryClient, ClientInfo, ClientInfoDetails, IggyDuration, IggyError, OptionSpec, OptionsScope,
+ Snapshot, SnapshotCompression, Stats, SystemClient, SystemSnapshotType,
};
use iggy_binary_protocol::codec::WireEncode;
use iggy_binary_protocol::codes::{
- GET_CLIENT_CODE, GET_CLIENTS_CODE, GET_ME_CODE, GET_SNAPSHOT_FILE_CODE, GET_STATS_CODE,
- PING_CODE,
+ DESCRIBE_OPTIONS_CODE, GET_CLIENT_CODE, GET_CLIENTS_CODE, GET_ME_CODE, GET_SNAPSHOT_FILE_CODE,
+ GET_STATS_CODE, PING_CODE,
};
use iggy_binary_protocol::requests::system::{
- GetClientRequest, GetClientsRequest, GetMeRequest, GetSnapshotRequest, GetStatsRequest,
- PingRequest,
+ DescribeOptionsRequest, GetClientRequest, GetClientsRequest, GetMeRequest, GetSnapshotRequest,
+ GetStatsRequest, PingRequest,
};
use iggy_binary_protocol::responses::clients::get_client::ClientDetailsResponse;
use iggy_binary_protocol::responses::clients::get_clients::GetClientsResponse;
+use iggy_binary_protocol::responses::system::DescribeOptionsResponse;
use iggy_binary_protocol::responses::system::get_stats::StatsResponse;
#[async_trait::async_trait]
@@ -44,6 +45,21 @@ impl<B: BinaryClient> SystemClient for B {
Ok(Stats::from(wire_resp))
}
+ async fn describe_options(&self, scope: OptionsScope) -> Result<Vec<OptionSpec>, IggyError> {
+ fail_if_not_authenticated(self).await?;
+ let response = self
+ .send_raw_with_response(
+ DESCRIBE_OPTIONS_CODE,
+ DescribeOptionsRequest {
+ scope: scope.as_code(),
+ }
+ .to_bytes(),
+ )
+ .await?;
+ let wire_resp = super::decode_response::<DescribeOptionsResponse>(&response)?;
+ crate::wire_conversions::option_specs_from_wire(wire_resp)
+ }
+
async fn get_me(&self) -> Result<ClientInfoDetails, IggyError> {
fail_if_not_authenticated(self).await?;
let response = self
diff --git a/core/common/src/traits/binary_impls/topics.rs b/core/common/src/traits/binary_impls/topics.rs
index b0bfd1a5d..976c8b9d1 100644
--- a/core/common/src/traits/binary_impls/topics.rs
+++ b/core/common/src/traits/binary_impls/topics.rs
@@ -18,8 +18,8 @@
use crate::traits::binary_auth::fail_if_not_authenticated;
use crate::wire_conversions::{identifier_to_wire, topics_from_wire};
use crate::{
- BinaryClient, CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize, Topic,
- TopicClient, TopicDetails,
+ BinaryClient, DEFAULT_PARTITIONS_COUNT, Identifier, IggyError, Topic, TopicClient,
+ TopicCreateOptions, TopicDetails, TopicUpdateOptions,
};
use iggy_binary_protocol::WireName;
use iggy_binary_protocol::codec::WireEncode;
@@ -84,11 +84,7 @@ impl<B: BinaryClient> TopicClient for B {
&self,
stream_id: &Identifier,
name: &str,
- partitions_count: u32,
- compression_algorithm: CompressionAlgorithm,
- replication_factor: Option<u8>,
- message_expiry: IggyExpiry,
- max_topic_size: MaxTopicSize,
+ options: &TopicCreateOptions,
) -> Result<TopicDetails, IggyError> {
fail_if_not_authenticated(self).await?;
let wire_stream_id = identifier_to_wire(stream_id)?;
@@ -98,12 +94,9 @@ impl<B: BinaryClient> TopicClient for B {
CREATE_TOPIC_CODE,
CreateTopicRequest {
stream_id: wire_stream_id,
- partitions_count,
- compression_algorithm: compression_algorithm.as_code(),
- message_expiry: u64::from(message_expiry),
- max_topic_size: u64::from(max_topic_size),
- replication_factor: replication_factor.unwrap_or(0),
+ partitions_count: options.partitions_count.unwrap_or(DEFAULT_PARTITIONS_COUNT),
name: wire_name,
+ options: options.to_wire()?,
}
.to_bytes(),
)
@@ -117,10 +110,7 @@ impl<B: BinaryClient> TopicClient for B {
stream_id: &Identifier,
topic_id: &Identifier,
name: &str,
- compression_algorithm: CompressionAlgorithm,
- replication_factor: Option<u8>,
- message_expiry: IggyExpiry,
- max_topic_size: MaxTopicSize,
+ options: &TopicUpdateOptions,
) -> Result<(), IggyError> {
fail_if_not_authenticated(self).await?;
let wire_stream_id = identifier_to_wire(stream_id)?;
@@ -131,11 +121,8 @@ impl<B: BinaryClient> TopicClient for B {
UpdateTopicRequest {
stream_id: wire_stream_id,
topic_id: wire_topic_id,
- compression_algorithm: compression_algorithm.as_code(),
- message_expiry: u64::from(message_expiry),
- max_topic_size: u64::from(max_topic_size),
- replication_factor: replication_factor.unwrap_or(0),
name: wire_name,
+ options: options.to_wire()?,
}
.to_bytes(),
)
diff --git a/core/common/src/traits/binary_impls/users.rs b/core/common/src/traits/binary_impls/users.rs
index a37580366..eb785109a 100644
--- a/core/common/src/traits/binary_impls/users.rs
+++ b/core/common/src/traits/binary_impls/users.rs
@@ -19,9 +19,8 @@ use crate::traits::binary_auth::fail_if_not_authenticated;
use crate::wire_conversions::{identifier_to_wire, permissions_to_wire, users_from_wire};
use crate::{
BinaryClient, ClientState, DiagnosticEvent, Identifier, IdentityInfo, IggyError, Permissions,
- UserClient, UserInfo, UserInfoDetails, UserStatus,
+ UserClient, UserInfo, UserInfoDetails, UserStatus, UserUpdateOptions,
};
-use iggy_binary_protocol::WireName;
use iggy_binary_protocol::codec::WireEncode;
use iggy_binary_protocol::codes::LOGIN_REGISTER_CODE;
use iggy_binary_protocol::codes::{
@@ -35,6 +34,7 @@ use iggy_binary_protocol::requests::users::{
};
use iggy_binary_protocol::responses::users::LoginRegisterResponse;
use iggy_binary_protocol::responses::users::{GetUsersResponse, UserDetailsResponse};
+use iggy_binary_protocol::{WireName, WireOptions};
use secrecy::SecretString;
#[async_trait::async_trait]
@@ -87,6 +87,7 @@ impl<B: BinaryClient> UserClient for B {
password: password.to_string(),
status: status.as_code(),
permissions: wire_perms,
+ options: WireOptions::empty(),
}
.to_bytes(),
)
@@ -111,6 +112,7 @@ impl<B: BinaryClient> UserClient for B {
user_id: &Identifier,
username: Option<&str>,
status: Option<UserStatus>,
+ options: &UserUpdateOptions,
) -> Result<(), IggyError> {
fail_if_not_authenticated(self).await?;
let wire_id = identifier_to_wire(user_id)?;
@@ -124,6 +126,7 @@ impl<B: BinaryClient> UserClient for B {
user_id: wire_id,
username: wire_username,
status: status.map(|s| s.as_code()),
+ options: options.to_wire()?,
}
.to_bytes(),
)
diff --git a/core/common/src/traits/stream_client.rs b/core/common/src/traits/stream_client.rs
index 41aa0aee4..e5fa1ea46 100644
--- a/core/common/src/traits/stream_client.rs
+++ b/core/common/src/traits/stream_client.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::{Identifier, IggyError, Stream, StreamDetails};
+use crate::{Identifier, IggyError, Stream, StreamDetails, StreamUpdateOptions};
use async_trait::async_trait;
/// This trait defines the methods to interact with the stream module.
@@ -36,7 +36,12 @@ pub trait StreamClient {
/// Update a stream by unique ID or name.
///
/// Authentication is required, and the permission to manage the streams.
- async fn update_stream(&self, stream_id: &Identifier, name: &str) -> Result<(), IggyError>;
+ async fn update_stream(
+ &self,
+ stream_id: &Identifier,
+ name: &str,
+ options: &StreamUpdateOptions,
+ ) -> Result<(), IggyError>;
/// Delete a stream by unique ID or name.
///
/// Authentication is required, and the permission to manage the streams.
diff --git a/core/common/src/traits/system_client.rs b/core/common/src/traits/system_client.rs
index 4ccee4de8..fe85ef907 100644
--- a/core/common/src/traits/system_client.rs
+++ b/core/common/src/traits/system_client.rs
@@ -16,8 +16,8 @@
// under the License.
use crate::{
- ClientInfo, ClientInfoDetails, IggyDuration, IggyError, Snapshot, SnapshotCompression, Stats,
- SystemSnapshotType,
+ ClientInfo, ClientInfoDetails, IggyDuration, IggyError, OptionSpec, OptionsScope, Snapshot,
+ SnapshotCompression, Stats, SystemSnapshotType,
};
use async_trait::async_trait;
@@ -40,6 +40,13 @@ pub trait SystemClient {
///
/// Authentication is required, and the permission to read the server info.
async fn get_clients(&self) -> Result<Vec<ClientInfo>, IggyError>;
+ /// Describe the option catalog for a resource scope: the keys its create
+ /// command accepts, their canonical kinds, and this server's current
+ /// defaults. Unknown keys are rejected at create, so this is the way to
+ /// discover support rather than probing.
+ ///
+ /// Authentication is required.
+ async fn describe_options(&self, scope: OptionsScope) -> Result<Vec<OptionSpec>, IggyError>;
/// Ping the server to check if it's alive.
async fn ping(&self) -> Result<(), IggyError>;
async fn heartbeat_interval(&self) -> IggyDuration;
diff --git a/core/common/src/traits/topic_client.rs b/core/common/src/traits/topic_client.rs
index 26b31be96..930fa1ead 100644
--- a/core/common/src/traits/topic_client.rs
+++ b/core/common/src/traits/topic_client.rs
@@ -15,9 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::{
- CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize, Topic, TopicDetails,
-};
+use crate::{Identifier, IggyError, Topic, TopicCreateOptions, TopicDetails, TopicUpdateOptions};
use async_trait::async_trait;
/// This trait defines the methods to interact with the topic module.
@@ -38,17 +36,15 @@ pub trait TopicClient {
async fn get_topics(&self, stream_id: &Identifier) -> Result<Vec<Topic>, IggyError>;
/// Create a new topic.
///
- /// Authentication is required, and the permission to manage the topics.
- #[allow(clippy::too_many_arguments)]
+ /// Every knob rides `options`; an absent key resolves against the
+ /// server's defaults at admission, so a future key costs no signature
+ /// change here. Authentication is required, and the permission to manage
+ /// the topics.
async fn create_topic(
&self,
stream_id: &Identifier,
name: &str,
- partitions_count: u32,
- compression_algorithm: CompressionAlgorithm,
- replication_factor: Option<u8>,
- message_expiry: IggyExpiry,
- max_topic_size: MaxTopicSize,
+ options: &TopicCreateOptions,
) -> Result<TopicDetails, IggyError>;
/// Update a topic by unique ID or name.
///
@@ -58,10 +54,7 @@ pub trait TopicClient {
stream_id: &Identifier,
topic_id: &Identifier,
name: &str,
- compression_algorithm: CompressionAlgorithm,
- replication_factor: Option<u8>,
- message_expiry: IggyExpiry,
- max_topic_size: MaxTopicSize,
+ options: &TopicUpdateOptions,
) -> Result<(), IggyError>;
/// Delete a topic by unique ID or name.
///
diff --git a/core/common/src/traits/user_client.rs b/core/common/src/traits/user_client.rs
index 43fdddbb8..97766964c 100644
--- a/core/common/src/traits/user_client.rs
+++ b/core/common/src/traits/user_client.rs
@@ -17,6 +17,7 @@
use crate::{
Identifier, IdentityInfo, IggyError, Permissions, UserInfo, UserInfoDetails, UserStatus,
+ UserUpdateOptions,
};
use async_trait::async_trait;
@@ -53,6 +54,7 @@ pub trait UserClient {
user_id: &Identifier,
username: Option<&str>,
status: Option<UserStatus>,
+ options: &UserUpdateOptions,
) -> Result<(), IggyError>;
/// Update the permissions of a user by unique ID or username.
///
diff --git a/core/common/src/types/message/user_headers.rs b/core/common/src/types/message/user_headers.rs
index 54e2522f4..07223ce80 100644
--- a/core/common/src/types/message/user_headers.rs
+++ b/core/common/src/types/message/user_headers.rs
@@ -616,6 +616,28 @@ impl<T> HeaderField<T> {
_marker: PhantomData,
}
}
+
+ /// Wrap raw wire bytes as a field of `kind`.
+ ///
+ /// The option catalog hands out each key's default as a kind plus bytes, so
+ /// anything rendering a catalog entry needs a way back to a typed field.
+ ///
+ /// # Errors
+ ///
+ /// `IggyError::InvalidHeaderValue` when a fixed-size kind's payload is the
+ /// wrong length, or the payload exceeds the field length limit.
+ pub fn from_raw(kind: HeaderKind, value: &[u8]) -> Result<Self, IggyError> {
+ // 255 is the field length the TLV codec's own `TryFrom` impls enforce.
+ if value.is_empty() || value.len() > 255 {
+ return Err(IggyError::InvalidHeaderValue);
+ }
+ if let Some(expected) = kind.expected_size()
+ && value.len() != expected
+ {
+ return Err(IggyError::InvalidHeaderValue);
+ }
+ Ok(Self::new_unchecked(kind, value))
+ }
}
impl<T: Eq> Ord for HeaderField<T> {
diff --git a/core/common/src/types/mod.rs b/core/common/src/types/mod.rs
index f1a2f37af..feb5b4b56 100644
--- a/core/common/src/types/mod.rs
+++ b/core/common/src/types/mod.rs
@@ -27,6 +27,7 @@ pub(crate) mod either;
pub(crate) mod http;
pub(crate) mod identifier;
pub(crate) mod message;
+pub(crate) mod options;
pub(crate) mod partition;
pub(crate) mod permissions;
pub(crate) mod personal_access_tokens;
diff --git a/core/common/src/types/options/mod.rs b/core/common/src/types/options/mod.rs
new file mode 100644
index 000000000..a7ff727f2
--- /dev/null
+++ b/core/common/src/types/options/mod.rs
@@ -0,0 +1,1531 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Key-value options attached to streams, topics, and users at creation.
+//!
+//! Options reuse the typed [`HeaderKey`] / [`HeaderValue`] machinery from
+//! message user headers. The map MUST stay a `BTreeMap`: metadata snapshots
+//! require deterministic ordering across replicas, and a hash map would make
+//! snapshot bytes diverge per replica.
+//!
+//! # SDK parity
+//!
+//! Every SDK can set any option key, read both blocks back, and ask the server
+//! which keys it accepts. What differs is only how each language spells the
+//! argument, since each reuses the type it already had for message user headers:
+//!
+//! | SDK | Arbitrary keys on create/update | Reads the blocks | `describe_options` |
+//! |--------|--------------------------------------|------------------|--------------------|
+//! | Rust | `TopicCreateOptions::raw` | yes | yes |
+//! | Node | `options?: OptionEntry[]` | yes | yes |
+//! | Python | `options=` string dict | yes | yes |
+//! | Go | trailing `options ...HeaderEntry` | yes | yes |
+//! | Java | `Map<String, HeaderValue>` overload | yes | yes |
+//! | C# | optional `IReadOnlyDictionary` | yes | yes |
+//! | C++ | trailing `Vec<HeaderEntry>` | yes | yes |
+//!
+//! One transport caveat everywhere: REST renders option values as readable
+//! strings rather than the typed TLV, so a value read back over HTTP is
+//! String-kinded while the same value over a binary transport carries the kind
+//! the server stored. REST also reports both provenances in one map with a
+//! per-entry flag, which each SDK splits into the two maps its binary path
+//! produces.
+//!
+//! Beyond the eight typed keys, Go, Java, C# and C++ ship typed constructors or
+//! builders for the five keys that have no named parameter of their own, so no
+//! caller has to spell an option key as a bare string.
+//!
+//! The block's byte layout is pinned by a golden vector that Rust, Node, Go and
+//! Java each assert independently, so a new encoder has a fixture to match
+//! rather than a description to interpret.
+
+use std::collections::BTreeMap;
+use std::str::FromStr;
+
+use iggy_binary_protocol::{WireOptions, WireUserHeaderEntry};
+use serde::{Deserialize, Serialize};
+
+use crate::types::compression::compression_algorithm::CompressionAlgorithm;
+use crate::types::message::{HeaderKey, HeaderKind, HeaderValue};
+use crate::{IggyByteSize, IggyError, IggyExpiry, MaxTopicSize};
+
+/// A single resolved option entry.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct OptionValue {
+ /// The effective value, resolved by the admitting primary at creation.
+ pub value: HeaderValue,
+ /// Whether the client explicitly sent this key. Derived entries
+ /// (`explicit == false`) were filled from server defaults at admission
+ /// and would have resolved differently under other server configs.
+ pub explicit: bool,
+}
+
+/// Which provenance classes an encoded options block carries.
+///
+/// Responses split a resource's options into two blocks so `GetTopic` can say
+/// which values the client chose and which admission filled in.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum OptionsProvenance {
+ /// Every entry, whatever its provenance.
+ All,
+ /// Only the keys the client sent.
+ Explicit,
+ /// Only the keys admission resolved from server defaults.
+ Derived,
+}
+
+impl OptionValue {
+ /// Whether this entry belongs in a block encoded for `provenance`.
+ #[must_use]
+ pub const fn matches(&self, provenance: OptionsProvenance) -> bool {
+ match provenance {
+ OptionsProvenance::All => true,
+ OptionsProvenance::Explicit => self.explicit,
+ OptionsProvenance::Derived => !self.explicit,
+ }
+ }
+
+ #[must_use]
+ pub fn explicit(value: HeaderValue) -> Self {
+ Self {
+ value,
+ explicit: true,
+ }
+ }
+
+ #[must_use]
+ pub fn derived(value: HeaderValue) -> Self {
+ Self {
+ value,
+ explicit: false,
+ }
+ }
+}
+
+/// Options attached to a stream, topic, or user, keyed by option name.
+pub type ResourceOptions = BTreeMap<HeaderKey, OptionValue>;
+
+/// JSON form for [`ResourceOptions`].
+///
+/// [`HeaderKey`] serializes as a struct (`kind` + base64 `value`), and JSON
+/// object keys must be strings, so the derived impl fails outright with
+/// "key must be a string" the moment a map is non-empty. Every HTTP response
+/// carrying options goes through here instead, rendering the key as its plain
+/// text. Binary transports are unaffected: they never touch serde.
+pub mod resource_options_json {
+ use super::{HeaderKey, HeaderValue, OptionValue, ResourceOptions};
+ use serde::de::Error as _;
+ use serde::{Deserialize, Deserializer, Serialize, Serializer};
+ use std::collections::BTreeMap;
+ use std::str::FromStr;
+
+ /// One entry's JSON form: the value in the same string form a config file
+ /// or a `POST` body would carry it in, plus its provenance.
+ ///
+ /// The stored value is a typed [`HeaderValue`], and serializing that gives
+ /// `{"kind":"uint64","value":"<base64>"}` - which no operator can read and
+ /// no client can feed back, since the create body takes options as a plain
+ /// string map. Rendering the string form makes the "re-send only the
+ /// explicit options" round trip actually expressible over HTTP.
+ #[derive(Serialize, Deserialize)]
+ struct OptionValueJson {
+ value: String,
+ explicit: bool,
+ }
+
+ /// # Errors
+ ///
+ /// Propagates the serializer's own errors.
+ pub fn serialize<S: Serializer>(
+ options: &ResourceOptions,
+ serializer: S,
+ ) -> Result<S::Ok, S::Error> {
+ let readable: BTreeMap<String, OptionValueJson> = options
+ .iter()
+ .map(|(key, option)| {
+ (
+ String::from_utf8_lossy(key.as_bytes()).into_owned(),
+ OptionValueJson {
+ value: option.value.to_string_value(),
+ explicit: option.explicit,
+ },
+ )
+ })
+ .collect();
+ readable.serialize(serializer)
+ }
+
+ /// # Errors
+ ///
+ /// Returns a deserializer error when a key is not a valid option name or a
+ /// value does not fit a header value.
+ pub fn deserialize<'de, D: Deserializer<'de>>(
+ deserializer: D,
+ ) -> Result<ResourceOptions, D::Error> {
+ let readable = BTreeMap::<String, OptionValueJson>::deserialize(deserializer)?;
+ readable
+ .into_iter()
+ .map(|(key, option)| {
+ let key = HeaderKey::from_str(&key).map_err(D::Error::custom)?;
+ let value = HeaderValue::from_str(&option.value).map_err(D::Error::custom)?;
+ Ok((
+ key,
+ OptionValue {
+ value,
+ explicit: option.explicit,
+ },
+ ))
+ })
+ .collect()
+ }
+}
+
+/// Topic option catalog: the keys `CreateTopic` accepts.
+pub mod topic_option_keys {
+ /// Compression algorithm name (`none`, `gzip`).
+ pub const COMPRESSION_ALGORITHM: &str = "compression_algorithm";
+ /// Message expiry: `Uint64` micros or a humantime string (`7 days`).
+ pub const MESSAGE_EXPIRY: &str = "message_expiry";
+ /// Topic size cap: `Uint64` bytes or a byte-size string (`1 GiB`).
+ pub const MAX_TOPIC_SIZE: &str = "max_topic_size";
+ /// Segment size: `Uint64` bytes or a byte-size string (`128 MiB`).
+ /// Bounded: a 512-byte multiple within
+ /// [`super::MIN_TOPIC_SEGMENT_SIZE`]..=the server's segment ceiling.
+ pub const SEGMENT_SIZE: &str = "segment_size";
+ /// Whether writes to this topic's partitions fsync: `Bool`, or the
+ /// strings `true` / `false`.
+ pub const ENFORCE_FSYNC: &str = "enforce_fsync";
+ /// Flush the journal once it holds this many messages: `Uint32`.
+ /// Must be non-zero.
+ pub const MESSAGES_REQUIRED_TO_SAVE: &str = "messages_required_to_save";
+ /// Flush the journal once it holds this many bytes: `Uint64` or a
+ /// byte-size string. Paired with [`Self::MESSAGES_REQUIRED_TO_SAVE`];
+ /// whichever threshold trips first flushes.
+ pub const SIZE_OF_MESSAGES_REQUIRED_TO_SAVE: &str = "size_of_messages_required_to_save";
+ /// Reserve the segment's bytes up front on a filesystem that supports it:
+ /// `Bool`, or the strings `true` / `false`. Pairs with
+ /// [`Self::SEGMENT_SIZE`] -- preallocation reserves exactly that much, so
+ /// the two only make sense decided together.
+ pub const PREALLOCATE_SEGMENTS: &str = "preallocate_segments";
+}
+
+/// Values an absent topic option resolves to at admission.
+///
+/// These are the knobs' single source of truth: they used to live in
+/// `config.toml` (`[system.topic]`, `[system.partition]`, `[system.segment]`),
+/// which meant every one of them had two homes and an operator could not tell
+/// which won. A topic carries whatever it was created with; anything the
+/// client did not send resolves to the constant here and is persisted as a
+/// derived entry, so the effective value is always visible on `GetTopic`.
+///
+/// Each value matches what the shipped `config.toml` carried, so removing the
+/// keys changed no behavior for a topic created without options.
+pub const DEFAULT_PARTITIONS_COUNT: u32 = 1;
+/// `MaxTopicSize::Unlimited` (was `[system.topic] max_size = "unlimited"`).
+pub const DEFAULT_MAX_TOPIC_SIZE: u64 = u64::MAX;
+/// `IggyExpiry::NeverExpire` (was `[system.topic] message_expiry = "none"`).
+pub const DEFAULT_MESSAGE_EXPIRY: u64 = u64::MAX;
+/// 1 GiB (was `[system.segment] size = "1 GiB"`).
+pub const DEFAULT_SEGMENT_SIZE: u64 = 1024 * 1024 * 1024;
+/// Was `[system.partition] enforce_fsync = false`.
+pub const DEFAULT_ENFORCE_FSYNC: bool = false;
+/// Was `[system.partition] messages_required_to_save = 1024`.
+pub const DEFAULT_MESSAGES_REQUIRED_TO_SAVE: u32 = 1024;
+/// Opt-in, unlike the `[system.segment] preallocate = true` this replaced.
+///
+/// That default was never actually in force: the reservation ran through
+/// `compio::spawn_blocking`, which panics the shard because shard executors
+/// disable the blocking pool, so any deployment that worked at all had
+/// preallocation off. With the call fixed to run inline it reserves real
+/// extents, and `FALLOC_FL_KEEP_SIZE` against the 1 GiB default segment size
+/// means 1 GiB of disk per partition the moment it is created -- a full test
+/// sweep reserved 393 GB before this was flipped. A topic that wants the
+/// latency benefit asks for it with `preallocate_segments`.
+pub const DEFAULT_PREALLOCATE_SEGMENTS: bool = false;
+/// 1 MiB (was `[system.partition] size_of_messages_required_to_save`).
+pub const DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE: u64 = 1024 * 1024;
+
+/// Every runtime knob at its default, for a partition built with no resolved
+/// topic options (simulator, unit tests).
+impl Default for TopicRuntimeDefaults {
+ fn default() -> Self {
+ Self {
+ segment_size: IggyByteSize::from(DEFAULT_SEGMENT_SIZE),
+ enforce_fsync: DEFAULT_ENFORCE_FSYNC,
+ messages_required_to_save: DEFAULT_MESSAGES_REQUIRED_TO_SAVE,
+ size_of_messages_required_to_save: IggyByteSize::from(
+ DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE,
+ ),
+ preallocate_segments: DEFAULT_PREALLOCATE_SEGMENTS,
+ }
+ }
+}
+
+/// Ceiling for `size_of_messages_required_to_save`.
+///
+/// A threshold above the largest a segment may be can never trip, so every
+/// committed message would sit in the journal until its segment rotates, and a
+/// crash does not preserve the journal. These two thresholds used to be
+/// operator-only config; anyone holding `create_topic` can set them now, so the
+/// ceilings are enforced at admission. Mirrors
+/// `configs::validators::SEGMENT_MAX_SIZE_BYTES`, which lives in the crate that
+/// depends on this one.
+pub const MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE: u64 = 1024 * 1024 * 1024;
+
+/// Ceiling for `messages_required_to_save`, derived from the byte ceiling: a
+/// segment that large cannot hold more messages than this, because a message
+/// costs at least its header on disk.
+pub const MAX_MESSAGES_REQUIRED_TO_SAVE: u32 = 16 * 1024 * 1024;
+
+/// Smallest per-topic segment size. Segments far below the shipped default
+/// explode the per-partition segment count, which state transfer bounds via
+/// its manifest entry cap; a partition that crosses it becomes unservable
+/// for transfer, so the floor is enforced at admission.
+pub const MIN_TOPIC_SEGMENT_SIZE: u64 = 1024 * 1024;
+
+/// Largest per-topic `segment_size` a node admits.
+///
+/// The ceiling used to be computed per node as the smaller of the global
+/// segment maximum and `transfer_artifact_bytes_max - max_message_size` (a
+/// received segment artifact can be one whole batch past the cap, and an
+/// artifact ceiling below that livelocks a partition's rejoin). Config
+/// validation now refuses boot unless that subtraction is at least the segment
+/// maximum, so the minimum is always the maximum and the per-node expression was
+/// dead. Mirrors `configs::validators::SEGMENT_MAX_SIZE_BYTES`, which lives in
+/// the crate that depends on this one and pins the two together in a test.
+pub const MAX_TOPIC_SEGMENT_SIZE: u64 = 1024 * 1024 * 1024;
+
+/// Ceiling on what one `preallocate_segments` topic may reserve when created:
+/// `segment_size * partitions_count`.
+///
+/// The reservation runs inline on the shard thread - once per partition as
+/// segments rotate, and again for every owned partition at boot - and
+/// `FALLOC_FL_KEEP_SIZE` makes it real disk rather than a sparse hole. Without a
+/// cap, one create at the 1 GiB default segment size and the maximum partition
+/// count reserves a terabyte before a single message arrives.
+pub const MAX_PREALLOCATED_TOPIC_BYTES: u64 = 64 * 1024 * 1024 * 1024;
+
+/// Validate what a preallocating topic would reserve at admission.
+///
+/// `segment_size_bytes` is the topic's resolved segment size. Callers skip this
+/// for a topic whose effective `preallocate_segments` is false: nothing is
+/// reserved up front there, so the product does not bound anything.
+///
+/// # Errors
+///
+/// Returns `IggyError::InvalidOptionValue("preallocate_segments")` when the
+/// reservation would exceed [`MAX_PREALLOCATED_TOPIC_BYTES`].
+pub fn validate_preallocated_topic_bytes(
+ segment_size_bytes: u64,
+ partitions_count: u32,
+) -> Result<(), IggyError> {
+ let reserved = segment_size_bytes.saturating_mul(u64::from(partitions_count));
+ if reserved > MAX_PREALLOCATED_TOPIC_BYTES {
+ return Err(IggyError::InvalidOptionValue(
+ topic_option_keys::PREALLOCATE_SEGMENTS.to_string(),
+ ));
+ }
+ Ok(())
+}
+
+/// Validate an explicit per-topic `segment_size` against its bounds.
+///
+/// `ceiling` is node-derived: the smaller of the global segment maximum and
+/// the state-transfer artifact budget minus one bus frame (a segment may
+/// close one whole batch past its cap; an artifact ceiling below that
+/// refuses a legal segment and livelocks the partition's rejoin).
+///
+/// # Errors
+///
+/// Returns `IggyError::InvalidOptionValue("segment_size")` when the value is
+/// below [`MIN_TOPIC_SEGMENT_SIZE`], above `ceiling`, or not a 512-byte
+/// multiple.
+pub fn validate_topic_segment_size(size_bytes: u64, ceiling: u64) -> Result<(), IggyError> {
+ if size_bytes < MIN_TOPIC_SEGMENT_SIZE
+ || size_bytes > ceiling
+ || !size_bytes.is_multiple_of(512)
+ {
+ return Err(IggyError::InvalidOptionValue(
+ topic_option_keys::SEGMENT_SIZE.to_string(),
+ ));
+ }
+ Ok(())
+}
+
+/// Resource whose option catalog `DescribeOptions` serves.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum OptionsScope {
+ Topic,
+ Stream,
+ User,
+}
+
+impl OptionsScope {
+ #[must_use]
+ pub fn as_code(&self) -> u8 {
+ match self {
+ Self::Topic => 1,
+ Self::Stream => 2,
+ Self::User => 3,
+ }
+ }
+
+ /// # Errors
+ ///
+ /// Returns `IggyError::InvalidCommand` for an unknown scope code.
+ pub fn from_code(code: u8) -> Result<Self, IggyError> {
+ match code {
+ 1 => Ok(Self::Topic),
+ 2 => Ok(Self::Stream),
+ 3 => Ok(Self::User),
+ _ => Err(IggyError::InvalidCommand),
+ }
+ }
+}
+
+impl std::fmt::Display for OptionsScope {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Self::Topic => write!(f, "topic"),
+ Self::Stream => write!(f, "stream"),
+ Self::User => write!(f, "user"),
+ }
+ }
+}
+
+impl FromStr for OptionsScope {
+ type Err = IggyError;
+
+ fn from_str(scope: &str) -> Result<Self, Self::Err> {
+ match scope {
+ "topic" => Ok(Self::Topic),
+ "stream" => Ok(Self::Stream),
+ "user" => Ok(Self::User),
+ _ => Err(IggyError::InvalidCommand),
+ }
+ }
+}
+
+/// One entry of a resource's option catalog, as served by `DescribeOptions`.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct OptionSpec {
+ /// Option key accepted by the create command.
+ pub key: String,
+ /// Canonical kind for this key: what the server encodes its default under,
+ /// and what a value set at CREATE is stored as whatever kind the client sent
+ /// it as. Create admission re-encodes the block from its own parse, so a
+ /// `--set segment_size=128MiB` string lands as a `Uint64`.
+ ///
+ /// An UPDATE is the exception: it stores the client's bytes verbatim, so a
+ /// key set that way keeps the kind it arrived in.
+ pub kind: HeaderKind,
+ /// The key's default in `kind`'s encoding; empty when the key has no
+ /// default. A build constant rather than a per-node value: these defaults
+ /// stopped being config-derived when the `[system.*]` keys became options.
+ #[serde(default)]
+ pub default_value: Vec<u8>,
+ /// Human-readable description.
+ #[serde(default)]
+ pub description: String,
+}
+
+/// Every key `CreateTopic` currently accepts. Unknown keys are rejected at
+/// the edge, never skipped: a silently ignored knob would hand the client
+/// server defaults without it ever learning.
+pub const TOPIC_OPTION_KEYS: &[&str] = &[
+ topic_option_keys::COMPRESSION_ALGORITHM,
+ topic_option_keys::MESSAGE_EXPIRY,
+ topic_option_keys::MAX_TOPIC_SIZE,
+ topic_option_keys::SEGMENT_SIZE,
+ topic_option_keys::ENFORCE_FSYNC,
+ topic_option_keys::MESSAGES_REQUIRED_TO_SAVE,
+ topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE,
+ topic_option_keys::PREALLOCATE_SEGMENTS,
+];
+
+/// The subset of [`TOPIC_OPTION_KEYS`] an `UpdateTopic` options block may
+/// carry.
+///
+/// These three used to be fixed fields of the update command, which meant one
+/// setting had two homes and an update always rewrote all three whether the
+/// caller meant to or not. As options they are patched: a key the client did
+/// not send keeps its current value.
+///
+/// The partition runtime knobs (`segment_size`, `enforce_fsync`, both flush
+/// thresholds, `preallocate_segments`) stay out, and not only because nothing
+/// re-pushes them to a live partition. They describe how a partition's storage
+/// was laid down: changing `segment_size` mid-segment leaves one segment sized
+/// by the old cap and the next by the new one, and `preallocate_segments` can
+/// only act on a file not yet opened. A topic gets them at creation and keeps
+/// them, so its segments stay uniform.
+pub const UPDATABLE_TOPIC_OPTION_KEYS: &[&str] = &[
+ topic_option_keys::COMPRESSION_ALGORITHM,
+ topic_option_keys::MESSAGE_EXPIRY,
+ topic_option_keys::MAX_TOPIC_SIZE,
+];
+
+/// Keys an `UpdateStream` options block may carry. Empty because streams have
+/// no catalog keys yet: the block exists so the first one costs a catalog
+/// entry instead of another wire change, and until then every key is rejected
+/// by name rather than stored and ignored.
+pub const UPDATABLE_STREAM_OPTION_KEYS: &[&str] = &[];
+
+/// Keys an `UpdateUser` options block may carry. Empty for the same reason as
+/// [`UPDATABLE_STREAM_OPTION_KEYS`].
+pub const UPDATABLE_USER_OPTION_KEYS: &[&str] = &[];
+
+/// Longest key echoed back in an option error. A rejected key is attacker-sized
+/// by definition, and over HTTP the error text rides the response body back to
+/// the client.
+///
+/// Binary transports carry only the error code, so a TCP, QUIC or WebSocket
+/// client rebuilds [`IggyError::UnsupportedOptionKey`] with an empty string and
+/// never sees which key was refused. `DescribeOptions` is the discovery path
+/// there - see the note on that variant.
+const ERROR_KEY_PREVIEW_LEN: usize = 64;
+
+fn key_preview(key: &str) -> String {
+ key.chars().take(ERROR_KEY_PREVIEW_LEN).collect()
+}
+
+/// Build an options map from string key-values, all marked explicit.
+///
+/// Shared by the update-options types: their keys are all client-sent, so none
+/// of the derived-provenance handling that create needs applies. Callers that
+/// also have typed fields insert those afterwards, so a typed value wins on
+/// collision and keeps its canonical kind.
+///
+/// # Errors
+///
+/// `UnsupportedOptionKey` when a key is empty or over 255 bytes,
+/// `InvalidOptionValue` when a value is. Both bounds come from the
+/// header-field codec these entries ride.
+fn raw_options_map(raw: &BTreeMap<String, String>) -> Result<ResourceOptions, IggyError> {
+ let mut options = ResourceOptions::new();
+ for (key, value) in raw {
+ let header_key = HeaderKey::from_str(key)
+ .map_err(|_| IggyError::UnsupportedOptionKey(key_preview(key)))?;
+ let header_value = HeaderValue::try_from(value.as_str())
+ .map_err(|_| IggyError::InvalidOptionValue(key_preview(key)))?;
+ options.insert(header_key, OptionValue::explicit(header_value));
+ }
+ Ok(options)
+}
+
+/// Encode string key-values into an options block, all marked explicit.
+///
+/// # Errors
+///
+/// See [`raw_options_map`].
+fn raw_options_to_wire(raw: &BTreeMap<String, String>) -> Result<WireOptions, IggyError> {
+ crate::wire_conversions::resource_options_to_wire(
+ &raw_options_map(raw)?,
+ OptionsProvenance::All,
+ )
+}
+
+/// The options an `UpdateStream` may carry.
+///
+/// Absent keys leave the stream's existing values alone -- an update patches
+/// the option map, it does not replace it. See [`TopicUpdateOptions`] for why.
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct StreamUpdateOptions {
+ /// Keys sent as `String` values, checked against
+ /// [`UPDATABLE_STREAM_OPTION_KEYS`] server-side. That list is empty, so
+ /// every key is currently refused by name; the field exists so the first
+ /// updatable stream key costs a catalog entry, not a wire change.
+ pub raw: BTreeMap<String, String>,
+}
+
+impl StreamUpdateOptions {
+ /// Encode the present keys into an options block.
+ ///
+ /// # Errors
+ ///
+ /// See [`raw_options_to_wire`].
+ pub fn to_wire(&self) -> Result<WireOptions, IggyError> {
+ raw_options_to_wire(&self.raw)
+ }
+}
+
+/// The options an `UpdateUser` may carry. Same patch semantics as
+/// [`StreamUpdateOptions`].
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct UserUpdateOptions {
+ /// Keys sent as `String` values, checked against
+ /// [`UPDATABLE_USER_OPTION_KEYS`] server-side. That list is empty, so every
+ /// key is currently refused by name; the field exists so the first
+ /// updatable user key costs a catalog entry, not a wire change.
+ pub raw: BTreeMap<String, String>,
+}
+
+impl UserUpdateOptions {
+ /// Encode the present keys into an options block.
+ ///
+ /// # Errors
+ ///
+ /// See [`raw_options_to_wire`].
+ pub fn to_wire(&self) -> Result<WireOptions, IggyError> {
+ raw_options_to_wire(&self.raw)
+ }
+}
+
+/// This node's configured fallbacks for the runtime knobs, used to fill the
+/// derived-options block for keys a client did not send.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct TopicRuntimeDefaults {
+ pub segment_size: IggyByteSize,
+ pub enforce_fsync: bool,
+ pub messages_required_to_save: u32,
+ pub size_of_messages_required_to_save: IggyByteSize,
+ pub preallocate_segments: bool,
+}
+
+/// A topic's resolved runtime knobs, as carried from the metadata plane to
+/// each of its partitions. `None` means "keep the shard-wide configured
+/// value": topics created without an options block (simulator, unit tests)
+/// have no resolved values to carry.
+#[derive(Debug, Clone, Copy, Default, PartialEq)]
+pub struct TopicRuntimeOptions {
+ pub segment_size: Option<IggyByteSize>,
+ pub enforce_fsync: Option<bool>,
+ pub messages_required_to_save: Option<u32>,
+ pub size_of_messages_required_to_save: Option<IggyByteSize>,
+ pub preallocate_segments: Option<bool>,
+}
+
+impl TopicRuntimeOptions {
+ /// Derive the runtime knobs from a topic's persisted options map.
+ ///
+ /// Degrades per key, not per map. An entry this build cannot interpret
+ /// leaves its own knob unset and every other knob intact, so one key a
+ /// newer node wrote cannot silently drop a topic's `enforce_fsync` or
+ /// reset its segment size along with it.
+ #[must_use]
+ pub fn from_resource_options(options: &ResourceOptions) -> Self {
+ let parsed = TopicCreateOptions::from_resource_options(options);
+ Self {
+ segment_size: parsed.segment_size,
+ enforce_fsync: parsed.enforce_fsync,
+ messages_required_to_save: parsed.messages_required_to_save,
+ size_of_messages_required_to_save: parsed.size_of_messages_required_to_save,
+ preallocate_segments: parsed.preallocate_segments,
+ }
+ }
+}
+
+/// The options an `UpdateTopic` may carry, as a type that cannot express a
+/// key the update path rejects (see [`UPDATABLE_TOPIC_OPTION_KEYS`]).
+///
+/// Separate from [`TopicCreateOptions`] on purpose: reusing that struct would
+/// let a caller set `segment_size` on an update, encode it, and only learn at
+/// the server that it was refused. Absent keys leave the topic's existing
+/// value alone -- an update patches the option map, it does not replace it.
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct TopicUpdateOptions {
+ /// `None` leaves the topic's current algorithm alone.
+ pub compression_algorithm: Option<CompressionAlgorithm>,
+ /// `None` leaves the topic's current expiry alone.
+ pub message_expiry: Option<IggyExpiry>,
+ /// `None` leaves the topic's current cap alone.
+ pub max_topic_size: Option<MaxTopicSize>,
+ /// Keys sent as `String` values, checked against
+ /// [`UPDATABLE_TOPIC_OPTION_KEYS`] server-side. Lets a client reach an
+ /// updatable key added to the catalog after this build shipped.
+ pub raw: BTreeMap<String, String>,
+}
+
+impl TopicUpdateOptions {
+ /// Encode the present keys into an options block, in canonical kinds.
+ ///
+ /// A typed field is inserted after the raw entries, so it wins on collision
+ /// and keeps its canonical kind.
+ ///
+ /// # Errors
+ ///
+ /// See [`raw_options_map`].
+ pub fn to_wire(&self) -> Result<WireOptions, IggyError> {
+ let mut options = raw_options_map(&self.raw)?;
+ if let Some(compression_algorithm) = self.compression_algorithm {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::COMPRESSION_ALGORITHM)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(
+ HeaderValue::try_from(compression_algorithm.to_string().as_str())
+ .expect("compression name fits a header value"),
+ ),
+ );
+ }
+ if let Some(message_expiry) = self.message_expiry {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(HeaderValue::from(u64::from(message_expiry))),
+ );
+ }
+ if let Some(max_topic_size) = self.max_topic_size {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(HeaderValue::from(u64::from(max_topic_size))),
+ );
+ }
+ crate::wire_conversions::resource_options_to_wire(&options, OptionsProvenance::All)
+ }
+}
+
+/// Typed view of the known topic option keys, parsed from a wire block.
+///
+/// `None` means the key was absent, which always means "resolve from server
+/// defaults at admission". Values that parse to their type's `ServerDefault`
+/// sentinel are normalized to `None` for the same reason.
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct TopicCreateOptions {
+ /// Partitions to allocate. NOT an option key: it fills the `CreateTopic`
+ /// command's own fixed field, because it is an argument to the operation
+ /// rather than a property of the topic (admission consumes it to compute
+ /// assignments, and a stored count would go stale on the first
+ /// `CreatePartitions`). Carried here so callers pass one bundle.
+ /// `None` means [`DEFAULT_PARTITIONS_COUNT`].
+ pub partitions_count: Option<u32>,
+ pub compression_algorithm: Option<CompressionAlgorithm>,
+ pub message_expiry: Option<IggyExpiry>,
+ pub max_topic_size: Option<MaxTopicSize>,
+ /// Per-topic segment size; `None` resolves against `[system.segment]
+ /// size` at admission. `0` is normalized to `None`.
+ pub segment_size: Option<IggyByteSize>,
+ /// Per-topic fsync enforcement; `None` resolves against
+ /// `[system.partition] enforce_fsync`.
+ pub enforce_fsync: Option<bool>,
+ /// Per-topic message-count flush threshold; `None` resolves against
+ /// `[system.partition] messages_required_to_save`. `0` is rejected.
+ pub messages_required_to_save: Option<u32>,
+ /// Per-topic byte flush threshold; `None` resolves against
+ /// [`DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE`].
+ pub size_of_messages_required_to_save: Option<IggyByteSize>,
+ /// Whether this topic's segments reserve their bytes on open; `None`
+ /// resolves against [`DEFAULT_PREALLOCATE_SEGMENTS`].
+ pub preallocate_segments: Option<bool>,
+ /// String-valued keys with no typed field above, parsed server-side
+ /// through each key's `FromStr`. Lets a client reach a key added to the
+ /// server catalog after this build shipped. Outbound only: a typed field
+ /// wins on collision in [`Self::to_wire`], and [`Self::parse`] never
+ /// populates it.
+ pub raw: BTreeMap<String, String>,
+}
+
+impl TopicCreateOptions {
+ /// Parse a wire options block against the topic catalog.
+ ///
+ /// # Errors
+ ///
+ /// `UnsupportedOptionKey` for a key outside [`TOPIC_OPTION_KEYS`];
+ /// `InvalidOptionValue` when a value has the wrong kind or fails its
+ /// type's `FromStr`.
+ pub fn parse(options: &WireOptions) -> Result<Self, IggyError> {
+ let mut parsed = Self::default();
+ for entry in options {
+ // Wire validation already enforced UTF-8 string keys.
+ let key = String::from_utf8_lossy(entry.key);
+ parsed.absorb_strict(&entry, &key)?;
+ }
+ Ok(parsed)
+ }
+
+ /// Parse a block that is already COMMITTED, skipping what this build
+ /// cannot interpret.
+ ///
+ /// Admission gated the block against the catalog on the primary, once.
+ /// Re-running that gate at apply would make the verdict depend on the
+ /// build running it, so a replica that predates a key would reject an
+ /// operation its peers accepted and diverge from the group permanently.
+ /// The catalog belongs at the edge; apply only reads what it knows.
+ #[must_use]
+ pub fn parse_committed(options: &WireOptions) -> Self {
+ let mut parsed = Self::default();
+ for entry in options {
+ let key = String::from_utf8_lossy(entry.key);
+ // Discarding the error is the skip: see `absorb_strict`.
+ let _ = parsed.absorb_strict(&entry, &key);
+ }
+ parsed
+ }
+
+ /// Fold one catalog entry into `self`, refusing an entry this build cannot
+ /// interpret: a key outside [`TOPIC_OPTION_KEYS`], or a catalog key whose
+ /// value kind or payload does not parse. Shared by [`Self::parse`] (wire
+ /// block) and [`Self::from_resource_options`] (persisted map) so both read
+ /// the identical key set, kinds and value bounds.
+ ///
+ /// Every arm assigns only after its own parse succeeds, which is what makes
+ /// discarding the error a safe skip: a refused entry leaves `self` exactly
+ /// as it found it.
+ fn absorb_strict(
+ &mut self,
+ entry: &WireUserHeaderEntry<'_>,
+ key: &str,
+ ) -> Result<(), IggyError> {
+ let parsed = self;
+ {
+ match key {
+ topic_option_keys::COMPRESSION_ALGORITHM => {
+ let value = parse_compression(entry, key)?;
+ parsed.compression_algorithm = Some(value);
+ }
+ topic_option_keys::MESSAGE_EXPIRY => {
+ let expiry = IggyExpiry::from(parse_u64_or(entry, key, IggyExpiry::from_str)?);
+ parsed.message_expiry = (expiry != IggyExpiry::ServerDefault).then_some(expiry);
+ }
+ topic_option_keys::MAX_TOPIC_SIZE => {
+ let size =
+ MaxTopicSize::from(parse_u64_or(entry, key, MaxTopicSize::from_str)?);
+ parsed.max_topic_size = (size != MaxTopicSize::ServerDefault).then_some(size);
+ }
+ topic_option_keys::SEGMENT_SIZE => {
+ let size = parse_byte_size(entry, key)?;
+ parsed.segment_size = (size != 0).then_some(IggyByteSize::from(size));
+ }
+ topic_option_keys::ENFORCE_FSYNC => {
+ parsed.enforce_fsync = Some(parse_bool(entry, key)?);
+ }
+ topic_option_keys::MESSAGES_REQUIRED_TO_SAVE => {
+ let messages = parse_u32(entry, key)?;
+ if messages == 0 || messages > MAX_MESSAGES_REQUIRED_TO_SAVE {
+ return Err(IggyError::InvalidOptionValue(key.to_string()));
+ }
+ parsed.messages_required_to_save = Some(messages);
+ }
+ topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE => {
+ let size = parse_byte_size(entry, key)?;
+ if size > MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE {
+ return Err(IggyError::InvalidOptionValue(key.to_string()));
+ }
+ parsed.size_of_messages_required_to_save =
+ (size != 0).then_some(IggyByteSize::from(size));
+ }
+ topic_option_keys::PREALLOCATE_SEGMENTS => {
+ parsed.preallocate_segments = Some(parse_bool(entry, key)?);
+ }
+ _ => return Err(IggyError::UnsupportedOptionKey(key.to_string())),
+ }
+ }
+ Ok(())
+ }
+
+ /// Field-wise overlay: every key this block sets wins, the rest fall back
+ /// to `defaults`.
+ ///
+ /// Apply resolves the client block over the derived one this way. `raw` is
+ /// dropped: by the time a block reaches apply, every key it named has
+ /// already been parsed into a typed field or refused by admission.
+ #[must_use]
+ pub fn resolved_over(&self, defaults: &Self) -> Self {
+ Self {
+ partitions_count: self.partitions_count.or(defaults.partitions_count),
+ compression_algorithm: self
+ .compression_algorithm
+ .or(defaults.compression_algorithm),
+ message_expiry: self.message_expiry.or(defaults.message_expiry),
+ max_topic_size: self.max_topic_size.or(defaults.max_topic_size),
+ segment_size: self.segment_size.or(defaults.segment_size),
+ enforce_fsync: self.enforce_fsync.or(defaults.enforce_fsync),
+ messages_required_to_save: self
+ .messages_required_to_save
+ .or(defaults.messages_required_to_save),
+ size_of_messages_required_to_save: self
+ .size_of_messages_required_to_save
+ .or(defaults.size_of_messages_required_to_save),
+ preallocate_segments: self.preallocate_segments.or(defaults.preallocate_segments),
+ raw: BTreeMap::new(),
+ }
+ }
+
+ /// Encode the present keys into a client options block, in canonical
+ /// kinds. The client-side counterpart of [`Self::parse`].
+ ///
+ /// # Errors
+ ///
+ /// See [`Self::to_option_map`].
+ pub fn to_wire(&self) -> Result<WireOptions, IggyError> {
+ crate::wire_conversions::resource_options_to_wire(
+ &self.to_option_map()?,
+ OptionsProvenance::All,
+ )
+ }
+
+ /// The present keys as a canonically-kinded option map.
+ ///
+ /// A typed field is inserted after the raw entries, so it wins on
+ /// collision and keeps its canonical kind.
+ ///
+ /// # Errors
+ ///
+ /// See [`raw_options_map`].
+ pub fn to_option_map(&self) -> Result<ResourceOptions, IggyError> {
+ let mut options = raw_options_map(&self.raw)?;
+ if let Some(compression_algorithm) = self.compression_algorithm {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::COMPRESSION_ALGORITHM)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(
+ HeaderValue::try_from(compression_algorithm.to_string().as_str())
+ .expect("compression name fits a header value"),
+ ),
+ );
+ }
+ if let Some(message_expiry) = self.message_expiry {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(HeaderValue::from(u64::from(message_expiry))),
+ );
+ }
+ if let Some(max_topic_size) = self.max_topic_size {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(HeaderValue::from(u64::from(max_topic_size))),
+ );
+ }
+ if let Some(segment_size) = self.segment_size {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::SEGMENT_SIZE)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(HeaderValue::from(segment_size.as_bytes_u64())),
+ );
+ }
+ if let Some(enforce_fsync) = self.enforce_fsync {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::ENFORCE_FSYNC)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(HeaderValue::from(enforce_fsync)),
+ );
+ }
+ if let Some(messages_required_to_save) = self.messages_required_to_save {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::MESSAGES_REQUIRED_TO_SAVE)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(HeaderValue::from(messages_required_to_save)),
+ );
+ }
+ if let Some(size_of_messages) = self.size_of_messages_required_to_save {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(HeaderValue::from(size_of_messages.as_bytes_u64())),
+ );
+ }
+ if let Some(preallocate_segments) = self.preallocate_segments {
+ options.insert(
+ HeaderKey::from_str(topic_option_keys::PREALLOCATE_SEGMENTS)
+ .expect("catalog key is a valid header key"),
+ OptionValue::explicit(HeaderValue::from(preallocate_segments)),
+ );
+ }
+ Ok(options)
+ }
+
+ /// Render the present keys as string key-values, for a transport that
+ /// carries options as a JSON object instead of a TLV block.
+ ///
+ /// `compression_algorithm`, `message_expiry` and `max_topic_size` are left
+ /// out: the JSON body has dedicated fields for those three, and sending
+ /// both would be the two-sources-for-one-setting the update path refuses.
+ /// The rest ride as strings and are parsed server-side by the same
+ /// `FromStr` rules a config file value goes through, so a typed field
+ /// survives the round trip rather than being silently dropped.
+ #[must_use]
+ pub fn to_string_options(&self) -> BTreeMap<String, String> {
+ // Typed fields are inserted over the raw entries, matching the
+ // collision rule `to_wire` applies.
+ let mut options = self.raw.clone();
+ if let Some(segment_size) = self.segment_size {
+ options.insert(
+ topic_option_keys::SEGMENT_SIZE.to_owned(),
+ segment_size.as_bytes_u64().to_string(),
+ );
+ }
+ if let Some(enforce_fsync) = self.enforce_fsync {
+ options.insert(
+ topic_option_keys::ENFORCE_FSYNC.to_owned(),
+ enforce_fsync.to_string(),
+ );
+ }
+ if let Some(messages_required_to_save) = self.messages_required_to_save {
+ options.insert(
+ topic_option_keys::MESSAGES_REQUIRED_TO_SAVE.to_owned(),
+ messages_required_to_save.to_string(),
+ );
+ }
+ if let Some(size_of_messages) = self.size_of_messages_required_to_save {
+ options.insert(
+ topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE.to_owned(),
+ size_of_messages.as_bytes_u64().to_string(),
+ );
+ }
+ if let Some(preallocate_segments) = self.preallocate_segments {
+ options.insert(
+ topic_option_keys::PREALLOCATE_SEGMENTS.to_owned(),
+ preallocate_segments.to_string(),
+ );
+ }
+ options
+ }
+
+ /// Encode the resolved values for every key the client did NOT send into
+ /// a derived-options wire block, in canonical kinds. `partitions_count`
+ /// is never included: it is consumed at admission.
+ ///
+ /// # Errors
+ ///
+ /// See [`crate::wire_conversions::resource_options_to_wire`].
+ pub fn derived_block(
+ &self,
+ compression_algorithm: CompressionAlgorithm,
+ message_expiry: IggyExpiry,
+ max_topic_size: MaxTopicSize,
+ runtime_defaults: TopicRuntimeDefaults,
+ ) -> Result<WireOptions, IggyError> {
+ let mut derived = ResourceOptions::new();
+ if self.compression_algorithm.is_none() {
+ derived.insert(
+ HeaderKey::from_str(topic_option_keys::COMPRESSION_ALGORITHM)
+ .expect("catalog key is a valid header key"),
+ OptionValue::derived(
+ HeaderValue::try_from(compression_algorithm.to_string().as_str())
+ .expect("compression name fits a header value"),
+ ),
+ );
+ }
+ if self.message_expiry.is_none() {
+ derived.insert(
+ HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY)
+ .expect("catalog key is a valid header key"),
+ OptionValue::derived(HeaderValue::from(u64::from(message_expiry))),
+ );
+ }
+ if self.max_topic_size.is_none() {
+ derived.insert(
+ HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE)
+ .expect("catalog key is a valid header key"),
+ OptionValue::derived(HeaderValue::from(u64::from(max_topic_size))),
+ );
+ }
+ if self.segment_size.is_none() {
+ derived.insert(
+ HeaderKey::from_str(topic_option_keys::SEGMENT_SIZE)
+ .expect("catalog key is a valid header key"),
+ OptionValue::derived(HeaderValue::from(
+ runtime_defaults.segment_size.as_bytes_u64(),
+ )),
+ );
+ }
+ if self.enforce_fsync.is_none() {
+ derived.insert(
+ HeaderKey::from_str(topic_option_keys::ENFORCE_FSYNC)
+ .expect("catalog key is a valid header key"),
+ OptionValue::derived(HeaderValue::from(runtime_defaults.enforce_fsync)),
+ );
+ }
+ if self.messages_required_to_save.is_none() {
+ derived.insert(
+ HeaderKey::from_str(topic_option_keys::MESSAGES_REQUIRED_TO_SAVE)
+ .expect("catalog key is a valid header key"),
+ OptionValue::derived(HeaderValue::from(
+ runtime_defaults.messages_required_to_save,
+ )),
+ );
+ }
+ if self.size_of_messages_required_to_save.is_none() {
+ derived.insert(
+ HeaderKey::from_str(topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE)
+ .expect("catalog key is a valid header key"),
+ OptionValue::derived(HeaderValue::from(
+ runtime_defaults
+ .size_of_messages_required_to_save
+ .as_bytes_u64(),
+ )),
+ );
+ }
+ if self.preallocate_segments.is_none() {
+ derived.insert(
+ HeaderKey::from_str(topic_option_keys::PREALLOCATE_SEGMENTS)
+ .expect("catalog key is a valid header key"),
+ OptionValue::derived(HeaderValue::from(runtime_defaults.preallocate_segments)),
+ );
+ }
+ crate::wire_conversions::resource_options_to_wire(&derived, OptionsProvenance::All)
+ }
+
+ /// Parse a topic's PERSISTED options map (explicit plus admission-derived
+ /// entries) back into typed values.
+ ///
+ /// The persisted map is the single source of truth for a topic's knobs:
+ /// nothing on the STM `Topic` duplicates it per key, so a new key costs
+ /// one catalog entry rather than one field on every stored type.
+ ///
+ /// Infallible by design. A persisted map can hold keys a newer build
+ /// wrote, and refusing the whole map for one of them is what would make
+ /// this replica read a topic differently from the node that stored it.
+ /// Unreadable entries are skipped; their knobs stay unset and resolve
+ /// against shard-wide config exactly as an absent key does.
+ #[must_use]
+ pub fn from_resource_options(options: &ResourceOptions) -> Self {
+ let mut parsed = Self::default();
+ for (key, option) in options {
+ let key = String::from_utf8_lossy(key.as_bytes());
+ let entry = WireUserHeaderEntry {
+ key_kind: iggy_binary_protocol::WireHeaderKind(HeaderKind::String.as_code()),
+ key: key.as_bytes(),
+ value_kind: iggy_binary_protocol::WireHeaderKind(option.value.kind().as_code()),
+ value: option.value.as_bytes(),
+ };
+ // Discarding the error is the skip: see `absorb_strict`.
+ let _ = parsed.absorb_strict(&entry, &key);
+ }
+ parsed
+ }
+}
+
+fn parse_u32(entry: &WireUserHeaderEntry<'_>, key: &str) -> Result<u32, IggyError> {
+ if entry.value_kind.0 == HeaderKind::Uint32.as_code() {
+ let bytes: [u8; 4] = entry
+ .value
+ .try_into()
+ .map_err(|_| IggyError::InvalidOptionValue(key.to_string()))?;
+ return Ok(u32::from_le_bytes(bytes));
+ }
+ if entry.value_kind.0 == HeaderKind::String.as_code() {
+ return std::str::from_utf8(entry.value)
+ .ok()
+ .and_then(|text| text.parse().ok())
+ .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string()));
+ }
+ Err(IggyError::InvalidOptionValue(key.to_string()))
+}
+
+/// `Uint64` verbatim, or a `String` routed through `from_str` and converted
+/// back to the type's `u64` sentinel encoding.
+fn parse_u64_or<T, E>(
+ entry: &WireUserHeaderEntry<'_>,
+ key: &str,
+ from_str: impl Fn(&str) -> Result<T, E>,
+) -> Result<u64, IggyError>
+where
+ u64: From<T>,
+{
+ if entry.value_kind.0 == HeaderKind::Uint64.as_code() {
+ let bytes: [u8; 8] = entry
+ .value
+ .try_into()
+ .map_err(|_| IggyError::InvalidOptionValue(key.to_string()))?;
+ return Ok(u64::from_le_bytes(bytes));
+ }
+ if entry.value_kind.0 == HeaderKind::String.as_code() {
+ return std::str::from_utf8(entry.value)
+ .ok()
+ .and_then(|text| from_str(text).ok())
+ .map(u64::from)
+ .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string()));
+ }
+ Err(IggyError::InvalidOptionValue(key.to_string()))
+}
+
+/// `Uint64` verbatim, or a byte-size `String` (`128MiB`).
+fn parse_byte_size(entry: &WireUserHeaderEntry<'_>, key: &str) -> Result<u64, IggyError> {
+ parse_u64_or(entry, key, |text| {
+ IggyByteSize::from_str(text).map(|size| size.as_bytes_u64())
+ })
+}
+
+fn parse_bool(entry: &WireUserHeaderEntry<'_>, key: &str) -> Result<bool, IggyError> {
+ if entry.value_kind.0 == HeaderKind::Bool.as_code() {
+ // Exactly one byte of 0 or 1. Anything looser admits a value that
+ // `HeaderValue::as_bool` later refuses to read, so the stored map would
+ // hold an entry its own public accessor rejects; a multi-byte payload
+ // would pass this gate and only fail at apply, turning a client
+ // mistake into a committed state-machine rejection.
+ return match entry.value {
+ [0] => Ok(false),
+ [1] => Ok(true),
+ _ => Err(IggyError::InvalidOptionValue(key.to_string())),
+ };
+ }
+ if entry.value_kind.0 == HeaderKind::String.as_code() {
+ return std::str::from_utf8(entry.value)
+ .ok()
+ .and_then(|text| text.parse().ok())
+ .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string()));
+ }
+ Err(IggyError::InvalidOptionValue(key.to_string()))
+}
+
+fn parse_compression(
+ entry: &WireUserHeaderEntry<'_>,
+ key: &str,
+) -> Result<CompressionAlgorithm, IggyError> {
+ if entry.value_kind.0 == HeaderKind::Uint8.as_code() {
+ // Exactly one byte, for the same reason as `parse_bool`.
+ let [code] = entry.value else {
+ return Err(IggyError::InvalidOptionValue(key.to_string()));
+ };
+ return CompressionAlgorithm::from_code(*code)
+ .map_err(|_| IggyError::InvalidOptionValue(key.to_string()));
+ }
+ if entry.value_kind.0 == HeaderKind::String.as_code() {
+ return std::str::from_utf8(entry.value)
+ .ok()
+ .and_then(|text| CompressionAlgorithm::from_str(text).ok())
+ .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string()));
+ }
+ Err(IggyError::InvalidOptionValue(key.to_string()))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn to_wire_parse_roundtrip_preserves_typed_fields() {
+ let options = TopicCreateOptions {
+ compression_algorithm: Some(CompressionAlgorithm::Gzip),
+ message_expiry: Some(IggyExpiry::from(5_000_000u64)),
+ max_topic_size: Some(MaxTopicSize::from(2_000_000_000u64)),
+ segment_size: Some(IggyByteSize::from(134_217_728u64)),
+ enforce_fsync: Some(true),
+ messages_required_to_save: Some(500),
+ size_of_messages_required_to_save: Some(IggyByteSize::from(2_097_152u64)),
+ preallocate_segments: Some(false),
+ partitions_count: None,
+ raw: BTreeMap::new(),
+ };
+ let parsed = TopicCreateOptions::parse(&options.to_wire().unwrap()).unwrap();
+ assert_eq!(parsed, options);
+ }
+
+ #[test]
+ fn partitions_count_never_enters_the_options_block() {
+ // It is a fixed field of the CreateTopic command, so encoding it as a
+ // TLV entry would both duplicate it and make it look like a stored
+ // topic setting.
+ let options = TopicCreateOptions {
+ partitions_count: Some(7),
+ ..TopicCreateOptions::default()
+ };
+ assert!(options.to_wire().unwrap().is_empty());
+ // ...and the key is rejected if a client hand-rolls it into the block.
+ let raw = TopicCreateOptions {
+ raw: BTreeMap::from([("partitions_count".to_string(), "7".to_string())]),
+ ..TopicCreateOptions::default()
+ };
+ assert_eq!(
+ TopicCreateOptions::parse(&raw.to_wire().unwrap()),
+ Err(IggyError::UnsupportedOptionKey(
+ "partitions_count".to_string()
+ ))
+ );
+ }
+
+ #[test]
+ fn resource_options_render_as_readable_json_and_round_trip() {
+ // `HeaderKey` serializes as a struct, and JSON object keys must be
+ // strings, so the derived impl fails with "key must be a string" for
+ // any non-empty map -- which 500'd every HTTP response carrying
+ // options. Keys must render as their plain text, and values in the
+ // string form the create body takes them in: a base64 payload behind a
+ // kind tag is neither readable nor re-sendable.
+ #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)]
+ struct Holder {
+ #[serde(default, with = "super::resource_options_json")]
+ options: ResourceOptions,
+ }
+
+ let holder = Holder {
+ options: ResourceOptions::from([(
+ HeaderKey::from_str(topic_option_keys::SEGMENT_SIZE).unwrap(),
+ OptionValue::derived(HeaderValue::from(1024u64 * 1024)),
+ )]),
+ };
+ let json = serde_json::to_string(&holder).expect("options must serialize as JSON");
+ assert!(
+ json.contains("\"segment_size\""),
+ "the key must render as plain text, got: {json}"
+ );
+ assert!(
+ json.contains("\"value\":\"1048576\""),
+ "the value must render as its string form, got: {json}"
+ );
+
+ let decoded: Holder = serde_json::from_str(&json).expect("options must round-trip");
+ let (key, option) = decoded.options.iter().next().unwrap();
+ assert_eq!(key.as_bytes(), topic_option_keys::SEGMENT_SIZE.as_bytes());
+ assert!(!option.explicit, "provenance survives the round trip");
+ // Re-read as a string-kinded value, which is exactly what the create
+ // body's string map produces, so the server parses it the same way.
+ assert_eq!(option.value.kind(), HeaderKind::String);
+ assert_eq!(option.value.as_bytes(), b"1048576");
+ assert!(
+ TopicCreateOptions::parse(
+ &crate::wire_conversions::resource_options_to_wire(
+ &decoded.options,
+ OptionsProvenance::All
+ )
+ .unwrap()
+ )
+ .is_ok(),
+ "the rendered value must parse back through the catalog"
+ );
+ }
+
+ #[test]
+ fn zero_valued_sentinels_normalize_to_unspecified() {
+ // 0 is the ServerDefault sentinel for both expiry and size on the
+ // wire, so a client sending it means "resolve from the default"
+ // rather than "expire immediately" / "no space". This used to be a
+ // boot-time config rejection; with the keys per-topic it is a
+ // normalization at parse.
+ let options = TopicCreateOptions {
+ message_expiry: Some(IggyExpiry::from(0u64)),
+ max_topic_size: Some(MaxTopicSize::from(0u64)),
+ segment_size: Some(IggyByteSize::from(0u64)),
+ ..TopicCreateOptions::default()
+ };
+ let parsed = TopicCreateOptions::parse(&options.to_wire().unwrap()).unwrap();
+ assert_eq!(parsed.message_expiry, None);
+ assert_eq!(parsed.max_topic_size, None);
+ assert_eq!(parsed.segment_size, None);
+ }
+
+ #[test]
+ fn sentinel_zeros_do_not_survive_re_encoding() {
+ // A client may send 0 to mean "resolve the default". Parsing normalizes
+ // it to absent, so admission puts the resolved value in the derived
+ // block -- but apply merges with explicit winning. If the literal 0
+ // were still in the explicit block it would land back on top as the
+ // stored effective value, and a restart would re-parse it to absent and
+ // fall back to the node default rather than the value resolved at
+ // creation. Re-encoding from the parse is what drops it.
+ let sent = TopicCreateOptions {
+ message_expiry: Some(IggyExpiry::from(0u64)),
+ max_topic_size: Some(MaxTopicSize::from(0u64)),
+ segment_size: Some(IggyByteSize::from(0u64)),
+ size_of_messages_required_to_save: Some(IggyByteSize::from(0u64)),
+ enforce_fsync: Some(true),
+ ..TopicCreateOptions::default()
+ };
+ let parsed = TopicCreateOptions::parse(&sent.to_wire().unwrap()).unwrap();
+ let re_encoded = parsed.to_wire().unwrap();
+ let stored =
+ crate::wire_conversions::resource_options_from_wire(&re_encoded, true).unwrap();
+
+ for key in [
+ topic_option_keys::MESSAGE_EXPIRY,
+ topic_option_keys::MAX_TOPIC_SIZE,
+ topic_option_keys::SEGMENT_SIZE,
+ topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE,
+ ] {
+ assert!(
+ !stored.contains_key(&HeaderKey::from_str(key).unwrap()),
+ "{key} sentinel must not be persisted as an explicit value"
+ );
+ }
+ // A non-sentinel key alongside them still rides through untouched.
+ assert!(
+ stored.contains_key(&HeaderKey::from_str(topic_option_keys::ENFORCE_FSYNC).unwrap())
+ );
+ }
+
+ #[test]
+ fn runtime_options_derive_from_the_persisted_map() {
+ let options = TopicCreateOptions {
+ segment_size: Some(IggyByteSize::from(2_097_152u64)),
+ enforce_fsync: Some(true),
+ messages_required_to_save: Some(9),
+ ..TopicCreateOptions::default()
+ };
+ // What admission persists: the client block merged with derived
+ // defaults, keyed by HeaderKey. The channel reads exactly this.
+ let persisted =
+ crate::wire_conversions::resource_options_from_wire(&options.to_wire().unwrap(), true)
+ .unwrap();
+ let runtime = TopicRuntimeOptions::from_resource_options(&persisted);
+ assert_eq!(runtime.segment_size, Some(IggyByteSize::from(2_097_152u64)));
+ assert_eq!(runtime.enforce_fsync, Some(true));
+ assert_eq!(runtime.messages_required_to_save, Some(9));
+ assert_eq!(runtime.size_of_messages_required_to_save, None);
+ }
+
+ #[test]
+ fn typed_field_wins_over_raw_entry_for_the_same_key() {
+ let options = TopicCreateOptions {
+ enforce_fsync: Some(true),
+ raw: BTreeMap::from([("enforce_fsync".to_string(), "false".to_string())]),
+ ..TopicCreateOptions::default()
+ };
+ let parsed = TopicCreateOptions::parse(&options.to_wire().unwrap()).unwrap();
+ assert_eq!(parsed.enforce_fsync, Some(true));
+ }
+
+ #[test]
+ fn raw_entries_ride_as_strings_and_unknown_keys_reject() {
+ // `prepare_queue_depth` stays server-side on purpose: it is a
+ // node-resource bound (the view-change wire caps it), so a
+ // client-chosen value is an amplifier rather than a topic property.
+ let options = TopicCreateOptions {
+ raw: BTreeMap::from([("prepare_queue_depth".to_string(), "64".to_string())]),
+ ..TopicCreateOptions::default()
+ };
+ assert_eq!(
+ TopicCreateOptions::parse(&options.to_wire().unwrap()),
+ Err(IggyError::UnsupportedOptionKey(
+ "prepare_queue_depth".to_string()
+ ))
+ );
+
+ // Raw entries for catalog keys parse via the config-file rules.
+ let options = TopicCreateOptions {
+ raw: BTreeMap::from([
+ ("segment_size".to_string(), "128MiB".to_string()),
+ ("enforce_fsync".to_string(), "true".to_string()),
+ (
+ "size_of_messages_required_to_save".to_string(),
+ "4KiB".to_string(),
+ ),
+ ]),
+ ..TopicCreateOptions::default()
+ };
+ let parsed = TopicCreateOptions::parse(&options.to_wire().unwrap()).unwrap();
+ assert_eq!(
+ parsed.segment_size,
+ Some(IggyByteSize::from(134_217_728u64))
+ );
+ assert_eq!(parsed.enforce_fsync, Some(true));
+ assert_eq!(
+ parsed.size_of_messages_required_to_save,
+ Some(IggyByteSize::from(4096u64))
+ );
+ }
+
+ #[test]
+ fn string_values_parse_via_config_rules() {
+ let options = TopicCreateOptions {
+ raw: BTreeMap::from([
+ ("message_expiry".to_string(), "5s".to_string()),
+ ("max_topic_size".to_string(), "2GB".to_string()),
+ ("compression_algorithm".to_string(), "gzip".to_string()),
+ ]),
+ ..TopicCreateOptions::default()
+ };
+ let parsed = TopicCreateOptions::parse(&options.to_wire().unwrap()).unwrap();
+ assert_eq!(parsed.message_expiry, Some(IggyExpiry::from(5_000_000u64)));
+ assert_eq!(
+ parsed.compression_algorithm,
+ Some(CompressionAlgorithm::Gzip)
+ );
+ assert!(parsed.max_topic_size.is_some());
+ }
+
+ #[test]
+ fn options_block_budget_matches_user_headers() {
+ // `MAX_OPTIONS_BYTES` duplicates `MAX_USER_HEADERS_SIZE` because
+ // `iggy_binary_protocol` cannot import this crate. This is the only
+ // place both are visible, so it is where they get tied together.
+ assert_eq!(
+ iggy_binary_protocol::MAX_OPTIONS_BYTES,
+ crate::MAX_USER_HEADERS_SIZE as usize,
+ "options inherit the user-headers byte budget; update both"
+ );
+ }
+
+ #[test]
+ fn max_options_is_reachable_within_the_byte_budget() {
+ // The cheapest entry is 12 bytes: kind + u32 length + one byte, for
+ // key and value each. If the budget ever drops below this product the
+ // entry cap becomes unreachable and `MAX_OPTIONS` stops being the
+ // limit that actually binds.
+ const MIN_ENTRY_BYTES: usize = 2 * (1 + 4 + 1);
+ assert!(
+ iggy_binary_protocol::MAX_OPTIONS as usize * MIN_ENTRY_BYTES
+ <= iggy_binary_protocol::MAX_OPTIONS_BYTES,
+ "MAX_OPTIONS entries must fit in MAX_OPTIONS_BYTES"
+ );
+ }
+
+ #[test]
+ fn flush_threshold_ceilings_are_derived_from_the_segment_maximum() {
+ // A threshold no segment can reach never trips, so committed messages
+ // sit in the journal - which a crash does not preserve - until the
+ // segment rotates. The message ceiling follows from the byte one: a
+ // message costs at least its header on disk.
+ assert_eq!(
+ MAX_MESSAGES_REQUIRED_TO_SAVE as u64,
+ MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE
+ / crate::IGGY_MESSAGE_HEADER_SIZE
+ .try_into()
+ .unwrap_or(u64::MAX),
+ "message ceiling must stay derived from the byte ceiling"
+ );
+ }
+
+ #[test]
+ fn parse_rejects_flush_thresholds_above_their_ceilings() {
+ let over_messages = TopicCreateOptions {
+ raw: BTreeMap::from([(
+ topic_option_keys::MESSAGES_REQUIRED_TO_SAVE.to_string(),
+ (u64::from(MAX_MESSAGES_REQUIRED_TO_SAVE) + 1).to_string(),
+ )]),
+ ..TopicCreateOptions::default()
+ };
+ assert!(
+ TopicCreateOptions::parse(&over_messages.to_wire().unwrap()).is_err(),
+ "a message threshold above the ceiling must be refused"
+ );
+
+ let over_bytes = TopicCreateOptions {
+ raw: BTreeMap::from([(
+ topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE.to_string(),
+ (MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE + 1).to_string(),
+ )]),
+ ..TopicCreateOptions::default()
+ };
+ assert!(
+ TopicCreateOptions::parse(&over_bytes.to_wire().unwrap()).is_err(),
+ "a byte threshold above the ceiling must be refused"
+ );
+
+ let at_ceiling = TopicCreateOptions {
+ messages_required_to_save: Some(MAX_MESSAGES_REQUIRED_TO_SAVE),
+ size_of_messages_required_to_save: Some(IggyByteSize::from(
+ MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE,
+ )),
+ ..TopicCreateOptions::default()
+ };
+ assert!(
+ TopicCreateOptions::parse(&at_ceiling.to_wire().unwrap()).is_ok(),
+ "the ceiling itself must remain settable"
+ );
+ }
+
+ #[test]
+ fn preallocation_cap_bounds_segment_size_times_partitions() {
+ assert!(validate_preallocated_topic_bytes(DEFAULT_SEGMENT_SIZE, 64).is_ok());
+ assert!(validate_preallocated_topic_bytes(DEFAULT_SEGMENT_SIZE, 65).is_err());
+ // The product saturates rather than wrapping into a passing value.
+ assert!(validate_preallocated_topic_bytes(u64::MAX, u32::MAX).is_err());
+ }
+}
diff --git a/core/common/src/types/stream/mod.rs b/core/common/src/types/stream/mod.rs
index 5a3c9ce0a..9a48f1aae 100644
--- a/core/common/src/types/stream/mod.rs
+++ b/core/common/src/types/stream/mod.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::{IggyByteSize, IggyTimestamp, Topic};
+use crate::{IggyByteSize, IggyTimestamp, ResourceOptions, Topic};
use serde::{Deserialize, Serialize};
/// `Stream` represents the highest level of logical separation of data.
@@ -40,6 +40,9 @@ pub struct Stream {
pub messages_count: u64,
/// The total number of topics in the stream.
pub topics_count: u32,
+ /// Creation options, all client-explicit (streams have no derived keys).
+ #[serde(default, with = "crate::resource_options_json")]
+ pub options: ResourceOptions,
}
/// `StreamDetails` represents the detailed information about the stream.
@@ -67,4 +70,7 @@ pub struct StreamDetails {
pub topics_count: u32,
/// The collection of topics in the stream.
pub topics: Vec<Topic>,
+ /// Creation options, all client-explicit (streams have no derived keys).
+ #[serde(default, with = "crate::resource_options_json")]
+ pub options: ResourceOptions,
}
diff --git a/core/common/src/types/topic/mod.rs b/core/common/src/types/topic/mod.rs
index 1135f305c..9379c84a9 100644
--- a/core/common/src/types/topic/mod.rs
+++ b/core/common/src/types/topic/mod.rs
@@ -17,6 +17,7 @@
use crate::CompressionAlgorithm;
use crate::Partition;
+use crate::ResourceOptions;
use crate::utils::byte_size::IggyByteSize;
use crate::utils::expiry::IggyExpiry;
use crate::utils::timestamp::IggyTimestamp;
@@ -31,7 +32,6 @@ use serde::{Deserialize, Serialize};
/// - `size`: the total size of the topic in bytes.
/// - `message_expiry`: the expiry of the messages in the topic.
/// - `max_topic_size`: the maximum size of the topic.
-/// - `replication_factor`: replication factor for the topic.
/// - `messages_count`: the total number of messages in the topic.
/// - `partitions_count`: the total number of partitions in the topic.
#[derive(Debug, Serialize, Deserialize)]
@@ -51,12 +51,14 @@ pub struct Topic {
/// The optional maximum size of the topic.
/// Can't be lower than segment size in the config.
pub max_topic_size: MaxTopicSize,
- /// Replication factor for the topic.
- pub replication_factor: u8,
/// The total number of messages in the topic.
pub messages_count: u64,
/// The total number of partitions in the topic.
pub partitions_count: u32,
+ /// Creation options: client-explicit keys plus admission-derived
+ /// defaults, distinguished by each entry's `explicit` flag.
+ #[serde(default, with = "crate::resource_options_json")]
+ pub options: ResourceOptions,
}
/// `TopicDetails` represents the detailed information about the topic.
@@ -67,7 +69,6 @@ pub struct Topic {
/// - `size`: the total size of the topic.
/// - `message_expiry`: the expiry of the messages in the topic.
/// - `max_topic_size`: the maximum size of the topic.
-/// - `replication_factor`: replication factor for the topic.
/// - `messages_count`: the total number of messages in the topic.
/// - `partitions_count`: the total number of partitions in the topic.
/// - `partitions`: the collection of partitions in the topic.
@@ -88,12 +89,14 @@ pub struct TopicDetails {
/// The optional maximum size of the topic.
/// Can't be lower than segment size in the config.
pub max_topic_size: MaxTopicSize,
- /// Replication factor for the topic.
- pub replication_factor: u8,
/// The total number of messages in the topic.
pub messages_count: u64,
/// The total number of partitions in the topic.
pub partitions_count: u32,
/// The collection of partitions in the topic.
pub partitions: Vec<Partition>,
+ /// Creation options: client-explicit keys plus admission-derived
+ /// defaults, distinguished by each entry's `explicit` flag.
+ #[serde(default, with = "crate::resource_options_json")]
+ pub options: ResourceOptions,
}
diff --git a/core/common/src/types/user/user_info.rs b/core/common/src/types/user/user_info.rs
index 4ec944ca7..1c6ddec85 100644
--- a/core/common/src/types/user/user_info.rs
+++ b/core/common/src/types/user/user_info.rs
@@ -16,6 +16,7 @@
// under the License.
use crate::Permissions;
+use crate::ResourceOptions;
use crate::types::user::user_status::UserStatus;
use crate::utils::timestamp::IggyTimestamp;
use serde::{Deserialize, Serialize};
@@ -43,6 +44,9 @@ pub struct UserInfo {
pub status: UserStatus,
/// The username of the user.
pub username: String,
+ /// Creation options, all client-explicit (users have no derived keys).
+ #[serde(default, with = "crate::resource_options_json")]
+ pub options: ResourceOptions,
}
/// `UserInfoDetails` represents the detailed information about the user.
@@ -64,4 +68,7 @@ pub struct UserInfoDetails {
pub username: String,
/// The optional permissions of the user.
pub permissions: Option<Permissions>,
+ /// Creation options, all client-explicit (users have no derived keys).
+ #[serde(default, with = "crate::resource_options_json")]
+ pub options: ResourceOptions,
}
diff --git a/core/common/src/wire_conversions.rs b/core/common/src/wire_conversions.rs
index 44975f159..d87e392a0 100644
--- a/core/common/src/wire_conversions.rs
+++ b/core/common/src/wire_conversions.rs
@@ -26,10 +26,13 @@ use crate::{
ClusterNodeRole, ClusterNodeStatus, CompressionAlgorithm, Consumer, ConsumerGroup,
ConsumerGroupDetails, ConsumerGroupInfo, ConsumerGroupMember, ConsumerOffsetInfo,
GlobalPermissions, HeaderKey, HeaderKind, HeaderValue, IdKind, IdentityInfo, IggyByteSize,
- IggyError, IggyExpiry, MaxTopicSize, Partition, Permissions, PersonalAccessTokenInfo,
- RawPersonalAccessToken, Stats, Stream, StreamDetails, StreamPermissions, Topic, TopicDetails,
- TopicPermissions, TransportEndpoints, UserInfo, UserInfoDetails, UserStatus,
+ IggyError, IggyExpiry, MaxTopicSize, OptionSpec, OptionValue, OptionsProvenance, Partition,
+ Permissions, PersonalAccessTokenInfo, RawPersonalAccessToken, ResourceOptions, Stats, Stream,
+ StreamDetails, StreamPermissions, Topic, TopicDetails, TopicPermissions, TransportEndpoints,
+ UserInfo, UserInfoDetails, UserStatus,
};
+use bytes::{BufMut, BytesMut};
+use iggy_binary_protocol::primitives::options::{MAX_OPTIONS, MAX_OPTIONS_BYTES, WireOptions};
use iggy_binary_protocol::primitives::permissions::{
WireGlobalPermissions, WirePermissions, WireStreamPermissions, WireTopicPermissions,
};
@@ -70,16 +73,19 @@ const WIRE_NO_USER_ID: u32 = u32::MAX;
// Streams
// ---------------------------------------------------------------------------
-impl From<StreamResponse> for Stream {
- fn from(w: StreamResponse) -> Self {
- Self {
+impl TryFrom<StreamResponse> for Stream {
+ type Error = IggyError;
+
+ fn try_from(w: StreamResponse) -> Result<Self, Self::Error> {
+ Ok(Self {
id: w.id,
created_at: w.created_at.into(),
name: w.name.to_string(),
size: IggyByteSize::from(w.size_bytes),
messages_count: w.messages_count,
topics_count: w.topics_count,
- }
+ options: resource_options_from_wire(&w.options, true)?,
+ })
}
}
@@ -101,14 +107,19 @@ impl TryFrom<GetStreamResponse> for StreamDetails {
messages_count: w.stream.messages_count,
topics_count: w.stream.topics_count,
topics,
+ options: resource_options_from_wire(&w.stream.options, true)?,
})
}
}
-pub fn streams_from_wire(w: GetStreamsResponse) -> Vec<Stream> {
- let mut streams: Vec<Stream> = w.streams.into_iter().map(Stream::from).collect();
+pub fn streams_from_wire(w: GetStreamsResponse) -> Result<Vec<Stream>, IggyError> {
+ let mut streams: Vec<Stream> = w
+ .streams
+ .into_iter()
+ .map(Stream::try_from)
+ .collect::<Result<_, _>>()?;
streams.sort_by_key(|s| s.id);
- streams
+ Ok(streams)
}
// ---------------------------------------------------------------------------
@@ -124,6 +135,7 @@ impl TryFrom<TopicHeader> for Topic {
v => v.into(),
};
let max_topic_size: MaxTopicSize = w.max_topic_size.into();
+ let options = resource_options_from_wire_split(&w.options, &w.derived_options)?;
Ok(Self {
id: w.id,
created_at: w.created_at.into(),
@@ -134,7 +146,7 @@ impl TryFrom<TopicHeader> for Topic {
message_expiry,
compression_algorithm: CompressionAlgorithm::from_code(w.compression_algorithm)?,
max_topic_size,
- replication_factor: w.replication_factor,
+ options,
})
}
}
@@ -169,9 +181,9 @@ impl TryFrom<GetTopicResponse> for TopicDetails {
message_expiry: topic.message_expiry,
compression_algorithm: topic.compression_algorithm,
max_topic_size: topic.max_topic_size,
- replication_factor: topic.replication_factor,
partitions_count: topic.partitions_count,
partitions,
+ options: topic.options,
})
}
}
@@ -199,6 +211,7 @@ impl TryFrom<UserResponse> for UserInfo {
created_at: w.created_at.into(),
status: UserStatus::from_code(w.status)?,
username: w.username.to_string(),
+ options: resource_options_from_wire(&w.options, true)?,
})
}
}
@@ -215,6 +228,7 @@ impl TryFrom<UserDetailsResponse> for UserInfoDetails {
status: user.status,
username: user.username,
permissions,
+ options: user.options,
})
}
}
@@ -771,20 +785,46 @@ pub fn user_headers_from_wire(
/// slices each TLV field without bounds checks, relying on that validation.
pub(crate) fn user_headers_from_validated_slice(
buf: &[u8],
+) -> Result<BTreeMap<HeaderKey, HeaderValue>, IggyError> {
+ headers_from_validated_slice(buf, UnknownKinds::Reject)
+}
+
+/// What a decode does with an entry whose kind code has no domain meaning.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum UnknownKinds {
+ /// Fail the whole decode. A message header the reader cannot interpret is
+ /// the caller's problem, not something to hide from it.
+ Reject,
+ /// Drop the entry and keep the rest.
+ Skip,
+}
+
+fn headers_from_validated_slice(
+ buf: &[u8],
+ unknown: UnknownKinds,
) -> Result<BTreeMap<HeaderKey, HeaderValue>, IggyError> {
if buf.is_empty() {
return Ok(BTreeMap::new());
}
let mut headers = BTreeMap::new();
for entry in WireUserHeaderIterator::new(buf) {
- let key_kind = HeaderKind::from_code(entry.key_kind.0)?;
+ let (key_kind, value_kind) = match (
+ HeaderKind::from_code(entry.key_kind.0),
+ HeaderKind::from_code(entry.value_kind.0),
+ ) {
+ (Ok(key_kind), Ok(value_kind)) => (key_kind, value_kind),
+ (Err(error), _) | (Ok(_), Err(error)) => {
+ if unknown == UnknownKinds::Skip {
+ continue;
+ }
+ return Err(error);
+ }
+ };
if let Some(expected) = key_kind.expected_size()
&& entry.key.len() != expected
{
return Err(IggyError::InvalidHeaderKey);
}
-
- let value_kind = HeaderKind::from_code(entry.value_kind.0)?;
if let Some(expected) = value_kind.expected_size()
&& entry.value.len() != expected
{
@@ -799,6 +839,159 @@ pub(crate) fn user_headers_from_validated_slice(
Ok(headers)
}
+// -- Options conversions --
+
+/// Decode a `DescribeOptions` response into domain option specs.
+///
+/// # Errors
+///
+/// Returns `IggyError::InvalidHeaderKind` when an entry carries an unknown
+/// canonical kind code.
+pub fn option_specs_from_wire(
+ wire: iggy_binary_protocol::responses::system::DescribeOptionsResponse,
+) -> Result<Vec<OptionSpec>, IggyError> {
+ wire.entries
+ .into_iter()
+ .map(|entry| {
+ Ok(OptionSpec {
+ key: entry.key.to_string(),
+ kind: HeaderKind::from_code(entry.kind)?,
+ default_value: entry.default_value.to_vec(),
+ description: entry.description,
+ })
+ })
+ .collect()
+}
+
+/// Decode a [`WireOptions`] block into domain resource options.
+///
+/// Every decoded entry is marked with the given `explicit` flag; the wire
+/// block carries no per-entry provenance. Admission calls this once for the
+/// client block (`explicit == true`) and fills defaults separately.
+///
+/// Entries whose kind code has no domain meaning are skipped, not rejected.
+/// The wire layer forwards unknown kinds verbatim so a mixed-version cluster
+/// can round-trip them, and this decode sits on the apply path, on the SDK's
+/// list-decode and on the server's own reads: refusing the entry would mean an
+/// old replica rejecting a commit a new one accepted, one unreadable entry
+/// failing a whole `get_topics()`, and a topic that cannot be read back.
+///
+/// # Errors
+///
+/// Returns `IggyError::InvalidHeaderKey` / `InvalidHeaderValue` when a known
+/// fixed-size kind carries a mismatched payload.
+pub fn resource_options_from_wire(
+ wire: &iggy_binary_protocol::WireOptions,
+ explicit: bool,
+) -> Result<ResourceOptions, IggyError> {
+ let headers = headers_from_validated_slice(wire.as_bytes(), UnknownKinds::Skip)?;
+ Ok(headers
+ .into_iter()
+ .map(|(key, value)| (key, OptionValue { value, explicit }))
+ .collect())
+}
+
+/// Merge response-side `(explicit, derived)` wire blocks back into domain
+/// resource options, restoring per-key provenance. Explicit wins on a key
+/// collision (which a well-formed response never produces).
+///
+/// # Errors
+///
+/// Same contract as [`resource_options_from_wire`].
+pub fn resource_options_from_wire_split(
+ explicit: &iggy_binary_protocol::WireOptions,
+ derived: &iggy_binary_protocol::WireOptions,
+) -> Result<ResourceOptions, IggyError> {
+ let mut options = resource_options_from_wire(derived, false)?;
+ options.extend(resource_options_from_wire(explicit, true)?);
+ Ok(options)
+}
+
+/// Split domain resource options into `(explicit, derived)` wire blocks,
+/// the response-side layout that preserves per-key provenance.
+///
+/// # Errors
+///
+/// Same contract as [`resource_options_to_wire`].
+pub fn resource_options_to_wire_split(
+ options: &ResourceOptions,
+) -> Result<(WireOptions, WireOptions), IggyError> {
+ Ok((
+ resource_options_to_wire(options, OptionsProvenance::Explicit)?,
+ resource_options_to_wire(options, OptionsProvenance::Derived)?,
+ ))
+}
+
+/// Fixed per-entry cost of the TLV encoding: a kind byte and a `u32` length
+/// for the key and for the value.
+const OPTION_ENTRY_OVERHEAD: usize = 2 * (1 + 4);
+
+/// Encode domain resource options into a [`WireOptions`] block, keeping only
+/// the entries matching `provenance`. Encoding just the explicit ones is what
+/// lets a client round-trip a create without pinning server defaults.
+///
+/// # Errors
+///
+/// Returns `IggyError::OptionsBlockTooLarge` when the selected entries exceed
+/// `MAX_OPTIONS` or `MAX_OPTIONS_BYTES`. [`WireOptions::from_validated`] skips
+/// revalidation, so the two caps have to hold here: a block written past them
+/// is one the receiving peer's `decode_options_prefixed` refuses, which would
+/// make the resource permanently unreadable rather than merely oversized.
+pub fn resource_options_to_wire(
+ options: &ResourceOptions,
+ provenance: OptionsProvenance,
+) -> Result<WireOptions, IggyError> {
+ let entries: Vec<(&HeaderKey, &OptionValue)> = options
+ .iter()
+ .filter(|(_, option)| option.matches(provenance))
+ .collect();
+ if entries.is_empty() {
+ return Ok(WireOptions::empty());
+ }
+ if entries.len() > MAX_OPTIONS as usize {
+ return Err(IggyError::OptionsBlockTooLarge(format!(
+ "{} entries, maximum {MAX_OPTIONS}",
+ entries.len()
+ )));
+ }
+ let size: usize = entries
+ .iter()
+ .map(|(key, option)| {
+ OPTION_ENTRY_OVERHEAD + key.as_bytes().len() + option.value.as_bytes().len()
+ })
+ .sum();
+ if size > MAX_OPTIONS_BYTES {
+ return Err(IggyError::OptionsBlockTooLarge(format!(
+ "{size} bytes, maximum {MAX_OPTIONS_BYTES}"
+ )));
+ }
+ let mut buf = BytesMut::with_capacity(size);
+ for (key, option) in entries {
+ // A non-string key would encode a block the receiving peer refuses,
+ // making the resource permanently unreadable. It holds because every
+ // producer went through `validate_options`, not because `HeaderKey`
+ // guarantees it, so the invariant is asserted where it is relied on.
+ debug_assert_eq!(
+ key.kind(),
+ HeaderKind::String,
+ "option key must be a string"
+ );
+ buf.put_u8(key.kind().as_code());
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(key.as_bytes().len() as u32);
+ buf.put_slice(key.as_bytes());
+ buf.put_u8(option.value.kind().as_code());
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(option.value.as_bytes().len() as u32);
+ buf.put_slice(option.value.as_bytes());
+ }
+ // Structural validity holds by construction: entries come from valid
+ // `HeaderKey`s walked in `BTreeMap` order, so keys are sorted and unique,
+ // and the loop above asserts they are string-kinded. The caps cover what
+ // construction cannot.
+ Ok(WireOptions::from_validated(buf.freeze()))
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -868,4 +1061,35 @@ mod tests {
assert!(user_headers_from_validated_slice(&buf).is_err());
assert!(user_headers_from_wire(&WireUserHeaders::from_slice(&buf).unwrap()).is_err());
}
+
+ fn put_option_entry(buf: &mut Vec<u8>, key: &str, value_kind: u8, value: &[u8]) {
+ buf.push(HeaderKind::String.as_code());
+ buf.extend_from_slice(&u32::try_from(key.len()).unwrap().to_le_bytes());
+ buf.extend_from_slice(key.as_bytes());
+ buf.push(value_kind);
+ buf.extend_from_slice(&u32::try_from(value.len()).unwrap().to_le_bytes());
+ buf.extend_from_slice(value);
+ }
+
+ #[test]
+ fn given_option_with_unknown_value_kind_when_decoded_should_skip_only_that_entry() {
+ // A newer peer's option value kind. The wire layer forwards it, so the
+ // domain decode has to drop the entry rather than fail the block: this
+ // decode runs on the apply path, where an error means one replica
+ // rejecting a commit another accepted.
+ let mut buf = Vec::new();
+ put_option_entry(&mut buf, "from_the_future", 200, b"opaque");
+ put_option_entry(
+ &mut buf,
+ "segment_size",
+ HeaderKind::String.as_code(),
+ b"1MB",
+ );
+ let wire = WireOptions::from_slice(&buf).expect("unknown value kinds stay wire-valid");
+
+ let options = resource_options_from_wire(&wire, true).unwrap();
+
+ assert_eq!(options.len(), 1);
+ assert!(options.contains_key(&HeaderKey::from_str("segment_size").unwrap()));
+ }
}
diff --git a/core/configs/src/common/defaults.rs b/core/configs/src/common/defaults.rs
index 2cc6d2200..e733c5b7c 100644
--- a/core/configs/src/common/defaults.rs
+++ b/core/configs/src/common/defaults.rs
@@ -23,8 +23,8 @@ use super::server::{
};
use super::system::{
BackupConfig, CompatibilityConfig, CompressionConfig, EncryptionConfig, LoggingConfig,
- MessageDeduplicationConfig, PartitionConfig, RecoveryConfig, RuntimeConfig, SegmentConfig,
- StateConfig, StreamConfig, SystemConfig, TopicConfig,
+ PartitionConfig, RecoveryConfig, RuntimeConfig, SegmentConfig, StateConfig, StreamConfig,
+ SystemConfig, TopicConfig,
};
use configs::ConfigEnvMappings;
@@ -197,7 +197,6 @@ impl<S: ConfigEnvMappings + Default> Default for SystemConfig<S> {
segment: SegmentConfig::default(),
state: StateConfig::default(),
compression: CompressionConfig::default(),
- message_deduplication: MessageDeduplicationConfig::default(),
recovery: RecoveryConfig::default(),
memory_pool: MemoryPoolConfig::default(),
sharding: S::default(),
@@ -322,8 +321,6 @@ impl Default for TopicConfig {
fn default() -> TopicConfig {
TopicConfig {
path: SERVER_CONFIG.system.topic.path.parse().unwrap(),
- max_size: SERVER_CONFIG.system.topic.max_size.parse().unwrap(),
- message_expiry: SERVER_CONFIG.system.topic.message_expiry.parse().unwrap(),
}
}
}
@@ -332,15 +329,6 @@ impl Default for PartitionConfig {
fn default() -> PartitionConfig {
PartitionConfig {
path: SERVER_CONFIG.system.partition.path.parse().unwrap(),
- size_of_messages_required_to_save: SERVER_CONFIG
- .system
- .partition
- .size_of_messages_required_to_save
- .parse()
- .unwrap(),
- messages_required_to_save: SERVER_CONFIG.system.partition.messages_required_to_save
- as u32,
- enforce_fsync: SERVER_CONFIG.system.partition.enforce_fsync,
validate_checksum: SERVER_CONFIG.system.partition.validate_checksum,
}
}
@@ -349,8 +337,6 @@ impl Default for PartitionConfig {
impl Default for SegmentConfig {
fn default() -> SegmentConfig {
SegmentConfig {
- size: SERVER_CONFIG.system.segment.size.parse().unwrap(),
- preallocate: SERVER_CONFIG.system.segment.preallocate,
cache_indexes: SERVER_CONFIG.system.segment.cache_indexes.parse().unwrap(),
archive_expired: SERVER_CONFIG.system.segment.archive_expired,
}
@@ -368,21 +354,6 @@ impl Default for StateConfig {
}
}
-impl Default for MessageDeduplicationConfig {
- fn default() -> MessageDeduplicationConfig {
- MessageDeduplicationConfig {
- enabled: SERVER_CONFIG.system.message_deduplication.enabled,
- max_entries: SERVER_CONFIG.system.message_deduplication.max_entries as u64,
- expiry: SERVER_CONFIG
- .system
- .message_deduplication
- .expiry
- .parse()
- .unwrap(),
- }
- }
-}
-
impl Default for RecoveryConfig {
fn default() -> RecoveryConfig {
RecoveryConfig {
diff --git a/core/configs/src/common/displays.rs b/core/configs/src/common/displays.rs
index 3d0b43201..234cc0412 100644
--- a/core/configs/src/common/displays.rs
+++ b/core/configs/src/common/displays.rs
@@ -19,7 +19,6 @@ use super::server::{
ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessagesMaintenanceConfig,
TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig,
};
-use super::system::MessageDeduplicationConfig;
use super::{
http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig},
server::MessageSaverConfig,
@@ -164,11 +163,7 @@ impl Display for StreamConfig {
impl Display for TopicConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- write!(
- f,
- "{{ path: {}, max_size: {}, message_expiry: {} }}",
- self.path, self.max_size, self.message_expiry
- )
+ write!(f, "{{ path: {} }}", self.path)
}
}
@@ -176,22 +171,8 @@ impl Display for PartitionConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
- "{{ path: {}, messages_required_to_save: {}, size_of_messages_required_to_save: {}, enforce_fsync: {}, validate_checksum: {} }}",
- self.path,
- self.messages_required_to_save,
- self.size_of_messages_required_to_save,
- self.enforce_fsync,
- self.validate_checksum
- )
- }
-}
-
-impl Display for MessageDeduplicationConfig {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- write!(
- f,
- "{{ enabled: {}, max_entries: {:?}, expiry: {:?} }}",
- self.enabled, self.max_entries, self.expiry
+ "{{ path: {}, validate_checksum: {} }}",
+ self.path, self.validate_checksum
)
}
}
@@ -200,8 +181,8 @@ impl Display for SegmentConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
- "{{ size_bytes: {}, preallocate: {}, cache_indexes: {}, archive_expired: {} }}",
- self.size, self.preallocate, self.cache_indexes, self.archive_expired,
+ "{{ cache_indexes: {}, archive_expired: {} }}",
+ self.cache_indexes, self.archive_expired,
)
}
}
diff --git a/core/configs/src/common/system.rs b/core/configs/src/common/system.rs
index d72913fc5..a9172b0fb 100644
--- a/core/configs/src/common/system.rs
+++ b/core/configs/src/common/system.rs
@@ -19,9 +19,6 @@ use super::cache_indexes::CacheIndexesConfig;
use super::server::MemoryPoolConfig;
use configs::{ConfigEnv, ConfigEnvMappings};
use iggy_common::IggyByteSize;
-use iggy_common::IggyError;
-use iggy_common::IggyExpiry;
-use iggy_common::MaxTopicSize;
use iggy_common::{CompressionAlgorithm, IggyDuration};
use serde::{Deserialize, Serialize};
use serde_with::DisplayFromStr;
@@ -48,7 +45,6 @@ pub struct SystemConfig<S: ConfigEnvMappings> {
pub segment: SegmentConfig,
pub encryption: EncryptionConfig,
pub compression: CompressionConfig,
- pub message_deduplication: MessageDeduplicationConfig,
pub recovery: RecoveryConfig,
pub memory_pool: MemoryPoolConfig,
pub sharding: S,
@@ -133,50 +129,33 @@ pub struct StreamConfig {
pub path: String,
}
-#[serde_as]
+/// Only the on-disk layout: a topic's size cap and message expiry are its own
+/// creation options now (`max_topic_size`, `message_expiry`), defaulting to
+/// `iggy_common::DEFAULT_MAX_TOPIC_SIZE` / `DEFAULT_MESSAGE_EXPIRY`.
#[derive(Debug, Deserialize, Serialize, ConfigEnv)]
pub struct TopicConfig {
pub path: String,
- #[config_env(leaf)]
- #[serde_as(as = "DisplayFromStr")]
- pub max_size: MaxTopicSize,
- #[config_env(leaf)]
- #[serde_as(as = "DisplayFromStr")]
- pub message_expiry: IggyExpiry,
}
+/// `enforce_fsync`, `messages_required_to_save` and
+/// `size_of_messages_required_to_save` are per-topic creation options now,
+/// defaulting to the `iggy_common::DEFAULT_*` constants.
#[derive(Debug, Deserialize, Serialize, ConfigEnv)]
pub struct PartitionConfig {
pub path: String,
- pub messages_required_to_save: u32,
- #[config_env(leaf)]
- pub size_of_messages_required_to_save: IggyByteSize,
- pub enforce_fsync: bool,
pub validate_checksum: bool,
}
-#[serde_as]
-#[derive(Debug, Deserialize, Serialize, ConfigEnv)]
-pub struct MessageDeduplicationConfig {
- pub enabled: bool,
- pub max_entries: u64,
- #[config_env(leaf)]
- #[serde_as(as = "DisplayFromStr")]
- pub expiry: IggyDuration,
-}
-
#[derive(Debug, Deserialize, Serialize, ConfigEnv)]
pub struct RecoveryConfig {
pub recreate_missing_state: bool,
}
-#[serde_as]
+/// `size` and `preallocate` are per-topic creation options now
+/// (`segment_size`, `preallocate_segments`), defaulting to
+/// `iggy_common::DEFAULT_SEGMENT_SIZE` / `DEFAULT_PREALLOCATE_SEGMENTS`.
#[derive(Debug, Deserialize, Serialize, ConfigEnv)]
pub struct SegmentConfig {
- #[config_env(leaf)]
- pub size: IggyByteSize,
- #[serde(default)]
- pub preallocate: bool,
#[config_env(leaf)]
pub cache_indexes: CacheIndexesConfig,
pub archive_expired: bool,
@@ -328,32 +307,6 @@ impl<S: ConfigEnvMappings> SystemConfig<S> {
let path = self.get_segment_path(stream_id, topic_id, partition_id, start_offset);
format!("{path}.{INDEX_EXTENSION}")
}
-
- pub fn resolve_max_topic_size(
- &self,
- max_topic_size: MaxTopicSize,
- ) -> Result<MaxTopicSize, IggyError> {
- match max_topic_size {
- MaxTopicSize::ServerDefault => Ok(self.topic.max_size),
- _ => {
- if max_topic_size.as_bytes_u64() < self.segment.size.as_bytes_u64() {
- Err(IggyError::InvalidTopicSize(
- max_topic_size,
- self.segment.size,
- ))
- } else {
- Ok(max_topic_size)
- }
- }
- }
- }
-
- pub fn resolve_message_expiry(&self, message_expiry: IggyExpiry) -> IggyExpiry {
- match message_expiry {
- IggyExpiry::ServerDefault => self.topic.message_expiry,
- _ => message_expiry,
- }
- }
}
impl<S: ConfigEnvMappings> SystemPaths for SystemConfig<S> {
diff --git a/core/configs/src/common/validators.rs b/core/configs/src/common/validators.rs
index 9c7d54c43..34bfb82a4 100644
--- a/core/configs/src/common/validators.rs
+++ b/core/configs/src/common/validators.rs
@@ -74,34 +74,17 @@ impl Validatable<ConfigurationError> for TelemetryConfig {
impl Validatable<ConfigurationError> for PartitionConfig {
fn validate(&self) -> Result<(), ConfigurationError> {
- if self.messages_required_to_save == 0 {
- eprintln!("Configured system.partition.messages_required_to_save cannot be 0");
- return Err(ConfigurationError::InvalidConfigurationValue);
- }
-
+ // The flush thresholds this used to check are per-topic creation
+ // options now; their bounds are enforced at admission.
Ok(())
}
}
impl Validatable<ConfigurationError> for SegmentConfig {
fn validate(&self) -> Result<(), ConfigurationError> {
- if self.size > SEGMENT_MAX_SIZE_BYTES {
- eprintln!(
- "Configured system.segment.size {} B is greater than maximum {} B",
- self.size.as_bytes_u64(),
- SEGMENT_MAX_SIZE_BYTES
- );
- return Err(ConfigurationError::InvalidConfigurationValue);
- }
-
- if !self.size.as_bytes_u64().is_multiple_of(512) {
- eprintln!(
- "Configured system.segment.size {} B is not a multiple of 512 B",
- self.size.as_bytes_u64()
- );
- return Err(ConfigurationError::InvalidConfigurationValue);
- }
-
+ // Segment size is a per-topic creation option now; its ceiling, floor
+ // and 512 B-multiple rule are enforced by
+ // `iggy_common::validate_topic_segment_size` at admission.
Ok(())
}
}
diff --git a/core/configs/src/configs_impl/file_provider.rs b/core/configs/src/configs_impl/file_provider.rs
index a404208b9..b6b8d67cf 100644
--- a/core/configs/src/configs_impl/file_provider.rs
+++ b/core/configs/src/configs_impl/file_provider.rs
@@ -28,12 +28,44 @@ use tracing::{error, info, warn};
const DISPLAY_CONFIG_ENV: &str = "IGGY_DISPLAY_CONFIG";
+/// A config key that no longer exists, and what took over from it.
+///
+/// Nothing else catches such a key. Its struct field is gone and no config
+/// table sets `deny_unknown_fields`, so figment drops an unrecognized key
+/// without a word from either source: a server still carrying
+/// `enforce_fsync = true` would boot reporting success and run without fsync.
+/// Every relocated key used to change behavior, so the boot is refused rather
+/// than warned about.
+#[derive(Debug, Clone, Copy)]
+pub struct RelocatedKey {
+ /// Dotted config path, for example `system.segment.size`. A deleted table
+ /// matches everything nested under it as well.
+ pub path: &'static str,
+ /// The per-topic option key that replaced it, or `None` when the feature
+ /// it configured was removed outright.
+ pub replacement: Option<&'static str>,
+}
+
+impl RelocatedKey {
+ /// Sentence telling the operator where the setting went.
+ fn guidance(&self) -> String {
+ match self.replacement {
+ Some(option) => {
+ format!("it is now the per-topic '{option}' option, set on CreateTopic")
+ }
+ None => "the feature it configured was removed".to_string(),
+ }
+ }
+}
+
/// File-based configuration provider that combines file, default, and environment configurations.
pub struct FileConfigProvider<P> {
file_path: String,
default_config: Option<Data<Toml>>,
env_provider: P,
display_config: bool,
+ env_prefix: &'static str,
+ relocated_keys: &'static [RelocatedKey],
}
impl<P: Provider> FileConfigProvider<P> {
@@ -55,14 +87,75 @@ impl<P: Provider> FileConfigProvider<P> {
env_provider,
default_config,
display_config,
+ env_prefix: "",
+ relocated_keys: &[],
}
}
+
+ /// Refuse to load when any of `keys` is still set, in the config file or in
+ /// the environment.
+ ///
+ /// `env_prefix` is the prefix this config's env provider reads, used to
+ /// derive the variable name for each path. The table is per-config on
+ /// purpose: a server key left over in a shared container environment must
+ /// not stop the connectors runtime or the MCP server from booting.
+ pub fn with_relocated_keys(
+ mut self,
+ env_prefix: &'static str,
+ keys: &'static [RelocatedKey],
+ ) -> Self {
+ self.env_prefix = env_prefix;
+ self.relocated_keys = keys;
+ self
+ }
+
+ fn reject_relocated_keys(&self) -> Result<(), ConfigurationError> {
+ if self.relocated_keys.is_empty() {
+ return Ok(());
+ }
+ let file = file_exists(&self.file_path).then(|| Figment::from(Toml::file(&self.file_path)));
+ let env_names = env::vars_os().filter_map(|(name, _)| name.into_string().ok());
+ let mut found = false;
+
+ for key in self.relocated_keys {
+ if file
+ .as_ref()
+ .is_some_and(|file| file.find_value(key.path).is_ok())
+ {
+ found = true;
+ error!(
+ "Config key '{}' no longer exists; {}. Remove the key to boot.",
+ key.path,
+ key.guidance()
+ );
+ }
+ }
+ for (name, key) in relocated_env_vars(env_names, self.env_prefix, self.relocated_keys) {
+ found = true;
+ error!(
+ "Environment variable '{name}' sets config key '{}', which no longer exists; {}. \
+ Unset it to boot.",
+ key.path,
+ key.guidance()
+ );
+ }
+
+ if found {
+ return Err(ConfigurationError::InvalidConfigurationValue);
+ }
+ Ok(())
+ }
}
impl<P: Provider + Clone> ConfigProvider for FileConfigProvider<P> {
async fn load_config<T: ConfigurationType>(&self) -> Result<T, ConfigurationError> {
info!("Loading config from path: '{}'...", self.file_path);
+ // Both sources are checked before either is merged: the env provider
+ // below is just as silent about a key no field reads, and the
+ // pure-env container never touches the file branch at all.
+ self.reject_relocated_keys()?;
+
// Start with the default configuration if provided
let mut config_builder = Figment::new();
let has_default = self.default_config.is_some();
@@ -113,6 +206,42 @@ impl<P: Provider + Clone> ConfigProvider for FileConfigProvider<P> {
}
}
+/// Pair every name in `names` that addresses a relocated key with that key.
+///
+/// A key's variable name is its path uppercased with dots turned into
+/// underscores, matching what `#[derive(ConfigEnv)]` generates. A deleted table
+/// also matches its children, so `system.message_deduplication` catches
+/// `IGGY_SYSTEM_MESSAGE_DEDUPLICATION_ENABLED`.
+fn relocated_env_vars<'keys>(
+ names: impl Iterator<Item = String>,
+ prefix: &str,
+ keys: &'keys [RelocatedKey],
+) -> Vec<(String, &'keys RelocatedKey)> {
+ let derived: Vec<(String, &RelocatedKey)> = keys
+ .iter()
+ .map(|key| {
+ (
+ format!("{prefix}{}", key.path.replace('.', "_").to_uppercase()),
+ key,
+ )
+ })
+ .collect();
+
+ let mut found = Vec::new();
+ for name in names {
+ for (env_name, key) in &derived {
+ let nested = name
+ .strip_prefix(env_name.as_str())
+ .is_some_and(|rest| rest.starts_with('_'));
+ if &name == env_name || nested {
+ found.push((name, *key));
+ break;
+ }
+ }
+ }
+ found
+}
+
fn file_exists<P: AsRef<Path>>(path: P) -> bool {
let path = path.as_ref();
@@ -138,3 +267,89 @@ fn file_exists<P: AsRef<Path>>(path: P) -> bool {
};
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const KEYS: &[RelocatedKey] = &[
+ RelocatedKey {
+ path: "system.partition.enforce_fsync",
+ replacement: Some("enforce_fsync"),
+ },
+ RelocatedKey {
+ path: "system.message_deduplication",
+ replacement: None,
+ },
+ ];
+
+ fn names(names: &[&str]) -> Vec<String> {
+ names.iter().map(|name| (*name).to_string()).collect()
+ }
+
+ #[test]
+ fn given_env_var_for_relocated_leaf_when_matching_then_should_report_it() {
+ let found = relocated_env_vars(
+ names(&["IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC"]).into_iter(),
+ "IGGY_",
+ KEYS,
+ );
+
+ assert_eq!(found.len(), 1);
+ assert_eq!(found[0].0, "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC");
+ assert_eq!(found[0].1.replacement, Some("enforce_fsync"));
+ }
+
+ #[test]
+ fn given_env_var_under_removed_table_when_matching_then_should_report_it() {
+ let found = relocated_env_vars(
+ names(&["IGGY_SYSTEM_MESSAGE_DEDUPLICATION_ENABLED"]).into_iter(),
+ "IGGY_",
+ KEYS,
+ );
+
+ assert_eq!(found.len(), 1);
+ assert_eq!(found[0].1.path, "system.message_deduplication");
+ assert_eq!(found[0].1.replacement, None);
+ }
+
+ #[test]
+ fn given_live_env_vars_when_matching_then_should_report_none() {
+ let found = relocated_env_vars(
+ names(&[
+ "IGGY_SYSTEM_PARTITION_PATH",
+ "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNCHRONIZATION",
+ "IGGY_TCP_ADDRESS",
+ "RUST_LOG",
+ ])
+ .into_iter(),
+ "IGGY_",
+ KEYS,
+ );
+
+ assert!(found.is_empty(), "unexpected matches: {found:?}");
+ }
+
+ #[test]
+ fn given_another_configs_prefix_when_matching_then_should_report_none() {
+ let found = relocated_env_vars(
+ names(&["IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC"]).into_iter(),
+ "IGGY_CONNECTORS_",
+ KEYS,
+ );
+
+ assert!(found.is_empty(), "unexpected matches: {found:?}");
+ }
+
+ #[test]
+ fn given_no_relocated_keys_when_rejecting_then_should_accept() {
+ let provider = FileConfigProvider::new(
+ "nonexistent-config.toml".to_string(),
+ Toml::string(""),
+ false,
+ None,
+ );
+
+ assert!(provider.reject_relocated_keys().is_ok());
+ }
+}
diff --git a/core/configs/src/configs_impl/mod.rs b/core/configs/src/configs_impl/mod.rs
index 6b16acc0b..905cb46d7 100644
--- a/core/configs/src/configs_impl/mod.rs
+++ b/core/configs/src/configs_impl/mod.rs
@@ -33,7 +33,7 @@ mod typed_env_provider;
pub use env_mapping::{ConfigEnvMappings, EnvVarMapping};
pub use error::ConfigurationError;
-pub use file_provider::FileConfigProvider;
+pub use file_provider::{FileConfigProvider, RelocatedKey};
pub use parsing::parse_env_value_to_json;
pub use traits::{ConfigProvider, ConfigurationType};
pub use typed_env_provider::TypedEnvProvider;
diff --git a/core/configs/src/lib.rs b/core/configs/src/lib.rs
index e9339b2aa..3b2c59541 100644
--- a/core/configs/src/lib.rs
+++ b/core/configs/src/lib.rs
@@ -24,7 +24,7 @@ pub use common::{COMPONENT, cache_indexes, defaults, displays, http, system, val
pub use configs_derive::ConfigEnv;
pub use configs_impl::{
ConfigEnvMappings, ConfigProvider, ConfigurationError, ConfigurationType, EnvVarMapping,
- FileConfigProvider, TypedEnvProvider, parse_env_value_to_json,
+ FileConfigProvider, RelocatedKey, TypedEnvProvider, parse_env_value_to_json,
};
pub use server_config::{
cluster, message_bus, metadata, partition, quic, server, sharding, tcp, websocket,
diff --git a/core/configs/src/server_config/server.rs b/core/configs/src/server_config/server.rs
index 4dbd7e81d..5f0522c1a 100644
--- a/core/configs/src/server_config/server.rs
+++ b/core/configs/src/server_config/server.rs
@@ -26,7 +26,10 @@ use super::websocket::WebSocketConfig;
use crate::ConfigurationError;
use crate::common::http::HttpConfig;
use crate::common::system::SystemConfig;
-use configs::{ConfigEnv, ConfigEnvMappings, ConfigProvider, FileConfigProvider, TypedEnvProvider};
+use configs::{
+ ConfigEnv, ConfigEnvMappings, ConfigProvider, FileConfigProvider, RelocatedKey,
+ TypedEnvProvider,
+};
use err_trail::ErrContext;
use figment::providers::{Format, Toml};
use figment::value::Dict;
@@ -46,6 +49,48 @@ pub use crate::common::server::{
const DEFAULT_CONFIG_PATH: &str = "core/server/config.toml";
+/// Server config keys that became per-topic options, or went away with the
+/// feature they configured.
+///
+/// The provider refuses to boot while any of them is still set, in the config
+/// file or in the environment. See [`RelocatedKey`] for why a warning is not
+/// enough. The partition knobs matter most: they are create-only options now,
+/// so a topic that boots without one can never be given it afterwards.
+const RELOCATED_CONFIG_KEYS: &[RelocatedKey] = &[
+ RelocatedKey {
+ path: "system.topic.max_size",
+ replacement: Some("max_topic_size"),
+ },
+ RelocatedKey {
+ path: "system.topic.message_expiry",
+ replacement: Some("message_expiry"),
+ },
+ RelocatedKey {
+ path: "system.partition.enforce_fsync",
+ replacement: Some("enforce_fsync"),
+ },
+ RelocatedKey {
+ path: "system.partition.messages_required_to_save",
+ replacement: Some("messages_required_to_save"),
+ },
+ RelocatedKey {
+ path: "system.partition.size_of_messages_required_to_save",
+ replacement: Some("size_of_messages_required_to_save"),
+ },
+ RelocatedKey {
+ path: "system.segment.size",
+ replacement: Some("segment_size"),
+ },
+ RelocatedKey {
+ path: "system.segment.preallocate",
+ replacement: Some("preallocate_segments"),
+ },
+ RelocatedKey {
+ path: "system.message_deduplication",
+ replacement: None,
+ },
+];
+
/// [`SystemConfig`] bound to this crate's own
/// [`super::sharding::ShardingConfig`]. `core/server` names this alias
/// wherever it refers to the system config.
@@ -139,6 +184,7 @@ impl ServerConfig {
true,
Some(default_config),
)
+ .with_relocated_keys(ServerConfig::ENV_PREFIX, RELOCATED_CONFIG_KEYS)
}
/// All recognised env var names for [`ServerConfig`].
diff --git a/core/configs/src/server_config/validators.rs b/core/configs/src/server_config/validators.rs
index 6784ef294..60541d449 100644
--- a/core/configs/src/server_config/validators.rs
+++ b/core/configs/src/server_config/validators.rs
@@ -26,8 +26,9 @@ use super::COMPONENT;
use super::cluster::STATE_CHUNK_HEADER_LEN;
use super::server::{ExtraConfig, NamespaceConfig, ServerConfig};
use crate::ConfigurationError;
+use crate::common::validators::SEGMENT_MAX_SIZE_BYTES;
use err_trail::ErrContext;
-use iggy_common::{IggyExpiry, MaxTopicSize, Validatable};
+use iggy_common::{IggyExpiry, Validatable};
use server_common::sharding::IggyNamespace;
use tracing::warn;
@@ -102,31 +103,6 @@ impl Validatable<ConfigurationError> for ServerConfig {
format!("{COMPONENT} (error: {e}) - failed to validate message saver config")
})?;
- let topic_size = match self.system.topic.max_size {
- MaxTopicSize::Custom(size) => Ok(size.as_bytes_u64()),
- MaxTopicSize::Unlimited => Ok(u64::MAX),
- MaxTopicSize::ServerDefault => {
- eprintln!("system.topic.max_size cannot be ServerDefault in the server config");
- Err(ConfigurationError::InvalidConfigurationValue)
- }
- }?;
-
- if let IggyExpiry::ServerDefault = self.system.topic.message_expiry {
- eprintln!("system.topic.message_expiry cannot be ServerDefault in the server config");
- return Err(ConfigurationError::InvalidConfigurationValue);
- }
-
- // A zero duration encodes to wire value 0, the same value the wire uses
- // for ServerDefault, so it would silently collide with that sentinel.
- if let IggyExpiry::ExpireDuration(duration) = self.system.topic.message_expiry
- && duration.as_micros() == 0
- {
- eprintln!(
- "system.topic.message_expiry is a zero duration, which collides with the server-default sentinel on the wire; use \"none\" to never expire or a positive duration"
- );
- return Err(ConfigurationError::InvalidConfigurationValue);
- }
-
if self.http.enabled
&& let IggyExpiry::ServerDefault = self.http.jwt.access_token_expiry
{
@@ -171,15 +147,6 @@ impl Validatable<ConfigurationError> for ServerConfig {
}
}
- if topic_size < self.system.segment.size.as_bytes_u64() {
- eprintln!(
- "system.topic.max_size ({} B) must be >= system.segment.size ({} B)",
- topic_size,
- self.system.segment.size.as_bytes_u64()
- );
- return Err(ConfigurationError::InvalidConfigurationValue);
- }
-
// A received segment artifact can be one whole batch larger than the
// segment cap (rotation checks the cap AFTER appending), and the real
// batch bound is the BUS frame cap -- the server never enforces
@@ -188,21 +155,18 @@ impl Validatable<ConfigurationError> for ServerConfig {
// partition livelocks re-requesting the same segment from every peer at
// the backoff ceiling. Caught here so it is a boot error rather than one
// partition that silently never rejoins.
- let artifact_floor = self
- .system
- .segment
- .size
- .as_bytes_u64()
- .saturating_add(self.message_bus.max_message_size.as_bytes_u64());
+ // Segment size is per topic now, so the floor is the LARGEST segment
+ // any topic may legally be created with, not a configured value.
+ let artifact_floor =
+ SEGMENT_MAX_SIZE_BYTES.saturating_add(self.message_bus.max_message_size.as_bytes_u64());
if self.partition.transfer_artifact_bytes_max.as_bytes_u64() < artifact_floor {
eprintln!(
- "{COMPONENT} partition.transfer_artifact_bytes_max ({} B) must be at least \
- system.segment.size ({} B) + message_bus.max_message_size ({} B) = \
- {artifact_floor} B: a segment may close one whole batch past its cap, and an \
- artifact ceiling below that refuses a legal segment and livelocks the \
- partition's rejoin",
+ "{COMPONENT} partition.transfer_artifact_bytes_max ({} B) must be at least the \
+ largest legal segment ({SEGMENT_MAX_SIZE_BYTES} B) + \
+ message_bus.max_message_size ({} B) = {artifact_floor} B: a segment may close \
+ one whole batch past its cap, and an artifact ceiling below that refuses a \
+ legal segment and livelocks the partition's rejoin",
self.partition.transfer_artifact_bytes_max.as_bytes_u64(),
- self.system.segment.size.as_bytes_u64(),
self.message_bus.max_message_size.as_bytes_u64(),
);
return Err(ConfigurationError::InvalidConfigurationValue);
@@ -363,10 +327,6 @@ impl Validatable<ConfigurationError> for ServerConfig {
/// pristine config.toml boots without noise. The guard test below pins the
/// compared knobs against drift.
fn reject_unsupported_and_warn_inert(config: &ServerConfig) -> Result<(), ConfigurationError> {
- if config.system.message_deduplication.enabled {
- eprintln!("system.message_deduplication.enabled is not supported");
- return Err(ConfigurationError::InvalidConfigurationValue);
- }
if config.system.segment.archive_expired {
eprintln!("system.segment.archive_expired is not supported");
return Err(ConfigurationError::InvalidConfigurationValue);
@@ -474,6 +434,33 @@ mod tests {
.expect("config deserializes")
}
+ /// The per-topic `segment_size` ceiling lives in `iggy_common`, which cannot
+ /// import this crate. Admission reads it from there; boot validation reads
+ /// the constant here. This is the only place both are visible.
+ #[test]
+ fn given_segment_maximum_when_compared_to_the_option_ceiling_should_match() {
+ assert_eq!(
+ SEGMENT_MAX_SIZE_BYTES,
+ iggy_common::MAX_TOPIC_SEGMENT_SIZE,
+ "the option ceiling and the segment maximum must move together"
+ );
+ }
+
+ /// Admission may cap a topic's `segment_size` at
+ /// [`iggy_common::MAX_TOPIC_SEGMENT_SIZE`] flat only while boot refuses any
+ /// config whose artifact budget cannot carry a segment that large plus one
+ /// bus frame. Without that refusal the ceiling would have to be per node.
+ #[test]
+ fn given_artifact_budget_below_the_segment_ceiling_when_validating_should_reject() {
+ let config = config_with_override(
+ "[partition]\ntransfer_artifact_bytes_max = \"1 GiB\"\n[message_bus]\nmax_message_size = \"1 MiB\"\n",
+ );
+ assert!(
+ config.validate().is_err(),
+ "an artifact budget under segment maximum + one frame must refuse boot"
+ );
+ }
+
#[test]
fn given_shipped_default_config_when_validating_should_pass() {
let config: ServerConfig = Figment::new()
@@ -483,12 +470,6 @@ mod tests {
config.validate().expect("pristine config must validate");
}
- #[test]
- fn given_message_deduplication_enabled_when_validating_should_reject() {
- let config = config_with_override("[system.message_deduplication]\nenabled = true\n");
- assert!(config.validate().is_err());
- }
-
#[test]
fn given_web_ui_enabled_when_validating_should_pass() {
let config = config_with_override("[http]\nweb_ui = true\n");
@@ -509,12 +490,6 @@ mod tests {
assert!(config.validate().is_err());
}
- #[test]
- fn given_zero_message_expiry_when_validating_should_reject() {
- let config = config_with_override("[system.topic]\nmessage_expiry = \"0s\"\n");
- assert!(config.validate().is_err());
- }
-
#[test]
fn given_peer_queue_capacity_not_above_repair_chunk_max_when_validating_should_reject() {
// The default repair_chunk_max (128) must stay strictly below
diff --git a/core/harness_derive/src/attrs.rs b/core/harness_derive/src/attrs.rs
index ee4ff391e..76fee823c 100644
--- a/core/harness_derive/src/attrs.rs
+++ b/core/harness_derive/src/attrs.rs
@@ -371,7 +371,7 @@ fn parse_transport_ident(ident: Ident) -> syn::Result<Transport> {
}
}
-/// Parses a dot-notation config key like `segment.size` or `partition.messages_required_to_save`.
+/// Parses a dot-notation config key like `segment.cache_indexes` or `metadata.journal_slots`.
fn parse_config_key(input: ParseStream) -> syn::Result<(String, Span)> {
let first: Ident = input.parse()?;
let span = first.span();
@@ -629,9 +629,9 @@ mod tests {
#[test]
fn parse_server_static() {
- let attrs: IggyTestAttrs = syn::parse_quote!(server(segment.size = "1MiB"));
- let segment_size = attrs.server.find_override("segment.size").unwrap();
- assert!(matches!(&segment_size.value, ConfigValue::Static(s) if s == "1MiB"));
+ let attrs: IggyTestAttrs = syn::parse_quote!(server(segment.cache_indexes = "all"));
+ let cache_indexes = attrs.server.find_override("segment.cache_indexes").unwrap();
+ assert!(matches!(&cache_indexes.value, ConfigValue::Static(s) if s == "all"));
}
#[test]
@@ -642,9 +642,10 @@ mod tests {
#[test]
fn parse_server_matrix() {
- let attrs: IggyTestAttrs = syn::parse_quote!(server(segment.size = ["512B", "1MiB"]));
- let segment_size = attrs.server.find_override("segment.size").unwrap();
- assert!(matches!(&segment_size.value, ConfigValue::Matrix(v) if v.len() == 2));
+ let attrs: IggyTestAttrs =
+ syn::parse_quote!(server(segment.cache_indexes = ["none", "all"]));
+ let cache_indexes = attrs.server.find_override("segment.cache_indexes").unwrap();
+ assert!(matches!(&cache_indexes.value, ConfigValue::Matrix(v) if v.len() == 2));
}
#[test]
@@ -652,17 +653,20 @@ mod tests {
let attrs: IggyTestAttrs = syn::parse_quote!(
test_client_transport = [Tcp, Http],
server(
- segment.size = ["512B", "1MiB"],
- segment.cache_indexes = "none",
+ segment.cache_indexes = ["none", "all"],
+ metadata.journal_slots = 1024,
tcp.socket.nodelay = true
)
);
assert_eq!(attrs.transports.len(), 2);
- let segment_size = attrs.server.find_override("segment.size").unwrap();
let cache_indexes = attrs.server.find_override("segment.cache_indexes").unwrap();
+ let journal_slots = attrs
+ .server
+ .find_override("metadata.journal_slots")
+ .unwrap();
let tcp_nodelay = attrs.server.find_override("tcp.socket.nodelay").unwrap();
- assert!(matches!(&segment_size.value, ConfigValue::Matrix(v) if v.len() == 2));
- assert!(matches!(&cache_indexes.value, ConfigValue::Static(s) if s == "none"));
+ assert!(matches!(&cache_indexes.value, ConfigValue::Matrix(v) if v.len() == 2));
+ assert!(matches!(&journal_slots.value, ConfigValue::Static(s) if s == "1024"));
assert!(matches!(&tcp_nodelay.value, ConfigValue::Static(s) if s == "true"));
}
@@ -718,11 +722,11 @@ mod tests {
#[test]
fn parse_mcp_combined() {
let attrs: IggyTestAttrs =
- syn::parse_quote!(seed = my_seed, server(mcp, segment.size = "1MiB"));
+ syn::parse_quote!(seed = my_seed, server(mcp, segment.cache_indexes = "all"));
assert!(attrs.server.mcp.is_some());
assert!(attrs.seed_fn.is_some());
- let segment_size = attrs.server.find_override("segment.size").unwrap();
- assert!(matches!(&segment_size.value, ConfigValue::Static(s) if s == "1MiB"));
+ let cache_indexes = attrs.server.find_override("segment.cache_indexes").unwrap();
+ assert!(matches!(&cache_indexes.value, ConfigValue::Static(s) if s == "all"));
}
#[test]
@@ -750,13 +754,13 @@ mod tests {
#[test]
fn parse_dot_notation_deep() {
let attrs: IggyTestAttrs = syn::parse_quote!(server(
- partition.messages_required_to_save = [32, 64],
+ metadata.journal_slots = [512, 1024],
system.encryption.enabled = true
));
assert_eq!(attrs.server.config_overrides.len(), 2);
let msgs = attrs
.server
- .find_override("partition.messages_required_to_save")
+ .find_override("metadata.journal_slots")
.unwrap();
assert!(matches!(&msgs.value, ConfigValue::Matrix(v) if v.len() == 2));
}
@@ -808,11 +812,11 @@ mod tests {
let attrs: IggyTestAttrs = syn::parse_quote!(
cluster_nodes = [3, 5],
test_client_transport = [Tcp, Http],
- server(segment.size = ["512B", "1MiB"])
+ server(segment.cache_indexes = ["none", "all"])
);
assert!(matches!(&attrs.cluster_nodes, ClusterNodesValue::Matrix(v) if v == &[3, 5]));
assert_eq!(attrs.transports.len(), 2);
- let segment_size = attrs.server.find_override("segment.size").unwrap();
- assert!(matches!(&segment_size.value, ConfigValue::Matrix(v) if v.len() == 2));
+ let cache_indexes = attrs.server.find_override("segment.cache_indexes").unwrap();
+ assert!(matches!(&cache_indexes.value, ConfigValue::Matrix(v) if v.len() == 2));
}
}
diff --git a/core/harness_derive/src/codegen.rs b/core/harness_derive/src/codegen.rs
index 76e2b664a..d6adb360a 100644
--- a/core/harness_derive/src/codegen.rs
+++ b/core/harness_derive/src/codegen.rs
@@ -753,12 +753,8 @@ mod tests {
transport: Transport::Http,
transport_explicit: true,
config_values: vec![
- ("segment.size".to_string(), "1MiB".to_string()),
("segment.cache_indexes".to_string(), "all".to_string()),
- (
- "partition.messages_required_to_save".to_string(),
- "64".to_string(),
- ),
+ ("metadata.journal_slots".to_string(), "1024".to_string()),
],
tls: None,
websocket_tls: None,
@@ -766,7 +762,7 @@ mod tests {
};
assert_eq!(
v.suffix(),
- "http_segment_size_1mib_segment_cache_indexes_all_partition_messages_required_to_save_64"
+ "http_segment_cache_indexes_all_metadata_journal_slots_1024"
);
}
@@ -828,14 +824,14 @@ mod tests {
transport_explicit: true,
server: crate::attrs::ServerAttrs {
config_overrides: vec![
- ConfigOverride {
- path: "segment.size".to_string(),
- value: ConfigValue::Matrix(vec!["512B".to_string(), "1MiB".to_string()]),
- },
ConfigOverride {
path: "segment.cache_indexes".to_string(),
value: ConfigValue::Matrix(vec!["none".to_string(), "all".to_string()]),
},
+ ConfigOverride {
+ path: "partition.validate_checksum".to_string(),
+ value: ConfigValue::Matrix(vec!["true".to_string(), "false".to_string()]),
+ },
],
..Default::default()
},
@@ -844,7 +840,7 @@ mod tests {
jwks_server: None,
};
let variants = generate_variants(&attrs);
- // 2 transports * 2 segment sizes * 2 cache modes = 8 variants
+ // 2 transports * 2 cache modes * 2 checksum modes = 8 variants
assert_eq!(variants.len(), 8);
}
@@ -857,18 +853,18 @@ mod tests {
#[test]
fn cartesian_product_single() {
let overrides = vec![ConfigOverride {
- path: "segment.size".to_string(),
- value: ConfigValue::Matrix(vec!["512B".to_string(), "1MiB".to_string()]),
+ path: "segment.cache_indexes".to_string(),
+ value: ConfigValue::Matrix(vec!["none".to_string(), "all".to_string()]),
}];
let result = cartesian_product(&overrides);
assert_eq!(result.len(), 2);
assert_eq!(
result[0],
- vec![("segment.size".to_string(), "512B".to_string())]
+ vec![("segment.cache_indexes".to_string(), "none".to_string())]
);
assert_eq!(
result[1],
- vec![("segment.size".to_string(), "1MiB".to_string())]
+ vec![("segment.cache_indexes".to_string(), "all".to_string())]
);
}
diff --git a/core/harness_derive/src/lib.rs b/core/harness_derive/src/lib.rs
index fb527932c..757e02127 100644
--- a/core/harness_derive/src/lib.rs
+++ b/core/harness_derive/src/lib.rs
@@ -40,11 +40,11 @@
//! Test with server config matrix:
//! ```ignore
//! #[iggy_harness(server(
-//! segment_size = ["512B", "1MiB"],
-//! cache_indexes = ["none", "all"],
+//! segment.cache_indexes = ["none", "all"],
+//! partition.validate_checksum = [true, false],
//! ))]
//! async fn test_caching(client: &IggyClient) {
-//! // 2 segment sizes × 2 cache modes = 4 tests
+//! // 2 cache modes × 2 checksum modes = 4 tests
//! }
//! ```
diff --git a/core/integration/src/harness/config/mod.rs b/core/integration/src/harness/config/mod.rs
index fa3aab6d6..81f4a79a9 100644
--- a/core/integration/src/harness/config/mod.rs
+++ b/core/integration/src/harness/config/mod.rs
@@ -28,5 +28,5 @@ pub use common::{EncryptionConfig, IpAddrKind, TlsConfig};
pub use connectors_runtime::ConnectorsRuntimeConfig;
pub use jwks::JwksConfig;
pub use mcp::McpConfig;
-pub use resolve::resolve_config_paths;
+pub use resolve::{resolve_config_paths, validate_env_var_names};
pub use server::TestServerConfig;
diff --git a/core/integration/src/harness/config/resolve.rs b/core/integration/src/harness/config/resolve.rs
index e5f36f60a..5ca0ccbc1 100644
--- a/core/integration/src/harness/config/resolve.rs
+++ b/core/integration/src/harness/config/resolve.rs
@@ -21,9 +21,20 @@ use configs::server::ServerConfig;
use configs::{ConfigEnvMappings, EnvVarMapping};
use std::collections::HashMap;
+/// `IGGY_`-prefixed variables the server reads outside the config struct, so
+/// `ServerConfig::all_env_var_names` cannot know them. `IGGY_CONFIG_PATH`
+/// selects the config file itself and the root credentials are consumed by
+/// `args.rs` before the config loads; `IGGY_TEST_VERBOSE` is harness-only.
+pub const NON_CONFIG_ENV_VARS: [&str; 4] = [
+ "IGGY_CONFIG_PATH",
+ "IGGY_ROOT_USERNAME",
+ "IGGY_ROOT_PASSWORD",
+ "IGGY_TEST_VERBOSE",
+];
+
/// Resolve config paths to environment variable names.
///
-/// Takes a map of dot-notation config paths (e.g., "segment.size") and their values,
+/// Takes a map of dot-notation config paths (e.g., "partition.validate_checksum") and their values,
/// validates them against the `ServerConfig` mappings, and returns the
/// corresponding environment variable names with values.
///
@@ -115,6 +126,69 @@ fn find_mapping(path: &str) -> Option<&'static EnvVarMapping> {
ServerConfig::find_by_config_path(path)
}
+/// Reject `IGGY_*` names that no config leaf reads.
+///
+/// `extra_envs` is a raw env map with no schema behind it, so a name that was
+/// valid before a config key moved or was deleted keeps being set and simply
+/// stops doing anything. That failure mode is silent and expensive: the server
+/// boots on defaults, the test appears to configure something it does not, and
+/// the resulting behavior change surfaces somewhere unrelated. Attribute
+/// overrides never had this problem (`resolve_config_paths` validates them);
+/// this closes the same gap for the direct path.
+///
+/// Names outside the `IGGY_` prefix are left alone: those address the process
+/// environment (`RUST_LOG`, test scaffolding), not the config schema.
+/// [`NON_CONFIG_ENV_VARS`] carries the `IGGY_`-prefixed names the server reads
+/// outside the config struct.
+///
+/// # Errors
+///
+/// Returns a message naming every unknown variable, with near-miss
+/// suggestions drawn from the live mapping table.
+pub fn validate_env_var_names(envs: &HashMap<String, String>) -> Result<(), String> {
+ let known: Vec<&'static str> = ServerConfig::all_env_var_names();
+ let mut unknown: Vec<&String> = envs
+ .keys()
+ .filter(|name| {
+ name.starts_with("IGGY_")
+ && !known.contains(&name.as_str())
+ && !NON_CONFIG_ENV_VARS.contains(&name.as_str())
+ })
+ .collect();
+ if unknown.is_empty() {
+ return Ok(());
+ }
+ unknown.sort();
+ let mut report = String::new();
+ for name in unknown {
+ report.push_str(&format!(" unknown config env var: {name}\n"));
+ let lowered = name.trim_start_matches("IGGY_").to_ascii_lowercase();
+ let mut near: Vec<&&str> = known
+ .iter()
+ .filter(|candidate| {
+ let candidate = candidate.trim_start_matches("IGGY_").to_ascii_lowercase();
+ candidate.ends_with(&lowered)
+ || lowered.ends_with(&candidate)
+ || levenshtein(&candidate, &lowered) <= 4
+ })
+ .collect();
+ near.sort();
+ near.truncate(5);
+ if !near.is_empty() {
+ report.push_str(" did you mean: ");
+ report.push_str(
+ &near
+ .iter()
+ .map(std::string::ToString::to_string)
+ .collect::<Vec<_>>()
+ .join(", "),
+ );
+ report.push('\n');
+ }
+ }
+ Err(report)
+}
+
fn levenshtein(a: &str, b: &str) -> usize {
let a_len = a.len();
let b_len = b.len();
@@ -193,27 +267,69 @@ mod tests {
#[test]
fn resolve_valid_path() {
let mut overrides = HashMap::new();
- overrides.insert("system.segment.size".to_string(), "1MiB".to_string());
+ overrides.insert(
+ "system.partition.validate_checksum".to_string(),
+ "false".to_string(),
+ );
let result = resolve_config_paths(&overrides);
assert!(result.is_ok());
let env_vars = result.unwrap();
- assert!(env_vars.contains_key("IGGY_SYSTEM_SEGMENT_SIZE"));
+ assert!(env_vars.contains_key("IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM"));
assert_eq!(
- env_vars.get("IGGY_SYSTEM_SEGMENT_SIZE"),
- Some(&"1MiB".to_string())
+ env_vars.get("IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM"),
+ Some(&"false".to_string())
);
}
#[test]
fn resolve_with_implicit_system_prefix() {
let mut overrides = HashMap::new();
- overrides.insert("segment.size".to_string(), "1MiB".to_string());
+ overrides.insert(
+ "partition.validate_checksum".to_string(),
+ "false".to_string(),
+ );
let result = resolve_config_paths(&overrides);
assert!(result.is_ok());
let env_vars = result.unwrap();
- assert!(env_vars.contains_key("IGGY_SYSTEM_SEGMENT_SIZE"));
+ assert!(env_vars.contains_key("IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM"));
+ }
+
+ #[test]
+ fn validate_env_var_names_accepts_live_names_and_passes_through_non_iggy() {
+ let envs = HashMap::from([
+ (
+ "IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM".to_string(),
+ "false".to_string(),
+ ),
+ ("RUST_LOG".to_string(), "debug".to_string()),
+ ]);
+ assert!(validate_env_var_names(&envs).is_ok());
+ }
+
+ #[test]
+ fn validate_env_var_names_accepts_the_non_config_variables() {
+ let envs: HashMap<String, String> = NON_CONFIG_ENV_VARS
+ .iter()
+ .map(|name| ((*name).to_string(), "value".to_string()))
+ .collect();
+ assert!(
+ validate_env_var_names(&envs).is_ok(),
+ "variables the server reads outside the config struct must pass"
+ );
+ }
+
+ #[test]
+ fn validate_env_var_names_rejects_a_name_no_config_leaf_reads() {
+ // The exact shape that went silent when these keys moved to per-topic
+ // options: a name that was valid before and now does nothing.
+ let envs = HashMap::from([("IGGY_SYSTEM_SEGMENT_SIZE".to_string(), "1MiB".to_string())]);
+ let error = validate_env_var_names(&envs).expect_err("deleted key must be rejected");
+ assert!(
+ error.contains("IGGY_SYSTEM_SEGMENT_SIZE"),
+ "the report must name the offending variable, got: {error}"
+ );
}
#[test]
diff --git a/core/integration/src/harness/handle/server.rs b/core/integration/src/harness/handle/server.rs
index 5965bf10b..bb6a6486d 100644
--- a/core/integration/src/harness/handle/server.rs
+++ b/core/integration/src/harness/handle/server.rs
@@ -350,7 +350,14 @@ impl ServerHandle {
self.set_tls_envs("WEBSOCKET", &tls);
}
- // Extra envs from config (includes resolved config paths from macro)
+ // Extra envs from config (includes resolved config paths from macro).
+ // Validated first: a name no config leaf reads is a silent no-op, and
+ // a test that believes it configured the server but did not is worse
+ // than one that fails to start.
+ if let Err(report) = crate::harness::config::validate_env_var_names(&self.config.extra_envs)
+ {
+ panic!("invalid extra_envs for the test server:\n{report}");
+ }
for (k, v) in &self.config.extra_envs {
self.envs.insert(k.clone(), v.clone());
}
diff --git a/core/integration/src/harness/orchestrator/builder.rs b/core/integration/src/harness/orchestrator/builder.rs
index 4eb8b3d78..23de6d23a 100644
--- a/core/integration/src/harness/orchestrator/builder.rs
+++ b/core/integration/src/harness/orchestrator/builder.rs
@@ -386,8 +386,8 @@ mod tests {
.server(
TestServerConfig::builder()
.extra_envs(HashMap::from([(
- "IGGY_SYSTEM_SEGMENT_SIZE".to_string(),
- "1MiB".to_string(),
+ "IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM".to_string(),
+ "false".to_string(),
)]))
.build(),
)
@@ -446,7 +446,10 @@ mod tests {
.quic_enabled(false)
.websocket_enabled(false)
.extra_envs(HashMap::from([
- ("IGGY_SYSTEM_SEGMENT_SIZE".to_string(), "2MiB".to_string()),
+ (
+ "IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM".to_string(),
+ "false".to_string(),
+ ),
("TEST".to_string(), "value".to_string()),
]))
.build(),
diff --git a/core/integration/src/harness/seeds.rs b/core/integration/src/harness/seeds.rs
index 60b6ae00b..5485dbc6d 100644
--- a/core/integration/src/harness/seeds.rs
+++ b/core/integration/src/harness/seeds.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use iggy::prelude::{IggyClient, StreamClient, TopicClient};
+use iggy::prelude::{IggyClient, StreamClient, TopicClient, TopicCreateOptions};
use iggy_common::{
- CompressionAlgorithm, Consumer, Identifier, IggyExpiry, IggyMessage, MaxTopicSize,
- Partitioning, PersonalAccessTokenExpiry, UserStatus,
+ Consumer, Identifier, IggyExpiry, IggyMessage, MaxTopicSize, Partitioning,
+ PersonalAccessTokenExpiry, UserStatus,
};
use iggy_common::{
ConsumerGroupClient, ConsumerOffsetClient, MessageClient, PersonalAccessTokenClient, UserClient,
@@ -55,11 +55,12 @@ pub async fn stream_with_topic(client: &IggyClient) -> Result<(), SeedError> {
.create_topic(
&"test_stream".try_into()?,
"test_topic",
- 1,
- CompressionAlgorithm::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::Unlimited,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ max_topic_size: Some(MaxTopicSize::Unlimited),
+ ..TopicCreateOptions::default()
+ },
)
.await?;
Ok(())
@@ -75,11 +76,10 @@ pub async fn connector_stream(client: &IggyClient) -> Result<(), SeedError> {
.create_topic(
&stream_id,
names::TOPIC,
- 1,
- CompressionAlgorithm::None,
- None,
- IggyExpiry::ServerDefault,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..TopicCreateOptions::default()
+ },
)
.await?;
@@ -98,11 +98,10 @@ pub async fn connector_multi_topic_stream(client: &IggyClient) -> Result<(), See
.create_topic(
&stream_id,
names::TOPIC,
- 1,
- CompressionAlgorithm::None,
- None,
- IggyExpiry::ServerDefault,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..TopicCreateOptions::default()
+ },
)
.await?;
@@ -110,11 +109,10 @@ pub async fn connector_multi_topic_stream(client: &IggyClient) -> Result<(), See
.create_topic(
&stream_id,
names::TOPIC_2,
- 1,
- CompressionAlgorithm::None,
- None,
- IggyExpiry::ServerDefault,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..TopicCreateOptions::default()
+ },
)
.await?;
@@ -132,11 +130,10 @@ pub async fn mcp_standard(client: &IggyClient) -> Result<(), SeedError> {
.create_topic(
&stream_id,
names::TOPIC,
- 1,
- CompressionAlgorithm::None,
- None,
- IggyExpiry::ServerDefault,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ ..TopicCreateOptions::default()
+ },
)
.await?;
diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs
index 2c9a62c11..f1400d2cc 100644
--- a/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs
+++ b/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs
@@ -21,7 +21,7 @@ use crate::cli::common::{
};
use assert_cmd::assert::Assert;
use async_trait::async_trait;
-use iggy::prelude::{Client, IggyExpiry, MaxTopicSize};
+use iggy::prelude::{Client, IggyExpiry, TopicCreateOptions};
use predicates::str::diff;
use serial_test::parallel;
@@ -94,11 +94,11 @@ impl IggyCmdTestCase for TestConsumerGroupCreateCmd {
.create_topic(
&self.stream_id.try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs
index 8012d7435..21fbdb1d1 100644
--- a/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs
+++ b/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs
@@ -23,7 +23,7 @@ use assert_cmd::assert::Assert;
use async_trait::async_trait;
use iggy::prelude::Client;
use iggy::prelude::IggyExpiry;
-use iggy::prelude::MaxTopicSize;
+use iggy::prelude::TopicCreateOptions;
use predicates::str::diff;
use serial_test::parallel;
@@ -97,11 +97,11 @@ impl IggyCmdTestCase for TestConsumerGroupDeleteCmd {
.create_topic(
&self.stream_id.try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs
index 57bb5aec3..6ceb3c94c 100644
--- a/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs
+++ b/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs
@@ -23,7 +23,7 @@ use assert_cmd::assert::Assert;
use async_trait::async_trait;
use iggy::prelude::Client;
use iggy::prelude::IggyExpiry;
-use iggy::prelude::MaxTopicSize;
+use iggy::prelude::TopicCreateOptions;
use predicates::str::{contains, starts_with};
use serial_test::parallel;
@@ -97,11 +97,11 @@ impl IggyCmdTestCase for TestConsumerGroupGetCmd {
.create_topic(
&self.stream_id.try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs
index bd4641089..8cf6e2f3b 100644
--- a/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs
+++ b/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs
@@ -23,7 +23,7 @@ use assert_cmd::assert::Assert;
use async_trait::async_trait;
use iggy::prelude::Client;
use iggy::prelude::IggyExpiry;
-use iggy::prelude::MaxTopicSize;
+use iggy::prelude::TopicCreateOptions;
use predicates::str::{contains, starts_with};
use serial_test::parallel;
@@ -94,11 +94,11 @@ impl IggyCmdTestCase for TestConsumerGroupListCmd {
.create_topic(
&stream.id.try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await
.expect("Failed to create topic");
diff --git a/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs b/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs
index 1aa27db00..41288c58b 100644
--- a/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs
+++ b/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs
@@ -104,11 +104,11 @@ impl IggyCmdTestCase for TestConsumerOffsetGetCmd {
.create_topic(
&stream.id.try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await
.expect("Failed to create topic");
diff --git a/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs b/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs
index 6bb2508dd..1de493808 100644
--- a/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs
+++ b/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs
@@ -105,11 +105,11 @@ impl IggyCmdTestCase for TestConsumerOffsetSetCmd {
.create_topic(
&self.stream_id.try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/general/test_help_command.rs b/core/integration/tests/cli/general/test_help_command.rs
index 5fbac9129..499073b54 100644
--- a/core/integration/tests/cli/general/test_help_command.rs
+++ b/core/integration/tests/cli/general/test_help_command.rs
@@ -38,6 +38,7 @@ Commands:
segment segments operations [alias: seg]
ping ping iggy server
me get current client info
+ options list the options a resource's create command accepts
stats get iggy server statistics
snapshot collect iggy server troubleshooting data
pat personal access token operations
diff --git a/core/integration/tests/cli/general/test_overview_command.rs b/core/integration/tests/cli/general/test_overview_command.rs
index 6b528208a..f8f447e04 100644
--- a/core/integration/tests/cli/general/test_overview_command.rs
+++ b/core/integration/tests/cli/general/test_overview_command.rs
@@ -49,6 +49,7 @@ Commands:
segment segments operations [alias: seg]
ping ping iggy server
me get current client info
+ options list the options a resource's create command accepts
stats get iggy server statistics
snapshot collect iggy server troubleshooting data
pat personal access token operations
diff --git a/core/integration/tests/cli/message/test_message_flush_command.rs b/core/integration/tests/cli/message/test_message_flush_command.rs
index 6b1dfde46..c07b18968 100644
--- a/core/integration/tests/cli/message/test_message_flush_command.rs
+++ b/core/integration/tests/cli/message/test_message_flush_command.rs
@@ -28,7 +28,7 @@ use async_trait::async_trait;
use iggy::prelude::Client;
use iggy::prelude::Identifier;
use iggy::prelude::IggyExpiry;
-use iggy::prelude::MaxTopicSize;
+use iggy::prelude::TopicCreateOptions;
use predicates::str::contains;
use serial_test::parallel;
use std::str::FromStr;
@@ -105,11 +105,11 @@ impl IggyCmdTestCase for TestMessageFetchCmd {
.create_topic(
&stream.id.try_into().unwrap(),
&self.topic_name,
- self.partitions_count,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(self.partitions_count),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await
.expect("Failed to create topic");
diff --git a/core/integration/tests/cli/message/test_message_poll_command.rs b/core/integration/tests/cli/message/test_message_poll_command.rs
index ccb4959fb..cc91fec1b 100644
--- a/core/integration/tests/cli/message/test_message_poll_command.rs
+++ b/core/integration/tests/cli/message/test_message_poll_command.rs
@@ -124,11 +124,11 @@ impl IggyCmdTestCase for TestMessagePollCmd {
.create_topic(
&stream.id.try_into().unwrap(),
&self.topic_name,
- self.partitions_count,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(self.partitions_count),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/message/test_message_poll_to_file_command.rs b/core/integration/tests/cli/message/test_message_poll_to_file_command.rs
index 7632cb3e5..a433191af 100644
--- a/core/integration/tests/cli/message/test_message_poll_to_file_command.rs
+++ b/core/integration/tests/cli/message/test_message_poll_to_file_command.rs
@@ -117,11 +117,11 @@ impl IggyCmdTestCase for TestMessagePollToFileCmd<'_> {
.create_topic(
&stream.id.try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/message/test_message_send_command.rs b/core/integration/tests/cli/message/test_message_send_command.rs
index 65fd96eae..a24a6c526 100644
--- a/core/integration/tests/cli/message/test_message_send_command.rs
+++ b/core/integration/tests/cli/message/test_message_send_command.rs
@@ -162,11 +162,11 @@ impl IggyCmdTestCase for TestMessageSendCmd {
.create_topic(
&stream.id.try_into().unwrap(),
&self.topic_name,
- self.partitions_count,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(self.partitions_count),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await
.expect("Failed to create topic");
diff --git a/core/integration/tests/cli/message/test_message_send_from_file_command.rs b/core/integration/tests/cli/message/test_message_send_from_file_command.rs
index d7b3b5ca1..cc67a0d79 100644
--- a/core/integration/tests/cli/message/test_message_send_from_file_command.rs
+++ b/core/integration/tests/cli/message/test_message_send_from_file_command.rs
@@ -105,11 +105,11 @@ impl IggyCmdTestCase for TestMessageSendFromFileCmd<'_> {
.create_topic(
&stream.id.try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await
.expect("Failed to create topic");
diff --git a/core/integration/tests/cli/partition/test_partition_create_command.rs b/core/integration/tests/cli/partition/test_partition_create_command.rs
index 90299bb9c..ff243f055 100644
--- a/core/integration/tests/cli/partition/test_partition_create_command.rs
+++ b/core/integration/tests/cli/partition/test_partition_create_command.rs
@@ -23,7 +23,7 @@ use async_trait::async_trait;
use iggy::prelude::Client;
use iggy::prelude::CompressionAlgorithm;
use iggy::prelude::IggyExpiry;
-use iggy::prelude::MaxTopicSize;
+use iggy::prelude::TopicCreateOptions;
use predicates::str::diff;
use serial_test::parallel;
@@ -91,11 +91,14 @@ impl IggyCmdTestCase for TestPartitionCreateCmd {
.create_topic(
&self.actual_stream_id.unwrap().try_into().unwrap(),
&self.topic_name,
- self.partitions_count,
- self.compression_algorithm,
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(self.partitions_count),
+ compression_algorithm: (self.compression_algorithm
+ != CompressionAlgorithm::default())
+ .then_some(self.compression_algorithm),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/partition/test_partition_delete_command.rs b/core/integration/tests/cli/partition/test_partition_delete_command.rs
index 0025666ee..d74424b02 100644
--- a/core/integration/tests/cli/partition/test_partition_delete_command.rs
+++ b/core/integration/tests/cli/partition/test_partition_delete_command.rs
@@ -22,7 +22,7 @@ use assert_cmd::assert::Assert;
use async_trait::async_trait;
use iggy::prelude::Client;
use iggy::prelude::IggyExpiry;
-use iggy::prelude::MaxTopicSize;
+use iggy::prelude::TopicCreateOptions;
use predicates::str::diff;
use serial_test::parallel;
@@ -79,11 +79,11 @@ impl IggyCmdTestCase for TestPartitionDeleteCmd {
.create_topic(
&self.actual_stream_id.unwrap().try_into().unwrap(),
&self.topic_name,
- self.partitions_count,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(self.partitions_count),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/stream/test_stream_purge_command.rs b/core/integration/tests/cli/stream/test_stream_purge_command.rs
index e0e7bace5..5a3339a14 100644
--- a/core/integration/tests/cli/stream/test_stream_purge_command.rs
+++ b/core/integration/tests/cli/stream/test_stream_purge_command.rs
@@ -61,11 +61,11 @@ impl IggyCmdTestCase for TestStreamPurgeCmd {
.create_topic(
&self.stream_name.clone().try_into().unwrap(),
&self.topic_name,
- 10,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(10),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/system/test_stats_command.rs b/core/integration/tests/cli/system/test_stats_command.rs
index 1510448ac..49575babc 100644
--- a/core/integration/tests/cli/system/test_stats_command.rs
+++ b/core/integration/tests/cli/system/test_stats_command.rs
@@ -24,8 +24,8 @@ use iggy::prelude::Client;
use iggy::prelude::Identifier;
use iggy::prelude::IggyExpiry;
use iggy::prelude::IggyMessage;
-use iggy::prelude::MaxTopicSize;
use iggy::prelude::Partitioning;
+use iggy::prelude::TopicCreateOptions;
use iggy_cli::commands::binary_system::stats::GetStatsOutput;
use iggy_common::Stats;
use predicates::str::{contains, starts_with};
@@ -74,11 +74,11 @@ impl IggyCmdTestCase for TestStatsCmd {
.create_topic(
&stream_id,
"topic",
- 5,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(5),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
@@ -184,11 +184,11 @@ impl IggyCmdTestCase for TestStatsCmdWithMessages {
.create_topic(
&self.stream_id.try_into().unwrap(),
"topic",
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/topic/test_topic_create_command.rs b/core/integration/tests/cli/topic/test_topic_create_command.rs
index b2d8437a1..af8dbb02a 100644
--- a/core/integration/tests/cli/topic/test_topic_create_command.rs
+++ b/core/integration/tests/cli/topic/test_topic_create_command.rs
@@ -40,7 +40,6 @@ struct TestTopicCreateCmd {
compression_algorithm: CompressionAlgorithm,
message_expiry: Option<Vec<String>>,
max_topic_size: MaxTopicSize,
- replication_factor: u8,
using_identifier: TestStreamId,
}
@@ -54,7 +53,6 @@ impl TestTopicCreateCmd {
compression_algorithm: CompressionAlgorithm,
message_expiry: Option<Vec<String>>,
max_topic_size: MaxTopicSize,
- replication_factor: u8,
using_identifier: TestStreamId,
) -> Self {
Self {
@@ -65,7 +63,6 @@ impl TestTopicCreateCmd {
compression_algorithm,
message_expiry,
max_topic_size,
- replication_factor,
using_identifier,
}
}
@@ -120,13 +117,12 @@ impl IggyCmdTestCase for TestTopicCreateCmd {
let max_topic_size = self.max_topic_size.to_string();
- let replication_factor = self.replication_factor;
-
let message = format!(
- "Executing create topic with name: {topic_name}, message expiry: {message_expiry}, compression algorithm: {compression_algorithm}, \
- max topic size: {max_topic_size}, replication factor: {replication_factor} in stream with ID: {stream_id}\n\
+ "Executing create topic with name: {topic_name}, message expiry: {message_expiry}, \
+ compression algorithm: {compression_algorithm}, max topic size: {max_topic_size} \
+ in stream with ID: {stream_id}\n\
Topic with name: {topic_name}, partitions count: {partitions_count}, compression algorithm: {compression_algorithm}, message expiry: {message_expiry}, \
- max topic size: {max_topic_size}, replication factor: {replication_factor} created in stream with ID: {stream_id}\n",
+ max topic size: {max_topic_size} created in stream with ID: {stream_id}\n",
);
command_state.success().stdout(diff(message));
@@ -189,7 +185,6 @@ pub async fn should_be_successful() {
Default::default(),
None,
MaxTopicSize::ServerDefault,
- 1,
TestStreamId::Named,
))
.await;
@@ -202,7 +197,6 @@ pub async fn should_be_successful() {
Default::default(),
None,
MaxTopicSize::ServerDefault,
- 1,
TestStreamId::Named,
))
.await;
@@ -215,7 +209,6 @@ pub async fn should_be_successful() {
Default::default(),
Some(vec![String::from("3days"), String::from("5s")]),
MaxTopicSize::Unlimited,
- 1,
TestStreamId::Named,
))
.await;
@@ -233,7 +226,6 @@ pub async fn should_be_successful() {
String::from("1s"),
]),
MaxTopicSize::Custom(IggyByteSize::from_str("2GiB").unwrap()),
- 1,
TestStreamId::Named,
))
.await;
@@ -295,10 +287,12 @@ Options:
{CLAP_INDENT}
[default: server_default]
- -r, --replication-factor <REPLICATION_FACTOR>
- Replication factor for the topic
+ --set <KEY=VALUE>
+ Additional topic option as key=value, repeatable
{CLAP_INDENT}
- [default: 1]
+ Values are sent as strings and parsed server-side through each option's
+ own FromStr (e.g. --set segment_size=128MiB). The server rejects keys it
+ does not support; run "iggy options topic" to list the ones it accepts.
-h, --help
Print help (see a summary with '-h')
@@ -328,10 +322,10 @@ Arguments:
[MESSAGE_EXPIRY]... Message expiry time in human-readable format like "unlimited" or "15days 2min 2s" [default: server_default]
Options:
- -t, --topic-id <TOPIC_ID> Topic ID to create
- -m, --max-topic-size <MAX_TOPIC_SIZE> Max topic size in human-readable format like "unlimited" or "15GB" [default: server_default]
- -r, --replication-factor <REPLICATION_FACTOR> Replication factor for the topic [default: 1]
- -h, --help Print help (see more with '--help')
+ -t, --topic-id <TOPIC_ID> Topic ID to create
+ -m, --max-topic-size <MAX_TOPIC_SIZE> Max topic size in human-readable format like "unlimited" or "15GB" [default: server_default]
+ --set <KEY=VALUE> Additional topic option as key=value, repeatable
+ -h, --help Print help (see more with '--help')
"#,
),
))
diff --git a/core/integration/tests/cli/topic/test_topic_delete_command.rs b/core/integration/tests/cli/topic/test_topic_delete_command.rs
index 9400f2ee0..24b572c4d 100644
--- a/core/integration/tests/cli/topic/test_topic_delete_command.rs
+++ b/core/integration/tests/cli/topic/test_topic_delete_command.rs
@@ -23,7 +23,7 @@ use assert_cmd::assert::Assert;
use async_trait::async_trait;
use iggy::prelude::Client;
use iggy::prelude::IggyExpiry;
-use iggy::prelude::MaxTopicSize;
+use iggy::prelude::TopicCreateOptions;
use predicates::str::diff;
use serial_test::parallel;
@@ -80,11 +80,11 @@ impl IggyCmdTestCase for TestTopicDeleteCmd {
.create_topic(
&self.stream_name.clone().try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/topic/test_topic_get_command.rs b/core/integration/tests/cli/topic/test_topic_get_command.rs
index 46435dde3..5715e9211 100644
--- a/core/integration/tests/cli/topic/test_topic_get_command.rs
+++ b/core/integration/tests/cli/topic/test_topic_get_command.rs
@@ -23,7 +23,7 @@ use assert_cmd::assert::Assert;
use async_trait::async_trait;
use iggy::prelude::Client;
use iggy::prelude::IggyExpiry;
-use iggy::prelude::MaxTopicSize;
+use iggy::prelude::TopicCreateOptions;
use predicates::str::{contains, starts_with};
use serial_test::parallel;
@@ -80,11 +80,11 @@ impl IggyCmdTestCase for TestTopicGetCmd {
.create_topic(
&self.stream_name.clone().try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/topic/test_topic_list_command.rs b/core/integration/tests/cli/topic/test_topic_list_command.rs
index a0162ae8c..08a277e43 100644
--- a/core/integration/tests/cli/topic/test_topic_list_command.rs
+++ b/core/integration/tests/cli/topic/test_topic_list_command.rs
@@ -23,7 +23,7 @@ use assert_cmd::assert::Assert;
use async_trait::async_trait;
use iggy::prelude::Client;
use iggy::prelude::IggyExpiry;
-use iggy::prelude::MaxTopicSize;
+use iggy::prelude::TopicCreateOptions;
use predicates::str::{contains, starts_with};
use serial_test::parallel;
@@ -74,11 +74,11 @@ impl IggyCmdTestCase for TestTopicListCmd {
.create_topic(
&self.stream_name.clone().try_into().unwrap(),
&self.topic_name,
- 1,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/topic/test_topic_purge_command.rs b/core/integration/tests/cli/topic/test_topic_purge_command.rs
index 4d1323c6f..e028f0dad 100644
--- a/core/integration/tests/cli/topic/test_topic_purge_command.rs
+++ b/core/integration/tests/cli/topic/test_topic_purge_command.rs
@@ -79,11 +79,11 @@ impl IggyCmdTestCase for TestTopicPurgeCmd {
.create_topic(
&self.stream_name.clone().try_into().unwrap(),
&self.topic_name,
- 10,
- Default::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(10),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
diff --git a/core/integration/tests/cli/topic/test_topic_update_command.rs b/core/integration/tests/cli/topic/test_topic_update_command.rs
index b259f06b8..9cad22689 100644
--- a/core/integration/tests/cli/topic/test_topic_update_command.rs
+++ b/core/integration/tests/cli/topic/test_topic_update_command.rs
@@ -27,6 +27,7 @@ use iggy::prelude::CompressionAlgorithm;
use iggy::prelude::IggyByteSize;
use iggy::prelude::IggyExpiry;
use iggy::prelude::MaxTopicSize;
+use iggy::prelude::TopicCreateOptions;
use predicates::str::diff;
use serial_test::parallel;
use std::str::FromStr;
@@ -40,12 +41,10 @@ struct TestTopicUpdateCmd {
compression_algorithm: CompressionAlgorithm,
message_expiry: Option<Vec<String>>,
max_topic_size: MaxTopicSize,
- replication_factor: u8,
topic_new_name: String,
topic_new_compression_algorithm: CompressionAlgorithm,
topic_new_message_expiry: Option<Vec<String>>,
topic_new_max_size: MaxTopicSize,
- topic_new_replication_factor: u8,
using_stream_id: TestStreamId,
using_topic_id: TestTopicId,
}
@@ -60,12 +59,10 @@ impl TestTopicUpdateCmd {
compression_algorithm: CompressionAlgorithm,
message_expiry: Option<Vec<String>>,
max_topic_size: MaxTopicSize,
- replication_factor: u8,
topic_new_name: String,
topic_new_compression_algorithm: CompressionAlgorithm,
topic_new_message_expiry: Option<Vec<String>>,
topic_new_max_size: MaxTopicSize,
- topic_new_replication_factor: u8,
using_stream_id: TestStreamId,
using_topic_id: TestTopicId,
) -> Self {
@@ -77,12 +74,10 @@ impl TestTopicUpdateCmd {
compression_algorithm,
message_expiry,
max_topic_size,
- replication_factor,
topic_new_name,
topic_new_compression_algorithm,
topic_new_message_expiry,
topic_new_max_size,
- topic_new_replication_factor,
using_stream_id,
using_topic_id,
}
@@ -103,13 +98,6 @@ impl TestTopicUpdateCmd {
command.push(self.topic_new_compression_algorithm.to_string());
command.push(format!("--max-topic-size={}", self.topic_new_max_size));
- if self.topic_new_replication_factor != 1 {
- command.push(format!(
- "--replication-factor={}",
- self.topic_new_replication_factor
- ));
- }
-
if let Some(message_expiry) = &self.topic_new_message_expiry {
command.extend(message_expiry.clone());
}
@@ -138,11 +126,17 @@ impl IggyCmdTestCase for TestTopicUpdateCmd {
.create_topic(
&self.stream_name.clone().try_into().unwrap(),
&self.topic_name,
- 1,
- self.compression_algorithm,
- Some(self.replication_factor),
- message_expiry,
- self.max_topic_size,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ compression_algorithm: (self.compression_algorithm
+ != CompressionAlgorithm::default())
+ .then_some(self.compression_algorithm),
+ message_expiry: (message_expiry != IggyExpiry::ServerDefault)
+ .then_some(message_expiry),
+ max_topic_size: (self.max_topic_size != MaxTopicSize::ServerDefault)
+ .then_some(self.max_topic_size),
+ ..TopicCreateOptions::default()
+ },
)
.await;
assert!(topic.is_ok());
@@ -175,17 +169,16 @@ impl IggyCmdTestCase for TestTopicUpdateCmd {
})
.to_string();
- let replication_factor = self.topic_new_replication_factor;
let new_topic_name = &self.topic_new_name;
let new_max_topic_size = self.topic_new_max_size.to_string();
let expected_message = format!(
"Executing update topic with ID: {topic_id}, name: {new_topic_name}, \
- message expiry: {message_expiry}, compression algorithm: {compression_algorithm}, max topic size: {new_max_topic_size}, \
- replication factor: {replication_factor}, in stream with ID: {stream_id}\n\
+ message expiry: {message_expiry}, compression algorithm: {compression_algorithm}, max topic size: \
+ {new_max_topic_size}, in stream with ID: {stream_id}\n\
Topic with ID: {topic_id} updated name: {new_topic_name}, updated message expiry: {message_expiry}, \
- updated compression algorithm: {compression_algorithm}, updated max topic size: {new_max_topic_size}, \
- updated replication factor: {replication_factor} in stream with ID: {stream_id}\n"
+ updated compression algorithm: {compression_algorithm}, \
+ updated max topic size: {new_max_topic_size} in stream with ID: {stream_id}\n"
);
command_state.success().stdout(diff(expected_message));
@@ -246,12 +239,10 @@ pub async fn should_be_successful() {
Default::default(),
None,
MaxTopicSize::ServerDefault,
- 1,
String::from("new_name"),
CompressionAlgorithm::Gzip,
None,
MaxTopicSize::Custom(IggyByteSize::from_str("2GiB").unwrap()),
- 1,
TestStreamId::Named,
TestTopicId::Named,
))
@@ -265,12 +256,10 @@ pub async fn should_be_successful() {
Default::default(),
None,
MaxTopicSize::ServerDefault,
- 1,
String::from("testing"),
CompressionAlgorithm::Gzip,
None,
MaxTopicSize::Unlimited,
- 1,
TestStreamId::Named,
TestTopicId::Named,
))
@@ -338,11 +327,6 @@ Options:
{CLAP_INDENT}
[default: server_default]
- -r, --replication-factor <REPLICATION_FACTOR>
- New replication factor for the topic
-{CLAP_INDENT}
- [default: 1]
-
-h, --help
Print help (see a summary with '-h')
"#,
@@ -372,9 +356,8 @@ Arguments:
[MESSAGE_EXPIRY]... New message expiry time in human-readable format like "unlimited" or "15days 2min 2s" [default: server_default]
Options:
- -m, --max-topic-size <MAX_TOPIC_SIZE> New max topic size in human-readable format like "unlimited" or "15GB" [default: server_default]
- -r, --replication-factor <REPLICATION_FACTOR> New replication factor for the topic [default: 1]
- -h, --help Print help (see more with '--help')
+ -m, --max-topic-size <MAX_TOPIC_SIZE> New max topic size in human-readable format like "unlimited" or "15GB" [default: server_default]
+ -h, --help Print help (see more with '--help')
"#,
),
))
diff --git a/core/integration/tests/cluster/client_table_restart.rs b/core/integration/tests/cluster/client_table_restart.rs
index b6fd026d1..2d93d106b 100644
--- a/core/integration/tests/cluster/client_table_restart.rs
+++ b/core/integration/tests/cluster/client_table_restart.rs
@@ -81,7 +81,9 @@ use iggy_binary_protocol::consensus::{
use iggy_binary_protocol::requests::streams::CreateStreamRequest;
use iggy_binary_protocol::requests::users::LoginRegisterRequest;
use iggy_binary_protocol::responses::users::LoginRegisterResponse;
-use iggy_binary_protocol::{ClientVersionInfo, HEADER_SIZE, IGGY_PROTOCOL_VERSION, WireName};
+use iggy_binary_protocol::{
+ ClientVersionInfo, HEADER_SIZE, IGGY_PROTOCOL_VERSION, WireName, WireOptions,
+};
use integration::harness::TestHarness;
use integration::iggy_harness;
use secrecy::SecretString;
@@ -258,6 +260,7 @@ pub(super) fn tcp_addrs(harness: &TestHarness) -> Vec<SocketAddr> {
pub(super) fn create_stream_payload(name: &str) -> Bytes {
CreateStreamRequest {
name: WireName::new(name).unwrap(),
+ options: WireOptions::empty(),
}
.to_bytes()
}
diff --git a/core/integration/tests/cluster/multi_shard_partition_convergence.rs b/core/integration/tests/cluster/multi_shard_partition_convergence.rs
index 5379074b9..88a782efa 100644
--- a/core/integration/tests/cluster/multi_shard_partition_convergence.rs
+++ b/core/integration/tests/cluster/multi_shard_partition_convergence.rs
@@ -67,11 +67,11 @@ async fn create_topic(client: &IggyClient, stream: &Identifier, name: &str) {
.create_topic(
stream,
name,
- 1,
- CompressionAlgorithm::default(),
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await
.unwrap_or_else(|error| panic!("create_topic {name}: {error}"));
diff --git a/core/integration/tests/cluster/partition_state_transfer.rs b/core/integration/tests/cluster/partition_state_transfer.rs
index fbc5c9391..daef6ed97 100644
--- a/core/integration/tests/cluster/partition_state_transfer.rs
+++ b/core/integration/tests/cluster/partition_state_transfer.rs
@@ -51,6 +51,10 @@ const STORED_CONSUMER_OFFSET: u64 = 17;
/// 16 MiB pull completed in ~120ms on a release build, inside one marker
/// poll). Also enough commits (> ring capacity 64) that the fresh
/// rejoiner's floor refuses.
+/// Segment cap for the multi-artifact spec: small enough that the 64 MiB bulky
+/// seed spans several sealed segments, so the install runs from a manifest with
+/// one spill per artifact rather than a single segment.
+const SEGMENT_SIZE_MULTI_ARTIFACT: u64 = 8 * 1024 * 1024;
const BULKY_MESSAGES_COUNT: u32 = 256;
const BULKY_PAYLOAD_LEN: usize = 256 * 1024;
/// Poll for the kill gate: the serving marker appears at descriptor-serve
@@ -85,8 +89,7 @@ const MARKER_POLL: Duration = Duration::from_millis(200);
cluster_nodes = 3,
server(
system.sharding.cpu_allocation = "0..1",
- partition.evicted_ring_capacity = "64",
- system.partition.messages_required_to_save = "1"
+ partition.evicted_ring_capacity = "64"
)
)]
async fn given_evicted_ring_when_fresh_node_joins_late_should_state_transfer_partition(
@@ -190,8 +193,7 @@ async fn given_evicted_ring_when_fresh_node_joins_late_should_state_transfer_par
cluster_nodes = 3,
server(
system.sharding.cpu_allocation = "0..1",
- partition.evicted_ring_capacity = "64",
- system.partition.messages_required_to_save = "1"
+ partition.evicted_ring_capacity = "64"
)
)]
async fn given_evicted_ring_when_node_restarts_with_data_should_state_transfer_partition(
@@ -204,7 +206,7 @@ async fn given_evicted_ring_when_node_restarts_with_data_should_state_transfer_p
.root_client_for_node(0)
.await
.expect("connect a root client to the node");
- seed_topic(&client).await;
+ seed_topic(&client, None).await;
produce(&client, 40).await;
sleep(Duration::from_secs(1)).await;
harness.stop_node(2).expect("stop node 2");
@@ -242,12 +244,10 @@ async fn given_evicted_ring_when_node_restarts_with_data_should_state_transfer_p
cluster_nodes = 3,
server(
system.sharding.cpu_allocation = "0..1",
- partition.evicted_ring_capacity = "64",
- system.partition.messages_required_to_save = "1",
- system.segment.size = "8MiB"
+ partition.evicted_ring_capacity = "64"
)
)]
-// The 8 MiB segment cap makes the 64 MiB bulky seed span several sealed
+// The topic's 8 MiB segment cap makes the 64 MiB bulky seed span several sealed
// segments, so unlike the other specs (single-segment under the default 1 GiB
// cap) this one installs from a MULTI-ARTIFACT manifest, one spill per artifact.
// The installed segment count at the end is what asserts that. The re-armed
@@ -261,7 +261,7 @@ async fn given_transfer_peer_dies_when_stalled_should_leave_dead_peer_and_recove
.root_client_for_node(0)
.await
.expect("connect a root client to the node");
- seed_topic(&client).await;
+ seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await;
// Bulky payloads so the pull spans many 256 KiB chunks: the kill below
// must land while the transfer is provably in flight, and a small
// partition finishes inside the marker-poll latency, leaving the
@@ -357,7 +357,12 @@ async fn connect_any(harness: &TestHarness, nodes: &[usize]) -> Option<IggyClien
None
}
-async fn seed_topic(client: &IggyClient) {
+/// `segment_size` is `None` for the specs that only need the default cap; the
+/// multi-artifact spec passes one small enough that its bulky seed spans
+/// several sealed segments. `messages_required_to_save = 1` is the forcing
+/// function every spec here shares: each commit flushes and ring-evicts, which
+/// is what marches `repair_retained_from` forward.
+async fn seed_topic(client: &IggyClient, segment_size: Option<u64>) {
client
.create_stream(STREAM_NAME)
.await
@@ -366,11 +371,13 @@ async fn seed_topic(client: &IggyClient) {
.create_topic(
&Identifier::named(STREAM_NAME).expect("stream identifier"),
TOPIC_NAME,
- 1,
- CompressionAlgorithm::None,
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ messages_required_to_save: Some(1),
+ segment_size: segment_size.map(IggyByteSize::from),
+ ..TopicCreateOptions::default()
+ },
)
.await
.expect("create topic with one partition");
@@ -413,7 +420,7 @@ async fn produce_bulky(client: &IggyClient, count: u32, payload_len: usize) {
}
async fn seed_partition(client: &IggyClient) {
- seed_topic(client).await;
+ seed_topic(client, None).await;
produce(client, MESSAGES_COUNT).await;
assert_eq!(
poll_count(client, MESSAGES_COUNT).await,
diff --git a/core/integration/tests/cluster/register_forwarding.rs b/core/integration/tests/cluster/register_forwarding.rs
index 4811956f3..f85d74db5 100644
--- a/core/integration/tests/cluster/register_forwarding.rs
+++ b/core/integration/tests/cluster/register_forwarding.rs
@@ -341,11 +341,11 @@ async fn given_a_backup_when_auto_login_dials_it_should_settle_on_the_leader(
.create_topic(
&stream_id,
TOPIC_NAME,
- 1,
- CompressionAlgorithm::None,
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await
.expect("create topic after a backup-dialed login");
diff --git a/core/integration/tests/config_provider/mod.rs b/core/integration/tests/config_provider/mod.rs
index 32d0d9aec..a3bfcbe83 100644
--- a/core/integration/tests/config_provider/mod.rs
+++ b/core/integration/tests/config_provider/mod.rs
@@ -32,7 +32,7 @@ async fn validate_config_env_override() {
let expected_http = true;
let expected_tcp = true;
let expected_message_saver = true;
- let expected_message_expiry = "1s";
+ let expected_validate_checksum = false;
unsafe {
env::set_var("IGGY_HTTP_ENABLED", expected_http.to_string());
@@ -41,7 +41,10 @@ async fn validate_config_env_override() {
"IGGY_MESSAGE_SAVER_ENABLED",
expected_message_saver.to_string(),
);
- env::set_var("IGGY_SYSTEM_TOPIC_MESSAGE_EXPIRY", expected_message_expiry);
+ env::set_var(
+ "IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM",
+ expected_validate_checksum.to_string(),
+ );
}
let config_path = get_root_path().join("../server/config.toml");
@@ -56,15 +59,15 @@ async fn validate_config_env_override() {
assert_eq!(config.tcp.enabled, expected_tcp);
assert_eq!(config.message_saver.enabled, expected_message_saver);
assert_eq!(
- config.system.topic.message_expiry.to_string(),
- expected_message_expiry
+ config.system.partition.validate_checksum,
+ expected_validate_checksum
);
unsafe {
env::remove_var("IGGY_HTTP_ENABLED");
env::remove_var("IGGY_TCP_ENABLED");
env::remove_var("IGGY_MESSAGE_SAVER_ENABLED");
- env::remove_var("IGGY_SYSTEM_TOPIC_MESSAGE_EXPIRY");
+ env::remove_var("IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM");
}
}
diff --git a/core/integration/tests/connectors/fixtures/doris/container.rs b/core/integration/tests/connectors/fixtures/doris/container.rs
index f435084ec..ae972af7b 100644
--- a/core/integration/tests/connectors/fixtures/doris/container.rs
+++ b/core/integration/tests/connectors/fixtures/doris/container.rs
@@ -194,6 +194,88 @@ fn check_vm_max_map_count() -> Result<(), TestBinaryError> {
Ok(())
}
+/// Attempts to create-or-attach the shared container before giving up. Only
+/// name conflicts are retried, and each one means another process already
+/// created it, so the bound is about how many tests can be racing at once
+/// rather than about how long Doris takes to boot.
+const CONTAINER_START_ATTEMPTS: u32 = 30;
+const CONTAINER_START_RETRY_DELAY: Duration = Duration::from_secs(1);
+
+/// Creates the shared Doris container, or attaches to it if another test won
+/// the race.
+///
+/// `ReuseDirective::Always` resolves reuse by inspecting the daemon and then
+/// creating when nothing matches, with no lock spanning the two steps. On a
+/// cold daemon with several doris tests in flight, every one of them inspects
+/// before any of them creates, so one wins and the rest get
+/// `409 Conflict: name already in use`. Retrying is what turns those losers
+/// into attachers: by the next attempt the winner's container exists, so the
+/// inspect half succeeds. Any other failure is returned as-is.
+async fn start_shared_container(
+ entrypoint_cmd: &str,
+) -> Result<ContainerAsync<GenericImage>, TestBinaryError> {
+ let mut conflict = String::new();
+ for attempt in 1..=CONTAINER_START_ATTEMPTS {
+ // FE HTTP and FE MySQL get ephemeral host ports (the connector and
+ // tests connect via the resolved mapping). BE HTTP must be pinned
+ // 1:1 — the FE always returns Location: http://127.0.0.1:8040/...
+ // for the Stream Load redirect, and that's only reachable from the
+ // host if container:8040 is bound to host:8040.
+ //
+ // `with_container_name` + `with_reuse(Always)` is what makes the
+ // container survive across nextest's per-test processes: the first
+ // test creates `iggy-test-doris`, every later test (in any process)
+ // attaches to it. The 1:1 BE port is therefore held continuously by
+ // one container, never racing with itself across container restarts.
+ //
+ // Rebuilt per attempt because `start` consumes the request.
+ let result = GenericImage::new(DORIS_IMAGE, DORIS_TAG)
+ // GenericImage's own with_entrypoint/with_wait_for must come before
+ // any ImageExt method, which turns GenericImage into ContainerRequest.
+ .with_entrypoint("bash")
+ .with_wait_for(WaitFor::http(
+ HttpWaitStrategy::new(FE_HEALTH_ENDPOINT)
+ .with_port(FE_HTTP_PORT.tcp())
+ .with_expected_status_code(200u16),
+ ))
+ .with_env_var("SKIP_CHECK_ULIMIT", "true")
+ .with_cmd(["-c", entrypoint_cmd])
+ .with_mapped_port(0, FE_HTTP_PORT.tcp())
+ .with_mapped_port(0, FE_MYSQL_PORT.tcp())
+ .with_mapped_port(BE_HTTP_PORT, BE_HTTP_PORT.tcp())
+ .with_container_name(DORIS_CONTAINER_NAME)
+ .with_reuse(ReuseDirective::Always)
+ .start()
+ .await;
+
+ match result {
+ Ok(container) => return Ok(container),
+ Err(error) => {
+ let message = error.to_string();
+ if !message.contains("is already in use") {
+ return Err(TestBinaryError::FixtureSetup {
+ fixture_type: "DorisContainer".to_string(),
+ message: format!("Failed to start container: {message}"),
+ });
+ }
+ info!(
+ "Doris container name taken by another test (attempt {attempt}), retrying to attach"
+ );
+ conflict = message;
+ sleep(CONTAINER_START_RETRY_DELAY).await;
+ }
+ }
+ }
+
+ Err(TestBinaryError::FixtureSetup {
+ fixture_type: "DorisContainer".to_string(),
+ message: format!(
+ "Failed to attach to container '{DORIS_CONTAINER_NAME}' after \
+ {CONTAINER_START_ATTEMPTS} attempts: {conflict}"
+ ),
+ })
+}
+
pub struct DorisContainer {
// Held only so testcontainers' Drop runs on test exit. ReuseDirective::Always
// makes that Drop a no-op (the container is left running for the next test
@@ -222,39 +304,7 @@ impl DorisContainer {
exec bash /usr/local/bin/entry_point.sh"
);
- // FE HTTP and FE MySQL get ephemeral host ports (the connector and
- // tests connect via the resolved mapping). BE HTTP must be pinned
- // 1:1 — the FE always returns Location: http://127.0.0.1:8040/...
- // for the Stream Load redirect, and that's only reachable from the
- // host if container:8040 is bound to host:8040.
- //
- // `with_container_name` + `with_reuse(Always)` is what makes the
- // container survive across nextest's per-test processes: the first
- // test creates `iggy-test-doris`, every later test (in any process)
- // attaches to it. The 1:1 BE port is therefore held continuously by
- // one container, never racing with itself across container restarts.
- let container = GenericImage::new(DORIS_IMAGE, DORIS_TAG)
- // GenericImage's own with_entrypoint/with_wait_for must come before
- // any ImageExt method, which turns GenericImage into ContainerRequest.
- .with_entrypoint("bash")
- .with_wait_for(WaitFor::http(
- HttpWaitStrategy::new(FE_HEALTH_ENDPOINT)
- .with_port(FE_HTTP_PORT.tcp())
- .with_expected_status_code(200u16),
- ))
- .with_env_var("SKIP_CHECK_ULIMIT", "true")
- .with_cmd(["-c", entrypoint_cmd.as_str()])
- .with_mapped_port(0, FE_HTTP_PORT.tcp())
- .with_mapped_port(0, FE_MYSQL_PORT.tcp())
- .with_mapped_port(BE_HTTP_PORT, BE_HTTP_PORT.tcp())
- .with_container_name(DORIS_CONTAINER_NAME)
- .with_reuse(ReuseDirective::Always)
- .start()
- .await
- .map_err(|e| TestBinaryError::FixtureSetup {
- fixture_type: "DorisContainer".to_string(),
- message: format!("Failed to start container: {e}"),
- })?;
+ let container = start_shared_container(&entrypoint_cmd).await?;
let ports = container
.ports()
diff --git a/core/integration/tests/connectors/fixtures/elasticsearch/container.rs b/core/integration/tests/connectors/fixtures/elasticsearch/container.rs
index 5749b1809..1fded111f 100644
--- a/core/integration/tests/connectors/fixtures/elasticsearch/container.rs
+++ b/core/integration/tests/connectors/fixtures/elasticsearch/container.rs
@@ -20,12 +20,14 @@ use reqwest_middleware::ClientWithMiddleware as HttpClient;
use reqwest_retry::RetryTransientMiddleware;
use reqwest_retry::policies::ExponentialBackoff;
use serde::Deserialize;
+use std::time::Duration;
use testcontainers_modules::testcontainers::core::wait::HttpWaitStrategy;
use testcontainers_modules::testcontainers::core::{IntoContainerPort, WaitFor};
use testcontainers_modules::testcontainers::runners::AsyncRunner;
use testcontainers_modules::testcontainers::{
ContainerAsync, GenericImage, ImageExt, ReuseDirective,
};
+use tokio::time::sleep;
use tracing::info;
const ELASTICSEARCH_IMAGE: &str = "docker.io/library/elasticsearch";
@@ -99,16 +101,30 @@ pub struct ElasticsearchContainer {
pub base_url: String,
}
-impl ElasticsearchContainer {
- pub async fn start() -> Result<Self, TestBinaryError> {
- let container = GenericImage::new(ELASTICSEARCH_IMAGE, ELASTICSEARCH_TAG)
+/// See [`start_shared_container`]: same create-or-attach race as the Doris
+/// fixture, same bound.
+const CONTAINER_START_ATTEMPTS: u32 = 30;
+const CONTAINER_START_RETRY_DELAY: Duration = Duration::from_secs(1);
+
+/// Creates the shared Elasticsearch container, or attaches to it if another
+/// test won the race.
+///
+/// `ReuseDirective::Always` inspects the daemon and then creates when nothing
+/// matches, with no lock across the two steps, so a cold daemon with several
+/// elasticsearch tests in flight leaves one winner and the rest holding
+/// `409 Conflict: name already in use`. Retrying converts those into attaches.
+async fn start_shared_container() -> Result<ContainerAsync<GenericImage>, TestBinaryError> {
+ let mut conflict = String::new();
+ for attempt in 1..=CONTAINER_START_ATTEMPTS {
+ // Rebuilt per attempt because `start` consumes the request.
+ let result = GenericImage::new(ELASTICSEARCH_IMAGE, ELASTICSEARCH_TAG)
.with_exposed_port(ELASTICSEARCH_PORT.tcp())
.with_wait_for(WaitFor::http(
HttpWaitStrategy::new(ELASTICSEARCH_HEALTH_ENDPOINT)
.with_port(ELASTICSEARCH_PORT.tcp())
.with_expected_status_code(200u16),
))
- .with_startup_timeout(std::time::Duration::from_secs(120))
+ .with_startup_timeout(Duration::from_secs(120))
.with_env_var("discovery.type", "single-node")
.with_env_var("xpack.security.enabled", "false")
.with_env_var("ES_JAVA_OPTS", "-Xms512m -Xmx512m")
@@ -116,11 +132,39 @@ impl ElasticsearchContainer {
.with_container_name(ELASTICSEARCH_CONTAINER_NAME)
.with_reuse(ReuseDirective::Always)
.start()
- .await
- .map_err(|e| TestBinaryError::FixtureSetup {
- fixture_type: "ElasticsearchContainer".to_string(),
- message: format!("Failed to start container: {e}"),
- })?;
+ .await;
+
+ match result {
+ Ok(container) => return Ok(container),
+ Err(error) => {
+ let message = error.to_string();
+ if !message.contains("is already in use") {
+ return Err(TestBinaryError::FixtureSetup {
+ fixture_type: "ElasticsearchContainer".to_string(),
+ message: format!("Failed to start container: {message}"),
+ });
+ }
+ info!(
+ "Elasticsearch container name taken by another test (attempt {attempt}), retrying to attach"
+ );
+ conflict = message;
+ sleep(CONTAINER_START_RETRY_DELAY).await;
+ }
+ }
+ }
+
+ Err(TestBinaryError::FixtureSetup {
+ fixture_type: "ElasticsearchContainer".to_string(),
+ message: format!(
+ "Failed to attach to container '{ELASTICSEARCH_CONTAINER_NAME}' after \
+ {CONTAINER_START_ATTEMPTS} attempts: {conflict}"
+ ),
+ })
+}
+
+impl ElasticsearchContainer {
+ pub async fn start() -> Result<Self, TestBinaryError> {
+ let container = start_shared_container().await?;
info!("Started Elasticsearch container");
diff --git a/core/integration/tests/connectors/fixtures/s3/fixture.rs b/core/integration/tests/connectors/fixtures/s3/fixture.rs
index adeb03405..3fdbf1781 100644
--- a/core/integration/tests/connectors/fixtures/s3/fixture.rs
+++ b/core/integration/tests/connectors/fixtures/s3/fixture.rs
@@ -21,6 +21,7 @@ use integration::harness::{TestBinaryError, TestFixture};
use s3::creds::Credentials;
use s3::{Bucket, Region};
use std::collections::HashMap;
+use std::time::Duration;
use testcontainers_modules::testcontainers::core::wait::HttpWaitStrategy;
use testcontainers_modules::testcontainers::core::{IntoContainerPort, WaitFor};
use testcontainers_modules::testcontainers::runners::AsyncRunner;
@@ -36,6 +37,9 @@ const MINIO_CONSOLE_PORT: u16 = 9001;
const MINIO_ACCESS_KEY: &str = "admin";
const MINIO_SECRET_KEY: &str = "password";
const MINIO_BUCKET: &str = "iggy-s3-test";
+/// Bounds the wait for MinIO's S3 API to come up behind its health endpoint.
+const BUCKET_CREATE_ATTEMPTS: u32 = 30;
+const BUCKET_CREATE_RETRY_DELAY: Duration = Duration::from_secs(1);
const ENV_SINK_PATH: &str = "IGGY_CONNECTORS_SINK_S3_PATH";
const ENV_SINK_STREAMS_0_STREAM: &str = "IGGY_CONNECTORS_SINK_S3_STREAMS_0_STREAM";
@@ -204,22 +208,44 @@ impl TestFixture for S3SinkFixture {
message: format!("Failed to create credentials: {e}"),
})?;
- let config = s3::BucketConfiguration::default();
- let response = Bucket::create_with_path_style(
- MINIO_BUCKET,
- region.clone(),
- credentials.clone(),
- config,
- )
- .await
- .map_err(|e| TestBinaryError::FixtureSetup {
- fixture_type: "S3SinkFixture".to_string(),
- message: format!("Failed to create bucket: {e}"),
- })?;
- info!(
- "S3 bucket '{}' ready (status: {})",
- MINIO_BUCKET, response.response_code
- );
+ // MinIO answers on its health endpoint before it can serve the S3 API,
+ // so bucket creation can come back 503 while it finishes starting. The
+ // call itself is `Ok` in that case -- the status lives in the response
+ // -- so taking it as success left the bucket absent, and the first
+ // `list_objects` then parsed an S3 error document as a listing and
+ // failed with the unhelpful `missing field 'Name'`. Retry until the
+ // status is a real one: 2xx created, 409 already owned by us.
+ let mut last_status = 0;
+ let mut created = false;
+ for _ in 0..BUCKET_CREATE_ATTEMPTS {
+ let response = Bucket::create_with_path_style(
+ MINIO_BUCKET,
+ region.clone(),
+ credentials.clone(),
+ s3::BucketConfiguration::default(),
+ )
+ .await
+ .map_err(|e| TestBinaryError::FixtureSetup {
+ fixture_type: "S3SinkFixture".to_string(),
+ message: format!("Failed to create bucket: {e}"),
+ })?;
+ last_status = response.response_code;
+ if (200..300).contains(&last_status) || last_status == 409 {
+ created = true;
+ break;
+ }
+ tokio::time::sleep(BUCKET_CREATE_RETRY_DELAY).await;
+ }
+ if !created {
+ return Err(TestBinaryError::FixtureSetup {
+ fixture_type: "S3SinkFixture".to_string(),
+ message: format!(
+ "Bucket '{MINIO_BUCKET}' not creatable after \
+ {BUCKET_CREATE_ATTEMPTS} attempts (last status: {last_status})"
+ ),
+ });
+ }
+ info!("S3 bucket '{MINIO_BUCKET}' ready (status: {last_status})");
let mut bucket = Bucket::new(MINIO_BUCKET, region, credentials).map_err(|e| {
TestBinaryError::FixtureSetup {
diff --git a/core/integration/tests/data_integrity/verify_after_server_restart.rs b/core/integration/tests/data_integrity/verify_after_server_restart.rs
index 1bed4408e..1ac9b91fa 100644
--- a/core/integration/tests/data_integrity/verify_after_server_restart.rs
+++ b/core/integration/tests/data_integrity/verify_after_server_restart.rs
@@ -40,31 +40,19 @@ fn build_server_config(cache_setting: &str) -> TestServerConfig {
"IGGY_SYSTEM_SEGMENT_CACHE_INDEXES".to_string(),
cache_setting.to_string(),
);
- // The server flushes on the journal thresholds (no flush primitive), so
- // force every committed batch straight to disk: the restart asserts
- // below need everything durable, and the explicit flush calls are
- // cfg'd out under vsr (`flush_unsaved_buffer` answers
- // FeatureUnavailable there and is slated for removal). Legacy keeps its
- // shipped buffered defaults; the flush loops below are its barrier.
- extra_envs.insert(
- "IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE".to_string(),
- "1".to_string(),
- );
- extra_envs.insert(
- "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC".to_string(),
- "true".to_string(),
- );
TestServerConfig::builder().extra_envs(extra_envs).build()
}
// TODO(numminex) - Move the message generation method from benchmark run to a special method.
//
-// The durability barrier is the eager-flush envs in `build_server_config`
-// (`flush_unsaved_buffer` answers FeatureUnavailable on VSR, so there is no
-// explicit flush loop), and `iggy-bench` must be freshly built: the harness
-// spawns the prebuilt binary, and a stale one never completes a frame
-// against the server, tripping the bench timeout in
-// `run_bench_and_wait_for_finish`.
+// The durability barrier is the graceful restart itself: shutdown force-flushes
+// the committed journal. The eager-flush knobs that used to stand in for it are
+// topic creation options now, and the topics here are created by `iggy-bench`,
+// which exposes no flag for them.
+//
+// `iggy-bench` must be freshly built: the harness spawns the prebuilt binary,
+// and a stale one never completes a frame against the server, tripping the
+// bench timeout in `run_bench_and_wait_for_finish`.
#[test_matrix(
[cache_all(), cache_open_segment(), cache_none()]
)]
@@ -331,11 +319,12 @@ async fn should_handle_resource_deletion_and_restart() {
.create_topic(
&stream_ident,
&format!("topic-{}", topic_idx),
- 3,
- CompressionAlgorithm::None,
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::Unlimited,
+ &TopicCreateOptions {
+ partitions_count: Some(3),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ max_topic_size: Some(MaxTopicSize::Unlimited),
+ ..TopicCreateOptions::default()
+ },
)
.await
.unwrap();
@@ -411,11 +400,12 @@ async fn should_handle_resource_deletion_and_restart() {
.create_topic(
&stream_0_ident,
"topic-reused",
- 1,
- CompressionAlgorithm::None,
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::Unlimited,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ max_topic_size: Some(MaxTopicSize::Unlimited),
+ ..TopicCreateOptions::default()
+ },
)
.await
.unwrap();
diff --git a/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs b/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs
index cb953b1b8..a2ec465bd 100644
--- a/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs
+++ b/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs
@@ -65,11 +65,11 @@ async fn run(harness: &TestHarness) {
.create_topic(
&stream,
TOPIC_NAME,
- 1,
- CompressionAlgorithm::None,
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ },
)
.await
.unwrap();
diff --git a/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs b/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs
index e242d81b3..1efe14593 100644
--- a/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs
+++ b/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs
@@ -76,12 +76,9 @@ const CONVERGENCE_STABLE_POLLS: u32 = 3;
// `messages_required_to_save = 1` forces every committed batch to persist to its
// segment immediately on every node, so each replica materialises the segment
// files while running (the VSR server serves no flush_unsaved_buffer, and
-// shutdown-flush would couple the test to drain behaviour). The harness applies
-// the `server(...)` config to every cluster node.
-#[iggy_harness(
- cluster_nodes = 3,
- server(partition.messages_required_to_save = 1)
-)]
+// shutdown-flush would couple the test to drain behaviour). It is a topic
+// creation option, so it travels with the topic to every replica.
+#[iggy_harness(cluster_nodes = 3)]
async fn should_persist_byte_identical_data_across_cluster_replicas(harness: &mut TestHarness) {
let client = harness.tcp_root_client().await.unwrap();
client.create_stream(STREAM_NAME).await.unwrap();
@@ -89,11 +86,12 @@ async fn should_persist_byte_identical_data_across_cluster_replicas(harness: &mu
.create_topic(
&Identifier::named(STREAM_NAME).unwrap(),
TOPIC_NAME,
- 1,
- CompressionAlgorithm::None,
- None,
- IggyExpiry::NeverExpire,
- MaxTopicSize::ServerDefault,
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ messages_required_to_save: Some(1),
+ ..TopicCreateOptions::default()
+ },
)
.await
.unwrap();
diff --git a/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs b/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs
index 8fac77b96..f4f935a82 100644
--- a/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs
+++ b/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs
... 22474 lines suppressed ...