You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2026/08/14 13:06:17 UTC
(iggy) 01/02: address review comments
This is an automated email from the ASF dual-hosted git repository.
numinnex pushed a commit to branch rework_topic_commands
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit b52b51ed39cddb706a6f6d4e08dbe040224ba83e
Author: Grzegorz Koszyk <g....@gmail.com>
AuthorDate: Fri Aug 14 13:16:38 2026 +0200
address review comments
---
.../features/step_definitions/messaging_steps.cpp | 2 +-
core/ai/mcp/src/service/mod.rs | 12 +-
core/ai/mcp/src/service/requests.rs | 17 +-
core/binary_protocol/src/codec.rs | 34 +-
core/binary_protocol/src/primitives/options.rs | 41 +++
.../topics/create_topic_with_assignments.rs | 33 +-
.../src/responses/clients/get_client.rs | 2 +-
.../consumer_groups/get_consumer_group.rs | 2 +-
.../consumer_groups/sync_consumer_group.rs | 4 +-
.../src/responses/messages/send_messages.rs | 4 +-
.../src/responses/streams/get_stream.rs | 2 +-
.../src/responses/system/describe_options.rs | 35 +-
.../src/responses/system/get_cluster_metadata.rs | 2 +-
.../src/responses/system/get_stats.rs | 2 +-
.../src/responses/topics/get_topics.rs | 16 +-
core/cli/src/args/mod.rs | 9 +-
core/cli/src/args/system.rs | 9 +-
core/cli/src/args/topic.rs | 2 +-
core/cli/src/commands/binary_system/mod.rs | 1 +
core/cli/src/commands/binary_system/options.rs | 86 +++++
.../cli/src/commands/binary_topics/update_topic.rs | 1 -
core/cli/src/main.rs | 3 +-
core/common/src/error/iggy_error.rs | 4 +
core/common/src/http/topics/create_topic.rs | 4 +
core/common/src/types/message/user_headers.rs | 22 ++
core/common/src/types/options/mod.rs | 352 +++++++++++++++++----
core/common/src/wire_conversions.rs | 92 +++++-
core/configs/src/configs_impl/file_provider.rs | 249 ++++++++++++---
core/configs/src/configs_impl/mod.rs | 2 +-
core/configs/src/lib.rs | 2 +-
core/configs/src/server_config/server.rs | 48 ++-
core/configs/src/server_config/validators.rs | 27 ++
.../tests/cli/general/test_help_command.rs | 1 +
.../tests/cli/general/test_overview_command.rs | 1 +
.../tests/cli/topic/test_topic_create_command.rs | 2 +-
.../tests/server/topic_admission_vsr.rs | 112 +++++++
core/metadata/src/impls/metadata.rs | 157 +--------
core/metadata/src/stm/consumer_group.rs | 4 +-
core/metadata/src/stm/snapshot.rs | 17 +
core/metadata/src/stm/stream.rs | 155 +++++++--
core/server/config.toml | 20 +-
core/server/src/bootstrap.rs | 24 --
core/server/src/dispatch.rs | 50 ++-
core/server/src/http/handlers.rs | 51 ++-
core/server/src/responses.rs | 212 +++++++------
core/server/src/segment_cleaner.rs | 8 +-
.../apache/iggy/examples/async/AsyncProducer.java | 2 -
.../producer/GettingStartedProducer.java | 11 +-
.../producer/MessageEnvelopeProducer.java | 8 +-
.../producer/MessageHeadersProducer.java | 8 +-
.../multitenant/producer/MultiTenantProducer.java | 1 -
.../sinkdataproducer/SinkDataProducer.java | 1 -
.../iggy/examples/streambuilder/StreamBasic.java | 8 +-
.../examples/tcptls/producer/TcpTlsProducer.java | 11 +-
foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs | 68 ++++
foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs | 18 ++
.../Implementations/HttpMessageStream.cs | 10 +
.../IggyClient/Implementations/TcpMessageStream.cs | 18 ++
foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 50 +++
.../binary_response_deserializer.go | 35 +-
.../binary_response_deserializer_test.go | 32 ++
foreign/go/contracts/compression_algorithm.go | 32 +-
foreign/go/contracts/compression_algorithm_test.go | 90 ++++++
foreign/go/internal/command/topic.go | 45 ++-
foreign/go/internal/command/update_user.go | 20 +-
foreign/go/internal/command/update_user_test.go | 76 +++++
.../org/apache/iggy/serde/BytesDeserializer.java | 67 +++-
.../java/org/apache/iggy/stream/StreamBase.java | 18 +-
.../src/main/java/org/apache/iggy/topic/Topic.java | 16 +-
.../apache/iggy/serde/BytesDeserializerTest.java | 50 ++-
.../iggy/serde/OptionsBlockGoldenVectorTest.java | 63 ++++
.../org/apache/iggy/stream/StreamDetailsTest.java | 7 +-
.../org/apache/iggy/topic/TopicDetailsTest.java | 5 +-
foreign/node/src/wire/command-set.ts | 4 +-
foreign/node/src/wire/options.utils.test.ts | 86 +++++
foreign/node/src/wire/options.utils.ts | 11 +-
.../src/wire/system/describe-options.command.ts | 102 ++++++
foreign/node/src/wire/system/index.ts | 1 +
foreign/python/src/topic.rs | 1 -
package-lock.json | 110 +++++++
package.json | 5 +
81 files changed, 2405 insertions(+), 620 deletions(-)
diff --git a/bdd/cpp/features/step_definitions/messaging_steps.cpp b/bdd/cpp/features/step_definitions/messaging_steps.cpp
index df4afc7db..d3964e6a3 100644
--- a/bdd/cpp/features/step_definitions/messaging_steps.cpp
+++ b/bdd/cpp/features/step_definitions/messaging_steps.cpp
@@ -78,7 +78,7 @@ WHEN("^I create a topic with name \"([^\"]{1,255})\" in stream ([0-9]+) with ([0
context->client->create_topic(bdd::make_numeric_identifier(static_cast<std::uint32_t>(stream_id)), topic_name,
static_cast<std::uint32_t>(partitions_count),
- std::string(compression.compression_algorithm_value()), 0,
+ std::string(compression.compression_algorithm_value()),
std::string(message_expiry.expiry_kind()), message_expiry.expiry_value(),
std::string(max_topic_size.max_topic_size()));
}
diff --git a/core/ai/mcp/src/service/mod.rs b/core/ai/mcp/src/service/mod.rs
index 0a36bbc39..7ba7b027d 100644
--- a/core/ai/mcp/src/service/mod.rs
+++ b/core/ai/mcp/src/service/mod.rs
@@ -156,6 +156,7 @@ impl IggyService {
compression_algorithm,
message_expiry,
max_size,
+ options,
}): Parameters<CreateTopic>,
) -> Result<CallToolResult, ErrorData> {
self.permissions.ensure_create()?;
@@ -180,6 +181,10 @@ impl IggyService {
.then_some(message_expiry),
max_topic_size: (max_size != MaxTopicSize::ServerDefault)
.then_some(max_size),
+ // Raw keys ride as strings and are parsed server-side by
+ // the same rules a config value goes through, so a key
+ // added after this build shipped is still reachable.
+ raw: options,
..TopicCreateOptions::default()
},
)
@@ -197,20 +202,21 @@ impl IggyService {
compression_algorithm,
message_expiry,
max_size,
+ options,
}): Parameters<UpdateTopic>,
) -> Result<CallToolResult, ErrorData> {
self.permissions.ensure_update()?;
// Absent means "leave alone" now, so an unparsable value must not
// silently become a reset to the server default.
- let options = TopicUpdateOptions {
+ let update_options = TopicUpdateOptions {
compression_algorithm: compression_algorithm.and_then(|ca| ca.parse().ok()),
message_expiry: message_expiry.and_then(|me| me.parse().ok()),
max_topic_size: max_size.and_then(|ms| ms.parse().ok()),
- ..TopicUpdateOptions::default()
+ raw: options,
};
request(
self.client
- .update_topic(&id(&stream_id)?, &id(&topic_id)?, &name, &options)
+ .update_topic(&id(&stream_id)?, &id(&topic_id)?, &name, &update_options)
.await,
)
}
diff --git a/core/ai/mcp/src/service/requests.rs b/core/ai/mcp/src/service/requests.rs
index af0f8dedd..0a498c65a 100644
--- a/core/ai/mcp/src/service/requests.rs
+++ b/core/ai/mcp/src/service/requests.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
use iggy::prelude;
use rmcp::schemars::{self, JsonSchema};
@@ -87,6 +87,13 @@ pub struct CreateTopic {
#[schemars(description = "maximum size (optional)")]
pub max_size: Option<String>,
+
+ #[schemars(
+ description = "additional options as string key-values, e.g. {\"segment_size\": \"128 MiB\"}; \
+ call describe_options for the keys the server accepts (optional)"
+ )]
+ #[serde(default)]
+ pub options: BTreeMap<String, String>,
}
#[derive(Debug, Deserialize, JsonSchema)]
@@ -108,6 +115,14 @@ pub struct UpdateTopic {
#[schemars(description = "maximum size (optional)")]
pub max_size: Option<String>,
+
+ #[schemars(
+ description = "options to change, as string key-values; only compression_algorithm, \
+ message_expiry and max_topic_size may be updated, and a key left out \
+ keeps its current value (optional)"
+ )]
+ #[serde(default)]
+ pub options: BTreeMap<String, String>,
}
#[derive(Debug, Deserialize, JsonSchema)]
diff --git a/core/binary_protocol/src/codec.rs b/core/binary_protocol/src/codec.rs
index 2dccf443a..4edb5304b 100644
--- a/core/binary_protocol/src/codec.rs
+++ b/core/binary_protocol/src/codec.rs
@@ -251,18 +251,6 @@ pub fn read_bytes(buf: &[u8], offset: usize, len: usize) -> Result<&[u8], WireEr
})
}
-/// Cap a pre-allocation hint so a bogus wire count cannot cause OOM.
-/// The actual count is validated by the decode loop - this only limits
-/// the upfront allocation.
-#[inline]
-#[must_use]
-pub fn capped_capacity(count: usize, remaining: usize, min_item_size: usize) -> usize {
- if min_item_size == 0 {
- return count;
- }
- count.min(remaining / min_item_size)
-}
-
/// Capacity to preallocate for `count` elements that still have to be decoded
/// out of `remaining` bytes, each at least `min_element_size` bytes on the
/// wire.
@@ -272,6 +260,10 @@ pub fn capped_capacity(count: usize, remaining: usize, min_item_size: usize) ->
/// or corrupt frame claiming `u32::MAX` elements therefore aborts the process
/// on allocation failure instead of returning a decode error. The bytes left
/// to decode are the honest ceiling on how many elements can exist.
+///
+/// A zero `min_element_size` is treated as one byte rather than passing the
+/// wire count through: an element that occupies no bytes cannot exist, and
+/// trusting the count is the exact hole this function was written to close.
#[must_use]
#[inline]
pub fn bounded_capacity(count: usize, remaining: usize, min_element_size: usize) -> usize {
@@ -283,17 +275,17 @@ mod tests {
use super::*;
#[test]
- fn capped_capacity_limits_allocation() {
- assert_eq!(capped_capacity(1_000_000, 100, 10), 10);
- assert_eq!(capped_capacity(5, 100, 10), 5);
- assert_eq!(capped_capacity(10, 100, 10), 10);
- assert_eq!(capped_capacity(11, 100, 10), 10);
- assert_eq!(capped_capacity(0, 100, 10), 0);
- assert_eq!(capped_capacity(100, 0, 10), 0);
+ fn bounded_capacity_limits_allocation() {
+ assert_eq!(bounded_capacity(1_000_000, 100, 10), 10);
+ assert_eq!(bounded_capacity(5, 100, 10), 5);
+ assert_eq!(bounded_capacity(10, 100, 10), 10);
+ assert_eq!(bounded_capacity(11, 100, 10), 10);
+ assert_eq!(bounded_capacity(0, 100, 10), 0);
+ assert_eq!(bounded_capacity(100, 0, 10), 0);
}
#[test]
- fn capped_capacity_zero_item_size_returns_count() {
- assert_eq!(capped_capacity(1_000_000, 100, 0), 1_000_000);
+ fn bounded_capacity_zero_element_size_still_bounds_by_the_buffer() {
+ assert_eq!(bounded_capacity(1_000_000, 100, 0), 100);
}
}
diff --git a/core/binary_protocol/src/primitives/options.rs b/core/binary_protocol/src/primitives/options.rs
index 66e1623e8..e7139fd8a 100644
--- a/core/binary_protocol/src/primitives/options.rs
+++ b/core/binary_protocol/src/primitives/options.rs
@@ -424,6 +424,47 @@ mod tests {
assert_eq!(decoded, options);
}
+ /// The cross-SDK golden vector for an options block.
+ ///
+ /// Every SDK writes this block from its own encoder, so "it round-trips
+ /// through my own decoder" proves nothing about interoperability. The bytes
+ /// here are the contract: `foreign/{node,go,java}` pin the identical vector
+ /// in their own unit tests, and a change to the TLV layout has to break all
+ /// of them together instead of leaving one SDK talking to itself.
+ ///
+ /// `enforce_fsync=true` (a `Bool`) and `segment_size=1 GiB` (a `Uint64`)
+ /// cover both a one-byte and an eight-byte value, in the sorted key order
+ /// every encoder has to produce.
+ const GOLDEN_OPTIONS_BLOCK: &[u8] = &[
+ 2, 13, 0, 0, 0, // key kind String, length 13
+ b'e', b'n', b'f', b'o', b'r', b'c', b'e', b'_', b'f', b's', b'y', b'n', b'c', 3, 1, 0, 0,
+ 0, 1, // value kind Bool, length 1, true
+ 2, 12, 0, 0, 0, // key kind String, length 12
+ b's', b'e', b'g', b'm', b'e', b'n', b't', b'_', b's', b'i', b'z', b'e', 12, 8, 0, 0,
+ 0, // value kind Uint64, length 8
+ 0, 0, 0, 64, 0, 0, 0, 0, // 1 GiB little-endian
+ ];
+
+ #[test]
+ fn golden_options_block_is_byte_stable() {
+ let encoded = encode(&[
+ (STRING, b"enforce_fsync", 3, &[1]),
+ (
+ STRING,
+ b"segment_size",
+ UINT64,
+ &1_073_741_824u64.to_le_bytes(),
+ ),
+ ]);
+
+ assert_eq!(
+ &encoded[..],
+ GOLDEN_OPTIONS_BLOCK,
+ "the options TLV layout changed; update every SDK's copy of this vector"
+ );
+ assert_eq!(validate_options(GOLDEN_OPTIONS_BLOCK).unwrap(), 2);
+ }
+
#[test]
fn prefixed_truncation_errors() {
let buf = encode(&[(STRING, b"key", STRING, b"value")]);
diff --git a/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs b/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs
index 078b42d91..b6665b618 100644
--- a/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs
+++ b/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs
@@ -17,10 +17,10 @@
use crate::WireError;
use crate::codec::{WireDecode, WireEncode, read_u32_le};
-use crate::primitives::options::WireOptions;
+use crate::primitives::options::{WireOptions, decode_options_prefixed};
use crate::primitives::partition_assignment::CreatedPartitionAssignment;
use crate::requests::topics::CreateTopicRequest;
-use bytes::{BufMut, Bytes, BytesMut};
+use bytes::{BufMut, BytesMut};
fn usize_to_u32(value: usize, context: &str) -> u32 {
u32::try_from(value).unwrap_or_else(|_| panic!("{context} exceeds u32"))
@@ -77,8 +77,19 @@ impl WireEncode for CreateTopicWithAssignmentsRequest {
impl WireDecode for CreateTopicWithAssignmentsRequest {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
let request_size = read_u32_le(buf, 0)? as usize;
- let request_start = 4;
- let request_end = request_start + request_size;
+ let request_start: usize = 4;
+ // `checked_add`: on a 32-bit target a wire-supplied length wraps, the
+ // truncation guard below then passes, and the slice panics instead.
+ // This decode runs on journal replay, so that panic would take a
+ // replica down rather than fail one client request.
+ let request_end =
+ request_start
+ .checked_add(request_size)
+ .ok_or_else(|| WireError::UnexpectedEof {
+ offset: request_start,
+ need: request_size,
+ have: buf.len().saturating_sub(request_start),
+ })?;
if buf.len() < request_end {
return Err(WireError::UnexpectedEof {
offset: request_start,
@@ -89,18 +100,8 @@ impl WireDecode for CreateTopicWithAssignmentsRequest {
let request = CreateTopicRequest::decode_from(&buf[request_start..request_end])?;
- let derived_size = read_u32_le(buf, request_end)? as usize;
- let derived_start = request_end + 4;
- let derived_end = derived_start + derived_size;
- if buf.len() < derived_end {
- return Err(WireError::UnexpectedEof {
- offset: derived_start,
- need: derived_size,
- have: buf.len().saturating_sub(derived_start),
- });
- }
- let derived_options =
- WireOptions::from_bytes(Bytes::copy_from_slice(&buf[derived_start..derived_end]))?;
+ let (derived_options, derived_consumed) = decode_options_prefixed(buf, request_end)?;
+ let derived_end = request_end + derived_consumed;
let partitions_count = read_u32_le(buf, derived_end)? as usize;
let mut offset = derived_end + 4;
diff --git a/core/binary_protocol/src/responses/clients/get_client.rs b/core/binary_protocol/src/responses/clients/get_client.rs
index 3fa4ffedc..aa721c0c3 100644
--- a/core/binary_protocol/src/responses/clients/get_client.rs
+++ b/core/binary_protocol/src/responses/clients/get_client.rs
@@ -57,7 +57,7 @@ impl WireDecode for ClientDetailsResponse {
let (client, mut pos) = ClientResponse::decode(buf)?;
let count = client.consumer_groups_count as usize;
let remaining = buf.len().saturating_sub(pos);
- let mut consumer_groups = Vec::with_capacity(crate::codec::capped_capacity(
+ let mut consumer_groups = Vec::with_capacity(crate::codec::bounded_capacity(
count,
remaining,
ConsumerGroupInfoResponse::SIZE,
diff --git a/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs b/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs
index 551cf0da1..cc4f175f3 100644
--- a/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs
+++ b/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs
@@ -52,7 +52,7 @@ impl WireDecode for ConsumerGroupMemberResponse {
let id = read_u32_le(buf, 0)?;
let partitions_count = read_u32_le(buf, 4)?;
let remaining = buf.len().saturating_sub(8);
- let mut partitions = Vec::with_capacity(crate::codec::capped_capacity(
+ let mut partitions = Vec::with_capacity(crate::codec::bounded_capacity(
partitions_count as usize,
remaining,
4,
diff --git a/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs b/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs
index 9f9c3283a..3fe5d6551 100644
--- a/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs
+++ b/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs
@@ -16,7 +16,7 @@
// under the License.
use crate::WireError;
-use crate::codec::{WireDecode, WireEncode, capped_capacity, read_u32_le, read_u64_le};
+use crate::codec::{WireDecode, WireEncode, bounded_capacity, read_u32_le, read_u64_le};
use bytes::{BufMut, BytesMut};
/// `SyncConsumerGroup` response.
@@ -53,7 +53,7 @@ impl WireDecode for SyncConsumerGroupResponse {
let partitions_count = read_u32_le(buf, 8)?;
let remaining = buf.len().saturating_sub(12);
let mut partitions =
- Vec::with_capacity(capped_capacity(partitions_count as usize, remaining, 4));
+ Vec::with_capacity(bounded_capacity(partitions_count as usize, remaining, 4));
let mut offset = 12;
for _ in 0..partitions_count {
partitions.push(read_u32_le(buf, offset)?);
diff --git a/core/binary_protocol/src/responses/messages/send_messages.rs b/core/binary_protocol/src/responses/messages/send_messages.rs
index 22a7a4624..32785b7be 100644
--- a/core/binary_protocol/src/responses/messages/send_messages.rs
+++ b/core/binary_protocol/src/responses/messages/send_messages.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::codec::{WireDecode, WireEncode, capped_capacity, read_u32_le, read_u64_le};
+use crate::codec::{WireDecode, WireEncode, bounded_capacity, read_u32_le, read_u64_le};
use crate::error::WireError;
use bytes::{BufMut, BytesMut};
use std::borrow::Cow;
@@ -122,7 +122,7 @@ impl WireDecode for SendMessagesResponse {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
let confirmations_count = read_u32_le(buf, 0)?;
let remaining = buf.len().saturating_sub(4);
- let mut confirmations = Vec::with_capacity(capped_capacity(
+ let mut confirmations = Vec::with_capacity(bounded_capacity(
confirmations_count as usize,
remaining,
CONFIRMATION_SIZE,
diff --git a/core/binary_protocol/src/responses/streams/get_stream.rs b/core/binary_protocol/src/responses/streams/get_stream.rs
index 2b09afe71..8966de31d 100644
--- a/core/binary_protocol/src/responses/streams/get_stream.rs
+++ b/core/binary_protocol/src/responses/streams/get_stream.rs
@@ -54,7 +54,7 @@ pub struct TopicHeader {
}
impl TopicHeader {
- const FIXED_SIZE: usize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1; // 50
+ pub(crate) const FIXED_SIZE: usize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1; // 50
}
impl WireEncode for TopicHeader {
diff --git a/core/binary_protocol/src/responses/system/describe_options.rs b/core/binary_protocol/src/responses/system/describe_options.rs
index 311282b82..8410699df 100644
--- a/core/binary_protocol/src/responses/system/describe_options.rs
+++ b/core/binary_protocol/src/responses/system/describe_options.rs
@@ -28,10 +28,12 @@ use bytes::{BufMut, Bytes, BytesMut};
/// [default_len:u32][default bytes][description_len:u32][description]
/// ```
///
-/// `kind` is the canonical header kind code the server persists the value
-/// under; clients may also send a `String` value parsed via the same rules as
-/// the server config file. `default` is the server's current resolved default
-/// in the canonical kind's encoding, empty when the key has no server default.
+/// `kind` is the header kind code the server encodes this key's own default
+/// under, and the kind a client should prefer; a `String` value parsed by the
+/// same rules as a config file value is accepted too, and an explicitly set
+/// entry keeps the kind the client sent. `default` is that default in `kind`'s
+/// encoding, empty when the key has none, and is a build constant rather than a
+/// per-node value.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OptionDescriptor {
pub key: WireName,
@@ -64,7 +66,15 @@ impl WireDecode for OptionDescriptor {
pos += 1;
let default_len = read_u32_le(buf, pos)? as usize;
pos += 4;
- let default_end = pos + default_len;
+ // `checked_add`: on a 32-bit target a wire-supplied length wraps, the
+ // truncation guard below then passes, and the slice panics instead.
+ let default_end = pos
+ .checked_add(default_len)
+ .ok_or_else(|| WireError::UnexpectedEof {
+ offset: pos,
+ need: default_len,
+ have: buf.len().saturating_sub(pos),
+ })?;
if buf.len() < default_end {
return Err(WireError::UnexpectedEof {
offset: pos,
@@ -76,7 +86,13 @@ impl WireDecode for OptionDescriptor {
pos = default_end;
let description_len = read_u32_le(buf, pos)? as usize;
pos += 4;
- let description_end = pos + description_len;
+ let description_end =
+ pos.checked_add(description_len)
+ .ok_or_else(|| WireError::UnexpectedEof {
+ offset: pos,
+ need: description_len,
+ have: buf.len().saturating_sub(pos),
+ })?;
if buf.len() < description_end {
return Err(WireError::UnexpectedEof {
offset: pos,
@@ -125,9 +141,10 @@ impl WireEncode for DescribeOptionsResponse {
}
}
-/// Smallest a descriptor can encode as: a name, a kind byte, a
-/// length-prefixed default and a length-prefixed description.
-const MIN_DESCRIPTOR_SIZE: usize = 4 + 1 + 1 + 4 + 4;
+/// Smallest a descriptor can encode as: a one-character `WireName` (a length
+/// byte plus at least one byte of name), a kind byte, and empty
+/// length-prefixed default and description.
+const MIN_DESCRIPTOR_SIZE: usize = 2 + 1 + 4 + 4;
impl WireDecode for DescribeOptionsResponse {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
diff --git a/core/binary_protocol/src/responses/system/get_cluster_metadata.rs b/core/binary_protocol/src/responses/system/get_cluster_metadata.rs
index 0b343e644..22748b4e7 100644
--- a/core/binary_protocol/src/responses/system/get_cluster_metadata.rs
+++ b/core/binary_protocol/src/responses/system/get_cluster_metadata.rs
@@ -138,7 +138,7 @@ impl WireDecode for ClusterMetadataResponse {
pos += 4;
let remaining = buf.len().saturating_sub(pos);
- let mut nodes = Vec::with_capacity(crate::codec::capped_capacity(
+ let mut nodes = Vec::with_capacity(crate::codec::bounded_capacity(
nodes_count,
remaining,
NODE_FIXED_SIZE,
diff --git a/core/binary_protocol/src/responses/system/get_stats.rs b/core/binary_protocol/src/responses/system/get_stats.rs
index 3e84bd439..7921876eb 100644
--- a/core/binary_protocol/src/responses/system/get_stats.rs
+++ b/core/binary_protocol/src/responses/system/get_stats.rs
@@ -207,7 +207,7 @@ impl WireDecode for StatsResponse {
pos += 4;
let remaining = buf.len().saturating_sub(pos);
- let mut cache_metrics = Vec::with_capacity(crate::codec::capped_capacity(
+ let mut cache_metrics = Vec::with_capacity(crate::codec::bounded_capacity(
cache_count,
remaining,
CacheMetricEntry::SIZE,
diff --git a/core/binary_protocol/src/responses/topics/get_topics.rs b/core/binary_protocol/src/responses/topics/get_topics.rs
index 5ce465bb9..6852866e3 100644
--- a/core/binary_protocol/src/responses/topics/get_topics.rs
+++ b/core/binary_protocol/src/responses/topics/get_topics.rs
@@ -27,9 +27,12 @@ use bytes::{BufMut, BytesMut};
/// [topics_count:u32_le][TopicHeader]*
/// ```
///
-/// The count prefix exists because a topic element carries variable-length
-/// options blocks: without it, a decoder has no framing to detect a short
-/// read against.
+/// The count prefix is what lets a decoder pre-size its `Vec`: an element
+/// carries two variable-length option blocks, so the payload length alone says
+/// nothing useful about how many elements are in it. Detecting a short read
+/// does not need it - the element decoder errors on one, which is how
+/// `GetStreams` and `GetUsers` still run their loops to end-of-payload. This
+/// response is the odd one out of the three, and every SDK special-cases it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GetTopicsResponse {
pub topics: Vec<TopicHeader>,
@@ -53,9 +56,10 @@ impl WireEncode for GetTopicsResponse {
}
}
-/// Smallest a topic header can encode as: the fixed ids, timestamps, sizes and
-/// counts, plus a name length and two length-prefixed option blocks.
-const MIN_TOPIC_HEADER_SIZE: usize = 45;
+/// Smallest a topic header can encode as: the fixed ids, timestamps, sizes,
+/// counts and name length, plus the shortest name `WireName` accepts (one byte,
+/// since it rejects an empty one) and two empty length-prefixed option blocks.
+const MIN_TOPIC_HEADER_SIZE: usize = TopicHeader::FIXED_SIZE + 1 + 4 + 4;
impl WireDecode for GetTopicsResponse {
fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
diff --git a/core/cli/src/args/mod.rs b/core/cli/src/args/mod.rs
index a7159addb..42d517810 100644
--- a/core/cli/src/args/mod.rs
+++ b/core/cli/src/args/mod.rs
@@ -37,7 +37,7 @@ use crate::args::{
partition::PartitionAction,
personal_access_token::PersonalAccessTokenAction,
stream::StreamAction,
- system::{PingArgs, StatsArgs},
+ system::{OptionsArgs, PingArgs, StatsArgs},
topic::TopicAction,
};
@@ -162,6 +162,13 @@ pub(crate) enum Command {
/// server address and protocol type.
#[clap(verbatim_doc_comment)]
Me,
+ /// list the options a resource's create command accepts
+ ///
+ /// Prints the server's option catalog for one scope (topic, stream, user):
+ /// every key, the kind its default is reported in, that default, and the
+ /// bounds each value is checked against. These are the keys `--set` takes.
+ #[clap(verbatim_doc_comment)]
+ Options(OptionsArgs),
/// get iggy server statistics
///
/// Collect basic Iggy server statistics like number of streams, topics, partitions, etc.
diff --git a/core/cli/src/args/system.rs b/core/cli/src/args/system.rs
index 39f58fc37..a7195a2f1 100644
--- a/core/cli/src/args/system.rs
+++ b/core/cli/src/args/system.rs
@@ -17,7 +17,7 @@
use crate::args::common::ListModeExt;
use clap::Args;
-use iggy::prelude::{SnapshotCompression, SystemSnapshotType};
+use iggy::prelude::{OptionsScope, SnapshotCompression, SystemSnapshotType};
use iggy_cli::commands::utils::login_session_expiry::LoginSessionExpiry;
#[derive(Debug, Clone, Args)]
@@ -38,6 +38,13 @@ pub(crate) struct LoginArgs {
pub(crate) expiry: Option<Vec<LoginSessionExpiry>>,
}
+#[derive(Debug, Clone, Args)]
+pub(crate) struct OptionsArgs {
+ /// Resource whose option catalog to print (topic, stream, user)
+ #[arg(value_parser = clap::value_parser!(OptionsScope))]
+ pub(crate) scope: OptionsScope,
+}
+
#[derive(Debug, Clone, Args)]
pub(crate) struct StatsArgs {
/// List mode (table, list, JSON, TOML)
diff --git a/core/cli/src/args/topic.rs b/core/cli/src/args/topic.rs
index 94285ca8e..875b3705f 100644
--- a/core/cli/src/args/topic.rs
+++ b/core/cli/src/args/topic.rs
@@ -126,7 +126,7 @@ pub(crate) struct TopicCreateArgs {
///
/// Values are sent as strings and parsed server-side through each option's
/// own FromStr (e.g. --set segment_size=128MiB). The server rejects keys it
- /// does not support; GET /options/topic lists the ones it accepts.
+ /// does not support; run "iggy options topic" to list the ones it accepts.
#[arg(long = "set", value_name = "KEY=VALUE", value_parser = parse_key_value, verbatim_doc_comment)]
pub(crate) set: Vec<(String, String)>,
}
diff --git a/core/cli/src/commands/binary_system/mod.rs b/core/cli/src/commands/binary_system/mod.rs
index 979ebeabe..2bbc45a3b 100644
--- a/core/cli/src/commands/binary_system/mod.rs
+++ b/core/cli/src/commands/binary_system/mod.rs
@@ -18,6 +18,7 @@
pub mod login;
pub mod logout;
pub mod me;
+pub mod options;
pub mod ping;
pub mod session;
pub mod session_status;
diff --git a/core/cli/src/commands/binary_system/options.rs b/core/cli/src/commands/binary_system/options.rs
new file mode 100644
index 000000000..5fc7f6beb
--- /dev/null
+++ b/core/cli/src/commands/binary_system/options.rs
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::commands::cli_command::{CliCommand, PRINT_TARGET};
+use anyhow::Context;
+use async_trait::async_trait;
+use comfy_table::Table;
+use iggy_common::{Client, HeaderValue, OptionSpec, OptionsScope};
+use tracing::{Level, event};
+
+/// `iggy options <scope>`: print the server's option catalog.
+///
+/// This is the discovery surface for `--set`. A key outside the catalog is
+/// refused at create, and the binary transports carry only the error code back,
+/// so without this command a CLI user has no way to see which keys their server
+/// knows or what each one is bounded by.
+pub struct DescribeOptionsCmd {
+ scope: OptionsScope,
+}
+
+impl DescribeOptionsCmd {
+ pub fn new(scope: OptionsScope) -> Self {
+ Self { scope }
+ }
+}
+
+#[async_trait]
+impl CliCommand for DescribeOptionsCmd {
+ fn explain(&self) -> String {
+ format!("describe {} options", self.scope)
+ }
+
+ async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> {
+ let specs = client
+ .describe_options(self.scope)
+ .await
+ .with_context(|| "Problem sending describe_options command".to_owned())?;
+
+ if specs.is_empty() {
+ event!(target: PRINT_TARGET, Level::INFO,
+ "{} resources accept no options yet", self.scope);
+ return Ok(());
+ }
+
+ let mut table = Table::new();
+ table.set_header(vec!["Key", "Kind", "Default", "Description"]);
+ for spec in &specs {
+ table.add_row(vec![
+ spec.key.as_str(),
+ spec.kind.to_string().as_str(),
+ render_default(spec).as_str(),
+ spec.description.as_str(),
+ ]);
+ }
+ event!(target: PRINT_TARGET, Level::INFO, "{table}");
+
+ Ok(())
+ }
+}
+
+/// The catalog's default in the string form `--set` takes, so the printed value
+/// can be handed straight back to a create.
+fn render_default(spec: &OptionSpec) -> String {
+ if spec.default_value.is_empty() {
+ return String::new();
+ }
+ match HeaderValue::from_raw(spec.kind, &spec.default_value) {
+ Ok(value) => value.to_string_value(),
+ // A kind this build cannot name still has a readable byte count.
+ Err(_) => format!("<{} bytes>", spec.default_value.len()),
+ }
+}
diff --git a/core/cli/src/commands/binary_topics/update_topic.rs b/core/cli/src/commands/binary_topics/update_topic.rs
index ab9ce8c9b..04f3b827a 100644
--- a/core/cli/src/commands/binary_topics/update_topic.rs
+++ b/core/cli/src/commands/binary_topics/update_topic.rs
@@ -34,7 +34,6 @@ pub struct UpdateTopicCmd {
}
impl UpdateTopicCmd {
- #[allow(clippy::too_many_arguments)]
pub fn new(
stream_id: Identifier,
topic_id: Identifier,
diff --git a/core/cli/src/main.rs b/core/cli/src/main.rs
index 9a87f8039..93a73618f 100644
--- a/core/cli/src/main.rs
+++ b/core/cli/src/main.rs
@@ -77,7 +77,7 @@ use iggy_cli::commands::{
create_stream::CreateStreamCmd, delete_stream::DeleteStreamCmd, get_stream::GetStreamCmd,
get_streams::GetStreamsCmd, purge_stream::PurgeStreamCmd, update_stream::UpdateStreamCmd,
},
- binary_system::{me::GetMeCmd, ping::PingCmd, stats::GetStatsCmd},
+ binary_system::{me::GetMeCmd, options::DescribeOptionsCmd, ping::PingCmd, stats::GetStatsCmd},
binary_topics::{
create_topic::CreateTopicCmd, delete_topic::DeleteTopicCmd, get_topic::GetTopicCmd,
get_topics::GetTopicsCmd, purge_topic::PurgeTopicCmd, update_topic::UpdateTopicCmd,
@@ -182,6 +182,7 @@ fn get_command(
},
Command::Ping(args) => Box::new(PingCmd::new(args.count)),
Command::Me => Box::new(GetMeCmd::new()),
+ Command::Options(args) => Box::new(DescribeOptionsCmd::new(args.scope)),
Command::Stats(args) => Box::new(GetStatsCmd::new(cli_options.quiet, args.output.into())),
Command::Snapshot(args) => Box::new(GetSnapshotCmd::new(
args.compression,
diff --git a/core/common/src/error/iggy_error.rs b/core/common/src/error/iggy_error.rs
index 45328ad16..d80006ade 100644
--- a/core/common/src/error/iggy_error.rs
+++ b/core/common/src/error/iggy_error.rs
@@ -400,6 +400,10 @@ pub enum IggyError {
InvalidBatchChecksum(u64, u64, u64) = 4039,
#[error("Invalid header kind code: {0}")]
InvalidHeaderKind(u8) = 4040,
+ /// The key is only populated where the error text itself travels, which
+ /// today means HTTP. A binary transport sends the code alone, so a client
+ /// rebuilding this variant from it renders an empty key; `DescribeOptions`
+ /// is what tells that client which keys exist.
#[error("Unsupported option key: {0}")]
UnsupportedOptionKey(String) = 4041,
#[error("Invalid option value for key: {0}")]
diff --git a/core/common/src/http/topics/create_topic.rs b/core/common/src/http/topics/create_topic.rs
index 7a217a5c7..99a18f706 100644
--- a/core/common/src/http/topics/create_topic.rs
+++ b/core/common/src/http/topics/create_topic.rs
@@ -51,6 +51,10 @@ pub struct CreateTopic {
pub name: String,
/// Additional topic options as string key-values, parsed by the server
/// with the same rules as its config file. Unknown keys are rejected.
+ ///
+ /// A response renders each option as `{"value": "<string>", "explicit":
+ /// bool}`, in this same string form, so the values a `GET` reports can be
+ /// sent straight back here.
#[serde(default)]
pub options: BTreeMap<String, String>,
}
diff --git a/core/common/src/types/message/user_headers.rs b/core/common/src/types/message/user_headers.rs
index 54e2522f4..07223ce80 100644
--- a/core/common/src/types/message/user_headers.rs
+++ b/core/common/src/types/message/user_headers.rs
@@ -616,6 +616,28 @@ impl<T> HeaderField<T> {
_marker: PhantomData,
}
}
+
+ /// Wrap raw wire bytes as a field of `kind`.
+ ///
+ /// The option catalog hands out each key's default as a kind plus bytes, so
+ /// anything rendering a catalog entry needs a way back to a typed field.
+ ///
+ /// # Errors
+ ///
+ /// `IggyError::InvalidHeaderValue` when a fixed-size kind's payload is the
+ /// wrong length, or the payload exceeds the field length limit.
+ pub fn from_raw(kind: HeaderKind, value: &[u8]) -> Result<Self, IggyError> {
+ // 255 is the field length the TLV codec's own `TryFrom` impls enforce.
+ if value.is_empty() || value.len() > 255 {
+ return Err(IggyError::InvalidHeaderValue);
+ }
+ if let Some(expected) = kind.expected_size()
+ && value.len() != expected
+ {
+ return Err(IggyError::InvalidHeaderValue);
+ }
+ Ok(Self::new_unchecked(kind, value))
+ }
}
impl<T: Eq> Ord for HeaderField<T> {
diff --git a/core/common/src/types/options/mod.rs b/core/common/src/types/options/mod.rs
index 45428723b..0f98a1633 100644
--- a/core/common/src/types/options/mod.rs
+++ b/core/common/src/types/options/mod.rs
@@ -21,6 +21,33 @@
//! message user headers. The map MUST stay a `BTreeMap`: metadata snapshots
//! require deterministic ordering across replicas, and a hash map would make
//! snapshot bytes diverge per replica.
+//!
+//! # SDK parity
+//!
+//! One feature, and the SDKs are at different points on it. What each can do
+//! today, so a gap is a known gap rather than a surprise:
+//!
+//! | SDK | Set typed keys | Set arbitrary keys | Read the blocks | `describe_options` |
+//! |--------|----------------|--------------------|-----------------|--------------------|
+//! | Rust | all | yes (`raw`) | yes | yes |
+//! | Node | all | yes | yes | yes |
+//! | Python | all | yes (string dict) | no | no |
+//! | Go | 3 legacy | no | yes | no |
+//! | C# | 3 legacy | no | yes (TCP) | yes (TCP) |
+//! | Java | 3 legacy | no | yes | no |
+//! | C++ | 3 legacy | no | no | no |
+//!
+//! "3 legacy" is `compression_algorithm`, `message_expiry` and `max_topic_size`,
+//! the three that used to be fixed fields of `CreateTopic`. Every SDK can
+//! therefore express what it could before this feature; what a lagging one
+//! cannot yet do is reach the partition runtime knobs or a key added to the
+//! server catalog after it shipped.
+//!
+//! The write side is what to close next, and it is one change per SDK: accept a
+//! string map alongside the typed arguments and encode it into the block the
+//! same way. The block's byte layout is pinned by a golden vector that Rust,
+//! Node, Go and Java each assert independently, so a new encoder has a fixture
+//! to match rather than a description to interpret.
use std::collections::BTreeMap;
use std::str::FromStr;
@@ -96,12 +123,26 @@ pub type ResourceOptions = BTreeMap<HeaderKey, OptionValue>;
/// carrying options goes through here instead, rendering the key as its plain
/// text. Binary transports are unaffected: they never touch serde.
pub mod resource_options_json {
- use super::{HeaderKey, OptionValue, ResourceOptions};
+ use super::{HeaderKey, HeaderValue, OptionValue, ResourceOptions};
use serde::de::Error as _;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::collections::BTreeMap;
use std::str::FromStr;
+ /// One entry's JSON form: the value in the same string form a config file
+ /// or a `POST` body would carry it in, plus its provenance.
+ ///
+ /// The stored value is a typed [`HeaderValue`], and serializing that gives
+ /// `{"kind":"uint64","value":"<base64>"}` - which no operator can read and
+ /// no client can feed back, since the create body takes options as a plain
+ /// string map. Rendering the string form makes the "re-send only the
+ /// explicit options" round trip actually expressible over HTTP.
+ #[derive(Serialize, Deserialize)]
+ struct OptionValueJson {
+ value: String,
+ explicit: bool,
+ }
+
/// # Errors
///
/// Propagates the serializer's own errors.
@@ -109,26 +150,41 @@ pub mod resource_options_json {
options: &ResourceOptions,
serializer: S,
) -> Result<S::Ok, S::Error> {
- let readable: BTreeMap<String, &OptionValue> = options
+ let readable: BTreeMap<String, OptionValueJson> = options
.iter()
- .map(|(key, value)| (String::from_utf8_lossy(key.as_bytes()).into_owned(), value))
+ .map(|(key, option)| {
+ (
+ String::from_utf8_lossy(key.as_bytes()).into_owned(),
+ OptionValueJson {
+ value: option.value.to_string_value(),
+ explicit: option.explicit,
+ },
+ )
+ })
.collect();
readable.serialize(serializer)
}
/// # Errors
///
- /// Returns a deserializer error when a key is not a valid option name.
+ /// Returns a deserializer error when a key is not a valid option name or a
+ /// value does not fit a header value.
pub fn deserialize<'de, D: Deserializer<'de>>(
deserializer: D,
) -> Result<ResourceOptions, D::Error> {
- let readable = BTreeMap::<String, OptionValue>::deserialize(deserializer)?;
+ let readable = BTreeMap::<String, OptionValueJson>::deserialize(deserializer)?;
readable
.into_iter()
- .map(|(key, value)| {
- HeaderKey::from_str(&key)
- .map(|key| (key, value))
- .map_err(D::Error::custom)
+ .map(|(key, option)| {
+ let key = HeaderKey::from_str(&key).map_err(D::Error::custom)?;
+ let value = HeaderValue::from_str(&option.value).map_err(D::Error::custom)?;
+ Ok((
+ key,
+ OptionValue {
+ value,
+ explicit: option.explicit,
+ },
+ ))
})
.collect()
}
@@ -215,12 +271,73 @@ impl Default for TopicRuntimeDefaults {
}
}
+/// Ceiling for `size_of_messages_required_to_save`.
+///
+/// A threshold above the largest a segment may be can never trip, so every
+/// committed message would sit in the journal until its segment rotates, and a
+/// crash does not preserve the journal. These two thresholds used to be
+/// operator-only config; anyone holding `create_topic` can set them now, so the
+/// ceilings are enforced at admission. Mirrors
+/// `configs::validators::SEGMENT_MAX_SIZE_BYTES`, which lives in the crate that
+/// depends on this one.
+pub const MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE: u64 = 1024 * 1024 * 1024;
+
+/// Ceiling for `messages_required_to_save`, derived from the byte ceiling: a
+/// segment that large cannot hold more messages than this, because a message
+/// costs at least its header on disk.
+pub const MAX_MESSAGES_REQUIRED_TO_SAVE: u32 = 16 * 1024 * 1024;
+
/// Smallest per-topic segment size. Segments far below the shipped default
/// explode the per-partition segment count, which state transfer bounds via
/// its manifest entry cap; a partition that crosses it becomes unservable
/// for transfer, so the floor is enforced at admission.
pub const MIN_TOPIC_SEGMENT_SIZE: u64 = 1024 * 1024;
+/// Largest per-topic `segment_size` a node admits.
+///
+/// The ceiling used to be computed per node as the smaller of the global
+/// segment maximum and `transfer_artifact_bytes_max - max_message_size` (a
+/// received segment artifact can be one whole batch past the cap, and an
+/// artifact ceiling below that livelocks a partition's rejoin). Config
+/// validation now refuses boot unless that subtraction is at least the segment
+/// maximum, so the minimum is always the maximum and the per-node expression was
+/// dead. Mirrors `configs::validators::SEGMENT_MAX_SIZE_BYTES`, which lives in
+/// the crate that depends on this one and pins the two together in a test.
+pub const MAX_TOPIC_SEGMENT_SIZE: u64 = 1024 * 1024 * 1024;
+
+/// Ceiling on what one `preallocate_segments` topic may reserve when created:
+/// `segment_size * partitions_count`.
+///
+/// The reservation runs inline on the shard thread - once per partition as
+/// segments rotate, and again for every owned partition at boot - and
+/// `FALLOC_FL_KEEP_SIZE` makes it real disk rather than a sparse hole. Without a
+/// cap, one create at the 1 GiB default segment size and the maximum partition
+/// count reserves a terabyte before a single message arrives.
+pub const MAX_PREALLOCATED_TOPIC_BYTES: u64 = 64 * 1024 * 1024 * 1024;
+
+/// Validate what a preallocating topic would reserve at admission.
+///
+/// `segment_size_bytes` is the topic's resolved segment size. Callers skip this
+/// for a topic whose effective `preallocate_segments` is false: nothing is
+/// reserved up front there, so the product does not bound anything.
+///
+/// # Errors
+///
+/// Returns `IggyError::InvalidOptionValue("preallocate_segments")` when the
+/// reservation would exceed [`MAX_PREALLOCATED_TOPIC_BYTES`].
+pub fn validate_preallocated_topic_bytes(
+ segment_size_bytes: u64,
+ partitions_count: u32,
+) -> Result<(), IggyError> {
+ let reserved = segment_size_bytes.saturating_mul(u64::from(partitions_count));
+ if reserved > MAX_PREALLOCATED_TOPIC_BYTES {
+ return Err(IggyError::InvalidOptionValue(
+ topic_option_keys::PREALLOCATE_SEGMENTS.to_string(),
+ ));
+ }
+ Ok(())
+}
+
/// Validate an explicit per-topic `segment_size` against its bounds.
///
/// `ceiling` is node-derived: the smaller of the global segment maximum and
@@ -305,11 +422,15 @@ impl FromStr for OptionsScope {
pub struct OptionSpec {
/// Option key accepted by the create command.
pub key: String,
- /// Canonical kind the server persists the value under. A `String` value
- /// parsed via the server-config rules is always accepted too.
+ /// Kind the server encodes this key's own default under, and the kind a
+ /// client should prefer. Not a promise about what ends up stored: an
+ /// explicit entry is persisted and echoed back with the kind the client
+ /// sent it as, and a `String` value parsed through the server-config rules
+ /// stays a `String` in the map - which is what every CLI `--set` produces.
pub kind: HeaderKind,
- /// This node's current resolved default in the canonical kind's
- /// encoding; empty when the key has no server default.
+ /// The key's default in `kind`'s encoding; empty when the key has no
+ /// default. A build constant rather than a per-node value: these defaults
+ /// stopped being config-derived when the `[system.*]` keys became options.
#[serde(default)]
pub default_value: Vec<u8>,
/// Human-readable description.
@@ -363,7 +484,13 @@ pub const UPDATABLE_STREAM_OPTION_KEYS: &[&str] = &[];
pub const UPDATABLE_USER_OPTION_KEYS: &[&str] = &[];
/// Longest key echoed back in an option error. A rejected key is attacker-sized
-/// by definition, and the error rides the wire back to the client.
+/// by definition, and over HTTP the error text rides the response body back to
+/// the client.
+///
+/// Binary transports carry only the error code, so a TCP, QUIC or WebSocket
+/// client rebuilds [`IggyError::UnsupportedOptionKey`] with an empty string and
+/// never sees which key was refused. `DescribeOptions` is the discovery path
+/// there - see the note on that variant.
const ERROR_KEY_PREVIEW_LEN: usize = 64;
fn key_preview(key: &str) -> String {
@@ -609,7 +736,7 @@ impl TopicCreateOptions {
for entry in options {
// Wire validation already enforced UTF-8 string keys.
let key = String::from_utf8_lossy(entry.key);
- parsed.absorb(&entry, &key, UnknownEntry::Reject)?;
+ parsed.absorb_strict(&entry, &key)?;
}
Ok(parsed)
}
@@ -627,34 +754,21 @@ impl TopicCreateOptions {
let mut parsed = Self::default();
for entry in options {
let key = String::from_utf8_lossy(entry.key);
- // Infallible: `Skip` turns every parse error into a no-op.
- let _ = parsed.absorb(&entry, &key, UnknownEntry::Skip);
+ // Discarding the error is the skip: see `absorb_strict`.
+ let _ = parsed.absorb_strict(&entry, &key);
}
parsed
}
- /// Fold one catalog entry into `self`, applying `unknown` to an entry this
- /// build cannot interpret. Shared by [`Self::parse`] (wire block) and
- /// [`Self::from_resource_options`] (persisted map) so both read the
- /// identical key set, kinds, and value bounds, and differ only in what
- /// they do with an entry outside it.
- fn absorb(
- &mut self,
- entry: &WireUserHeaderEntry<'_>,
- key: &str,
- unknown: UnknownEntry,
- ) -> Result<(), IggyError> {
- // Every arm assigns only after its parse succeeds, so a skipped entry
- // leaves `self` exactly as it found it.
- match self.absorb_strict(entry, key) {
- Ok(()) => Ok(()),
- Err(error) => match unknown {
- UnknownEntry::Reject => Err(error),
- UnknownEntry::Skip => Ok(()),
- },
- }
- }
-
+ /// Fold one catalog entry into `self`, refusing an entry this build cannot
+ /// interpret: a key outside [`TOPIC_OPTION_KEYS`], or a catalog key whose
+ /// value kind or payload does not parse. Shared by [`Self::parse`] (wire
+ /// block) and [`Self::from_resource_options`] (persisted map) so both read
+ /// the identical key set, kinds and value bounds.
+ ///
+ /// Every arm assigns only after its own parse succeeds, which is what makes
+ /// discarding the error a safe skip: a refused entry leaves `self` exactly
+ /// as it found it.
fn absorb_strict(
&mut self,
entry: &WireUserHeaderEntry<'_>,
@@ -685,13 +799,16 @@ impl TopicCreateOptions {
}
topic_option_keys::MESSAGES_REQUIRED_TO_SAVE => {
let messages = parse_u32(entry, key)?;
- if messages == 0 {
+ if messages == 0 || messages > MAX_MESSAGES_REQUIRED_TO_SAVE {
return Err(IggyError::InvalidOptionValue(key.to_string()));
}
parsed.messages_required_to_save = Some(messages);
}
topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE => {
let size = parse_byte_size(entry, key)?;
+ if size > MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE {
+ return Err(IggyError::InvalidOptionValue(key.to_string()));
+ }
parsed.size_of_messages_required_to_save =
(size != 0).then_some(IggyByteSize::from(size));
}
@@ -704,16 +821,56 @@ impl TopicCreateOptions {
Ok(())
}
+ /// Field-wise overlay: every key this block sets wins, the rest fall back
+ /// to `defaults`.
+ ///
+ /// Apply resolves the client block over the derived one this way. `raw` is
+ /// dropped: by the time a block reaches apply, every key it named has
+ /// already been parsed into a typed field or refused by admission.
+ #[must_use]
+ pub fn resolved_over(&self, defaults: &Self) -> Self {
+ Self {
+ partitions_count: self.partitions_count.or(defaults.partitions_count),
+ compression_algorithm: self
+ .compression_algorithm
+ .or(defaults.compression_algorithm),
+ message_expiry: self.message_expiry.or(defaults.message_expiry),
+ max_topic_size: self.max_topic_size.or(defaults.max_topic_size),
+ segment_size: self.segment_size.or(defaults.segment_size),
+ enforce_fsync: self.enforce_fsync.or(defaults.enforce_fsync),
+ messages_required_to_save: self
+ .messages_required_to_save
+ .or(defaults.messages_required_to_save),
+ size_of_messages_required_to_save: self
+ .size_of_messages_required_to_save
+ .or(defaults.size_of_messages_required_to_save),
+ preallocate_segments: self.preallocate_segments.or(defaults.preallocate_segments),
+ raw: BTreeMap::new(),
+ }
+ }
+
/// Encode the present keys into a client options block, in canonical
/// kinds. The client-side counterpart of [`Self::parse`].
///
+ /// # Errors
+ ///
+ /// See [`Self::to_option_map`].
+ pub fn to_wire(&self) -> Result<WireOptions, IggyError> {
+ crate::wire_conversions::resource_options_to_wire(
+ &self.to_option_map()?,
+ OptionsProvenance::All,
+ )
+ }
+
+ /// The present keys as a canonically-kinded option map.
+ ///
/// A typed field is inserted after the raw entries, so it wins on
/// collision and keeps its canonical kind.
///
/// # Errors
///
/// See [`raw_options_map`].
- pub fn to_wire(&self) -> Result<WireOptions, IggyError> {
+ pub fn to_option_map(&self) -> Result<ResourceOptions, IggyError> {
let mut options = raw_options_map(&self.raw)?;
if let Some(compression_algorithm) = self.compression_algorithm {
options.insert(
@@ -774,7 +931,7 @@ impl TopicCreateOptions {
OptionValue::explicit(HeaderValue::from(preallocate_segments)),
);
}
- crate::wire_conversions::resource_options_to_wire(&options, OptionsProvenance::All)
+ Ok(options)
}
/// Render the present keys as string key-values, for a transport that
@@ -932,27 +1089,13 @@ impl TopicCreateOptions {
value_kind: iggy_binary_protocol::WireHeaderKind(option.value.kind().as_code()),
value: option.value.as_bytes(),
};
- // Infallible: `Skip` turns every parse error into a no-op.
- let _ = parsed.absorb(&entry, &key, UnknownEntry::Skip);
+ // Discarding the error is the skip: see `absorb_strict`.
+ let _ = parsed.absorb_strict(&entry, &key);
}
parsed
}
}
-/// What a parse does with an entry this build cannot interpret: a key outside
-/// [`TOPIC_OPTION_KEYS`], or a catalog key whose value kind or payload does
-/// not parse.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum UnknownEntry {
- /// Refuse the block. Used at admission, where the client is present to be
- /// told which key it got wrong, and where the catalog gate belongs.
- Reject,
- /// Leave the knob unset and keep going. Used when reading state a newer
- /// build wrote, where refusing would diverge this replica from the group
- /// over a key it merely cannot read.
- Skip,
-}
-
fn parse_u32(entry: &WireUserHeaderEntry<'_>, key: &str) -> Result<u32, IggyError> {
if entry.value_kind.0 == HeaderKind::Uint32.as_code() {
let bytes: [u8; 4] = entry
@@ -1093,11 +1236,13 @@ mod tests {
}
#[test]
- fn resource_options_round_trip_through_json_with_string_keys() {
+ fn resource_options_render_as_readable_json_and_round_trip() {
// `HeaderKey` serializes as a struct, and JSON object keys must be
// strings, so the derived impl fails with "key must be a string" for
// any non-empty map -- which 500'd every HTTP response carrying
- // options. Keys must render as their plain text.
+ // options. Keys must render as their plain text, and values in the
+ // string form the create body takes them in: a base64 payload behind a
+ // kind tag is neither readable nor re-sendable.
#[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)]
struct Holder {
#[serde(default, with = "super::resource_options_json")]
@@ -1115,8 +1260,30 @@ mod tests {
json.contains("\"segment_size\""),
"the key must render as plain text, got: {json}"
);
+ assert!(
+ json.contains("\"value\":\"1048576\""),
+ "the value must render as its string form, got: {json}"
+ );
+
let decoded: Holder = serde_json::from_str(&json).expect("options must round-trip");
- assert_eq!(decoded, holder);
+ let (key, option) = decoded.options.iter().next().unwrap();
+ assert_eq!(key.as_bytes(), topic_option_keys::SEGMENT_SIZE.as_bytes());
+ assert!(!option.explicit, "provenance survives the round trip");
+ // Re-read as a string-kinded value, which is exactly what the create
+ // body's string map produces, so the server parses it the same way.
+ assert_eq!(option.value.kind(), HeaderKind::String);
+ assert_eq!(option.value.as_bytes(), b"1048576");
+ assert!(
+ TopicCreateOptions::parse(
+ &crate::wire_conversions::resource_options_to_wire(
+ &decoded.options,
+ OptionsProvenance::All
+ )
+ .unwrap()
+ )
+ .is_ok(),
+ "the rendered value must parse back through the catalog"
+ );
}
#[test]
@@ -1292,4 +1459,67 @@ mod tests {
"MAX_OPTIONS entries must fit in MAX_OPTIONS_BYTES"
);
}
+
+ #[test]
+ fn flush_threshold_ceilings_are_derived_from_the_segment_maximum() {
+ // A threshold no segment can reach never trips, so committed messages
+ // sit in the journal - which a crash does not preserve - until the
+ // segment rotates. The message ceiling follows from the byte one: a
+ // message costs at least its header on disk.
+ assert_eq!(
+ MAX_MESSAGES_REQUIRED_TO_SAVE as u64,
+ MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE
+ / crate::IGGY_MESSAGE_HEADER_SIZE
+ .try_into()
+ .unwrap_or(u64::MAX),
+ "message ceiling must stay derived from the byte ceiling"
+ );
+ }
+
+ #[test]
+ fn parse_rejects_flush_thresholds_above_their_ceilings() {
+ let over_messages = TopicCreateOptions {
+ raw: BTreeMap::from([(
+ topic_option_keys::MESSAGES_REQUIRED_TO_SAVE.to_string(),
+ (u64::from(MAX_MESSAGES_REQUIRED_TO_SAVE) + 1).to_string(),
+ )]),
+ ..TopicCreateOptions::default()
+ };
+ assert!(
+ TopicCreateOptions::parse(&over_messages.to_wire().unwrap()).is_err(),
+ "a message threshold above the ceiling must be refused"
+ );
+
+ let over_bytes = TopicCreateOptions {
+ raw: BTreeMap::from([(
+ topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE.to_string(),
+ (MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE + 1).to_string(),
+ )]),
+ ..TopicCreateOptions::default()
+ };
+ assert!(
+ TopicCreateOptions::parse(&over_bytes.to_wire().unwrap()).is_err(),
+ "a byte threshold above the ceiling must be refused"
+ );
+
+ let at_ceiling = TopicCreateOptions {
+ messages_required_to_save: Some(MAX_MESSAGES_REQUIRED_TO_SAVE),
+ size_of_messages_required_to_save: Some(IggyByteSize::from(
+ MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE,
+ )),
+ ..TopicCreateOptions::default()
+ };
+ assert!(
+ TopicCreateOptions::parse(&at_ceiling.to_wire().unwrap()).is_ok(),
+ "the ceiling itself must remain settable"
+ );
+ }
+
+ #[test]
+ fn preallocation_cap_bounds_segment_size_times_partitions() {
+ assert!(validate_preallocated_topic_bytes(DEFAULT_SEGMENT_SIZE, 64).is_ok());
+ assert!(validate_preallocated_topic_bytes(DEFAULT_SEGMENT_SIZE, 65).is_err());
+ // The product saturates rather than wrapping into a passing value.
+ assert!(validate_preallocated_topic_bytes(u64::MAX, u32::MAX).is_err());
+ }
}
diff --git a/core/common/src/wire_conversions.rs b/core/common/src/wire_conversions.rs
index 72e808fb8..d87e392a0 100644
--- a/core/common/src/wire_conversions.rs
+++ b/core/common/src/wire_conversions.rs
@@ -785,20 +785,46 @@ pub fn user_headers_from_wire(
/// slices each TLV field without bounds checks, relying on that validation.
pub(crate) fn user_headers_from_validated_slice(
buf: &[u8],
+) -> Result<BTreeMap<HeaderKey, HeaderValue>, IggyError> {
+ headers_from_validated_slice(buf, UnknownKinds::Reject)
+}
+
+/// What a decode does with an entry whose kind code has no domain meaning.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum UnknownKinds {
+ /// Fail the whole decode. A message header the reader cannot interpret is
+ /// the caller's problem, not something to hide from it.
+ Reject,
+ /// Drop the entry and keep the rest.
+ Skip,
+}
+
+fn headers_from_validated_slice(
+ buf: &[u8],
+ unknown: UnknownKinds,
) -> Result<BTreeMap<HeaderKey, HeaderValue>, IggyError> {
if buf.is_empty() {
return Ok(BTreeMap::new());
}
let mut headers = BTreeMap::new();
for entry in WireUserHeaderIterator::new(buf) {
- let key_kind = HeaderKind::from_code(entry.key_kind.0)?;
+ let (key_kind, value_kind) = match (
+ HeaderKind::from_code(entry.key_kind.0),
+ HeaderKind::from_code(entry.value_kind.0),
+ ) {
+ (Ok(key_kind), Ok(value_kind)) => (key_kind, value_kind),
+ (Err(error), _) | (Ok(_), Err(error)) => {
+ if unknown == UnknownKinds::Skip {
+ continue;
+ }
+ return Err(error);
+ }
+ };
if let Some(expected) = key_kind.expected_size()
&& entry.key.len() != expected
{
return Err(IggyError::InvalidHeaderKey);
}
-
- let value_kind = HeaderKind::from_code(entry.value_kind.0)?;
if let Some(expected) = value_kind.expected_size()
&& entry.value.len() != expected
{
@@ -843,17 +869,22 @@ pub fn option_specs_from_wire(
/// block carries no per-entry provenance. Admission calls this once for the
/// client block (`explicit == true`) and fills defaults separately.
///
+/// Entries whose kind code has no domain meaning are skipped, not rejected.
+/// The wire layer forwards unknown kinds verbatim so a mixed-version cluster
+/// can round-trip them, and this decode sits on the apply path, on the SDK's
+/// list-decode and on the server's own reads: refusing the entry would mean an
+/// old replica rejecting a commit a new one accepted, one unreadable entry
+/// failing a whole `get_topics()`, and a topic that cannot be read back.
+///
/// # Errors
///
-/// Returns `IggyError::InvalidHeaderKey` / `InvalidHeaderValue` when a kind
-/// code has no domain meaning or a fixed-size kind carries a mismatched
-/// payload. The wire layer accepts unknown kinds for forwarding; interpreting
-/// them into the domain requires known semantics.
+/// Returns `IggyError::InvalidHeaderKey` / `InvalidHeaderValue` when a known
+/// fixed-size kind carries a mismatched payload.
pub fn resource_options_from_wire(
wire: &iggy_binary_protocol::WireOptions,
explicit: bool,
) -> Result<ResourceOptions, IggyError> {
- let headers = user_headers_from_validated_slice(wire.as_bytes())?;
+ let headers = headers_from_validated_slice(wire.as_bytes(), UnknownKinds::Skip)?;
Ok(headers
.into_iter()
.map(|(key, value)| (key, OptionValue { value, explicit }))
@@ -936,6 +967,15 @@ pub fn resource_options_to_wire(
}
let mut buf = BytesMut::with_capacity(size);
for (key, option) in entries {
+ // A non-string key would encode a block the receiving peer refuses,
+ // making the resource permanently unreadable. It holds because every
+ // producer went through `validate_options`, not because `HeaderKey`
+ // guarantees it, so the invariant is asserted where it is relied on.
+ debug_assert_eq!(
+ key.kind(),
+ HeaderKind::String,
+ "option key must be a string"
+ );
buf.put_u8(key.kind().as_code());
#[allow(clippy::cast_possible_truncation)]
buf.put_u32_le(key.as_bytes().len() as u32);
@@ -946,8 +986,9 @@ pub fn resource_options_to_wire(
buf.put_slice(option.value.as_bytes());
}
// Structural validity holds by construction: entries come from valid
- // `HeaderKey`s walked in `BTreeMap` order, so keys are string-kinded,
- // sorted and unique. The caps above cover what construction cannot.
+ // `HeaderKey`s walked in `BTreeMap` order, so keys are sorted and unique,
+ // and the loop above asserts they are string-kinded. The caps cover what
+ // construction cannot.
Ok(WireOptions::from_validated(buf.freeze()))
}
@@ -1020,4 +1061,35 @@ mod tests {
assert!(user_headers_from_validated_slice(&buf).is_err());
assert!(user_headers_from_wire(&WireUserHeaders::from_slice(&buf).unwrap()).is_err());
}
+
+ fn put_option_entry(buf: &mut Vec<u8>, key: &str, value_kind: u8, value: &[u8]) {
+ buf.push(HeaderKind::String.as_code());
+ buf.extend_from_slice(&u32::try_from(key.len()).unwrap().to_le_bytes());
+ buf.extend_from_slice(key.as_bytes());
+ buf.push(value_kind);
+ buf.extend_from_slice(&u32::try_from(value.len()).unwrap().to_le_bytes());
+ buf.extend_from_slice(value);
+ }
+
+ #[test]
+ fn given_option_with_unknown_value_kind_when_decoded_should_skip_only_that_entry() {
+ // A newer peer's option value kind. The wire layer forwards it, so the
+ // domain decode has to drop the entry rather than fail the block: this
+ // decode runs on the apply path, where an error means one replica
+ // rejecting a commit another accepted.
+ let mut buf = Vec::new();
+ put_option_entry(&mut buf, "from_the_future", 200, b"opaque");
+ put_option_entry(
+ &mut buf,
+ "segment_size",
+ HeaderKind::String.as_code(),
+ b"1MB",
+ );
+ let wire = WireOptions::from_slice(&buf).expect("unknown value kinds stay wire-valid");
+
+ let options = resource_options_from_wire(&wire, true).unwrap();
+
+ assert_eq!(options.len(), 1);
+ assert!(options.contains_key(&HeaderKey::from_str("segment_size").unwrap()));
+ }
}
diff --git a/core/configs/src/configs_impl/file_provider.rs b/core/configs/src/configs_impl/file_provider.rs
index 1c664701e..b6b8d67cf 100644
--- a/core/configs/src/configs_impl/file_provider.rs
+++ b/core/configs/src/configs_impl/file_provider.rs
@@ -28,47 +28,34 @@ use tracing::{error, info, warn};
const DISPLAY_CONFIG_ENV: &str = "IGGY_DISPLAY_CONFIG";
-/// Config keys that became per-topic options, paired with the option key that
-/// replaced each one.
+/// A config key that no longer exists, and what took over from it.
///
-/// Nothing else catches them. Their struct fields are gone and no config table
-/// sets `deny_unknown_fields`, so figment drops an unrecognized key without a
-/// word: a server still carrying `enforce_fsync = true` would boot reporting
-/// success and run without fsync. Every entry here used to change behavior, so
-/// the boot is refused rather than warned about.
-const RELOCATED_CONFIG_KEYS: &[(&str, &str)] = &[
- ("system.topic.max_size", "max_topic_size"),
- ("system.topic.message_expiry", "message_expiry"),
- ("system.partition.enforce_fsync", "enforce_fsync"),
- (
- "system.partition.messages_required_to_save",
- "messages_required_to_save",
- ),
- (
- "system.partition.size_of_messages_required_to_save",
- "size_of_messages_required_to_save",
- ),
- ("system.segment.size", "segment_size"),
- ("system.segment.preallocate", "preallocate_segments"),
-];
-
-/// Refuse a config file that still sets a key which moved to a topic option.
-fn reject_relocated_keys(file_path: &str) -> Result<(), ConfigurationError> {
- let file = Figment::from(Toml::file(file_path));
- let mut found = false;
- for (key, option) in RELOCATED_CONFIG_KEYS {
- if file.find_value(key).is_ok() {
- found = true;
- error!(
- "Config key '{key}' no longer exists; it is now the per-topic '{option}' option, \
- set on CreateTopic. Remove the key to boot."
- );
+/// Nothing else catches such a key. Its struct field is gone and no config
+/// table sets `deny_unknown_fields`, so figment drops an unrecognized key
+/// without a word from either source: a server still carrying
+/// `enforce_fsync = true` would boot reporting success and run without fsync.
+/// Every relocated key used to change behavior, so the boot is refused rather
+/// than warned about.
+#[derive(Debug, Clone, Copy)]
+pub struct RelocatedKey {
+ /// Dotted config path, for example `system.segment.size`. A deleted table
+ /// matches everything nested under it as well.
+ pub path: &'static str,
+ /// The per-topic option key that replaced it, or `None` when the feature
+ /// it configured was removed outright.
+ pub replacement: Option<&'static str>,
+}
+
+impl RelocatedKey {
+ /// Sentence telling the operator where the setting went.
+ fn guidance(&self) -> String {
+ match self.replacement {
+ Some(option) => {
+ format!("it is now the per-topic '{option}' option, set on CreateTopic")
+ }
+ None => "the feature it configured was removed".to_string(),
}
}
- if found {
- return Err(ConfigurationError::InvalidConfigurationValue);
- }
- Ok(())
}
/// File-based configuration provider that combines file, default, and environment configurations.
@@ -77,6 +64,8 @@ pub struct FileConfigProvider<P> {
default_config: Option<Data<Toml>>,
env_provider: P,
display_config: bool,
+ env_prefix: &'static str,
+ relocated_keys: &'static [RelocatedKey],
}
impl<P: Provider> FileConfigProvider<P> {
@@ -98,7 +87,63 @@ impl<P: Provider> FileConfigProvider<P> {
env_provider,
default_config,
display_config,
+ env_prefix: "",
+ relocated_keys: &[],
+ }
+ }
+
+ /// Refuse to load when any of `keys` is still set, in the config file or in
+ /// the environment.
+ ///
+ /// `env_prefix` is the prefix this config's env provider reads, used to
+ /// derive the variable name for each path. The table is per-config on
+ /// purpose: a server key left over in a shared container environment must
+ /// not stop the connectors runtime or the MCP server from booting.
+ pub fn with_relocated_keys(
+ mut self,
+ env_prefix: &'static str,
+ keys: &'static [RelocatedKey],
+ ) -> Self {
+ self.env_prefix = env_prefix;
+ self.relocated_keys = keys;
+ self
+ }
+
+ fn reject_relocated_keys(&self) -> Result<(), ConfigurationError> {
+ if self.relocated_keys.is_empty() {
+ return Ok(());
}
+ let file = file_exists(&self.file_path).then(|| Figment::from(Toml::file(&self.file_path)));
+ let env_names = env::vars_os().filter_map(|(name, _)| name.into_string().ok());
+ let mut found = false;
+
+ for key in self.relocated_keys {
+ if file
+ .as_ref()
+ .is_some_and(|file| file.find_value(key.path).is_ok())
+ {
+ found = true;
+ error!(
+ "Config key '{}' no longer exists; {}. Remove the key to boot.",
+ key.path,
+ key.guidance()
+ );
+ }
+ }
+ for (name, key) in relocated_env_vars(env_names, self.env_prefix, self.relocated_keys) {
+ found = true;
+ error!(
+ "Environment variable '{name}' sets config key '{}', which no longer exists; {}. \
+ Unset it to boot.",
+ key.path,
+ key.guidance()
+ );
+ }
+
+ if found {
+ return Err(ConfigurationError::InvalidConfigurationValue);
+ }
+ Ok(())
}
}
@@ -106,6 +151,11 @@ impl<P: Provider + Clone> ConfigProvider for FileConfigProvider<P> {
async fn load_config<T: ConfigurationType>(&self) -> Result<T, ConfigurationError> {
info!("Loading config from path: '{}'...", self.file_path);
+ // Both sources are checked before either is merged: the env provider
+ // below is just as silent about a key no field reads, and the
+ // pure-env container never touches the file branch at all.
+ self.reject_relocated_keys()?;
+
// Start with the default configuration if provided
let mut config_builder = Figment::new();
let has_default = self.default_config.is_some();
@@ -118,7 +168,6 @@ impl<P: Provider + Clone> ConfigProvider for FileConfigProvider<P> {
// If the config file exists, merge it into the configuration
if file_exists(&self.file_path) {
info!("Found configuration file at path: '{}'.", self.file_path);
- reject_relocated_keys(&self.file_path)?;
config_builder = config_builder.merge(Toml::file(&self.file_path));
} else {
warn!(
@@ -157,6 +206,42 @@ impl<P: Provider + Clone> ConfigProvider for FileConfigProvider<P> {
}
}
+/// Pair every name in `names` that addresses a relocated key with that key.
+///
+/// A key's variable name is its path uppercased with dots turned into
+/// underscores, matching what `#[derive(ConfigEnv)]` generates. A deleted table
+/// also matches its children, so `system.message_deduplication` catches
+/// `IGGY_SYSTEM_MESSAGE_DEDUPLICATION_ENABLED`.
+fn relocated_env_vars<'keys>(
+ names: impl Iterator<Item = String>,
+ prefix: &str,
+ keys: &'keys [RelocatedKey],
+) -> Vec<(String, &'keys RelocatedKey)> {
+ let derived: Vec<(String, &RelocatedKey)> = keys
+ .iter()
+ .map(|key| {
+ (
+ format!("{prefix}{}", key.path.replace('.', "_").to_uppercase()),
+ key,
+ )
+ })
+ .collect();
+
+ let mut found = Vec::new();
+ for name in names {
+ for (env_name, key) in &derived {
+ let nested = name
+ .strip_prefix(env_name.as_str())
+ .is_some_and(|rest| rest.starts_with('_'));
+ if &name == env_name || nested {
+ found.push((name, *key));
+ break;
+ }
+ }
+ }
+ found
+}
+
fn file_exists<P: AsRef<Path>>(path: P) -> bool {
let path = path.as_ref();
@@ -182,3 +267,89 @@ fn file_exists<P: AsRef<Path>>(path: P) -> bool {
};
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const KEYS: &[RelocatedKey] = &[
+ RelocatedKey {
+ path: "system.partition.enforce_fsync",
+ replacement: Some("enforce_fsync"),
+ },
+ RelocatedKey {
+ path: "system.message_deduplication",
+ replacement: None,
+ },
+ ];
+
+ fn names(names: &[&str]) -> Vec<String> {
+ names.iter().map(|name| (*name).to_string()).collect()
+ }
+
+ #[test]
+ fn given_env_var_for_relocated_leaf_when_matching_then_should_report_it() {
+ let found = relocated_env_vars(
+ names(&["IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC"]).into_iter(),
+ "IGGY_",
+ KEYS,
+ );
+
+ assert_eq!(found.len(), 1);
+ assert_eq!(found[0].0, "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC");
+ assert_eq!(found[0].1.replacement, Some("enforce_fsync"));
+ }
+
+ #[test]
+ fn given_env_var_under_removed_table_when_matching_then_should_report_it() {
+ let found = relocated_env_vars(
+ names(&["IGGY_SYSTEM_MESSAGE_DEDUPLICATION_ENABLED"]).into_iter(),
+ "IGGY_",
+ KEYS,
+ );
+
+ assert_eq!(found.len(), 1);
+ assert_eq!(found[0].1.path, "system.message_deduplication");
+ assert_eq!(found[0].1.replacement, None);
+ }
+
+ #[test]
+ fn given_live_env_vars_when_matching_then_should_report_none() {
+ let found = relocated_env_vars(
+ names(&[
+ "IGGY_SYSTEM_PARTITION_PATH",
+ "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNCHRONIZATION",
+ "IGGY_TCP_ADDRESS",
+ "RUST_LOG",
+ ])
+ .into_iter(),
+ "IGGY_",
+ KEYS,
+ );
+
+ assert!(found.is_empty(), "unexpected matches: {found:?}");
+ }
+
+ #[test]
+ fn given_another_configs_prefix_when_matching_then_should_report_none() {
+ let found = relocated_env_vars(
+ names(&["IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC"]).into_iter(),
+ "IGGY_CONNECTORS_",
+ KEYS,
+ );
+
+ assert!(found.is_empty(), "unexpected matches: {found:?}");
+ }
+
+ #[test]
+ fn given_no_relocated_keys_when_rejecting_then_should_accept() {
+ let provider = FileConfigProvider::new(
+ "nonexistent-config.toml".to_string(),
+ Toml::string(""),
+ false,
+ None,
+ );
+
+ assert!(provider.reject_relocated_keys().is_ok());
+ }
+}
diff --git a/core/configs/src/configs_impl/mod.rs b/core/configs/src/configs_impl/mod.rs
index 6b16acc0b..905cb46d7 100644
--- a/core/configs/src/configs_impl/mod.rs
+++ b/core/configs/src/configs_impl/mod.rs
@@ -33,7 +33,7 @@ mod typed_env_provider;
pub use env_mapping::{ConfigEnvMappings, EnvVarMapping};
pub use error::ConfigurationError;
-pub use file_provider::FileConfigProvider;
+pub use file_provider::{FileConfigProvider, RelocatedKey};
pub use parsing::parse_env_value_to_json;
pub use traits::{ConfigProvider, ConfigurationType};
pub use typed_env_provider::TypedEnvProvider;
diff --git a/core/configs/src/lib.rs b/core/configs/src/lib.rs
index e9339b2aa..3b2c59541 100644
--- a/core/configs/src/lib.rs
+++ b/core/configs/src/lib.rs
@@ -24,7 +24,7 @@ pub use common::{COMPONENT, cache_indexes, defaults, displays, http, system, val
pub use configs_derive::ConfigEnv;
pub use configs_impl::{
ConfigEnvMappings, ConfigProvider, ConfigurationError, ConfigurationType, EnvVarMapping,
- FileConfigProvider, TypedEnvProvider, parse_env_value_to_json,
+ FileConfigProvider, RelocatedKey, TypedEnvProvider, parse_env_value_to_json,
};
pub use server_config::{
cluster, message_bus, metadata, partition, quic, server, sharding, tcp, websocket,
diff --git a/core/configs/src/server_config/server.rs b/core/configs/src/server_config/server.rs
index 4dbd7e81d..5f0522c1a 100644
--- a/core/configs/src/server_config/server.rs
+++ b/core/configs/src/server_config/server.rs
@@ -26,7 +26,10 @@ use super::websocket::WebSocketConfig;
use crate::ConfigurationError;
use crate::common::http::HttpConfig;
use crate::common::system::SystemConfig;
-use configs::{ConfigEnv, ConfigEnvMappings, ConfigProvider, FileConfigProvider, TypedEnvProvider};
+use configs::{
+ ConfigEnv, ConfigEnvMappings, ConfigProvider, FileConfigProvider, RelocatedKey,
+ TypedEnvProvider,
+};
use err_trail::ErrContext;
use figment::providers::{Format, Toml};
use figment::value::Dict;
@@ -46,6 +49,48 @@ pub use crate::common::server::{
const DEFAULT_CONFIG_PATH: &str = "core/server/config.toml";
+/// Server config keys that became per-topic options, or went away with the
+/// feature they configured.
+///
+/// The provider refuses to boot while any of them is still set, in the config
+/// file or in the environment. See [`RelocatedKey`] for why a warning is not
+/// enough. The partition knobs matter most: they are create-only options now,
+/// so a topic that boots without one can never be given it afterwards.
+const RELOCATED_CONFIG_KEYS: &[RelocatedKey] = &[
+ RelocatedKey {
+ path: "system.topic.max_size",
+ replacement: Some("max_topic_size"),
+ },
+ RelocatedKey {
+ path: "system.topic.message_expiry",
+ replacement: Some("message_expiry"),
+ },
+ RelocatedKey {
+ path: "system.partition.enforce_fsync",
+ replacement: Some("enforce_fsync"),
+ },
+ RelocatedKey {
+ path: "system.partition.messages_required_to_save",
+ replacement: Some("messages_required_to_save"),
+ },
+ RelocatedKey {
+ path: "system.partition.size_of_messages_required_to_save",
+ replacement: Some("size_of_messages_required_to_save"),
+ },
+ RelocatedKey {
+ path: "system.segment.size",
+ replacement: Some("segment_size"),
+ },
+ RelocatedKey {
+ path: "system.segment.preallocate",
+ replacement: Some("preallocate_segments"),
+ },
+ RelocatedKey {
+ path: "system.message_deduplication",
+ replacement: None,
+ },
+];
+
/// [`SystemConfig`] bound to this crate's own
/// [`super::sharding::ShardingConfig`]. `core/server` names this alias
/// wherever it refers to the system config.
@@ -139,6 +184,7 @@ impl ServerConfig {
true,
Some(default_config),
)
+ .with_relocated_keys(ServerConfig::ENV_PREFIX, RELOCATED_CONFIG_KEYS)
}
/// All recognised env var names for [`ServerConfig`].
diff --git a/core/configs/src/server_config/validators.rs b/core/configs/src/server_config/validators.rs
index 41e382c0b..60541d449 100644
--- a/core/configs/src/server_config/validators.rs
+++ b/core/configs/src/server_config/validators.rs
@@ -434,6 +434,33 @@ mod tests {
.expect("config deserializes")
}
+ /// The per-topic `segment_size` ceiling lives in `iggy_common`, which cannot
+ /// import this crate. Admission reads it from there; boot validation reads
+ /// the constant here. This is the only place both are visible.
+ #[test]
+ fn given_segment_maximum_when_compared_to_the_option_ceiling_should_match() {
+ assert_eq!(
+ SEGMENT_MAX_SIZE_BYTES,
+ iggy_common::MAX_TOPIC_SEGMENT_SIZE,
+ "the option ceiling and the segment maximum must move together"
+ );
+ }
+
+ /// Admission may cap a topic's `segment_size` at
+ /// [`iggy_common::MAX_TOPIC_SEGMENT_SIZE`] flat only while boot refuses any
+ /// config whose artifact budget cannot carry a segment that large plus one
+ /// bus frame. Without that refusal the ceiling would have to be per node.
+ #[test]
+ fn given_artifact_budget_below_the_segment_ceiling_when_validating_should_reject() {
+ let config = config_with_override(
+ "[partition]\ntransfer_artifact_bytes_max = \"1 GiB\"\n[message_bus]\nmax_message_size = \"1 MiB\"\n",
+ );
+ assert!(
+ config.validate().is_err(),
+ "an artifact budget under segment maximum + one frame must refuse boot"
+ );
+ }
+
#[test]
fn given_shipped_default_config_when_validating_should_pass() {
let config: ServerConfig = Figment::new()
diff --git a/core/integration/tests/cli/general/test_help_command.rs b/core/integration/tests/cli/general/test_help_command.rs
index 5fbac9129..499073b54 100644
--- a/core/integration/tests/cli/general/test_help_command.rs
+++ b/core/integration/tests/cli/general/test_help_command.rs
@@ -38,6 +38,7 @@ Commands:
segment segments operations [alias: seg]
ping ping iggy server
me get current client info
+ options list the options a resource's create command accepts
stats get iggy server statistics
snapshot collect iggy server troubleshooting data
pat personal access token operations
diff --git a/core/integration/tests/cli/general/test_overview_command.rs b/core/integration/tests/cli/general/test_overview_command.rs
index 6b528208a..f8f447e04 100644
--- a/core/integration/tests/cli/general/test_overview_command.rs
+++ b/core/integration/tests/cli/general/test_overview_command.rs
@@ -49,6 +49,7 @@ Commands:
segment segments operations [alias: seg]
ping ping iggy server
me get current client info
+ options list the options a resource's create command accepts
stats get iggy server statistics
snapshot collect iggy server troubleshooting data
pat personal access token operations
diff --git a/core/integration/tests/cli/topic/test_topic_create_command.rs b/core/integration/tests/cli/topic/test_topic_create_command.rs
index ffa7a0387..af8dbb02a 100644
--- a/core/integration/tests/cli/topic/test_topic_create_command.rs
+++ b/core/integration/tests/cli/topic/test_topic_create_command.rs
@@ -292,7 +292,7 @@ Options:
{CLAP_INDENT}
Values are sent as strings and parsed server-side through each option's
own FromStr (e.g. --set segment_size=128MiB). The server rejects keys it
- does not support; GET /options/topic lists the ones it accepts.
+ does not support; run "iggy options topic" to list the ones it accepts.
-h, --help
Print help (see a summary with '-h')
diff --git a/core/integration/tests/server/topic_admission_vsr.rs b/core/integration/tests/server/topic_admission_vsr.rs
index 80f1ed9c4..eb8e2977b 100644
--- a/core/integration/tests/server/topic_admission_vsr.rs
+++ b/core/integration/tests/server/topic_admission_vsr.rs
@@ -189,6 +189,118 @@ async fn given_updated_topic_when_getting_topic_should_echo_stored_values(harnes
);
}
+#[iggy_harness(
+ test_client_transport = [Tcp],
+ server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true)
+)]
+async fn given_update_below_one_segment_when_updating_topic_should_reject_typed(
+ harness: &TestHarness,
+) {
+ let client = harness.tcp_root_client().await.expect("tcp root client");
+ client
+ .create_stream("update-bounds-stream")
+ .await
+ .expect("create stream");
+ let stream_id = Identifier::from_str_value("update-bounds-stream").expect("stream identifier");
+ create_topic_with(
+ &client,
+ &stream_id,
+ "update-bounds-topic",
+ 1,
+ MaxTopicSize::Unlimited,
+ )
+ .await
+ .expect("create topic");
+ let topic_id = Identifier::from_str_value("update-bounds-topic").expect("topic identifier");
+
+ // Create refuses a cap under one segment; an update has to refuse the same
+ // value, or the stored map reports a size the topic can never enforce.
+ let tiny = MaxTopicSize::Custom(IggyByteSize::from_str("10KiB").expect("byte size"));
+ let result = client
+ .update_topic(
+ &stream_id,
+ &topic_id,
+ "update-bounds-topic",
+ &TopicUpdateOptions {
+ max_topic_size: Some(tiny),
+ ..TopicUpdateOptions::default()
+ },
+ )
+ .await;
+ let invalid_size = IggyError::InvalidTopicSize(tiny, IggyByteSize::default()).as_code();
+ assert!(
+ matches!(&result, Err(error) if error.as_code() == invalid_size),
+ "update to a cap below one segment must deny with InvalidTopicSize, got {result:?}"
+ );
+
+ let topic = client
+ .get_topic(&stream_id, &topic_id)
+ .await
+ .expect("get topic")
+ .expect("topic exists");
+ assert_eq!(
+ topic.max_topic_size,
+ MaxTopicSize::Unlimited,
+ "a denied update must leave the stored cap alone"
+ );
+}
+
+#[iggy_harness(
+ test_client_transport = [Tcp],
+ server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true)
+)]
+async fn given_sentinel_option_when_creating_topic_should_store_the_resolved_default(
+ harness: &TestHarness,
+) {
+ let client = harness.tcp_root_client().await.expect("tcp root client");
+ client
+ .create_stream("sentinel-stream")
+ .await
+ .expect("create stream");
+ let stream_id = Identifier::from_str_value("sentinel-stream").expect("stream identifier");
+
+ // `server_default` reaches the wire as a literal 0 through the raw map, the
+ // same shape the CLI's `--set max_topic_size=server_default` produces. The
+ // stored map has to report the resolved default rather than that 0, or one
+ // GetTopic response contradicts itself.
+ client
+ .create_topic(
+ &stream_id,
+ "sentinel-topic",
+ &TopicCreateOptions {
+ partitions_count: Some(1),
+ raw: std::collections::BTreeMap::from([(
+ "max_topic_size".to_string(),
+ "server_default".to_string(),
+ )]),
+ ..TopicCreateOptions::default()
+ },
+ )
+ .await
+ .expect("create topic with a sentinel option");
+
+ let topic_id = Identifier::from_str_value("sentinel-topic").expect("topic identifier");
+ let topic = client
+ .get_topic(&stream_id, &topic_id)
+ .await
+ .expect("get topic")
+ .expect("topic exists");
+ let stored = topic
+ .options
+ .get(&HeaderKey::from_str("max_topic_size").expect("option key"))
+ .expect("max_topic_size is stored");
+ assert_eq!(
+ u64::from(topic.max_topic_size),
+ iggy_common::DEFAULT_MAX_TOPIC_SIZE,
+ "the typed field resolves to the node default"
+ );
+ assert_eq!(
+ stored.value.as_bytes(),
+ &iggy_common::DEFAULT_MAX_TOPIC_SIZE.to_le_bytes(),
+ "the options map must agree with the typed field"
+ );
+}
+
#[iggy_harness(
test_client_transport = [Tcp],
server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true)
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs
index 632af893a..94b372262 100644
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@ -748,34 +748,6 @@ pub struct IggyMetadata<C, J, S, M, SB = PingPongSuperblock> {
/// [`Self::set_commit_notifier`] runs (the server bootstrap on shard
/// 0 sets it; peer shards and tests leave it `None`).
commit_notifier: RefCell<Option<CommitNotifier>>,
- /// Resolved byte value for `MaxTopicSize::ServerDefault` (`0` on the
- /// wire). Primary admission rewrites the sentinel to this value before
- /// replication so the committed state carries a concrete size and every
- /// replica resolves identically regardless of local config. Set from
- /// server config at bootstrap ([`Self::set_default_max_topic_size`]);
- /// defaults to unlimited, matching the shipped server config.
- default_max_topic_size: Cell<u64>,
- /// Resolved micros value for `IggyExpiry::ServerDefault` (`0` on the wire).
- /// Same admission-time sentinel resolution as [`Self::default_max_topic_size`];
- /// set from server config at bootstrap ([`Self::set_default_message_expiry`]).
- /// Defaults to never-expire, matching the shipped server config.
- default_message_expiry: Cell<u64>,
- /// Resolved byte value an absent `segment_size` option takes at
- /// admission. Same resolution discipline as the two defaults above; set
- /// from `system.segment.size` at bootstrap
- /// ([`Self::set_default_segment_size`]). Defaults to 1 GiB, matching the
- /// shipped server config.
- default_segment_size: Cell<u64>,
- /// Ceiling for an explicit per-topic `segment_size`
- /// ([`Self::set_max_topic_segment_size`]). Defaults to 1 GiB.
- max_topic_segment_size: Cell<u64>,
- /// Configured fallbacks for the remaining per-topic runtime knobs,
- /// resolved into the derived-options block at admission exactly like the
- /// sizes above. Seeded from `[system.partition]` at bootstrap.
- default_enforce_fsync: Cell<bool>,
- default_messages_required_to_save: Cell<u32>,
- default_size_of_messages_required_to_save: Cell<u64>,
- default_preallocate_segments: Cell<bool>,
/// Client-table mutations at or below this op are already reflected in a
/// state-transferred table, so the tail-repair commit walk must skip
/// them (re-running `commit_register` would double-bump epochs). `0`
@@ -823,23 +795,6 @@ where
journal_gate: LocalGate::new(),
client_table: RefCell::new(ClientTable::new(CLIENTS_TABLE_MAX)),
commit_notifier: RefCell::new(None),
- // Seeded from the shared constants, not from literals: a node that
- // never reaches `set_partition_runtime_defaults` (simulator, unit
- // tests) still derives the documented defaults. Drifting from them
- // here re-arms preallocation, which reserves a whole segment per
- // partition on a `fallocate` that now runs inline on the shard.
- default_max_topic_size: Cell::new(iggy_common::DEFAULT_MAX_TOPIC_SIZE),
- default_message_expiry: Cell::new(iggy_common::DEFAULT_MESSAGE_EXPIRY),
- default_segment_size: Cell::new(iggy_common::DEFAULT_SEGMENT_SIZE),
- max_topic_segment_size: Cell::new(iggy_common::DEFAULT_SEGMENT_SIZE),
- default_enforce_fsync: Cell::new(iggy_common::DEFAULT_ENFORCE_FSYNC),
- default_messages_required_to_save: Cell::new(
- iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE,
- ),
- default_size_of_messages_required_to_save: Cell::new(
- iggy_common::DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE,
- ),
- default_preallocate_segments: Cell::new(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS),
client_table_frontier: Cell::new(0),
transfer_offer_cache: RefCell::new(None),
}
@@ -927,23 +882,6 @@ impl<C, J, S, M, SB> IggyMetadata<C, J, S, M, SB> {
op > self.client_table_frontier.get()
}
- /// Install the resolved byte value used for `MaxTopicSize::ServerDefault`.
- /// Server-ng bootstrap calls this with `system.topic.max_size` on every
- /// shard (responses read it too); only shard 0's copy feeds admission.
- pub fn set_default_max_topic_size(&self, max_topic_size_bytes: u64) {
- self.default_max_topic_size.set(max_topic_size_bytes);
- }
-
- /// Byte value a stored `MaxTopicSize::ServerDefault` resolves to on this
- /// node. Read by the per-shard segment cleaner, which enforces retention
- /// locally and so must resolve the sentinel at enforcement time: create
- /// admission rewrites it before replication, but an UPDATE back to
- /// `ServerDefault` leaves the sentinel in committed state.
- #[must_use]
- pub const fn default_max_topic_size(&self) -> u64 {
- self.default_max_topic_size.get()
- }
-
/// Raise the forced-checkpoint margin to cover a configured
/// prepare-queue depth (`[metadata] prepare_queue_depth`). Clamped to
/// the built-in floor by the coordinator; no-op on shards without a
@@ -962,77 +900,6 @@ impl<C, J, S, M, SB> IggyMetadata<C, J, S, M, SB> {
self.client_table.borrow_mut().set_capacity(max_clients);
}
- /// Install the resolved micros value used for `IggyExpiry::ServerDefault`.
- /// Server-ng bootstrap calls this with `system.topic.message_expiry`; only
- /// shard 0's copy feeds admission.
- pub fn set_default_message_expiry(&self, message_expiry_micros: u64) {
- self.default_message_expiry.set(message_expiry_micros);
- }
-
- /// Micros value an absent `message_expiry` option resolves to on this
- /// node. Read by the options catalog (`DescribeOptions`).
- #[must_use]
- pub const fn default_message_expiry(&self) -> u64 {
- self.default_message_expiry.get()
- }
-
- /// Install the resolved byte value an absent `segment_size` option
- /// resolves to at admission. The server bootstrap calls this with
- /// `system.segment.size`; only shard 0's copy feeds admission.
- pub fn set_default_segment_size(&self, segment_size_bytes: u64) {
- self.default_segment_size.set(segment_size_bytes);
- }
-
- /// Byte value an absent `segment_size` option resolves to on this node.
- /// Read by admission and the options catalog (`DescribeOptions`).
- #[must_use]
- pub const fn default_segment_size(&self) -> u64 {
- self.default_segment_size.get()
- }
-
- /// Install the largest per-topic `segment_size` this node admits: the
- /// smaller of the global segment maximum and the state-transfer artifact
- /// budget minus one bus frame. Seeded at bootstrap.
- pub fn set_max_topic_segment_size(&self, ceiling_bytes: u64) {
- self.max_topic_segment_size.set(ceiling_bytes);
- }
-
- /// Largest per-topic `segment_size` this node admits.
- #[must_use]
- pub const fn max_topic_segment_size(&self) -> u64 {
- self.max_topic_segment_size.get()
- }
-
- /// Install this node's `[system.partition]` fallbacks for the remaining
- /// per-topic runtime knobs. Seeded at bootstrap on every shard;
- /// only shard 0's copy feeds admission.
- pub fn set_partition_runtime_defaults(
- &self,
- enforce_fsync: bool,
- messages_required_to_save: u32,
- size_of_messages_required_to_save: u64,
- preallocate_segments: bool,
- ) {
- self.default_enforce_fsync.set(enforce_fsync);
- self.default_messages_required_to_save
- .set(messages_required_to_save);
- self.default_size_of_messages_required_to_save
- .set(size_of_messages_required_to_save);
- self.default_preallocate_segments.set(preallocate_segments);
- }
-
- /// This node's configured fallbacks for the per-topic runtime knobs.
- /// Read by admission and the options catalog (`DescribeOptions`).
- #[must_use]
- pub const fn partition_runtime_defaults(&self) -> (bool, u32, u64, bool) {
- (
- self.default_enforce_fsync.get(),
- self.default_messages_required_to_save.get(),
- self.default_size_of_messages_required_to_save.get(),
- self.default_preallocate_segments.get(),
- )
- }
-
/// Fire post-commit notifier. Clones the `Rc` out under a short
/// borrow so a re-entrant `set_commit_notifier` from inside the
/// closure cannot panic on `borrow_mut`.
@@ -3425,17 +3292,17 @@ where
request.options = explicit.to_wire()?;
let resolved_segment_size = explicit
.segment_size
- .unwrap_or_else(|| IggyByteSize::from(self.default_segment_size.get()));
+ .unwrap_or_else(|| IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE));
let resolved_max_topic_size = explicit
.max_topic_size
- .unwrap_or_else(|| MaxTopicSize::from(self.default_max_topic_size.get()));
+ .unwrap_or_else(|| MaxTopicSize::from(iggy_common::DEFAULT_MAX_TOPIC_SIZE));
// Backstop for the transport-side typed checks: an explicit
// segment size outside its bounds, or a topic cap below one
// segment, must never enter the WAL.
if let Some(segment_size) = explicit.segment_size {
validate_topic_segment_size(
segment_size.as_bytes_u64(),
- self.max_topic_segment_size.get(),
+ iggy_common::MAX_TOPIC_SEGMENT_SIZE,
)?;
}
if resolved_max_topic_size.as_bytes_u64() < resolved_segment_size.as_bytes_u64() {
@@ -3447,26 +3314,26 @@ where
explicit.compression_algorithm.unwrap_or_default(),
explicit
.message_expiry
- .unwrap_or_else(|| IggyExpiry::from(self.default_message_expiry.get())),
+ .unwrap_or_else(|| IggyExpiry::from(iggy_common::DEFAULT_MESSAGE_EXPIRY)),
resolved_max_topic_size,
TopicRuntimeDefaults {
segment_size: resolved_segment_size,
enforce_fsync: explicit
.enforce_fsync
- .unwrap_or_else(|| self.default_enforce_fsync.get()),
+ .unwrap_or(iggy_common::DEFAULT_ENFORCE_FSYNC),
messages_required_to_save: explicit
.messages_required_to_save
- .unwrap_or_else(|| self.default_messages_required_to_save.get()),
+ .unwrap_or(iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE),
size_of_messages_required_to_save: explicit
.size_of_messages_required_to_save
.unwrap_or_else(|| {
IggyByteSize::from(
- self.default_size_of_messages_required_to_save.get(),
+ iggy_common::DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE,
)
}),
preallocate_segments: explicit
.preallocate_segments
- .unwrap_or_else(|| self.default_preallocate_segments.get()),
+ .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS),
},
)?;
let partitions = self
@@ -4530,15 +4397,13 @@ mod tests {
#[test]
fn prepare_request_stamps_create_topic_message_expiry_default() {
// A `CreateTopic` without an explicit `message_expiry` option must be
- // resolved at primary admission to the configured default, riding the
+ // resolved at primary admission to the build default, riding the
// derived block, so the replicated prepare -- and thus every
// replica's commit -- holds a concrete expiry.
const CLIENT: u128 = 1;
const SESSION: u64 = 10;
const ACTING_USER: u32 = 7;
- const CONFIGURED_EXPIRY_MICROS: u64 = 7_200_000_000;
let plane = metadata_plane();
- plane.set_default_message_expiry(CONFIGURED_EXPIRY_MICROS);
plane.client_table.borrow_mut().commit_register(
CLIENT,
ACTING_USER,
@@ -4560,7 +4425,9 @@ mod tests {
.expect("derived block parses against the catalog");
assert_eq!(
derived.message_expiry,
- Some(iggy_common::IggyExpiry::from(CONFIGURED_EXPIRY_MICROS)),
+ Some(iggy_common::IggyExpiry::from(
+ iggy_common::DEFAULT_MESSAGE_EXPIRY
+ )),
"ServerDefault expiry must be resolved into the derived block at admission"
);
assert_eq!(
diff --git a/core/metadata/src/stm/consumer_group.rs b/core/metadata/src/stm/consumer_group.rs
index b83504f8c..fd757c2d4 100644
--- a/core/metadata/src/stm/consumer_group.rs
+++ b/core/metadata/src/stm/consumer_group.rs
@@ -37,7 +37,7 @@ use bytes::Bytes;
use bytes::{BufMut, BytesMut};
use iggy_binary_protocol::WireIdentifier;
use iggy_binary_protocol::codec::{
- WireDecode, WireEncode, capped_capacity, read_u32_le, read_u64_le, read_u128_le,
+ WireDecode, WireEncode, bounded_capacity, read_u32_le, read_u64_le, read_u128_le,
};
use iggy_binary_protocol::requests::consumer_groups::{
CreateConsumerGroupRequest, DeleteConsumerGroupRequest,
@@ -435,7 +435,7 @@ impl WireDecode for JoinConsumerGroupRequest {
// corrupt/bit-rotted count (near u32::MAX) allocate gigabytes and abort
// every backup that applies it.
let mut in_flight =
- Vec::with_capacity(capped_capacity(count, buf.len().saturating_sub(pos), 4));
+ Vec::with_capacity(bounded_capacity(count, buf.len().saturating_sub(pos), 4));
for _ in 0..count {
in_flight.push(read_u32_le(buf, pos)?);
pos += 4;
diff --git a/core/metadata/src/stm/snapshot.rs b/core/metadata/src/stm/snapshot.rs
index 0a6df6aa9..89ef62b87 100644
--- a/core/metadata/src/stm/snapshot.rs
+++ b/core/metadata/src/stm/snapshot.rs
@@ -582,6 +582,7 @@ mod tests {
// actually grows need their own pins.
const TOPIC_FIELD_COUNT: u32 = 11;
const STREAM_FIELD_COUNT: u32 = 6;
+ const USER_FIELD_COUNT: u32 = 7;
let topic = TopicSnapshot {
id: 0,
@@ -625,6 +626,22 @@ mod tests {
STREAM_FIELD_COUNT,
"StreamSnapshot's field count changed; bump SNAPSHOT_FORMAT_VERSION with it"
);
+
+ let user = crate::stm::user::UserSnapshot {
+ id: 0,
+ username: String::new(),
+ password_hash: String::new(),
+ status: iggy_common::UserStatus::Active,
+ created_at: IggyTimestamp::default(),
+ permissions: None,
+ options: ResourceOptions::new(),
+ };
+ let encoded = rmp_serde::to_vec(&user).unwrap();
+ assert_eq!(
+ rmp::decode::read_array_len(&mut encoded.as_slice()).unwrap(),
+ USER_FIELD_COUNT,
+ "UserSnapshot's field count changed; bump SNAPSHOT_FORMAT_VERSION with it"
+ );
}
#[test]
diff --git a/core/metadata/src/stm/stream.rs b/core/metadata/src/stm/stream.rs
index db0d4e02b..e1a7b3572 100644
--- a/core/metadata/src/stm/stream.rs
+++ b/core/metadata/src/stm/stream.rs
@@ -62,8 +62,8 @@ use iggy_binary_protocol::responses::topics::get_topic::PartitionResponse;
use iggy_binary_protocol::{WireIdentifier, WireName};
use iggy_common::wire_conversions::{resource_options_from_wire, resource_options_to_wire_split};
use iggy_common::{
- CompressionAlgorithm, IggyExpiry, IggyTimestamp, MaxTopicSize, PartitionStats, ResourceOptions,
- StreamStats, TopicCreateOptions, TopicStats,
+ CompressionAlgorithm, IggyByteSize, IggyExpiry, IggyTimestamp, MaxTopicSize, PartitionStats,
+ ResourceOptions, StreamStats, TopicCreateOptions, TopicRuntimeOptions, TopicStats,
};
use serde::{Deserialize, Serialize};
use server_common::sharding::IggyNamespace;
@@ -883,6 +883,26 @@ impl Streams {
})
}
+ /// A topic's explicitly set segment size, or `None` when the stream or
+ /// topic is unknown or the topic left the key to the node default.
+ ///
+ /// Update admission reads it for the floor `max_topic_size` has to clear:
+ /// `segment_size` is create-only, so the stored value is the one every one
+ /// of the topic's partitions is already rotating at.
+ #[must_use]
+ pub fn topic_segment_size(
+ &self,
+ stream_id: &WireIdentifier,
+ topic_id: &WireIdentifier,
+ ) -> Option<IggyByteSize> {
+ self.inner.read(|inner| {
+ let stream_slab = inner.resolve_stream_id(stream_id)?;
+ let topic_slab = inner.resolve_topic_id(stream_slab, topic_id)?;
+ let topic = inner.items.get(stream_slab)?.topics.get(topic_slab)?;
+ TopicRuntimeOptions::from_resource_options(&topic.options).segment_size
+ })
+ }
+
/// Build the `ConsumerGroupDetailsResponse` for a group (members + their
/// round-robin partition assignment). `partitions_count` is the topic's
/// total partition count. `None` if the stream/topic/group is unknown.
@@ -1661,6 +1681,42 @@ impl StateHandler for CreateTopicWithAssignmentsRequest {
}
}
+ // Both blocks were validated and resolved at admission, so apply reads
+ // them leniently: a key this build does not know is skipped, not
+ // refused. Refusing would make the verdict depend on the build, and a
+ // replica that predates a key would then be missing a topic its peers
+ // committed. Decoded before the revision bump and the stats-registry
+ // insert below, so a block this build cannot read at all leaves no
+ // orphaned registry entry and wakes no reconciler.
+ let explicit = TopicCreateOptions::parse_committed(&self.request.options);
+ let derived = TopicCreateOptions::parse_committed(&self.derived_options);
+ let (Ok(explicit_map), Ok(derived_map)) = (
+ resource_options_from_wire(&self.request.options, true),
+ resource_options_from_wire(&self.derived_options, false),
+ ) else {
+ return ApplyReply::err(CreateTopicResult::InvalidOptionValue);
+ };
+ let resolved = explicit.resolved_over(&derived);
+ let Ok(resolved_map) = resolved.to_option_map() else {
+ return ApplyReply::err(CreateTopicResult::InvalidOptionValue);
+ };
+
+ // Explicit wins on collision. `partitions_count` cannot appear here at
+ // all: it is a fixed field of the command, not an option key.
+ let mut options = derived_map;
+ options.extend(explicit_map);
+ // A client may send the literal server-default sentinel (0) for a typed
+ // key. The typed fields normalize it to absent and resolve the node
+ // default, so the map has to report the resolved value too: otherwise
+ // the merge above drops the derived entry and one `GetTopic` response
+ // carries the resolved value in the fixed field and 0 in the options
+ // block. Provenance is left alone - the client did name the key.
+ for (key, resolved_value) in resolved_map {
+ if let Some(option) = options.get_mut(&key) {
+ option.value = resolved_value.value;
+ }
+ }
+
// Past validation: this commit adds partitions, so bump the
// monotonic revision and stamp every new partition with it.
let new_revision = state.revision.wrapping_add(1);
@@ -1685,39 +1741,14 @@ impl StateHandler for CreateTopicWithAssignmentsRequest {
return ApplyReply::err(CreateTopicResult::StreamNotFound);
};
- // Both blocks were validated and resolved at admission, so apply reads
- // them leniently: a key this build does not know is skipped, not
- // refused. Refusing would make the verdict depend on the build, and a
- // replica that predates a key would then be missing a topic its peers
- // committed.
- let explicit = TopicCreateOptions::parse_committed(&self.request.options);
- let derived = TopicCreateOptions::parse_committed(&self.derived_options);
- let (Ok(explicit_map), Ok(derived_map)) = (
- resource_options_from_wire(&self.request.options, true),
- resource_options_from_wire(&self.derived_options, false),
- ) else {
- return ApplyReply::err(CreateTopicResult::InvalidOptionValue);
- };
- // Explicit wins on collision. `partitions_count` cannot appear here at
- // all: it is a fixed field of the command, not an option key.
- let mut options = derived_map;
- options.extend(explicit_map);
-
let topic = Topic {
id: topic_id,
name: name_arc.clone(),
created_at: timestamp,
- message_expiry: explicit
- .message_expiry
- .or(derived.message_expiry)
- .unwrap_or(IggyExpiry::ServerDefault),
- compression_algorithm: explicit
- .compression_algorithm
- .or(derived.compression_algorithm)
- .unwrap_or_default(),
- max_topic_size: explicit
+ message_expiry: resolved.message_expiry.unwrap_or(IggyExpiry::ServerDefault),
+ compression_algorithm: resolved.compression_algorithm.unwrap_or_default(),
+ max_topic_size: resolved
.max_topic_size
- .or(derived.max_topic_size)
.unwrap_or(MaxTopicSize::ServerDefault),
options,
stats: topic_stats,
@@ -2396,6 +2427,70 @@ mod tests {
);
}
+ /// A client may send the literal 0 that means "resolve the default". The
+ /// merge lets that explicit entry win over the derived one, so the map has
+ /// to be rewritten from the resolved value: otherwise the persisted map
+ /// reports 0 while the typed field reports the resolved default, and the
+ /// single `GetTopic` response contradicts itself.
+ #[test]
+ fn create_topic_replaces_a_client_sent_sentinel_with_the_resolved_default() {
+ use iggy_common::{HeaderKey, TopicCreateOptions, topic_option_keys};
+ use std::str::FromStr;
+
+ let mut inner = StreamsInner::new();
+ create_stream(&mut inner, "s");
+
+ let explicit = TopicCreateOptions {
+ max_topic_size: Some(MaxTopicSize::from(0u64)),
+ ..TopicCreateOptions::default()
+ };
+ let derived = TopicCreateOptions {
+ max_topic_size: Some(MaxTopicSize::from(10_000_000_000u64)),
+ ..TopicCreateOptions::default()
+ };
+ // `to_wire` normalizes the sentinel away, so the block is hand-built to
+ // carry the literal 0 the CLI's `--set max_topic_size=server_default`
+ // puts on the wire through the raw map.
+ let mut sentinel = bytes::BytesMut::new();
+ iggy_binary_protocol::primitives::user_headers::encode_user_headers(
+ &[(
+ 2,
+ topic_option_keys::MAX_TOPIC_SIZE.as_bytes(),
+ 12,
+ &0u64.to_le_bytes(),
+ )],
+ &mut sentinel,
+ );
+ let request = CreateTopicWithAssignmentsRequest {
+ request: WireCreateTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ partitions_count: 1,
+ name: WireName::new("t").unwrap(),
+ options: WireOptions::from_bytes(sentinel.freeze()).unwrap(),
+ },
+ derived_options: derived.to_wire().unwrap(),
+ partitions: vec![CreatedPartitionAssignment {
+ partition_id: 0,
+ consensus_group_id: 1,
+ }],
+ };
+ assert!(explicit.max_topic_size.is_some(), "sentinel was sent");
+
+ let reply = StateHandler::apply(&request, &mut inner, IggyTimestamp::from(1));
+ assert_eq!(reply.code, 0, "create must succeed");
+
+ let stream = inner.items.get(0).unwrap();
+ let (_, topic) = stream.topics.iter().next().unwrap();
+ assert_eq!(topic.max_topic_size, MaxTopicSize::from(10_000_000_000u64));
+ let size_key = HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE).unwrap();
+ let stored = topic.options.get(&size_key).unwrap();
+ assert_eq!(
+ stored.value.as_bytes(),
+ &10_000_000_000u64.to_le_bytes(),
+ "the map must carry the resolved value, not the sentinel"
+ );
+ }
+
#[test]
fn create_stream_options_survive_snapshot_roundtrip() {
use crate::stm::snapshot::FillSnapshot;
diff --git a/core/server/config.toml b/core/server/config.toml
index 8a5c93752..6b2c423f7 100644
--- a/core/server/config.toml
+++ b/core/server/config.toml
@@ -952,13 +952,19 @@ transfer_served_cache_bytes_max = "2176 MiB"
# Alloc ceiling for ONE received state-transfer artifact, per shard. The
# receiver holds it resident through verify, walk and staging write, and up to
-# four transfers run at once. MUST cover system.segment.size plus
-# message_bus.max_message_size (a segment may close one whole batch past its
-# cap): under that, a legal segment is refused, the whole manifest with it, and
-# the partition livelocks re-requesting it from every peer. Boot validates the
-# floor. Raising this above the floor for headroom also DIVIDES the serving
-# concurrency derived from transfer_served_cache_bytes_max above, so raise that
-# in step. Must be > 0 and <= "64 GiB".
+# four transfers run at once. MUST cover the largest segment any topic may be
+# created with (a fixed 1 GiB, since segment_size is a per-topic option now)
+# plus message_bus.max_message_size, because a segment may close one whole batch
+# past its cap: under that, a legal segment is refused, the whole manifest with
+# it, and the partition livelocks re-requesting it from every peer. Boot
+# validates the floor.
+#
+# The shipped value sits EXACTLY at that floor: 1024 MiB + the shipped 64 MiB
+# max_message_size. Raising max_message_size alone therefore refuses boot --
+# raise this one by the same amount in the same edit. Raising it above the floor
+# for headroom also DIVIDES the serving concurrency derived from
+# transfer_served_cache_bytes_max above, so raise that in step. Must be > 0 and
+# <= "64 GiB".
transfer_artifact_bytes_max = "1088 MiB"
# Message bus configuration.
diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs
index c29c382ed..475a8b36c 100644
--- a/core/server/src/bootstrap.rs
+++ b/core/server/src/bootstrap.rs
@@ -1127,30 +1127,6 @@ async fn shard_main(
// view-change superblock write records the real (checkpoint_op, checksum)
// instead of (0, 0). No-op on peer shards, which have no coordinator.
metadata.seed_checkpoint_ref(checkpoint_seed.0, checkpoint_seed.1);
- // Shard 0's copy resolves the `ServerDefault` sentinels (max topic size and
- // message expiry) at create admission; responses echo stored values verbatim.
- metadata.set_default_max_topic_size(iggy_common::DEFAULT_MAX_TOPIC_SIZE);
- metadata.set_default_message_expiry(iggy_common::DEFAULT_MESSAGE_EXPIRY);
- metadata.set_default_segment_size(iggy_common::DEFAULT_SEGMENT_SIZE);
- // Per-topic segment ceiling: the smaller of the global segment maximum
- // and the state-transfer artifact budget minus one bus frame (a segment
- // may close one whole batch past its cap; an artifact ceiling below that
- // refuses a legal segment and livelocks the partition's rejoin).
- metadata.set_max_topic_segment_size(
- configs::validators::SEGMENT_MAX_SIZE_BYTES.min(
- config
- .partition
- .transfer_artifact_bytes_max
- .as_bytes_u64()
- .saturating_sub(config.message_bus.max_message_size.as_bytes_u64()),
- ),
- );
- metadata.set_partition_runtime_defaults(
- iggy_common::DEFAULT_ENFORCE_FSYNC,
- iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE,
- iggy_common::DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE,
- iggy_common::DEFAULT_PREALLOCATE_SEGMENTS,
- );
// Keep the forced-checkpoint margin >= the configured prepare-queue
// depth: ops already pipelined while a checkpoint runs append into that
// margin (config validation keeps journal_slots >= 4x this).
diff --git a/core/server/src/dispatch.rs b/core/server/src/dispatch.rs
index 85fb87937..8ad3c146c 100644
--- a/core/server/src/dispatch.rs
+++ b/core/server/src/dispatch.rs
@@ -96,7 +96,8 @@ use iggy_binary_protocol::{
use iggy_common::{
IggyByteSize, IggyError, MaxTopicSize, PollingStrategy, SnapshotCompression,
SystemSnapshotType, TopicCreateOptions, UPDATABLE_STREAM_OPTION_KEYS,
- UPDATABLE_TOPIC_OPTION_KEYS, UPDATABLE_USER_OPTION_KEYS, validate_topic_segment_size,
+ UPDATABLE_TOPIC_OPTION_KEYS, UPDATABLE_USER_OPTION_KEYS, validate_preallocated_topic_bytes,
+ validate_topic_segment_size,
};
use journal::superblock::SuperblockStore;
use journal::{Journal, JournalHandle};
@@ -822,6 +823,16 @@ pub(crate) fn validate_topic_bounds(
segment_size_bytes: u64,
) -> Result<(), IggyError> {
validate_partitions_count(partitions_count)?;
+ validate_topic_size_floor(max_topic_size, segment_size_bytes)
+}
+
+/// A topic cap below one segment can never be enforced: the first segment
+/// already exceeds it. Split out of [`validate_topic_bounds`] because update
+/// admission checks the cap without a partitions count to check.
+pub(crate) fn validate_topic_size_floor(
+ max_topic_size: MaxTopicSize,
+ segment_size_bytes: u64,
+) -> Result<(), IggyError> {
if !matches!(max_topic_size, MaxTopicSize::ServerDefault)
&& max_topic_size.as_bytes_u64() < segment_size_bytes
{
@@ -1148,22 +1159,28 @@ async fn handle_client_request<B, MJ, S, SB>(
// `parse` doubles as the catalog gate: an unknown key or a
// malformed value denies typed here, pre-consensus.
let options = TopicCreateOptions::parse(&create_topic.options)?;
- let metadata = shard.plane.metadata();
if let Some(segment_size) = options.segment_size {
validate_topic_segment_size(
segment_size.as_bytes_u64(),
- metadata.max_topic_segment_size(),
+ iggy_common::MAX_TOPIC_SEGMENT_SIZE,
)?;
}
+ let segment_size = options.segment_size.map_or_else(
+ || iggy_common::DEFAULT_SEGMENT_SIZE,
+ |segment_size| segment_size.as_bytes_u64(),
+ );
+ if options
+ .preallocate_segments
+ .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS)
+ {
+ validate_preallocated_topic_bytes(segment_size, create_topic.partitions_count)?;
+ }
validate_topic_bounds(
create_topic.partitions_count,
options
.max_topic_size
.unwrap_or(MaxTopicSize::ServerDefault),
- options.segment_size.map_or_else(
- || metadata.default_segment_size(),
- |segment_size| segment_size.as_bytes_u64(),
- ),
+ segment_size,
)
}),
Operation::CreatePartitions => CreatePartitionsRequest::decode_from(request_body(&request))
@@ -1183,7 +1200,24 @@ async fn handle_client_request<B, MJ, S, SB>(
.map_err(|_| IggyError::InvalidCommand)
.and_then(|update_topic| {
validate_option_keys(&update_topic.options, UPDATABLE_TOPIC_OPTION_KEYS)?;
- TopicCreateOptions::parse(&update_topic.options).map(|_| ())
+ let options = TopicCreateOptions::parse(&update_topic.options)?;
+ let Some(max_topic_size) = options.max_topic_size else {
+ return Ok(());
+ };
+ // An update can lower the cap below one segment just as a
+ // create can, and the stored map would then report a size the
+ // topic can never enforce. The floor is this topic's own
+ // segment size, since that key is create-only.
+ let metadata = shard.plane.metadata();
+ let segment_size = metadata
+ .mux_stm
+ .streams()
+ .topic_segment_size(&update_topic.stream_id, &update_topic.topic_id)
+ .map_or_else(
+ || iggy_common::DEFAULT_SEGMENT_SIZE,
+ |segment_size| segment_size.as_bytes_u64(),
+ );
+ validate_topic_size_floor(max_topic_size, segment_size)
}),
Operation::UpdateStream => UpdateStreamRequest::decode_from(request_body(&request))
.map_err(|_| IggyError::InvalidCommand)
diff --git a/core/server/src/http/handlers.rs b/core/server/src/http/handlers.rs
index 0668ac32c..f63549491 100644
--- a/core/server/src/http/handlers.rs
+++ b/core/server/src/http/handlers.rs
@@ -108,7 +108,8 @@ use iggy_common::{
RawPersonalAccessToken, SendMessages, SendMessagesConfirmations, Stats, Stream, StreamDetails,
StreamUpdateOptions, TokenInfo, Topic, TopicCreateOptions, TopicDetails, TopicUpdateOptions,
UPDATABLE_STREAM_OPTION_KEYS, UPDATABLE_TOPIC_OPTION_KEYS, UPDATABLE_USER_OPTION_KEYS,
- UserInfo, UserInfoDetails, UserUpdateOptions, Validatable, validate_topic_segment_size,
+ UserInfo, UserInfoDetails, UserUpdateOptions, Validatable, validate_preallocated_topic_bytes,
+ validate_topic_segment_size,
};
use metadata::impls::metadata::StreamsFrontend;
use metadata::permissioner::Permissioner;
@@ -120,7 +121,7 @@ use shard::{PartitionRead, PartitionReadReply};
use crate::auth::{verify_login_credentials, verify_pat_credentials};
use crate::dispatch::{
resolve_consumer_offset_request, resolve_poll_request, validate_option_keys,
- validate_topic_bounds,
+ validate_topic_bounds, validate_topic_size_floor,
};
use crate::http::error::{
Consistency, ConsistencyQuery, CustomError, PartitionWriteError, ProduceAck, ProduceQuery,
@@ -865,21 +866,28 @@ pub(in crate::http) async fn create_topic(
// same typed pre-consensus checks as native fields; unknown keys deny
// here with the key name.
let parsed = TopicCreateOptions::parse(&wire_options).map_err(WriteError::Rejected)?;
- let metadata = state.shard.plane.metadata();
if let Some(segment_size) = parsed.segment_size {
validate_topic_segment_size(
segment_size.as_bytes_u64(),
- metadata.max_topic_segment_size(),
+ iggy_common::MAX_TOPIC_SEGMENT_SIZE,
)
.map_err(WriteError::Rejected)?;
}
+ let segment_size = parsed.segment_size.map_or_else(
+ || iggy_common::DEFAULT_SEGMENT_SIZE,
+ |segment_size| segment_size.as_bytes_u64(),
+ );
+ if parsed
+ .preallocate_segments
+ .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS)
+ {
+ validate_preallocated_topic_bytes(segment_size, command.partitions_count)
+ .map_err(WriteError::Rejected)?;
+ }
validate_topic_bounds(
command.partitions_count,
parsed.max_topic_size.unwrap_or(MaxTopicSize::ServerDefault),
- parsed.segment_size.map_or_else(
- || metadata.default_segment_size(),
- |segment_size| segment_size.as_bytes_u64(),
- ),
+ segment_size,
)
.map_err(WriteError::Rejected)?;
let request = CreateTopicRequest {
@@ -920,13 +928,32 @@ pub(in crate::http) async fn update_topic(
}
.to_wire()
.map_err(WriteError::Rejected)?;
- // Same pre-consensus gate the TCP ingress applies: a key this command may
- // not change is denied here by name rather than riding a log entry.
+ // Same pre-consensus gates the TCP ingress applies: a key this command may
+ // not change is denied here by name rather than riding a log entry, and a
+ // cap below one of this topic's segments is denied for the same reason it is
+ // on create.
validate_option_keys(&wire_options, UPDATABLE_TOPIC_OPTION_KEYS)
.map_err(WriteError::Rejected)?;
+ let stream_wire = identifier_to_wire(&stream_id).map_err(WriteError::Rejected)?;
+ let topic_wire = identifier_to_wire(&topic_id).map_err(WriteError::Rejected)?;
+ if let Some(max_topic_size) = TopicCreateOptions::parse(&wire_options)
+ .map_err(WriteError::Rejected)?
+ .max_topic_size
+ {
+ let metadata = state.shard.plane.metadata();
+ let segment_size = metadata
+ .mux_stm
+ .streams()
+ .topic_segment_size(&stream_wire, &topic_wire)
+ .map_or_else(
+ || iggy_common::DEFAULT_SEGMENT_SIZE,
+ |segment_size| segment_size.as_bytes_u64(),
+ );
+ validate_topic_size_floor(max_topic_size, segment_size).map_err(WriteError::Rejected)?;
+ }
let request = UpdateTopicRequest {
- stream_id: identifier_to_wire(&stream_id).map_err(WriteError::Rejected)?,
- topic_id: identifier_to_wire(&topic_id).map_err(WriteError::Rejected)?,
+ stream_id: stream_wire,
+ topic_id: topic_wire,
name: WireName::new(command.name)
.map_err(|_| WriteError::Rejected(IggyError::InvalidTopicName))?,
options: wire_options,
diff --git a/core/server/src/responses.rs b/core/server/src/responses.rs
index 1ef8acfcc..5431d677f 100644
--- a/core/server/src/responses.rs
+++ b/core/server/src/responses.rs
@@ -537,7 +537,7 @@ where
{
match code {
DESCRIBE_OPTIONS_CODE => Ok(NonReplicatedResponse::Bytes(
- build_describe_options_response(shard, body)?.to_bytes(),
+ build_describe_options_response(body)?.to_bytes(),
)),
GET_CLUSTER_METADATA_CODE => Ok(NonReplicatedResponse::Bytes(
build_cluster_metadata_response(roster, shard, client_ip).to_bytes(),
@@ -978,108 +978,120 @@ where
})
}
-/// Serve the option catalog for one resource scope. Static specs plus this
-/// node's resolved defaults; streams and users have no catalog keys yet, so
-/// their scopes return empty (every key is rejected at create until one
-/// lands).
-fn build_describe_options_response<B, MJ, S, SB>(
- shard: &Rc<ShellShard<B, MJ, S, SB>>,
- body: &[u8],
-) -> Result<DescribeOptionsResponse, IggyError>
-where
- B: ShellBus,
- MJ: JournalHandle + 'static,
- MJ::Target: Journal<MJ::Storage, Entry = Message<PrepareHeader>, Header = PrepareHeader>,
- S: 'static,
- SB: SuperblockStore + 'static,
-{
+/// Every key `CreateTopic` accepts, with the kind, default and bounds of each.
+///
+/// Split out of [`build_describe_options_response`] so the descriptions have room
+/// to state the bounds each value is checked against: this catalog is the only
+/// place an operator learns them.
+///
+/// Every default is a build constant: these knobs stopped being config-derived
+/// when the `[system.*]` keys became topic options, so the catalog reads them
+/// straight from `iggy_common`.
+fn topic_option_descriptors() -> Result<Vec<OptionDescriptor>, IggyError> {
+ Ok(vec![
+ OptionDescriptor {
+ key: WireName::new(topic_option_keys::COMPRESSION_ALGORITHM)
+ .map_err(|_| IggyError::InvalidFormat)?,
+ kind: HeaderKind::String.as_code(),
+ default_value: Bytes::from_static(b"none"),
+ description: "Compression algorithm (none, gzip)".to_string(),
+ },
+ OptionDescriptor {
+ key: WireName::new(topic_option_keys::MESSAGE_EXPIRY)
+ .map_err(|_| IggyError::InvalidFormat)?,
+ kind: HeaderKind::Uint64.as_code(),
+ default_value: Bytes::copy_from_slice(
+ &iggy_common::DEFAULT_MESSAGE_EXPIRY.to_le_bytes(),
+ ),
+ description: "Message expiry in microseconds, or a humantime string \
+ (e.g. 7 days)"
+ .to_string(),
+ },
+ OptionDescriptor {
+ key: WireName::new(topic_option_keys::MAX_TOPIC_SIZE)
+ .map_err(|_| IggyError::InvalidFormat)?,
+ kind: HeaderKind::Uint64.as_code(),
+ default_value: Bytes::copy_from_slice(
+ &iggy_common::DEFAULT_MAX_TOPIC_SIZE.to_le_bytes(),
+ ),
+ description: "Topic size cap in bytes, or a byte-size string (e.g. 1 GiB); \
+ must be at least the segment size"
+ .to_string(),
+ },
+ OptionDescriptor {
+ key: WireName::new(topic_option_keys::SEGMENT_SIZE)
+ .map_err(|_| IggyError::InvalidFormat)?,
+ kind: HeaderKind::Uint64.as_code(),
+ default_value: Bytes::copy_from_slice(&iggy_common::DEFAULT_SEGMENT_SIZE.to_le_bytes()),
+ description: format!(
+ "Segment size in bytes, or a byte-size string (e.g. 128 MiB); a 512-byte \
+ multiple within {}..={}",
+ iggy_common::MIN_TOPIC_SEGMENT_SIZE,
+ iggy_common::MAX_TOPIC_SEGMENT_SIZE
+ ),
+ },
+ OptionDescriptor {
+ key: WireName::new(topic_option_keys::ENFORCE_FSYNC)
+ .map_err(|_| IggyError::InvalidFormat)?,
+ kind: HeaderKind::Bool.as_code(),
+ default_value: Bytes::copy_from_slice(&[u8::from(iggy_common::DEFAULT_ENFORCE_FSYNC)]),
+ description: "Whether writes to this topic's partitions fsync".to_string(),
+ },
+ OptionDescriptor {
+ key: WireName::new(topic_option_keys::MESSAGES_REQUIRED_TO_SAVE)
+ .map_err(|_| IggyError::InvalidFormat)?,
+ kind: HeaderKind::Uint32.as_code(),
+ default_value: Bytes::copy_from_slice(
+ &iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE.to_le_bytes(),
+ ),
+ description: format!(
+ "Flush the journal once it holds this many messages; \
+ 1..={}. A threshold no segment can reach leaves committed \
+ messages in the journal, which a crash does not preserve",
+ iggy_common::MAX_MESSAGES_REQUIRED_TO_SAVE
+ ),
+ },
+ OptionDescriptor {
+ key: WireName::new(topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE)
+ .map_err(|_| IggyError::InvalidFormat)?,
+ kind: HeaderKind::Uint64.as_code(),
+ default_value: Bytes::copy_from_slice(
+ &iggy_common::DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE.to_le_bytes(),
+ ),
+ description: format!(
+ "Flush the journal once it holds this many bytes, or a byte-size \
+ string; whichever threshold trips first flushes. At most {}",
+ iggy_common::MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE
+ ),
+ },
+ OptionDescriptor {
+ key: WireName::new(topic_option_keys::PREALLOCATE_SEGMENTS)
+ .map_err(|_| IggyError::InvalidFormat)?,
+ kind: HeaderKind::Bool.as_code(),
+ default_value: Bytes::copy_from_slice(&[u8::from(
+ iggy_common::DEFAULT_PREALLOCATE_SEGMENTS,
+ )]),
+ description: format!(
+ "Reserve each segment's bytes up front where the filesystem supports \
+ it; pairs with segment_size. The reservation is real disk and runs \
+ inline on the owning shard, at every rotation and once per owned \
+ partition at boot, so segment_size * partitions_count is capped at \
+ {} bytes",
+ iggy_common::MAX_PREALLOCATED_TOPIC_BYTES
+ ),
+ },
+ ])
+}
+
+/// Serve the option catalog for one resource scope.
+///
+/// Streams and users have no catalog keys yet, so their scopes return empty
+/// (every key is rejected at create until one lands).
+fn build_describe_options_response(body: &[u8]) -> Result<DescribeOptionsResponse, IggyError> {
let request =
DescribeOptionsRequest::decode_from(body).map_err(|_| IggyError::InvalidCommand)?;
let entries = match request.scope {
- OPTIONS_SCOPE_TOPIC => {
- let metadata = shard.plane.metadata();
- let (enforce_fsync, messages_required_to_save, size_of_messages, preallocate) =
- metadata.partition_runtime_defaults();
- vec![
- OptionDescriptor {
- key: WireName::new(topic_option_keys::COMPRESSION_ALGORITHM)
- .map_err(|_| IggyError::InvalidFormat)?,
- kind: HeaderKind::String.as_code(),
- default_value: Bytes::from_static(b"none"),
- description: "Compression algorithm (none, gzip)".to_string(),
- },
- OptionDescriptor {
- key: WireName::new(topic_option_keys::MESSAGE_EXPIRY)
- .map_err(|_| IggyError::InvalidFormat)?,
- kind: HeaderKind::Uint64.as_code(),
- default_value: Bytes::copy_from_slice(
- &metadata.default_message_expiry().to_le_bytes(),
- ),
- description: "Message expiry in microseconds, or a humantime string \
- (e.g. 7 days)"
- .to_string(),
- },
- OptionDescriptor {
- key: WireName::new(topic_option_keys::MAX_TOPIC_SIZE)
- .map_err(|_| IggyError::InvalidFormat)?,
- kind: HeaderKind::Uint64.as_code(),
- default_value: Bytes::copy_from_slice(
- &metadata.default_max_topic_size().to_le_bytes(),
- ),
- description: "Topic size cap in bytes, or a byte-size string (e.g. 1 GiB); \
- must be at least the segment size"
- .to_string(),
- },
- OptionDescriptor {
- key: WireName::new(topic_option_keys::SEGMENT_SIZE)
- .map_err(|_| IggyError::InvalidFormat)?,
- kind: HeaderKind::Uint64.as_code(),
- default_value: Bytes::copy_from_slice(
- &metadata.default_segment_size().to_le_bytes(),
- ),
- description: format!(
- "Segment size in bytes, or a byte-size string (e.g. 128 MiB); a 512-byte \
- multiple within {}..={}",
- iggy_common::MIN_TOPIC_SEGMENT_SIZE,
- metadata.max_topic_segment_size()
- ),
- },
- OptionDescriptor {
- key: WireName::new(topic_option_keys::ENFORCE_FSYNC)
- .map_err(|_| IggyError::InvalidFormat)?,
- kind: HeaderKind::Bool.as_code(),
- default_value: Bytes::copy_from_slice(&[u8::from(enforce_fsync)]),
- description: "Whether writes to this topic's partitions fsync".to_string(),
- },
- OptionDescriptor {
- key: WireName::new(topic_option_keys::MESSAGES_REQUIRED_TO_SAVE)
- .map_err(|_| IggyError::InvalidFormat)?,
- kind: HeaderKind::Uint32.as_code(),
- default_value: Bytes::copy_from_slice(&messages_required_to_save.to_le_bytes()),
- description: "Flush the journal once it holds this many messages; must be \
- non-zero"
- .to_string(),
- },
- OptionDescriptor {
- key: WireName::new(topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE)
- .map_err(|_| IggyError::InvalidFormat)?,
- kind: HeaderKind::Uint64.as_code(),
- default_value: Bytes::copy_from_slice(&size_of_messages.to_le_bytes()),
- description: "Flush the journal once it holds this many bytes, or a \
- byte-size string; whichever threshold trips first flushes"
- .to_string(),
- },
- OptionDescriptor {
- key: WireName::new(topic_option_keys::PREALLOCATE_SEGMENTS)
- .map_err(|_| IggyError::InvalidFormat)?,
- kind: HeaderKind::Bool.as_code(),
- default_value: Bytes::copy_from_slice(&[u8::from(preallocate)]),
- description: "Reserve each segment's bytes up front where the filesystem \
- supports it; pairs with segment_size"
- .to_string(),
- },
- ]
- }
+ OPTIONS_SCOPE_TOPIC => topic_option_descriptors()?,
OPTIONS_SCOPE_STREAM | OPTIONS_SCOPE_USER => Vec::new(),
_ => return Err(IggyError::InvalidCommand),
};
diff --git a/core/server/src/segment_cleaner.rs b/core/server/src/segment_cleaner.rs
index 49bc338ff..45795988e 100644
--- a/core/server/src/segment_cleaner.rs
+++ b/core/server/src/segment_cleaner.rs
@@ -63,7 +63,6 @@ fn stage_owned_partitions(shard: &Rc<ServerShard>) {
let streams = shard.plane.metadata().mux_stm.streams();
// Resolved once per pass, not per partition: the node default is a `Cell`
// written at bootstrap and never after.
- let default_max_topic_size = shard.plane.metadata().default_max_topic_size();
for namespace in namespaces {
let Some((message_expiry, max_topic_size, partition_count)) =
streams.topic_retention_config(namespace.stream_id(), namespace.topic_id())
@@ -78,8 +77,11 @@ fn stage_owned_partitions(shard: &Rc<ServerShard>) {
message_expiry,
IggyExpiry::NeverExpire | IggyExpiry::ServerDefault
);
- let max_bytes =
- per_partition_size_budget(max_topic_size, default_max_topic_size, partition_count);
+ let max_bytes = per_partition_size_budget(
+ max_topic_size,
+ iggy_common::DEFAULT_MAX_TOPIC_SIZE,
+ partition_count,
+ );
if !has_expiry && max_bytes.is_none() {
continue;
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java
index 710089ce7..1cf7b8280 100644
--- a/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java
+++ b/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java
@@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
-import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicInteger;
@@ -142,7 +141,6 @@ public final class AsyncProducer {
CompressionAlgorithm.None,
BigInteger.ZERO,
BigInteger.ZERO,
- Optional.empty(),
TOPIC_NAME)
.thenAccept(created -> log.info("Topic created: {}", created.name()));
});
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java
index cfd81e26a..8f3899e45 100644
--- a/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java
+++ b/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java
@@ -35,8 +35,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
-import static java.util.Optional.empty;
-
public final class GettingStartedProducer {
private static final String STREAM_NAME = "sample-stream";
@@ -120,14 +118,7 @@ public final class GettingStartedProducer {
return;
}
client.topics()
- .createTopic(
- STREAM_ID,
- 1L,
- CompressionAlgorithm.None,
- BigInteger.ZERO,
- BigInteger.ZERO,
- empty(),
- TOPIC_NAME);
+ .createTopic(STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME);
log.info("Topic {} was created.", TOPIC_NAME);
}
}
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java
index 9e55bc27e..7ef9c3001 100644
--- a/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java
+++ b/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java
@@ -78,13 +78,7 @@ public final class MessageEnvelopeProducer {
} else {
client.topics()
.createTopic(
- STREAM_ID,
- 1L,
- CompressionAlgorithm.None,
- BigInteger.ZERO,
- BigInteger.ZERO,
- Optional.empty(),
- TOPIC_NAME);
+ STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME);
log.info("Topic {} was created.", TOPIC_NAME);
}
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java
index 1039a8bbf..e56abd592 100644
--- a/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java
+++ b/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java
@@ -79,13 +79,7 @@ public final class MessageHeadersProducer {
} else {
client.topics()
.createTopic(
- STREAM_ID,
- 1L,
- CompressionAlgorithm.None,
- BigInteger.ZERO,
- BigInteger.ZERO,
- Optional.empty(),
- TOPIC_NAME);
+ STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME);
log.info("Topic {} was created.", TOPIC_NAME);
}
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java
index eb87bac3e..d2f40942f 100644
--- a/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java
+++ b/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java
@@ -260,7 +260,6 @@ public final class MultiTenantProducer {
CompressionAlgorithm.None,
BigInteger.ZERO,
BigInteger.ZERO,
- Optional.empty(),
topicId.getName());
log.info("Created topic {} for stream {}", topicId.getName(), streamId.getName());
}
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java
index c7ffb5c4e..bd9278314 100644
--- a/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java
+++ b/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java
@@ -117,7 +117,6 @@ public final class SinkDataProducer {
CompressionAlgorithm.None,
java.math.BigInteger.ZERO,
java.math.BigInteger.ZERO,
- java.util.Optional.empty(),
topicId.getName());
log.info("Created topic {}.", topicId.getName());
});
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java b/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java
index b8c9929f1..b2523f743 100644
--- a/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java
+++ b/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java
@@ -129,13 +129,7 @@ public final class StreamBasic {
} else {
client.topics()
.createTopic(
- STREAM_ID,
- 1L,
- CompressionAlgorithm.None,
- BigInteger.ZERO,
- BigInteger.ZERO,
- Optional.empty(),
- TOPIC_NAME);
+ STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME);
log.info("Topic {} was created.", TOPIC_NAME);
}
}
diff --git a/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java
index 4661bacf2..043970d72 100644
--- a/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java
+++ b/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java
@@ -35,8 +35,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
-import static java.util.Optional.empty;
-
/**
* TCP/TLS Producer Example
*
@@ -139,14 +137,7 @@ public final class TcpTlsProducer {
return;
}
client.topics()
- .createTopic(
- STREAM_ID,
- 1L,
- CompressionAlgorithm.None,
- BigInteger.ZERO,
- BigInteger.ZERO,
- empty(),
- TOPIC_NAME);
+ .createTopic(STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME);
log.info("Topic {} was created.", TOPIC_NAME);
}
}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs b/foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs
new file mode 100644
index 000000000..4dcf1cfb6
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.Iggy.Contracts;
+
+/// <summary>
+/// Resource whose option catalog the server describes.
+/// </summary>
+public enum OptionsScope
+{
+ /// <summary>
+ /// Keys a CreateTopic accepts.
+ /// </summary>
+ Topic = 1,
+
+ /// <summary>
+ /// Keys a CreateStream accepts; none yet.
+ /// </summary>
+ Stream = 2,
+
+ /// <summary>
+ /// Keys a CreateUser accepts; none yet.
+ /// </summary>
+ User = 3
+}
+
+/// <summary>
+/// One entry of a resource's option catalog, as served by DescribeOptions.
+/// </summary>
+public class OptionSpec
+{
+ /// <summary>
+ /// Option key the create command accepts.
+ /// </summary>
+ public required string Key { get; set; }
+
+ /// <summary>
+ /// Header kind code the server encodes this key's default under, and the
+ /// kind a client should prefer. Not a guarantee about storage: an
+ /// explicitly set entry keeps the kind the client sent.
+ /// </summary>
+ public required byte Kind { get; set; }
+
+ /// <summary>
+ /// The key's default in <see cref="Kind" />'s encoding; empty when the key
+ /// has no default.
+ /// </summary>
+ public required byte[] DefaultValue { get; set; } = [];
+
+ /// <summary>
+ /// What the option does, including any bounds it is checked against.
+ /// </summary>
+ public required string Description { get; set; }
+}
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs
index aab60f1c6..4b505eb10 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs
@@ -72,6 +72,24 @@ public interface IIggySystem
/// <returns>A task that represents the asynchronous operation and returns server statistics, or null if unavailable.</returns>
Task<StatsResponse?> GetStatsAsync(CancellationToken token = default);
+ /// <summary>
+ /// Retrieves the option catalog for a resource scope: every key its create
+ /// command accepts, with the kind, default and bounds of each.
+ /// </summary>
+ /// <remarks>
+ /// This is the discovery surface for options. A key outside the catalog is
+ /// refused at create, and binary transports carry only the error code, so a
+ /// client has no other way to learn which keys this server knows. Available
+ /// only for TCP.
+ /// </remarks>
+ /// <param name="scope">The resource whose catalog to describe.</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>
+ /// A task that represents the asynchronous operation and returns the catalog
+ /// entries, empty for a scope with no keys yet.
+ /// </returns>
+ Task<IReadOnlyList<OptionSpec>> DescribeOptionsAsync(OptionsScope scope, CancellationToken token = default);
+
/// <summary>
/// Retrieves cluster metadata including node information and connection information.
/// </summary>
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
index 1c0b10771..3e8ede596 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
@@ -469,6 +469,16 @@ public class HttpMessageStream : IIggyClient
throw new FeatureUnavailableException();
}
+ /// <inheritdoc />
+ public Task<IReadOnlyList<OptionSpec>> DescribeOptionsAsync(OptionsScope scope,
+ CancellationToken token = default)
+ {
+ // The REST catalog renders `kind` as a name and `default_value` as a JSON
+ // array, a second shape for the same contract. Until the two agree, the
+ // binary transports are the ones that carry it.
+ throw new FeatureUnavailableException();
+ }
+
/// <inheritdoc />
public async Task<StatsResponse?> GetStatsAsync(CancellationToken token = default)
{
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
index ece3e9c21..f772f9ac6 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
@@ -577,6 +577,24 @@ public sealed partial class TcpMessageStream : IIggyClient
return BinaryMapper.MapStats(responseBuffer.Memory.Span);
}
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<OptionSpec>> DescribeOptionsAsync(OptionsScope scope,
+ CancellationToken token = default)
+ {
+ var message = new[] { (byte)scope };
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DESCRIBE_OPTIONS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapOptionSpecs(responseBuffer.Memory.Span);
+ }
+
/// <inheritdoc />
public async Task<ClusterMetadata?> GetClusterMetadataAsync(CancellationToken token = default)
{
diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
index 914c1af9a..af2da9fb8 100644
--- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
+++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
@@ -1231,6 +1231,56 @@ internal static class BinaryMapper
}, 13 + name.Length);
}
+ internal static IReadOnlyList<OptionSpec> MapOptionSpecs(ReadOnlySpan<byte> payload)
+ {
+ var count = BinaryPrimitives.ReadUInt32LittleEndian(payload[..4]);
+ var position = 4;
+ var specs = new List<OptionSpec>();
+ for (var i = 0; i < count; i++)
+ {
+ var keyLength = payload[position];
+ position += 1;
+ EnsureFits(payload, position, keyLength, "option key");
+ var key = Encoding.UTF8.GetString(payload[position..(position + keyLength)]);
+ position += keyLength;
+
+ var kind = payload[position];
+ position += 1;
+
+ var defaultLength = (int)BinaryPrimitives.ReadUInt32LittleEndian(payload[position..(position + 4)]);
+ position += 4;
+ EnsureFits(payload, position, defaultLength, "option default value");
+ var defaultValue = payload[position..(position + defaultLength)].ToArray();
+ position += defaultLength;
+
+ var descriptionLength = (int)BinaryPrimitives.ReadUInt32LittleEndian(payload[position..(position + 4)]);
+ position += 4;
+ EnsureFits(payload, position, descriptionLength, "option description");
+ var description = Encoding.UTF8.GetString(payload[position..(position + descriptionLength)]);
+ position += descriptionLength;
+
+ specs.Add(new OptionSpec
+ {
+ Key = key,
+ Kind = kind,
+ DefaultValue = defaultValue,
+ Description = description
+ });
+ }
+
+ return specs;
+ }
+
+ private static void EnsureFits(ReadOnlySpan<byte> payload, int position, int length, string what)
+ {
+ if (position + length > payload.Length)
+ {
+ throw new InvalidOperationException(
+ $"Malformed DescribeOptions response: {what} of {length} bytes at offset {position} " +
+ $"overruns the {payload.Length}-byte payload");
+ }
+ }
+
internal static ClusterMetadata MapClusterMetadata(ReadOnlySpan<byte> payload)
{
var nameLength = BinaryPrimitives.ReadUInt32LittleEndian(payload[..4]);
diff --git a/foreign/go/binary_serialization/binary_response_deserializer.go b/foreign/go/binary_serialization/binary_response_deserializer.go
index 9b8edda62..60481a64b 100644
--- a/foreign/go/binary_serialization/binary_response_deserializer.go
+++ b/foreign/go/binary_serialization/binary_response_deserializer.go
@@ -58,8 +58,10 @@ func DeserializeStream(payload []byte) (*iggcon.StreamDetails, error) {
return nil, err
}
// Count-driven: a topic element carries variable-length options blocks,
- // so "consume until the buffer ends" no longer delimits it.
- topics := make([]iggcon.Topic, 0, stream.TopicsCount)
+ // so "consume until the buffer ends" no longer delimits it. The declared
+ // count is an unvalidated wire u32, so the allocation hint is capped by
+ // what the remaining body could possibly hold.
+ topics := make([]iggcon.Topic, 0, boundedCapacity(stream.TopicsCount, len(payload)-pos, topicMinimumSize))
for i := uint32(0); i < stream.TopicsCount; i++ {
topic, readBytes, err := DeserializeToTopic(payload, pos)
if err != nil {
@@ -264,11 +266,7 @@ func DeserializeTopics(payload []byte) ([]iggcon.Topic, error) {
// The declared count is server-controlled; the allocation hint is capped
// by what the body could possibly hold.
- maxTopics := (len(payload) - position) / topicMinimumSize
- if int(topicsCount) < maxTopics {
- maxTopics = int(topicsCount)
- }
- topics := make([]iggcon.Topic, 0, maxTopics)
+ topics := make([]iggcon.Topic, 0, boundedCapacity(topicsCount, len(payload)-position, topicMinimumSize))
for i := uint32(0); i < topicsCount; i++ {
topic, readBytes, err := DeserializeToTopic(payload, position)
if err != nil {
@@ -306,9 +304,26 @@ func DeserializeTopic(payload []byte) (*iggcon.TopicDetails, error) {
// max_topic_size + size_bytes + messages_count + name_len.
const topicFixedSize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1 // 50 bytes
-// topicMinimumSize is the smallest possible topic element: fixed fields plus
-// the two u32 options-length prefixes (explicit and derived), both zero.
-const topicMinimumSize = topicFixedSize + 4 + 4
+// topicMinimumSize is the smallest possible topic element: fixed fields, a
+// one-character name (the server rejects an empty one), and the two u32
+// options-length prefixes (explicit and derived), both zero.
+const topicMinimumSize = topicFixedSize + 1 + 4 + 4
+
+// boundedCapacity caps a wire-declared element count by what the remaining
+// bytes could possibly hold.
+//
+// The count is an unvalidated u32: at max it asks for a multi-hundred-gigabyte
+// reservation, and a Go allocation failure cannot be recovered from.
+func boundedCapacity(declared uint32, remaining int, minItemSize int) int {
+ if remaining <= 0 || minItemSize <= 0 {
+ return 0
+ }
+ capacity := remaining / minItemSize
+ if uint64(declared) < uint64(capacity) {
+ return int(declared)
+ }
+ return capacity
+}
func DeserializeToTopic(payload []byte, position int) (iggcon.Topic, int, error) {
remaining := len(payload) - position
diff --git a/foreign/go/binary_serialization/binary_response_deserializer_test.go b/foreign/go/binary_serialization/binary_response_deserializer_test.go
index b94dab665..010a64ef1 100644
--- a/foreign/go/binary_serialization/binary_response_deserializer_test.go
+++ b/foreign/go/binary_serialization/binary_response_deserializer_test.go
@@ -282,3 +282,35 @@ func TestDeserializeStreams_MaxLengthName(t *testing.T) {
t.Errorf("Name length = %d, want 255", len(streams[0].Name))
}
}
+
+func TestBoundedCapacity_CapsDeclaredCount(t *testing.T) {
+ tests := []struct {
+ name string
+ declared uint32
+ remaining int
+ minItemSize int
+ want int
+ }{
+ {"declared below what fits", 2, 10 * topicMinimumSize, topicMinimumSize, 2},
+ {"declared above what fits", 1 << 31, 3 * topicMinimumSize, topicMinimumSize, 3},
+ {"nothing left to read", 1 << 31, 0, topicMinimumSize, 0},
+ {"zero item size", 1 << 31, 1024, 0, 0},
+ }
+ for _, test := range tests {
+ got := boundedCapacity(test.declared, test.remaining, test.minItemSize)
+ if got != test.want {
+ t.Errorf("%s: boundedCapacity(%d, %d, %d) = %d, want %d",
+ test.name, test.declared, test.remaining, test.minItemSize, got, test.want)
+ }
+ }
+}
+
+func TestDeserializeStream_LyingTopicsCountDoesNotPreallocate(t *testing.T) {
+ // A u32 topics count with no topics behind it: the decode must fail on the
+ // missing element rather than reserve a slice for the declared count.
+ payload := encodeStream(1, 100, 1<<31, 0, 0, "s")
+
+ if _, err := DeserializeStream(payload); err == nil {
+ t.Fatal("expected error for topics count with no elements, got nil")
+ }
+}
diff --git a/foreign/go/contracts/compression_algorithm.go b/foreign/go/contracts/compression_algorithm.go
index e91da788b..76dc6c7f8 100644
--- a/foreign/go/contracts/compression_algorithm.go
+++ b/foreign/go/contracts/compression_algorithm.go
@@ -17,18 +17,44 @@
package iggcon
+import "fmt"
+
type CompressionAlgorithm uint8
const (
- CompressionAlgorithmNone CompressionAlgorithm = 1
- CompressionAlgorithmGzip CompressionAlgorithm = 2
+ // CompressionAlgorithmDefault is the zero value of the field: the server
+ // resolves the algorithm from its own configuration.
+ CompressionAlgorithmDefault CompressionAlgorithm = 0
+ CompressionAlgorithmNone CompressionAlgorithm = 1
+ CompressionAlgorithmGzip CompressionAlgorithm = 2
)
+// OptionValue renders the algorithm for the server's `compression_algorithm`
+// topic option, or "" when the option should be omitted so the server resolves
+// its own default.
+//
+// An unrecognized code is an error rather than a fall back to the default: the
+// topic would otherwise be created with the server's algorithm and no
+// diagnostic, where the old fixed-field wire format put the raw byte on the
+// wire and let the server reject it.
+func (c CompressionAlgorithm) OptionValue() (string, error) {
+ switch c {
+ case CompressionAlgorithmDefault, CompressionAlgorithmNone:
+ return "", nil
+ case CompressionAlgorithmGzip:
+ return "gzip", nil
+ default:
+ return "", fmt.Errorf("unknown compression algorithm code %d", uint8(c))
+ }
+}
+
func (c CompressionAlgorithm) String() string {
switch c {
+ case CompressionAlgorithmDefault, CompressionAlgorithmNone:
+ return "none"
case CompressionAlgorithmGzip:
return "gzip"
default:
- return "none"
+ return fmt.Sprintf("unknown(%d)", uint8(c))
}
}
diff --git a/foreign/go/contracts/compression_algorithm_test.go b/foreign/go/contracts/compression_algorithm_test.go
new file mode 100644
index 000000000..693fb0f1e
--- /dev/null
+++ b/foreign/go/contracts/compression_algorithm_test.go
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iggcon
+
+import (
+ "bytes"
+ "encoding/binary"
+ "testing"
+)
+
+func TestCompressionAlgorithm_OptionValue(t *testing.T) {
+ tests := []struct {
+ algorithm CompressionAlgorithm
+ want string
+ wantErr bool
+ }{
+ {CompressionAlgorithmDefault, "", false},
+ {CompressionAlgorithmNone, "", false},
+ {CompressionAlgorithmGzip, "gzip", false},
+ {CompressionAlgorithm(200), "", true},
+ }
+ for _, test := range tests {
+ got, err := test.algorithm.OptionValue()
+ if (err != nil) != test.wantErr {
+ t.Errorf("CompressionAlgorithm(%d).OptionValue() error = %v, wantErr %v",
+ uint8(test.algorithm), err, test.wantErr)
+ }
+ if got != test.want {
+ t.Errorf("CompressionAlgorithm(%d).OptionValue() = %q, want %q",
+ uint8(test.algorithm), got, test.want)
+ }
+ }
+}
+
+func TestCompressionAlgorithm_StringNamesUnknownCode(t *testing.T) {
+ if got := CompressionAlgorithm(200).String(); got != "unknown(200)" {
+ t.Errorf("String() = %q, want %q", got, "unknown(200)")
+ }
+}
+
+// goldenOptionsBlock is the cross-SDK golden vector for an options block.
+//
+// Rust pins the identical bytes in core/binary_protocol/src/primitives/options.rs,
+// as do the Node and Java SDKs. Round-tripping through this SDK's own decoder
+// proves nothing about interoperability; these bytes are the contract.
+var goldenOptionsBlock = []byte{
+ 2, 13, 0, 0, 0,
+ 'e', 'n', 'f', 'o', 'r', 'c', 'e', '_', 'f', 's', 'y', 'n', 'c',
+ 3, 1, 0, 0, 0, 1,
+ 2, 12, 0, 0, 0,
+ 's', 'e', 'g', 'm', 'e', 'n', 't', '_', 's', 'i', 'z', 'e',
+ 12, 8, 0, 0, 0,
+ 0, 0, 0, 64, 0, 0, 0, 0,
+}
+
+func TestGetHeadersBytes_MatchesTheCrossSdkGoldenVector(t *testing.T) {
+ segmentSize := make([]byte, 8)
+ binary.LittleEndian.PutUint64(segmentSize, 1073741824)
+ entries := []HeaderEntry{
+ {
+ Key: HeaderKey{Kind: String, Value: []byte("enforce_fsync")},
+ Value: HeaderValue{Kind: Bool, Value: []byte{1}},
+ },
+ {
+ Key: HeaderKey{Kind: String, Value: []byte("segment_size")},
+ Value: HeaderValue{Kind: Uint64, Value: segmentSize},
+ },
+ }
+
+ got := GetHeadersBytes(entries)
+
+ if !bytes.Equal(got, goldenOptionsBlock) {
+ t.Errorf("GetHeadersBytes() = %v, want %v", got, goldenOptionsBlock)
+ }
+}
diff --git a/foreign/go/internal/command/topic.go b/foreign/go/internal/command/topic.go
index 45140c9b7..46fa20c19 100644
--- a/foreign/go/internal/command/topic.go
+++ b/foreign/go/internal/command/topic.go
@@ -48,7 +48,11 @@ func (t *CreateTopic) MarshalBinary() ([]byte, error) {
return nil, err
}
nameBytes := []byte(t.Name)
- optionsBytes := iggcon.GetHeadersBytes(t.options())
+ options, err := t.options()
+ if err != nil {
+ return nil, err
+ }
+ optionsBytes := iggcon.GetHeadersBytes(options)
bytes := make([]byte, 0, len(streamIdBytes)+4+1+len(nameBytes)+len(optionsBytes))
bytes = append(bytes, streamIdBytes...)
@@ -64,10 +68,14 @@ func (t *CreateTopic) MarshalBinary() ([]byte, error) {
// option: it fills the command's own fixed field. Keys carrying the
// server-default sentinel (expiry 0, size 0, compression none) are omitted so
// the server derives them and returns them as derived entries.
-func (t *CreateTopic) options() []iggcon.HeaderEntry {
+func (t *CreateTopic) options() ([]iggcon.HeaderEntry, error) {
var options []iggcon.HeaderEntry
- if name := t.CompressionAlgorithm.String(); name != "none" {
- options = append(options, stringOption(topicOptionCompressionAlgorithm, name))
+ compression, err := t.CompressionAlgorithm.OptionValue()
+ if err != nil {
+ return nil, err
+ }
+ if compression != "" {
+ options = append(options, stringOption(topicOptionCompressionAlgorithm, compression))
}
if t.MessageExpiry != 0 {
options = append(options, uint64Option(topicOptionMessageExpiry, uint64(t.MessageExpiry)))
@@ -75,7 +83,7 @@ func (t *CreateTopic) options() []iggcon.HeaderEntry {
if t.MaxTopicSize != 0 {
options = append(options, uint64Option(topicOptionMaxTopicSize, t.MaxTopicSize))
}
- return options
+ return options, nil
}
func uint64Option(key string, value uint64) iggcon.HeaderEntry {
@@ -147,15 +155,18 @@ func (u *UpdateTopic) Code() Code {
return UpdateTopicCode
}
-// options builds the trailing options block. Only keys an update may change
-// are allowed; the server rejects the create-time knobs by name.
-// options builds the trailing options block. A zero value means the caller did
-// not set the key, so it is omitted and the server leaves the topic's current
-// value alone.
-func (u *UpdateTopic) options() []iggcon.HeaderEntry {
+// options builds the trailing options block. Only keys an update may change are
+// allowed; the server rejects the create-time knobs by name. A zero value means
+// the caller did not set the key, so it is omitted and the server leaves the
+// topic's current value alone.
+func (u *UpdateTopic) options() ([]iggcon.HeaderEntry, error) {
var options []iggcon.HeaderEntry
- if name := u.CompressionAlgorithm.String(); name != "none" {
- options = append(options, stringOption(topicOptionCompressionAlgorithm, name))
+ compression, err := u.CompressionAlgorithm.OptionValue()
+ if err != nil {
+ return nil, err
+ }
+ if compression != "" {
+ options = append(options, stringOption(topicOptionCompressionAlgorithm, compression))
}
if u.MessageExpiry != 0 {
options = append(options, uint64Option(topicOptionMessageExpiry, uint64(u.MessageExpiry)))
@@ -163,7 +174,7 @@ func (u *UpdateTopic) options() []iggcon.HeaderEntry {
if u.MaxTopicSize != 0 {
options = append(options, uint64Option(topicOptionMaxTopicSize, u.MaxTopicSize))
}
- return options
+ return options, nil
}
func (u *UpdateTopic) MarshalBinary() ([]byte, error) {
@@ -176,7 +187,11 @@ func (u *UpdateTopic) MarshalBinary() ([]byte, error) {
return nil, err
}
- optionsBytes := iggcon.GetHeadersBytes(u.options())
+ options, err := u.options()
+ if err != nil {
+ return nil, err
+ }
+ optionsBytes := iggcon.GetHeadersBytes(options)
buffer := make([]byte, 1+len(streamIdBytes)+len(topicIdBytes)+len(u.Name)+len(optionsBytes))
offset := 0
diff --git a/foreign/go/internal/command/update_user.go b/foreign/go/internal/command/update_user.go
index ba5454843..452b4db77 100644
--- a/foreign/go/internal/command/update_user.go
+++ b/foreign/go/internal/command/update_user.go
@@ -38,24 +38,24 @@ func (u *UpdateUser) MarshalBinary() ([]byte, error) {
if err != nil {
return nil, err
}
- length := len(userIdBytes)
-
- if u.Username == nil {
- u.Username = new(string)
+ username := ""
+ if u.Username != nil {
+ username = *u.Username
}
- username := *u.Username
-
+ // Both presence flags are always written; only the payload behind a flag is
+ // conditional. The options block decodes to the end of the payload, so a
+ // byte of slack here is a stray zero-kind entry the server rejects.
+ length := len(userIdBytes) + 2
if len(username) != 0 {
- length += 2 + len(username)
+ length += 1 + len(username)
}
-
if u.Status != nil {
- length += 2
+ length++
}
optionsBytes := iggcon.GetHeadersBytes(u.Options)
- bytes := make([]byte, length+1+len(optionsBytes))
+ bytes := make([]byte, length+len(optionsBytes))
position := 0
copy(bytes[position:position+len(userIdBytes)], userIdBytes)
diff --git a/foreign/go/internal/command/update_user_test.go b/foreign/go/internal/command/update_user_test.go
new file mode 100644
index 000000000..5cead9644
--- /dev/null
+++ b/foreign/go/internal/command/update_user_test.go
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package command
+
+import (
+ "testing"
+
+ iggcon "github.com/apache/iggy/foreign/go/contracts"
+)
+
+// The options block decodes to the end of the payload, so any slack the sizing
+// leaves behind reaches the server as an entry with kind 0, which it rejects.
+func TestUpdateUser_MarshalBinaryLeavesNoSlack(t *testing.T) {
+ username := "user"
+ status := iggcon.Active
+ userID, err := iggcon.NewIdentifier(uint32(7))
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ userIDBytes, err := userID.MarshalBinary()
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+
+ tests := []struct {
+ name string
+ username *string
+ status *iggcon.UserStatus
+ want int
+ }{
+ {"both set", &username, &status, len(userIDBytes) + 2 + len(username) + 2},
+ {"username only", &username, nil, len(userIDBytes) + 2 + len(username) + 1},
+ {"status only", nil, &status, len(userIDBytes) + 1 + 2},
+ {"neither set", nil, nil, len(userIDBytes) + 2},
+ }
+ for _, test := range tests {
+ command := UpdateUser{UserID: userID, Username: test.username, Status: test.status}
+ payload, err := command.MarshalBinary()
+ if err != nil {
+ t.Fatalf("%s: unexpected error: %v", test.name, err)
+ }
+ if len(payload) != test.want {
+ t.Errorf("%s: payload length = %d, want %d", test.name, len(payload), test.want)
+ }
+ }
+}
+
+func TestUpdateUser_MarshalBinaryDoesNotMutateUsername(t *testing.T) {
+ userID, err := iggcon.NewIdentifier(uint32(1))
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ command := UpdateUser{UserID: userID}
+
+ if _, err = command.MarshalBinary(); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if command.Username != nil {
+ t.Errorf("Username = %v, want nil", command.Username)
+ }
+}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java
index 6cd218806..8bd03c127 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java
@@ -31,6 +31,7 @@ import org.apache.iggy.consumergroup.ConsumerGroupAssignment;
import org.apache.iggy.consumergroup.ConsumerGroupDetails;
import org.apache.iggy.consumergroup.ConsumerGroupMember;
import org.apache.iggy.consumeroffset.ConsumerOffsetInfo;
+import org.apache.iggy.exception.IggyInvalidArgumentException;
import org.apache.iggy.exception.IggyMalformedResponseException;
import org.apache.iggy.message.BytesMessageId;
import org.apache.iggy.message.HeaderKey;
@@ -67,6 +68,7 @@ import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -80,8 +82,9 @@ public final class BytesDeserializer {
private static final int CONSUMER_GROUP_ASSIGNMENT_ENTRY_BYTES = Integer.BYTES;
private static final int SEND_CONFIRMATION_BYTES = 3 * Integer.BYTES + Long.BYTES;
private static final int MIN_CLUSTER_NODE_BYTES = 18;
- // 50-byte fixed part + empty name + two u32 options-block length prefixes.
- private static final int MIN_TOPIC_BYTES = 58;
+ // 50-byte fixed part + a one-character name (the server rejects an empty
+ // one) + two u32 options-block length prefixes.
+ private static final int MIN_TOPIC_BYTES = 59;
private BytesDeserializer() {}
@@ -93,9 +96,9 @@ public final class BytesDeserializer {
var messagesCount = readU64AsBigInteger(response);
var nameLength = response.readByte();
var name = response.readCharSequence(nameLength, StandardCharsets.UTF_8).toString();
- skipOptionsBlock(response, "stream options");
+ var options = readOptionsBlock(response, "stream options");
- return new StreamBase(streamId, createdAt, name, size.toString(), messagesCount, topicsCount);
+ return new StreamBase(streamId, createdAt, name, size.toString(), messagesCount, topicsCount, options);
}
public static StreamDetails readStreamDetails(ByteBuf response) {
@@ -145,8 +148,8 @@ public final class BytesDeserializer {
var messagesCount = readU64AsBigInteger(response);
var nameLength = response.readByte();
var name = response.readCharSequence(nameLength, StandardCharsets.UTF_8).toString();
- skipOptionsBlock(response, "topic explicit options");
- skipOptionsBlock(response, "topic derived options");
+ var options = readOptionsBlock(response, "topic explicit options");
+ var derivedOptions = readOptionsBlock(response, "topic derived options");
return new Topic(
topicId,
createdAt,
@@ -156,7 +159,9 @@ public final class BytesDeserializer {
CompressionAlgorithm.fromCode(compressionAlgorithmCode),
maxTopicSize,
messagesCount,
- partitionsCount);
+ partitionsCount,
+ options,
+ derivedOptions);
}
public static ConsumerGroupDetails readConsumerGroupDetails(ByteBuf response) {
@@ -520,7 +525,9 @@ public final class BytesDeserializer {
var usernameLength = response.readByte();
var username = response.readCharSequence(usernameLength, StandardCharsets.UTF_8)
.toString();
- skipOptionsBlock(response, "user options");
+ // Validated and dropped: users have no catalog keys yet, so the server
+ // refuses every one and the block is always empty.
+ readOptionsBlock(response, "user options");
return new UserInfo(userId, createdAt, status, username);
}
@@ -539,7 +546,14 @@ public final class BytesDeserializer {
return new PersonalAccessTokenInfo(name, expiryOptional);
}
- private static void skipOptionsBlock(ByteBuf buffer, String field) {
+ /**
+ * Reads a {@code u32}-length-prefixed options block into its entries.
+ *
+ * <p>Keys are always UTF-8 strings; values keep the kind the server sent, so a kind this
+ * build has no name for is dropped rather than failing the whole response - the wire
+ * contract forwards unknown value kinds so a mixed-version cluster can round-trip them.
+ */
+ private static Map<String, HeaderValue> readOptionsBlock(ByteBuf buffer, String field) {
if (buffer.readableBytes() < Integer.BYTES) {
throw new IggyMalformedResponseException("Missing length prefix for " + field);
}
@@ -548,7 +562,40 @@ public final class BytesDeserializer {
throw new IggyMalformedResponseException("Length " + optionsLength + " for " + field
+ " exceeds remaining payload of " + buffer.readableBytes() + " bytes");
}
- buffer.skipBytes(toInt(optionsLength));
+ if (optionsLength == 0) {
+ return Map.of();
+ }
+ ByteBuf options = buffer.readSlice(toInt(optionsLength));
+ Map<String, HeaderValue> entries = new LinkedHashMap<>();
+ while (options.isReadable()) {
+ if (options.readableBytes() < 1 + Integer.BYTES) {
+ throw new IggyMalformedResponseException("Truncated key header in " + field);
+ }
+ options.readUnsignedByte();
+ var keyLength = options.readUnsignedIntLE();
+ if (keyLength > options.readableBytes()) {
+ throw new IggyMalformedResponseException("Truncated key in " + field);
+ }
+ var key = options.readCharSequence(toInt(keyLength), StandardCharsets.UTF_8)
+ .toString();
+
+ if (options.readableBytes() < 1 + Integer.BYTES) {
+ throw new IggyMalformedResponseException("Truncated value header for '" + key + "' in " + field);
+ }
+ var valueKindCode = options.readUnsignedByte();
+ var valueLength = options.readUnsignedIntLE();
+ if (valueLength > options.readableBytes()) {
+ throw new IggyMalformedResponseException("Truncated value for '" + key + "' in " + field);
+ }
+ byte[] value = newByteArray(valueLength);
+ options.readBytes(value);
+ try {
+ entries.put(key, new HeaderValue(HeaderKind.fromCode(valueKindCode), value));
+ } catch (IggyInvalidArgumentException unknownKind) {
+ // A newer peer's value kind: keep every other entry readable.
+ }
+ }
+ return entries;
}
private static String readU32PrefixedString(ByteBuf buffer, String field) {
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java
index 4efc17f35..16aa3c589 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java
@@ -19,7 +19,23 @@
package org.apache.iggy.stream;
+import org.apache.iggy.message.HeaderValue;
+
import java.math.BigInteger;
+import java.util.Map;
+/**
+ * A stream as the server reports it.
+ *
+ * <p>{@code options} carries the creation options, keyed by option name. Streams have no
+ * catalog keys yet, so it is empty until one lands; the field is read rather than skipped
+ * so the first key needs no client change.
+ */
public record StreamBase(
- Long id, BigInteger createdAt, String name, String size, BigInteger messagesCount, Long topicsCount) {}
+ Long id,
+ BigInteger createdAt,
+ String name,
+ String size,
+ BigInteger messagesCount,
+ Long topicsCount,
+ Map<String, HeaderValue> options) {}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java
index 205f2d4fb..beac73674 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java
@@ -19,8 +19,20 @@
package org.apache.iggy.topic;
+import org.apache.iggy.message.HeaderValue;
+
import java.math.BigInteger;
+import java.util.Map;
+/**
+ * A topic as the server reports it.
+ *
+ * <p>{@code options} carries the keys the creating client set explicitly;
+ * {@code derivedOptions} carries the values admission resolved for the keys it did not.
+ * Both are keyed by option name, and the split is the provenance: a value in
+ * {@code derivedOptions} would have resolved differently under another server config.
+ * The fixed fields above repeat three of those keys and stay for compatibility.
+ */
public record Topic(
Long id,
BigInteger createdAt,
@@ -30,4 +42,6 @@ public record Topic(
CompressionAlgorithm compressionAlgorithm,
BigInteger maxTopicSize,
BigInteger messagesCount,
- Long partitionsCount) {}
+ Long partitionsCount,
+ Map<String, HeaderValue> options,
+ Map<String, HeaderValue> derivedOptions) {}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java
index cac2a0d6f..d36cdb5b8 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java
@@ -93,11 +93,38 @@ class BytesDeserializerTest {
writeU64(buffer, BigInteger.valueOf(50)); // messages count
buffer.writeByte(4); // name length
buffer.writeBytes("test".getBytes());
- // Non-empty explicit options block: skipping it must leave the buffer
- // positioned at the derived block.
writeOptionsBlock(
buffer, Map.of(HeaderKey.fromString("max_topic_size"), HeaderValue.fromUint64(BigInteger.TEN)));
- writeOptionsBlock(buffer, Map.of()); // derived options
+ writeOptionsBlock(buffer, Map.of(HeaderKey.fromString("segment_size"), HeaderValue.fromString("1 GiB")));
+ }
+
+ private static void writeTopicDataWithUnknownOptionKind(ByteBuf buffer) {
+ buffer.writeIntLE(10);
+ writeU64(buffer, BigInteger.valueOf(1000));
+ buffer.writeIntLE(4);
+ writeU64(buffer, BigInteger.ZERO);
+ buffer.writeByte(CompressionAlgorithm.None.asCode());
+ writeU64(buffer, BigInteger.valueOf(10000));
+ writeU64(buffer, BigInteger.valueOf(500));
+ writeU64(buffer, BigInteger.valueOf(50));
+ buffer.writeByte(4);
+ buffer.writeBytes("test".getBytes());
+
+ ByteBuf options = Unpooled.buffer();
+ var known = BytesSerializer.toBytes(
+ Map.of(HeaderKey.fromString("max_topic_size"), HeaderValue.fromUint64(BigInteger.TEN)));
+ options.writeBytes(known);
+ // Hand-rolled entry: a string key with a value kind no `HeaderKind` names.
+ options.writeByte(HeaderKind.String.asCode());
+ options.writeIntLE("from_the_future".length());
+ options.writeBytes("from_the_future".getBytes());
+ options.writeByte(200);
+ options.writeIntLE(2);
+ options.writeBytes(new byte[] {1, 2});
+ buffer.writeIntLE(options.readableBytes());
+ buffer.writeBytes(options);
+
+ writeOptionsBlock(buffer, Map.of());
}
private static void writeOptionsBlock(ByteBuf buffer, Map<HeaderKey, HeaderValue> options) {
@@ -228,6 +255,23 @@ class BytesDeserializerTest {
assertThat(topic.id()).isEqualTo(10L);
assertThat(topic.name()).isEqualTo("test");
assertThat(topic.partitionsCount()).isEqualTo(4L);
+ assertThat(topic.options()).containsOnlyKeys("max_topic_size");
+ assertThat(topic.options().get("max_topic_size").kind()).isEqualTo(HeaderKind.Uint64);
+ assertThat(topic.derivedOptions()).containsOnlyKeys("segment_size");
+ assertThat(new String(topic.derivedOptions().get("segment_size").value()))
+ .isEqualTo("1 GiB");
+ }
+
+ @Test
+ void shouldKeepReadableOptionsWhenOneValueKindIsUnknown() {
+ // The wire contract forwards value kinds a client build has no name
+ // for, so one of them must not cost the whole response.
+ ByteBuf buffer = Unpooled.buffer();
+ writeTopicDataWithUnknownOptionKind(buffer);
+
+ var topic = readTopic(buffer);
+
+ assertThat(topic.options()).containsOnlyKeys("max_topic_size");
}
@Test
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/OptionsBlockGoldenVectorTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/OptionsBlockGoldenVectorTest.java
new file mode 100644
index 000000000..ad327861c
--- /dev/null
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/OptionsBlockGoldenVectorTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iggy.serde;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.iggy.message.HeaderKey;
+import org.apache.iggy.message.HeaderValue;
+import org.junit.jupiter.api.Test;
+
+import java.math.BigInteger;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * The cross-SDK golden vector for an options block.
+ *
+ * <p>Rust pins the identical bytes in {@code core/binary_protocol/src/primitives/options.rs}, as do
+ * the Node and Go SDKs. Round-tripping a block through this SDK's own decoder proves nothing about
+ * interoperability; these bytes are the contract, and a change to the TLV layout has to break every
+ * copy of them together.
+ *
+ * <p>{@code enforce_fsync} (a one-byte {@code Bool}) and {@code segment_size} (an eight-byte
+ * {@code Uint64}) cover both value widths, in the sorted key order every encoder has to produce.
+ */
+class OptionsBlockGoldenVectorTest {
+
+ private static final byte[] GOLDEN_OPTIONS_BLOCK = {
+ 2, 13, 0, 0, 0, 'e', 'n', 'f', 'o', 'r', 'c', 'e', '_', 'f', 's', 'y', 'n', 'c', 3, 1, 0, 0, 0, 1, 2, 12, 0, 0,
+ 0, 's', 'e', 'g', 'm', 'e', 'n', 't', '_', 's', 'i', 'z', 'e', 12, 8, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0
+ };
+
+ @Test
+ void shouldEncodeTheCrossSdkGoldenVector() {
+ Map<HeaderKey, HeaderValue> options = new LinkedHashMap<>();
+ options.put(HeaderKey.fromString("enforce_fsync"), HeaderValue.fromBool(true));
+ options.put(HeaderKey.fromString("segment_size"), HeaderValue.fromUint64(BigInteger.valueOf(1_073_741_824L)));
+
+ ByteBuf encoded = BytesSerializer.toBytes(options);
+
+ byte[] bytes = new byte[encoded.readableBytes()];
+ encoded.readBytes(bytes);
+ assertThat(bytes).isEqualTo(GOLDEN_OPTIONS_BLOCK);
+ }
+}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java
index 48de3acd3..e8bc7fb98 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java
@@ -25,13 +25,14 @@ import org.junit.jupiter.api.Test;
import java.math.BigInteger;
import java.util.List;
+import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
class StreamDetailsTest {
@Test
void constructorWithStreamBaseCreatesExpectedStreamDetails() {
- var base = new StreamBase(10L, BigInteger.valueOf(500L), "name", "size", BigInteger.ZERO, 1L);
+ var base = new StreamBase(10L, BigInteger.valueOf(500L), "name", "size", BigInteger.ZERO, 1L, Map.of());
var topics = List.of(new Topic(
1L,
BigInteger.ZERO,
@@ -41,7 +42,9 @@ class StreamDetailsTest {
CompressionAlgorithm.None,
BigInteger.ONE,
BigInteger.ZERO,
- 2L));
+ 2L,
+ Map.of(),
+ Map.of()));
var streamDetails = new StreamDetails(base, topics);
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java
index ac9dfcde5..49eabd9e9 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java
@@ -24,6 +24,7 @@ import org.junit.jupiter.api.Test;
import java.math.BigInteger;
import java.util.List;
+import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@@ -39,7 +40,9 @@ class TopicDetailsTest {
CompressionAlgorithm.Gzip,
BigInteger.TWO,
BigInteger.ZERO,
- 1L);
+ 1L,
+ Map.of(),
+ Map.of());
var partitions = List.of(new Partition(1L, BigInteger.TEN, 2L, BigInteger.ZERO, "size", BigInteger.ONE));
var topicDetails = new TopicDetails(topic, partitions);
diff --git a/foreign/node/src/wire/command-set.ts b/foreign/node/src/wire/command-set.ts
index a7789cb7f..1588306d6 100644
--- a/foreign/node/src/wire/command-set.ts
+++ b/foreign/node/src/wire/command-set.ts
@@ -68,6 +68,7 @@ import { deletePartition } from './partition/delete-partition.command.js';
import { deleteSegments } from './segment/delete-segments.command.js';
+import { describeOptions } from './system/describe-options.command.js';
import { getStats } from './system/get-stats.command.js';
import { ping } from './system/ping.command.js';
@@ -190,7 +191,8 @@ type MessageAPI = ReturnType<typeof messageAPI>;
const systemAPI = (c: ClientProvider) => ({
ping: ping(c),
- getStats: getStats(c)
+ getStats: getStats(c),
+ describeOptions: describeOptions(c)
});
type SystemAPI = ReturnType<typeof systemAPI>;
diff --git a/foreign/node/src/wire/options.utils.test.ts b/foreign/node/src/wire/options.utils.test.ts
new file mode 100644
index 000000000..423e8577f
--- /dev/null
+++ b/foreign/node/src/wire/options.utils.test.ts
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+ serializeOptions,
+ deserializePrefixedOptions
+} from './options.utils.js';
+import { HeaderValue } from './message/header.utils.js';
+
+const prefixed = (block: Buffer): Buffer => {
+ const length = Buffer.alloc(4);
+ length.writeUInt32LE(block.length, 0);
+ return Buffer.concat([length, block]);
+};
+
+/**
+ * The cross-SDK golden vector for an options block.
+ *
+ * Rust pins the identical bytes in `core/binary_protocol/src/primitives/options.rs`,
+ * as do the Go and Java SDKs. Round-tripping through this SDK's own decoder proves
+ * nothing about interoperability; these bytes are the contract.
+ */
+const GOLDEN_OPTIONS_BLOCK = Buffer.from([
+ 2, 13, 0, 0, 0,
+ ...Buffer.from('enforce_fsync'),
+ 3, 1, 0, 0, 0, 1,
+ 2, 12, 0, 0, 0,
+ ...Buffer.from('segment_size'),
+ 12, 8, 0, 0, 0,
+ 0, 0, 0, 64, 0, 0, 0, 0
+]);
+
+describe('serializeOptions', () => {
+ it('encodes the cross-SDK golden vector byte for byte', () => {
+ const encoded = serializeOptions([
+ { key: 'enforce_fsync', value: HeaderValue.Bool(true) },
+ { key: 'segment_size', value: HeaderValue.Uint64(1_073_741_824n) }
+ ]);
+
+ assert.deepEqual(encoded, GOLDEN_OPTIONS_BLOCK);
+ });
+});
+
+describe('deserializePrefixedOptions', () => {
+ it('reads a whole block and reports the bytes it consumed', () => {
+ const block = prefixed(serializeOptions([
+ { key: 'segment_size', value: HeaderValue.Uint64(1_048_576n) }
+ ]));
+
+ const { bytesRead, options } = deserializePrefixedOptions(block);
+
+ assert.equal(bytesRead, block.length);
+ assert.deepEqual(options, { segment_size: 1_048_576n });
+ });
+
+ it('rejects a block whose declared length runs past the payload', () => {
+ // `subarray` clamps instead of throwing, so without the bounds check the
+ // truncated value comes back as raw bytes through the forward-compat catch
+ // and `bytesRead` over-reports, shifting every later field.
+ const block = prefixed(serializeOptions([
+ { key: 'segment_size', value: HeaderValue.Uint64(1_048_576n) }
+ ]));
+
+ assert.throws(
+ () => deserializePrefixedOptions(block.subarray(0, block.length - 4)),
+ /overruns the payload/
+ );
+ });
+});
diff --git a/foreign/node/src/wire/options.utils.ts b/foreign/node/src/wire/options.utils.ts
index cb38b2589..724276bb5 100644
--- a/foreign/node/src/wire/options.utils.ts
+++ b/foreign/node/src/wire/options.utils.ts
@@ -145,6 +145,15 @@ export const deserializePrefixedOptions = (
p: Buffer, pos = 0
): OptionsDeserialized => {
const length = p.readUInt32LE(pos);
- const options = deserializeOptions(p, pos + 4, pos + 4 + length);
+ const end = pos + 4 + length;
+ // Without this, `subarray` clamps a truncated block silently: a known-kind
+ // value throws inside `deserializeHeaderValue`, the forward-compat catch
+ // swallows it and hands back raw bytes, and `bytesRead` over-reports so every
+ // later field decodes from the wrong offset.
+ if (end > p.length)
+ throw new Error(
+ `Options block overruns the payload: ${length} bytes declared at ${pos}, ` +
+ `${p.length - pos - 4} available`);
+ const options = deserializeOptions(p, pos + 4, end);
return { bytesRead: 4 + length, options };
};
diff --git a/foreign/node/src/wire/system/describe-options.command.ts b/foreign/node/src/wire/system/describe-options.command.ts
new file mode 100644
index 000000000..275fd7922
--- /dev/null
+++ b/foreign/node/src/wire/system/describe-options.command.ts
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+import type { CommandResponse } from '../../client/index.js';
+import { COMMAND_CODE } from '../command.code.js';
+import { wrapCommand } from '../command.utils.js';
+
+/** Resource whose option catalog the server serves. */
+export const OptionsScope = {
+ Topic: 1,
+ Stream: 2,
+ User: 3
+} as const;
+
+export type OptionsScope = typeof OptionsScope[keyof typeof OptionsScope];
+
+/**
+ * One catalog entry: the key a create command accepts, the kind the server
+ * encodes its default under, that default, and what the option does.
+ *
+ * `kind` is the kind to prefer, not a guarantee: an explicit entry is stored
+ * with whatever kind the client sent, and a string value parsed through the
+ * server's config rules stays a string.
+ */
+export type OptionSpec = {
+ key: string,
+ kind: number,
+ defaultValue: Buffer,
+ description: string
+};
+
+export type DescribeOptions = {
+ scope: OptionsScope
+};
+
+const deserializeDescribeOptions = (b: Buffer): OptionSpec[] => {
+ const count = b.readUInt32LE(0);
+ let position = 4;
+ const specs: OptionSpec[] = [];
+ for (let i = 0; i < count; i++) {
+ const keyLength = b.readUInt8(position);
+ position += 1;
+ if (position + keyLength > b.length)
+ throw new Error(`Option key overruns the response at offset ${position}`);
+ const key = b.subarray(position, position + keyLength).toString();
+ position += keyLength;
+
+ const kind = b.readUInt8(position);
+ position += 1;
+
+ const defaultLength = b.readUInt32LE(position);
+ position += 4;
+ if (position + defaultLength > b.length)
+ throw new Error(`Default value for '${key}' overruns the response`);
+ const defaultValue = Buffer.from(
+ b.subarray(position, position + defaultLength)
+ );
+ position += defaultLength;
+
+ const descriptionLength = b.readUInt32LE(position);
+ position += 4;
+ if (position + descriptionLength > b.length)
+ throw new Error(`Description for '${key}' overruns the response`);
+ const description = b.subarray(
+ position, position + descriptionLength
+ ).toString();
+ position += descriptionLength;
+
+ specs.push({ key, kind, defaultValue, description });
+ }
+ return specs;
+};
+
+export const DESCRIBE_OPTIONS = {
+ code: COMMAND_CODE.DescribeOptions,
+
+ serialize: ({ scope }: DescribeOptions) => {
+ return Buffer.from([scope]);
+ },
+
+ deserialize: (r: CommandResponse): OptionSpec[] =>
+ deserializeDescribeOptions(r.data)
+};
+
+export const describeOptions = wrapCommand<DescribeOptions, OptionSpec[]>(
+ DESCRIBE_OPTIONS
+);
diff --git a/foreign/node/src/wire/system/index.ts b/foreign/node/src/wire/system/index.ts
index 5decb6ef4..3326db31a 100644
--- a/foreign/node/src/wire/system/index.ts
+++ b/foreign/node/src/wire/system/index.ts
@@ -16,5 +16,6 @@
// under the License.
//
+export * from './describe-options.command.js';
export * from './get-stats.command.js';
export * from './ping.command.js';
diff --git a/foreign/python/src/topic.rs b/foreign/python/src/topic.rs
index 38879a6f7..61adfe53f 100644
--- a/foreign/python/src/topic.rs
+++ b/foreign/python/src/topic.rs
@@ -218,7 +218,6 @@ impl Topic {
pub fn max_topic_size(&self) -> MaxTopicSize {
self.inner.max_topic_size.into()
}
-
}
#[gen_stub_pyclass]
diff --git a/package-lock.json b/package-lock.json
new file mode 100644
index 000000000..26f23d6c0
--- /dev/null
+++ b/package-lock.json
@@ -0,0 +1,110 @@
+{
+ "name": "iggy",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "dependencies": {
+ "ccusage": "^20.0.19"
+ }
+ },
+ "node_modules/@ccusage/ccusage-darwin-arm64": {
+ "version": "20.0.19",
+ "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-arm64/-/ccusage-darwin-arm64-20.0.19.tgz",
+ "integrity": "sha512-8b69h4tuMH1KNW+EYjFRJWpbWHXbMUtFu8cOCyFOQsR6lGxq37g0kzEGHSqG+iZEoUTHaabR5CE7ZHLhW9Q64A==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ]
+ },
+ "node_modules/@ccusage/ccusage-darwin-x64": {
+ "version": "20.0.19",
+ "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-x64/-/ccusage-darwin-x64-20.0.19.tgz",
+ "integrity": "sha512-YTsqIfsPo3qR9OMrwJtnJoq/fil7Jk5mzXV8k0Iejpiq7zysw4FfkjCeHr0UVJG8ihdqtE18o+xIk99p3v+5Ew==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ]
+ },
+ "node_modules/@ccusage/ccusage-linux-arm64": {
+ "version": "20.0.19",
+ "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-arm64/-/ccusage-linux-arm64-20.0.19.tgz",
+ "integrity": "sha512-4e+7hV3WrEpM7hQPQkh4/zNLTUSqzpd+vSSWl2y659+xQ+JCDLWqE6gzvXLMOAT/LTtcFtlrduIWDZt8VPceqQ==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@ccusage/ccusage-linux-x64": {
+ "version": "20.0.19",
+ "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-x64/-/ccusage-linux-x64-20.0.19.tgz",
+ "integrity": "sha512-VPbB6onrwOGojTKutFzeahttb98ZgmdsiQpOr/BJet1i3QdrmUGXGmJSt591xXrvVOVVgsC7/5wBELdWM9/bKA==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@ccusage/ccusage-win32-arm64": {
+ "version": "20.0.19",
+ "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-arm64/-/ccusage-win32-arm64-20.0.19.tgz",
+ "integrity": "sha512-oOv4mZkapxzvQeAMI+jqfTOv/zN2x2AYPAD+F4AnYHQQL4n898axhH9uPd2Ls0U5CDlvfh1zDTyNILz8X9M15A==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ]
+ },
+ "node_modules/@ccusage/ccusage-win32-x64": {
+ "version": "20.0.19",
+ "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-x64/-/ccusage-win32-x64-20.0.19.tgz",
+ "integrity": "sha512-KtDlTk07uv8cu6TlizB/RptcDiQePE1YSQe6uNNZTUnwVy9zq5R3fvUUFr9/d3FYHwimSGduoHXRS8Ta+Kmkrw==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ]
+ },
+ "node_modules/ccusage": {
+ "version": "20.0.19",
+ "resolved": "https://registry.npmjs.org/ccusage/-/ccusage-20.0.19.tgz",
+ "integrity": "sha512-vyNIyctcvTKFbrsU/WJuWiII3CVw8WopU/0NcR+lKmRbh62wm0jccmkWrFCG6j7R8agc8mIBz5+YlmGbXH1RnQ==",
+ "license": "MIT",
+ "bin": {
+ "ccusage": "src/cli.js"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ryoppippi"
+ },
+ "optionalDependencies": {
+ "@ccusage/ccusage-darwin-arm64": "20.0.19",
+ "@ccusage/ccusage-darwin-x64": "20.0.19",
+ "@ccusage/ccusage-linux-arm64": "20.0.19",
+ "@ccusage/ccusage-linux-x64": "20.0.19",
+ "@ccusage/ccusage-win32-arm64": "20.0.19",
+ "@ccusage/ccusage-win32-x64": "20.0.19"
+ }
+ }
+ }
+}
diff --git a/package.json b/package.json
new file mode 100644
index 000000000..b916f62ae
--- /dev/null
+++ b/package.json
@@ -0,0 +1,5 @@
+{
+ "dependencies": {
+ "ccusage": "^20.0.19"
+ }
+}