You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by pi...@apache.org on 2026/10/24 09:11:57 UTC
(iggy) branch master updated: feat(metadata)!: let clients follow metadata changes without re-listing (#4625)
This is an automated email from the ASF dual-hosted git repository.
spetz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new b818d646b feat(metadata)!: let clients follow metadata changes without re-listing (#4625)
b818d646b is described below
commit b818d646b43c041c6bf9d1bdea9866748685901f
Author: Piotr Gankiewicz <pi...@gmail.com>
AuthorDate: Sat Oct 24 11:11:52 2026 +0200
feat(metadata)!: let clients follow metadata changes without re-listing (#4625)
Anything that mirrors Iggy metadata had to list every stream, then
the topics of every stream, then the consumer groups of every topic,
on every refresh. The metadata journal could not replace that: a
checkpoint drains it, it holds requests rather than results, it
carries password, token and wrapped stream key material, and it hides
side effects such as the grants a delete prunes. Nothing recorded who
changed what.
Keep a bounded log of the public results of committed metadata
operations on every node. Handlers describe each result in the apply
reply, both apply paths append the records before the applied
frontier moves, and the log rides in the metadata snapshot, so it
survives restarts and moves with state transfer. Records are encoded
once through the option projection the read routes use, so stream
keys, wrapped keys and every secret or internal option stay on the
server. Topics now keep the op that created them, as streams do, so a
recreated id is never mistaken for its predecessor.
Serve the log as metadata.changes (code 17) on TCP, QUIC and
WebSocket and as GET /metadata/changes, answered by the receiving
node and gated by manage_servers. A caller pages records after a
cursor, filtered by groups and a prepare timestamp range. When its
cursor expires it pages the redacted current state, anchored to a
cursor taken before the scan, and replays the real changes after it.
Every SDK supports the command on each of its existing client
transports, the CLI follows it, and the web UI lists it for accounts
that may read it.
Records come from the handlers rather than from diffing state,
because only a handler knows the assigned id, the previous values and
the deleted resource. Only committed and applied operations are
recorded, so a cursor from one replica is valid on another, and a
lagging node answers behind instead of skipping records.
BREAKING CHANGE: data directories need snapshot format 14 and the new
storage tag, topics persist their creating op, and the Rust
SystemClient trait gains get_metadata_changes.
---
.github/actions/php/pre-merge/action.yml | 2 +-
.typos.toml | 2 +
Cargo.lock | 16 +-
Cargo.toml | 8 +-
README.md | 4 +-
.../step_definitions/metadata_changes_steps.cpp | 333 +++
.../step_definitions/resource_options_steps.cpp | 40 +-
bdd/cpp/features/step_definitions/world.hpp | 20 +-
bdd/cpp/scripts/entrypoint.sh | 1 +
bdd/docker-compose.coverage.yml | 4 +-
bdd/docker-compose.server.yml | 2 +
bdd/docker-compose.yml | 11 +
bdd/go/tests/metadata_changes.go | 471 ++++
bdd/go/tests/resource_options.go | 36 +-
bdd/go/tests/suite_test.go | 12 +
.../org/apache/iggy/bdd/MetadataChangesSteps.java | 302 +++
.../org/apache/iggy/bdd/ResourceOptionsSteps.java | 33 +
bdd/node/Dockerfile | 1 +
bdd/php/tests/MetadataChangesFeatureTest.php | 329 +++
bdd/php/tests/StreamOptionsFeatureTest.php | 63 +-
bdd/php/tests/StreamSteps.php | 99 +
bdd/python/tests/conftest.py | 134 +-
bdd/python/tests/test_metadata_changes.py | 315 +++
bdd/python/tests/test_stream_options.py | 115 +-
bdd/python/uv.lock | 2 +-
bdd/rust/Cargo.toml | 5 +
bdd/rust/tests/common/global_context.rs | 9 +-
.../rust/tests/metadata_changes.rs | 15 +-
bdd/rust/tests/steps/metadata_changes.rs | 339 +++
bdd/rust/tests/steps/mod.rs | 1 +
bdd/scenarios/metadata_changes.feature | 70 +
core/ai/mcp/Cargo.toml | 2 +-
core/bench/Cargo.toml | 2 +-
core/binary_protocol/Cargo.toml | 2 +-
core/binary_protocol/src/codes.rs | 2 +
core/binary_protocol/src/dispatch.rs | 3 +
.../src/primitives/metadata_changes/body.rs | 1771 +++++++++++++++
.../src/primitives/metadata_changes/mod.rs | 454 ++++
.../src/primitives/metadata_changes/position.rs | 384 ++++
core/binary_protocol/src/primitives/mod.rs | 1 +
.../src/requests/system/get_metadata_changes.rs | 322 +++
core/binary_protocol/src/requests/system/mod.rs | 2 +
.../src/responses/system/get_metadata_changes.rs | 488 +++++
core/binary_protocol/src/responses/system/mod.rs | 4 +
core/cli/Cargo.toml | 2 +-
core/cli/src/args/metadata.rs | 117 +
core/cli/src/args/mod.rs | 5 +
.../src/commands/binary_system/metadata_changes.rs | 337 +++
core/cli/src/commands/binary_system/mod.rs | 1 +
core/cli/src/commands/cli_command.rs | 4 +
core/cli/src/main.rs | 19 +-
core/common/Cargo.toml | 4 +-
.../common/examples/metadata_changes_fixtures.rs | 26 +-
.../examples/metadata_changes_fixtures/schema.rs | 1692 +++++++++++++++
core/common/fixtures/metadata_changes.json | 1997 +++++++++++++++++
core/common/src/lib.rs | 1 +
core/common/src/traits/binary_impls/system.rs | 239 ++-
core/common/src/traits/system_client.rs | 13 +-
core/common/src/types/metadata_changes/answer.rs | 114 +
core/common/src/types/metadata_changes/json.rs | 117 +
core/common/src/types/metadata_changes/mod.rs | 586 +++++
core/common/src/types/metadata_changes/position.rs | 381 ++++
core/common/src/types/metadata_changes/query.rs | 429 ++++
core/common/src/types/metadata_changes/records.rs | 1205 +++++++++++
core/common/src/types/metadata_changes/wire.rs | 731 +++++++
core/common/src/types/mod.rs | 1 +
core/common/src/utils/timestamp.rs | 8 +
core/common/tests/metadata_changes_fixtures.rs | 267 +++
core/configs/src/server_config/defaults.rs | 2 +
core/configs/src/server_config/displays.rs | 4 +-
core/configs/src/server_config/metadata.rs | 132 +-
core/configs/src/server_config/server.rs | 19 +
core/connectors/runtime/Cargo.toml | 2 +-
core/consensus/src/client_table.rs | 26 +
.../tests/cli/general/test_help_command.rs | 1 +
.../tests/cli/general/test_overview_command.rs | 1 +
.../integration/tests/cli/metadata/mod.rs | 4 +-
.../cli/metadata/test_metadata_changes_command.rs | 219 ++
core/integration/tests/cli/mod.rs | 1 +
core/integration/tests/cluster/metadata_changes.rs | 492 +++++
core/integration/tests/cluster/mod.rs | 1 +
core/integration/tests/sdk/metadata_changes.rs | 1467 +++++++++++++
core/integration/tests/sdk/mod.rs | 1 +
.../server/scenarios/authentication_scenario.rs | 82 +-
core/metadata/Cargo.toml | 6 +
core/metadata/benches/change_log.rs | 415 ++++
core/metadata/src/change_log/current.rs | 618 ++++++
core/metadata/src/change_log/mod.rs | 304 +++
core/metadata/src/change_log/query.rs | 682 ++++++
core/metadata/src/change_log/snapshot.rs | 471 ++++
core/metadata/src/change_log/store.rs | 717 +++++++
core/metadata/src/change_log/tests.rs | 2217 +++++++++++++++++++
core/metadata/src/impls/metadata.rs | 510 ++++-
core/metadata/src/impls/recovery.rs | 414 +++-
core/metadata/src/lib.rs | 5 +-
.../src/permissioner/permissioner_rules/system.rs | 37 +
core/metadata/src/stm/authz.rs | 96 +-
core/metadata/src/stm/consumer_group.rs | 407 +++-
core/metadata/src/stm/id_slab.rs | 27 +
core/metadata/src/stm/lifecycle.rs | 9 +-
core/metadata/src/stm/mod.rs | 11 +
core/metadata/src/stm/result.rs | 14 +-
core/metadata/src/stm/snapshot.rs | 57 +-
core/metadata/src/stm/stream.rs | 240 ++-
core/metadata/src/stm/user.rs | 230 +-
core/sdk/Cargo.toml | 2 +-
core/sdk/README.md | 76 +
.../src/client_wrappers/binary_system_client.rs | 18 +-
core/sdk/src/clients/binary_system.rs | 12 +-
core/sdk/src/http/system.rs | 16 +
core/sdk/src/prelude.rs | 13 +
core/server/Cargo.toml | 2 +-
core/server/README.md | 28 +-
core/server/config.toml | 14 +
core/server/server.http | 22 +
core/server/src/boot/handoff.rs | 38 +-
core/server/src/boot/mod.rs | 137 +-
core/server/src/boot/recovery.rs | 11 +
core/server/src/consumer_group/liveness.rs | 2 +
core/server/src/dispatch/authz.rs | 5 +-
core/server/src/dispatch/metadata_changes.rs | 127 ++
core/server/src/dispatch/mod.rs | 79 +-
core/server/src/dispatch/reads.rs | 37 +-
core/server/src/dispatch/submit.rs | 8 +
core/server/src/http.rs | 7 +-
core/server/src/http/handlers.rs | 47 +-
core/server/src/http/reads.rs | 96 +-
core/server/src/partition_reconciler.rs | 17 +-
core/server/src/responses.rs | 1 +
core/server/src/shell.rs | 15 +-
core/shard/src/lib.rs | 18 +-
core/shard/src/poll/completion_tests.rs | 18 +-
core/simulator/src/bin/workload-fuzz.rs | 5 +-
core/simulator/src/lib.rs | 8 +
core/simulator/src/replica.rs | 44 +-
core/simulator/src/workload/change_log_oracle.rs | 434 ++++
core/simulator/src/workload/invariants.rs | 5 +
core/simulator/src/workload/mod.rs | 1 +
core/simulator/src/workload/oracle.rs | 12 +
examples/node/package-lock.json | 4 +-
examples/python/uv.lock | 2 +-
foreign/cpp/BUILD.bazel | 4 +
foreign/cpp/Cargo.toml | 2 +-
foreign/cpp/MODULE.bazel | 2 +-
foreign/cpp/README.md | 88 +-
foreign/cpp/include/iggy.hpp | 2006 +++++++++++++++--
foreign/cpp/src/client.cpp | 73 +-
foreign/cpp/src/client.rs | 123 +-
foreign/cpp/src/lib.rs | 187 +-
foreign/cpp/src/type_conversion.rs | 661 +++++-
foreign/cpp/src/type_conversions.cpp | 448 ++++
foreign/cpp/tests/e2e/metadata_changes.cpp | 423 ++++
foreign/cpp/tests/fixtures/metadata_changes.json | 1997 +++++++++++++++++
foreign/cpp/tests/unit/metadata_changes.cpp | 820 +++++++
.../Iggy_SDK.Tests.BDD/Context/TestContext.cs | 1 +
.../csharp/Iggy_SDK.Tests.BDD/Context/TestHooks.cs | 1 +
.../Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj | 3 +
.../StepDefinitions/MetadataChangesSteps.cs | 306 +++
.../StepDefinitions/ResourceOptionsSteps.cs | 8 +
.../IggyTlsConnectionTests.cs | 39 +
.../MetadataChangesTests.cs | 483 +++++
.../Iggy_SDK/Contracts/MetadataChangeBodies.cs | 570 +++++
.../Iggy_SDK/Contracts/MetadataChangeCursor.cs | 103 +
.../Iggy_SDK/Contracts/MetadataChangeGroups.cs | 116 +
.../csharp/Iggy_SDK/Contracts/MetadataChanges.cs | 122 ++
.../Iggy_SDK/Contracts/MetadataChangesQuery.cs | 187 ++
.../Iggy_SDK/Contracts/MetadataCurrentPageToken.cs | 186 ++
foreign/csharp/Iggy_SDK/Contracts/OptionValues.cs | 13 +
.../Iggy_SDK/Contracts/SnakeCaseNames.cs} | 28 +-
.../csharp/Iggy_SDK/Contracts/StreamResponse.cs | 8 +-
.../csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs | 40 +
.../csharp/Iggy_SDK/Enums/MetadataChangeGroup.cs | 48 +
.../csharp/Iggy_SDK/Enums/MetadataChangeKind.cs | 88 +
.../Iggy_SDK/Enums/MetadataChangePhase.cs} | 14 +-
.../Iggy_SDK/Enums/MetadataChangeStartMode.cs} | 25 +-
.../Iggy_SDK/Enums/MetadataChangeStatus.cs} | 25 +-
.../Iggy_SDK/Enums/MetadataMemberLeftReason.cs} | 14 +-
.../Enums/MetadataPermissionsUpdatedReason.cs} | 14 +-
.../Iggy_SDK/Enums/MetadataSessionClosedReason.cs} | 17 +-
.../Iggy_SDK/Enums/MetadataTokenDeletedReason.cs} | 14 +-
foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs | 20 +
.../Implementations/HttpMessageStream.cs | 17 +
.../IggyClient/Implementations/TcpMessageStream.cs | 13 +
foreign/csharp/Iggy_SDK/Iggy_SDK.csproj | 2 +-
.../JsonConverters/MetadataChangesConverter.cs | 499 +++++
.../Mappers/BinaryMapper.MetadataChanges.cs | 773 +++++++
foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 92 +-
foreign/csharp/Iggy_SDK/Utils/Base64Url.cs | 78 +
foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs | 1 +
foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs | 1 +
.../ClientTests/HttpMetadataChangesTests.cs | 194 ++
.../Iggy_SDK_Tests/Fixtures/metadata_changes.json | 1997 +++++++++++++++++
.../csharp/Iggy_SDK_Tests/Iggy_SDK_Tests.csproj | 3 +
.../Iggy_SDK_Tests/MapperTests/BinaryMapper.cs | 15 +
.../MapperTests/MetadataChangesTests.cs | 844 ++++++++
foreign/csharp/README.md | 83 +
foreign/go/README.md | 79 +
.../metadata_changes_deserializer.go | 636 ++++++
foreign/go/client/example_metadata_changes_test.go | 112 +
foreign/go/client/tcp/tcp_utilities.go | 11 +
foreign/go/contracts/client.go | 10 +
foreign/go/contracts/metadata_change_position.go | 323 +++
foreign/go/contracts/metadata_changes.go | 762 +++++++
foreign/go/contracts/metadata_changes_test.go | 936 ++++++++
.../go/contracts/testdata/metadata_changes.json | 1997 +++++++++++++++++
foreign/go/contracts/version.go | 2 +-
foreign/go/internal/command/code.go | 1 +
foreign/go/internal/command/system.go | 68 +
foreign/go/internal/command/system_test.go | 112 +
foreign/go/internal/vsr/operation_test.go | 1 +
foreign/go/tests/e2e_metadata_changes_test.go | 399 ++++
foreign/go/tests/e2e_resource_options_test.go | 1 +
foreign/java/README.md | 70 +-
foreign/java/gradle.properties | 2 +-
.../org/apache/iggy/client/async/SystemClient.java | 15 +
.../iggy/client/async/tcp/SystemTcpClient.java | 25 +
.../apache/iggy/client/blocking/SystemClient.java | 16 +
.../client/blocking/http/MetadataChangesJson.java | 517 +++++
.../client/blocking/http/SystemHttpClient.java | 11 +
.../iggy/client/blocking/tcp/SystemTcpClient.java | 7 +
.../org/apache/iggy/serde/BytesDeserializer.java | 46 +-
.../org/apache/iggy/serde/BytesSerializer.java | 35 +
.../java/org/apache/iggy/serde/CommandCode.java | 1 +
.../apache/iggy/serde/MetadataChangesReader.java | 620 ++++++
.../apache/iggy/stream/EncryptionProjection.java | 7 +-
.../org/apache/iggy/system/MetadataChangeBody.java | 475 ++++
.../apache/iggy/system/MetadataChangeCursor.java | 108 +
.../apache/iggy/system/MetadataChangeEncoding.java | 92 +
.../apache/iggy/system/MetadataChangeGroup.java | 123 ++
.../org/apache/iggy/system/MetadataChangeKind.java | 91 +
.../apache/iggy/system/MetadataChangePhase.java | 66 +
.../apache/iggy/system/MetadataChangeRecord.java | 51 +
.../apache/iggy/system/MetadataChangeStart.java | 95 +
.../apache/iggy/system/MetadataChangeStatus.java | 70 +
.../org/apache/iggy/system/MetadataChanges.java | 60 +
.../apache/iggy/system/MetadataChangesQuery.java | 194 ++
.../apache/iggy/system/MetadataCurrentItem.java | 31 +
.../iggy/system/MetadataCurrentPageToken.java | 169 ++
.../org/apache/iggy/MetadataChangesFixture.java | 322 +++
.../java/org/apache/iggy/ProtocolParityTest.java | 1 +
.../iggy/client/async/MetadataChangesTest.java | 248 +++
.../iggy/client/blocking/SystemClientBaseTest.java | 412 ++++
.../blocking/http/MetadataChangesJsonTest.java | 113 +
.../client/blocking/tcp/TlsConnectionTest.java | 40 +
.../org/apache/iggy/serde/MetadataChangesTest.java | 410 ++++
.../src/test/resources/metadata_changes.json | 1997 +++++++++++++++++
foreign/node/README.md | 67 +-
foreign/node/package-lock.json | 4 +-
foreign/node/package.json | 2 +-
foreign/node/src/bdd/README.md | 2 +
foreign/node/src/bdd/metadata-changes.ts | 252 +++
foreign/node/src/bdd/world.ts | 15 +-
foreign/node/src/client/client.socket.test.ts | 34 +
foreign/node/src/client/client.socket.ts | 4 +-
...urce-options.e2e.ts => metadata-changes.e2e.ts} | 12 +-
foreign/node/src/e2e/metadata-changes.suite.ts | 369 ++++
foreign/node/src/e2e/resource-options.e2e.ts | 5 +
foreign/node/src/e2e/tls.system.e2e.ts | 9 +
foreign/node/src/index.ts | 48 +
foreign/node/src/wire/command-set.ts | 4 +-
foreign/node/src/wire/command.code.ts | 1 +
.../node/src/wire/fixtures/metadata_changes.json | 1997 +++++++++++++++++
.../system/get-metadata-changes.command.test.ts | 623 ++++++
.../wire/system/get-metadata-changes.command.ts | 259 +++
foreign/node/src/wire/system/index.ts | 17 +
.../node/src/wire/system/metadata-changes.utils.ts | 1164 ++++++++++
foreign/php/Cargo.toml | 4 +-
foreign/php/README.md | 73 +
foreign/php/iggy-php.stubs.php | 1139 ++++++++++
foreign/php/src/client.rs | 56 +-
foreign/php/src/lib.rs | 50 +
foreign/php/src/metadata_changes.rs | 2005 +++++++++++++++++
foreign/php/tests/MetadataChangesTest.php | 840 ++++++++
foreign/php/tests/TlsTest.php | 35 +
foreign/php/tests/bootstrap.php | 66 +-
foreign/php/tests/fixtures/metadata_changes.json | 1997 +++++++++++++++++
foreign/python/Cargo.toml | 4 +-
foreign/python/README.md | 70 +
foreign/python/apache_iggy.pyi | 1419 ++++++++++--
foreign/python/pyproject.toml | 2 +-
foreign/python/src/bin/stub_gen.rs | 67 +-
foreign/python/src/client.rs | 51 +
foreign/python/src/lib.rs | 50 +
foreign/python/src/metadata_changes.rs | 2257 ++++++++++++++++++++
.../python/tests/fixtures/metadata_changes.json | 1997 +++++++++++++++++
foreign/python/tests/test_metadata_changes.py | 455 ++++
foreign/python/tests/test_resource_options.py | 40 +
foreign/python/tests/test_stub.py | 65 +
foreign/python/tests/test_tls.py | 32 +-
foreign/python/tests/utils.py | 69 +-
foreign/python/uv.lock | 2 +-
foreign/swift/README.md | 65 +-
foreign/swift/Sources/Iggy/Client/IggyClient.swift | 17 +
foreign/swift/Sources/Iggy/Iggy.swift | 2 +-
.../Sources/Iggy/Models/MetadataChanges.swift | 1510 +++++++++++++
.../swift/Sources/Iggy/Protocol/CommandCode.swift | 2 +
foreign/swift/Sources/Iggy/Protocol/Requests.swift | 36 +
.../swift/Sources/Iggy/Protocol/Responses.swift | 83 +
.../IggyE2ETests/MetadataChangesE2ETests.swift | 375 ++++
foreign/swift/Tests/IggyTests/Fixtures/golden.json | 10 +-
.../Tests/IggyTests/Fixtures/metadata_changes.json | 1997 +++++++++++++++++
.../Tests/IggyTests/MetadataChangesTests.swift | 678 ++++++
.../Tests/IggyTests/ResponseVectorTests.swift | 26 +
.../IggyTests/Support/MetadataChangesFixture.swift | 107 +
.../swift/Tests/IggyTests/WireVectorTests.swift | 6 +
foreign/swift/Tools/golden-vectors/src/main.rs | 70 +-
scripts/run-bdd-tests.sh | 6 +-
web/docs/server-api.http | 16 +
web/package-lock.json | 4 +-
web/package.json | 2 +-
web/src/lib/api/fetchMetadataChanges.ts | 65 +
web/src/lib/api/metadataChanges.test.mjs | 279 +++
web/src/lib/api/metadataChanges.ts | 251 +++
web/src/lib/components/Navbar.svelte | 16 +
web/src/lib/types/appRoutes.ts | 1 +
web/src/routes/dashboard/+layout.ts | 4 +-
web/src/routes/dashboard/changes/+page.svelte | 564 +++++
317 files changed, 74460 insertions(+), 1243 deletions(-)
diff --git a/.github/actions/php/pre-merge/action.yml b/.github/actions/php/pre-merge/action.yml
index 3b8ba1475..604084ee3 100644
--- a/.github/actions/php/pre-merge/action.yml
+++ b/.github/actions/php/pre-merge/action.yml
@@ -109,7 +109,7 @@ runs:
printf '<?php\n'
awk 'NF { print "// " $0; next } { print "//" }' ASF_LICENSE.txt
printf '\n'
- tail -n +2 "$raw_stubs" | sed -E 's/(public readonly \?string \$[A-Za-z_][A-Za-z0-9_]*) = null;/\1;/'
+ tail -n +2 "$raw_stubs" | sed -E 's/(public readonly [^$]*\$[A-Za-z_][A-Za-z0-9_]*) = null;/\1;/'
} > foreign/php/iggy-php.stubs.php
rm -f "$raw_stubs"
git diff --exit-code -- foreign/php/iggy-php.stubs.php
diff --git a/.typos.toml b/.typos.toml
index 420315787..ce0fe9b1e 100644
--- a/.typos.toml
+++ b/.typos.toml
@@ -164,5 +164,7 @@ extend-exclude = [
"foreign/swift/Tests/IggyTests/Fixtures/golden.json",
# Resource options fixture copies in every SDK test tree (hex payloads)
"**/resource_options.json",
+ # Metadata change fixture copies in every SDK test tree (hex and base64url payloads)
+ "**/metadata_changes.json",
"**/.python-version",
]
diff --git a/Cargo.lock b/Cargo.lock
index b2155b6a6..ad0b60756 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7089,7 +7089,7 @@ checksum = "cd62e6b5e86ea8eeeb8db1de02880a6abc01a397b2ebb64b5d74ac255318f5cb"
[[package]]
name = "iggy"
-version = "0.11.1-edge.2"
+version = "0.11.1-edge.3"
dependencies = [
"async-broadcast",
"async-dropper",
@@ -7127,7 +7127,7 @@ dependencies = [
[[package]]
name = "iggy-bench"
-version = "0.6.1-edge.2"
+version = "0.6.1-edge.3"
dependencies = [
"async-trait",
"bench-report",
@@ -7185,7 +7185,7 @@ dependencies = [
[[package]]
name = "iggy-cli"
-version = "0.14.1-edge.2"
+version = "0.14.1-edge.3"
dependencies = [
"anyhow",
"apple-native-keyring-store",
@@ -7219,7 +7219,7 @@ dependencies = [
[[package]]
name = "iggy-connectors"
-version = "0.5.1-edge.2"
+version = "0.5.1-edge.3"
dependencies = [
"apache-avro 0.22.0",
"async-trait",
@@ -7319,7 +7319,7 @@ dependencies = [
[[package]]
name = "iggy-mcp"
-version = "0.5.1-edge.2"
+version = "0.5.1-edge.3"
dependencies = [
"axum",
"axum-server",
@@ -7354,7 +7354,7 @@ dependencies = [
[[package]]
name = "iggy_binary_protocol"
-version = "0.11.1-edge.2"
+version = "0.11.1-edge.3"
dependencies = [
"aligned-vec",
"bytemuck",
@@ -7369,7 +7369,7 @@ dependencies = [
[[package]]
name = "iggy_common"
-version = "0.11.1-edge.2"
+version = "0.11.1-edge.3"
dependencies = [
"aes-gcm 0.11.1",
"aes-gcm-siv",
@@ -12982,7 +12982,7 @@ dependencies = [
[[package]]
name = "server"
-version = "0.9.1-edge.2"
+version = "0.9.1-edge.3"
dependencies = [
"ahash",
"arc-swap",
diff --git a/Cargo.toml b/Cargo.toml
index d93a38834..f726d0c31 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -213,11 +213,11 @@ hyper-util = { version = "0.1.21", features = ["server-auto", "service"] }
iceberg = "0.10.1"
iceberg-catalog-rest = "0.10.1"
iceberg-storage-opendal = "0.10.1"
-iggy = { path = "core/sdk", version = "0.11.1-edge.2" }
-iggy-cli = { path = "core/cli", version = "0.14.1-edge.2" }
+iggy = { path = "core/sdk", version = "0.11.1-edge.3" }
+iggy-cli = { path = "core/cli", version = "0.14.1-edge.3" }
iggy-gateway-kafka = { path = "gateways/kafka" }
-iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.1-edge.2" }
-iggy_common = { path = "core/common", version = "0.11.1-edge.2" }
+iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.1-edge.3" }
+iggy_common = { path = "core/common", version = "0.11.1-edge.3" }
iggy_connector_doris_sink = { path = "core/connectors/sinks/doris_sink" }
iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.5.1-edge.2" }
indexmap = "2.14.2"
diff --git a/README.md b/README.md
index 4e4f84871..871cd1868 100644
--- a/README.md
+++ b/README.md
@@ -146,7 +146,7 @@ The server, SDKs and CLI are versioned independently. For example, server 0.9.0
We also publish edge releases (versions with an `-edge.N` suffix, and `apache/iggy:edge`) for the SDKs and the Docker images. They track the latest changes, but are not guaranteed to be stable and are not recommended for production use.
-This checkout moves the binary protocol to 0.12.0 and the metadata snapshot format to 13. Both are breaking: older clients and servers do not interoperate with it, and data directories written by older releases are refused. See [Upgrade recovery](core/server/README.md#upgrade-recovery).
+This checkout moves the binary protocol to 0.12.0 and the metadata snapshot format to 14. Both are breaking: older clients and servers do not interoperate with it, and data directories written by older releases are refused. See [Upgrade recovery](core/server/README.md#upgrade-recovery).
---
@@ -309,6 +309,8 @@ The CLI exposes `--durability persisted` and `--consumer-offset-durability persi
Streams, topics and users carry options from one catalog, listed by `iggy options <scope>` and changed by one sparse update. Each encrypted stream has its own data key, wrapped under the server's `[encryption] key`, and a stream created without encryption stays plaintext for life. Rotating the server key is a two-phase change through `previous_keys` and `retired_kek_fingerprints`. See [Resource options](core/server/README.md#resource-options) and [Stream encryption](core/server/README.md [...]
+Each node keeps a bounded log of the committed metadata changes, such as a stream created, a topic renamed, a user's permissions changed or a session closed. A client with `manage_servers` follows it with `metadata.changes` on every transport, `GET /metadata/changes` over HTTP or `iggy metadata changes`, and rebuilds from current state pages when its position falls out of the log. See [Metadata changes](core/server/README.md#metadata-changes).
+
For the detailed documentation of the configuration file, please refer to the [configuration](https://iggy.apache.org/docs/server/configuration) section.
---
diff --git a/bdd/cpp/features/step_definitions/metadata_changes_steps.cpp b/bdd/cpp/features/step_definitions/metadata_changes_steps.cpp
new file mode 100644
index 000000000..967c41d60
--- /dev/null
+++ b/bdd/cpp/features/step_definitions/metadata_changes_steps.cpp
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Unique step-class prefix so the generated CukeObject<n> symbols do not clash with the other
+// step files when linked together (see background_steps.cpp for details).
+#define CUKE_OBJECT_PREFIX IggyBddMetadataChanges
+
+#include <gtest/gtest.h>
+
+#include <cucumber-cpp/autodetect.hpp>
+
+#include <algorithm>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <variant>
+#include <vector>
+
+#include "world.hpp"
+
+namespace {
+
+constexpr std::uint32_t kRootUserId = 0;
+constexpr std::uint32_t kUnauthorized = 41;
+constexpr int kFollowAttempts = 100;
+constexpr auto kFollowInterval = std::chrono::milliseconds(100);
+constexpr const char *kHiddenOptions[] = {"encryption_key", "encryption_key_wrapped", "critical_keys"};
+
+// Walks the wire codes until to_string names the value, so no second name table is needed.
+template <typename Code>
+Code CodeNamed(const std::string &name) {
+ for (unsigned value = 0; value <= 255; ++value) {
+ try {
+ if (iggy::to_string(static_cast<Code>(value)) == name) {
+ return static_cast<Code>(value);
+ }
+ } catch (const std::invalid_argument &) {
+ }
+ }
+ throw std::invalid_argument("Unknown name " + name);
+}
+
+template <typename Code>
+std::vector<Code> CodesNamed(const std::string &names) {
+ std::vector<Code> codes;
+ std::stringstream stream(names);
+ std::string name;
+ while (std::getline(stream, name, ',')) {
+ codes.push_back(CodeNamed<Code>(name));
+ }
+ return codes;
+}
+
+iggy::MetadataChanges ReadChanges(bdd::GlobalContext &context, iggy::MetadataChangesQuery query) {
+ return bdd::sdk_client(context).GetMetadataChanges(
+ query.SetGroups(context.last_change_groups).SetLimit(iggy::MetadataChangesQuery::kMaxLimit));
+}
+
+std::optional<std::uint32_t> ChangeStreamId(const iggy::MetadataChangeRecord &record) {
+ if (const auto *stream = std::get_if<iggy::MetadataStreamChange>(&record.Change())) {
+ return stream->StreamId();
+ }
+ if (const auto *stream = std::get_if<iggy::MetadataStreamDeleted>(&record.Change())) {
+ return stream->StreamId();
+ }
+ if (const auto *topic = std::get_if<iggy::MetadataTopicChange>(&record.Change())) {
+ return topic->StreamId();
+ }
+ if (const auto *topic = std::get_if<iggy::MetadataTopicDeleted>(&record.Change())) {
+ return topic->StreamId();
+ }
+ if (const auto *partitions = std::get_if<iggy::MetadataPartitionsChanged>(&record.Change())) {
+ return partitions->StreamId();
+ }
+ return std::nullopt;
+}
+
+const iggy::MetadataChangeRecord &Followed(const bdd::GlobalContext &context, const std::string &kind) {
+ const auto wanted = CodeNamed<iggy::MetadataChangeKind>(kind);
+ for (const auto &record : context.last_changes) {
+ if (record.Kind() == wanted) {
+ return record;
+ }
+ }
+ throw std::runtime_error("No followed change of kind " + kind);
+}
+
+const iggy::ResourceOptions &FollowedStreamOptions(const bdd::GlobalContext &context, const std::string &kind) {
+ const auto &stream = std::get<iggy::MetadataStreamChange>(Followed(context, kind).Change());
+ if (!stream.Options()) {
+ throw std::runtime_error("Stream options must stay readable");
+ }
+ return *stream.Options();
+}
+
+std::optional<std::string> OptionText(const iggy::ResourceOptions &options, const std::string &key) {
+ for (const auto *entries : {&options.Explicit(), &options.Derived()}) {
+ const auto entry = entries->find(key);
+ if (entry != entries->end()) {
+ const auto &bytes = entry->second.Value();
+ return std::string(bytes.begin(), bytes.end());
+ }
+ }
+ return std::nullopt;
+}
+
+} // namespace
+
+GIVEN("^I hold the latest metadata change cursor for the groups \"([^\"]+)\"$") {
+ REGEX_PARAM(std::string, groups);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ context->last_change_groups = CodesNamed<iggy::MetadataChangeGroup>(groups);
+ const auto page = ReadChanges(*context, iggy::MetadataChangesQuery::Latest());
+ ASSERT_EQ(page.Status(), iggy::MetadataChangeStatus::Ok);
+ ASSERT_TRUE(page.Next().has_value());
+ context->last_change_cursor = page.Next();
+}
+
+WHEN("^I delete the stream$") {
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ bdd::sdk_client(*context).DeleteStream(iggy::Identifier::Numeric(context->last_stream_id.value()));
+}
+
+WHEN("^I create and delete ([0-9]+) streams named \"([^\"]+)\"$") {
+ REGEX_PARAM(int, count);
+ REGEX_PARAM(std::string, prefix);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ auto &client = bdd::sdk_client(*context);
+ for (int index = 0; index < count; ++index) {
+ const auto stream = client.CreateStream(prefix + "-" + std::to_string(index));
+ client.DeleteStream(iggy::Identifier::Numeric(stream.Id()));
+ }
+}
+
+THEN("^following the metadata changes should return the kinds \"([^\"]+)\" for the stream$") {
+ REGEX_PARAM(std::string, kinds);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ const auto expected = CodesNamed<iggy::MetadataChangeKind>(kinds);
+ const auto stream_id = context->last_stream_id.value();
+ auto cursor = context->last_change_cursor.value();
+ std::vector<iggy::MetadataChangeRecord> records;
+ // A two phase delete commits its record after the call returns.
+ for (int attempt = 0; attempt < kFollowAttempts; ++attempt) {
+ const auto page = ReadChanges(*context, iggy::MetadataChangesQuery::After(cursor));
+ ASSERT_EQ(page.Status(), iggy::MetadataChangeStatus::Ok);
+ cursor = page.Next().value();
+ records.insert(records.end(), page.Records().begin(), page.Records().end());
+ if (page.HasMore()) {
+ continue;
+ }
+ const auto mine = std::count_if(records.begin(), records.end(), [stream_id](const auto &record) {
+ return ChangeStreamId(record) == stream_id;
+ });
+ if (static_cast<std::size_t>(mine) >= expected.size()) {
+ break;
+ }
+ std::this_thread::sleep_for(kFollowInterval);
+ }
+
+ context->last_changes.clear();
+ std::vector<iggy::MetadataChangeKind> followed;
+ for (const auto &record : records) {
+ const auto group = record.Group();
+ ASSERT_TRUE(group.has_value());
+ EXPECT_NE(std::find(context->last_change_groups.begin(), context->last_change_groups.end(), *group),
+ context->last_change_groups.end())
+ << "record of op " << record.Op() << " is outside the requested groups";
+ if (ChangeStreamId(record) == stream_id) {
+ followed.push_back(record.Kind().value());
+ context->last_changes.push_back(record);
+ }
+ }
+ EXPECT_EQ(followed, expected);
+}
+
+THEN("^each followed change should name the root user as its actor$") {
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ for (const auto &record : context->last_changes) {
+ EXPECT_EQ(record.ActorUserId(), kRootUserId) << "record of op " << record.Op();
+ }
+}
+
+THEN("^the followed \"([^\"]+)\" change should carry a requested op before its own op$") {
+ REGEX_PARAM(std::string, kind);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ const auto &record = Followed(*context, kind);
+ const auto &deleted = std::get<iggy::MetadataStreamDeleted>(record.Change());
+ ASSERT_TRUE(deleted.RequestedOp().has_value());
+ EXPECT_LT(*deleted.RequestedOp(), record.Op());
+}
+
+THEN("^the followed \"([^\"]+)\" change should report encryption key source \"([^\"]+)\"$") {
+ REGEX_PARAM(std::string, kind);
+ REGEX_PARAM(std::string, source);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ EXPECT_EQ(OptionText(FollowedStreamOptions(*context, kind), "encryption_key_source"), source);
+}
+
+THEN("^the followed \"([^\"]+)\" change should carry the stream key fingerprint$") {
+ REGEX_PARAM(std::string, kind);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ ASSERT_TRUE(context->last_stream_key.has_value());
+ EXPECT_EQ(OptionText(FollowedStreamOptions(*context, kind), "encryption_key_fingerprint"),
+ iggy::EncryptionKeyFingerprint(*context->last_stream_key));
+}
+
+THEN("^the followed changes should expose no encryption key material$") {
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ for (const auto &record : context->last_changes) {
+ const auto *stream = std::get_if<iggy::MetadataStreamChange>(&record.Change());
+ if (stream == nullptr) {
+ continue;
+ }
+ ASSERT_TRUE(stream->Options().has_value()) << "stream options must stay readable";
+ for (const auto *hidden : kHiddenOptions) {
+ EXPECT_EQ(stream->Options()->Explicit().count(hidden), 0u) << hidden << " must never be returned";
+ EXPECT_EQ(stream->Options()->Derived().count(hidden), 0u) << hidden << " must never be returned";
+ }
+ }
+}
+
+THEN("^following the metadata changes should answer \"([^\"]+)\"$") {
+ REGEX_PARAM(std::string, status);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ const auto expected = CodeNamed<iggy::MetadataChangeStatus>(status);
+ const auto page = ReadChanges(*context, iggy::MetadataChangesQuery::After(context->last_change_cursor.value()));
+ EXPECT_EQ(page.Status(), expected);
+ if (expected != iggy::MetadataChangeStatus::Ok) {
+ EXPECT_TRUE(page.Records().empty());
+ EXPECT_FALSE(page.Next().has_value());
+ }
+}
+
+WHEN("^I read the current state for the groups \"([^\"]+)\"$") {
+ REGEX_PARAM(std::string, groups);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ context->last_change_groups = CodesNamed<iggy::MetadataChangeGroup>(groups);
+ context->last_current_items.clear();
+ auto query = iggy::MetadataChangesQuery::Current();
+ while (true) {
+ const auto page = ReadChanges(*context, query);
+ ASSERT_EQ(page.Status(), iggy::MetadataChangeStatus::Ok);
+ context->last_current_items.insert(context->last_current_items.end(), page.Items().begin(), page.Items().end());
+ if (!page.CurrentPageToken()) {
+ ASSERT_TRUE(page.Next().has_value());
+ context->last_change_cursor = page.Next();
+ break;
+ }
+ query = iggy::MetadataChangesQuery::Current(*page.CurrentPageToken());
+ }
+}
+
+THEN("^the current state should list the same streams as the stream list$") {
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ std::map<std::uint32_t, std::string> listed;
+ for (const auto &stream : bdd::sdk_client(*context).GetStreams()) {
+ listed.emplace(stream.Id(), stream.Name());
+ }
+ std::map<std::uint32_t, std::string> projected;
+ for (const auto &item : context->last_current_items) {
+ if (const auto *stream = std::get_if<iggy::MetadataStreamChange>(&item.Value())) {
+ projected.emplace(stream->StreamId(), stream->Name());
+ }
+ }
+ EXPECT_EQ(projected, listed);
+}
+
+GIVEN("^a user named \"([^\"]+)\" with password \"([^\"]+)\" and no server permissions exists$") {
+ REGEX_PARAM(std::string, username);
+ REGEX_PARAM(std::string, password);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ const auto user = bdd::sdk_client(*context).CreateUser(
+ username, password, iggy::UserStatus::Active,
+ iggy::Permissions().SetGlobal(iggy::GlobalPermissions().SetReadServers(true)), iggy::UserCreateOptions());
+ context->last_user_id = user.Id();
+}
+
+WHEN("^I authenticate as \"([^\"]+)\" with password \"([^\"]+)\"$") {
+ REGEX_PARAM(std::string, username);
+ REGEX_PARAM(std::string, password);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ auto &client = bdd::sdk_client(*context);
+ client.Logout();
+ client.Login(username, password);
+}
+
+THEN("^reading the metadata changes should be refused as unauthorized$") {
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ context->last_change_groups.clear();
+ try {
+ ReadChanges(*context, iggy::MetadataChangesQuery::Earliest());
+ FAIL() << "manage_servers is required";
+ } catch (const iggy::IggyException &error) {
+ EXPECT_EQ(error.Code(), kUnauthorized) << error.what();
+ }
+}
diff --git a/bdd/cpp/features/step_definitions/resource_options_steps.cpp b/bdd/cpp/features/step_definitions/resource_options_steps.cpp
index 598fab9af..81ef226f3 100644
--- a/bdd/cpp/features/step_definitions/resource_options_steps.cpp
+++ b/bdd/cpp/features/step_definitions/resource_options_steps.cpp
@@ -64,19 +64,6 @@ constexpr iggy::TopicOptionKey kTopicOptionKeys[] = {
iggy::TopicOptionKey::PreallocateSegments,
};
-iggy::IggyBlockingClient &Sdk(bdd::GlobalContext &context) {
- if (!context.sdk) {
- auto client =
- iggy::IggyBlockingClient::Builder()
- .WithServerAddress(bdd::required_env("IGGY_TCP_ADDRESS"))
- .WithAutoLogin(bdd::required_env("IGGY_ROOT_USERNAME"), bdd::required_env("IGGY_ROOT_PASSWORD"))
- .Build();
- client.Connect();
- context.sdk.emplace(std::move(client));
- }
- return *context.sdk;
-}
-
template <typename Key, std::size_t Count>
Key KeyFromName(const Key (&keys)[Count], const std::string &name) {
for (const auto key : keys) {
@@ -106,15 +93,15 @@ iggy::Identifier TopicIdentifier(const bdd::GlobalContext &context) {
}
iggy::StreamDetails FetchStream(bdd::GlobalContext &context) {
- return Sdk(context).GetStream(StreamIdentifier(context));
+ return bdd::sdk_client(context).GetStream(StreamIdentifier(context));
}
iggy::TopicDetails FetchTopic(bdd::GlobalContext &context) {
- return Sdk(context).GetTopic(StreamIdentifier(context), TopicIdentifier(context));
+ return bdd::sdk_client(context).GetTopic(StreamIdentifier(context), TopicIdentifier(context));
}
iggy::UserInfoDetails FetchUser(bdd::GlobalContext &context) {
- return Sdk(context).GetUser(iggy::Identifier::Numeric(context.last_user_id.value()));
+ return bdd::sdk_client(context).GetUser(iggy::Identifier::Numeric(context.last_user_id.value()));
}
void CreateStream(bdd::GlobalContext &context,
@@ -124,13 +111,13 @@ void CreateStream(bdd::GlobalContext &context,
if (key) {
options.SetEncryptionKey(*key);
}
- const auto stream = Sdk(context).CreateStream(name, options);
+ const auto stream = bdd::sdk_client(context).CreateStream(name, options);
context.last_stream_id = stream.Id();
context.last_stream_key = key;
}
void CreateTopic(bdd::GlobalContext &context, const std::string &name, iggy::TopicCreateOptions options) {
- const auto topic = Sdk(context).CreateTopic(StreamIdentifier(context), name, options);
+ const auto topic = bdd::sdk_client(context).CreateTopic(StreamIdentifier(context), name, options);
context.last_topic_id = topic.Id();
}
@@ -172,7 +159,7 @@ const iggy::OptionSpec &StreamOptionSpec(const bdd::GlobalContext &context, cons
}
void UpdateTopic(bdd::GlobalContext &context, const iggy::TopicUpdateOptions &options) {
- Sdk(context).UpdateTopic(StreamIdentifier(context), TopicIdentifier(context), options);
+ bdd::sdk_client(context).UpdateTopic(StreamIdentifier(context), TopicIdentifier(context), options);
}
} // namespace
@@ -311,7 +298,7 @@ WHEN("^I rename the stream to \"([^\"]+)\"$") {
REGEX_PARAM(std::string, name);
cucumber::ScenarioScope<bdd::GlobalContext> context;
- Sdk(*context).UpdateStream(StreamIdentifier(*context), iggy::StreamUpdateOptions().SetName(name));
+ bdd::sdk_client(*context).UpdateStream(StreamIdentifier(*context), iggy::StreamUpdateOptions().SetName(name));
}
THEN("^the stream name should be \"([^\"]+)\"$") {
@@ -327,7 +314,7 @@ WHEN("^I request a reset of the stream option \"([^\"]+)\"$") {
const auto options = iggy::StreamUpdateOptions().Reset(KeyFromName(kStreamOptionKeys, key));
try {
- Sdk(*context).UpdateStream(StreamIdentifier(*context), options);
+ bdd::sdk_client(*context).UpdateStream(StreamIdentifier(*context), options);
context->last_update_rejected = false;
} catch (const iggy::IggyException &) {
context->last_update_rejected = true;
@@ -343,7 +330,7 @@ THEN("^the stream update should be rejected$") {
WHEN("^I describe the stream options$") {
cucumber::ScenarioScope<bdd::GlobalContext> context;
- context->last_option_specs = Sdk(*context).DescribeOptions(iggy::OptionsScope::Stream);
+ context->last_option_specs = bdd::sdk_client(*context).DescribeOptions(iggy::OptionsScope::Stream);
}
THEN("^the stream option \"([^\"]+)\" should be updatable and not secret$") {
@@ -446,8 +433,8 @@ GIVEN("^a user named \"([^\"]+)\" with password \"([^\"]+)\" exists$") {
REGEX_PARAM(std::string, password);
cucumber::ScenarioScope<bdd::GlobalContext> context;
- const auto user =
- Sdk(*context).CreateUser(username, password, iggy::UserStatus::Active, std::nullopt, iggy::UserCreateOptions());
+ const auto user = bdd::sdk_client(*context).CreateUser(username, password, iggy::UserStatus::Active, std::nullopt,
+ iggy::UserCreateOptions());
context->last_user_id = user.Id();
}
@@ -456,8 +443,9 @@ WHEN("^I update the user username to \"([^\"]+)\" and status to \"([^\"]+)\"$")
REGEX_PARAM(std::string, status);
cucumber::ScenarioScope<bdd::GlobalContext> context;
- Sdk(*context).UpdateUser(iggy::Identifier::Numeric(context->last_user_id.value()),
- iggy::UserUpdateOptions().SetUsername(username).SetStatus(UserStatusFromName(status)));
+ bdd::sdk_client(*context).UpdateUser(
+ iggy::Identifier::Numeric(context->last_user_id.value()),
+ iggy::UserUpdateOptions().SetUsername(username).SetStatus(UserStatusFromName(status)));
}
THEN("^the user should have username \"([^\"]+)\"$") {
diff --git a/bdd/cpp/features/step_definitions/world.hpp b/bdd/cpp/features/step_definitions/world.hpp
index a10214778..9ddd576be 100644
--- a/bdd/cpp/features/step_definitions/world.hpp
+++ b/bdd/cpp/features/step_definitions/world.hpp
@@ -24,6 +24,7 @@
#include <optional>
#include <stdexcept>
#include <string>
+#include <utility>
#include <vector>
#include "iggy.hpp"
@@ -51,7 +52,7 @@ struct GlobalContext {
PolledData polled;
std::vector<std::uint8_t> raw_response;
std::string raw_error;
- // Public client for the option steps, connected on first use.
+ // Public client for the option and metadata change steps, see sdk_client().
std::optional<iggy::IggyBlockingClient> sdk;
std::optional<std::uint32_t> last_stream_id;
std::optional<std::uint32_t> last_topic_id;
@@ -60,6 +61,10 @@ struct GlobalContext {
bool last_update_rejected = false;
std::vector<iggy::OptionSpec> last_option_specs;
std::string call_error;
+ std::vector<iggy::MetadataChangeGroup> last_change_groups;
+ std::optional<iggy::MetadataChangeCursor> last_change_cursor;
+ std::vector<iggy::MetadataChangeRecord> last_changes;
+ std::vector<iggy::MetadataCurrentItem> last_current_items;
GlobalContext() = default;
GlobalContext(const GlobalContext &) = delete;
@@ -84,6 +89,19 @@ inline std::string required_env(const char *name) {
return std::string(value);
}
+// Public client for the option and metadata change steps, connected as root on first use.
+inline iggy::IggyBlockingClient &sdk_client(GlobalContext &context) {
+ if (!context.sdk) {
+ auto client = iggy::IggyBlockingClient::Builder()
+ .WithServerAddress(required_env("IGGY_TCP_ADDRESS"))
+ .WithAutoLogin(required_env("IGGY_ROOT_USERNAME"), required_env("IGGY_ROOT_PASSWORD"))
+ .Build();
+ client.Connect();
+ context.sdk.emplace(std::move(client));
+ }
+ return *context.sdk;
+}
+
inline iggy::ffi::Identifier make_string_identifier(const std::string &value) {
iggy::ffi::Identifier identifier;
identifier.set_string(value);
diff --git a/bdd/cpp/scripts/entrypoint.sh b/bdd/cpp/scripts/entrypoint.sh
index f04fcfd4e..0be4d5918 100755
--- a/bdd/cpp/scripts/entrypoint.sh
+++ b/bdd/cpp/scripts/entrypoint.sh
@@ -60,6 +60,7 @@ case "${BDD_FEATURE:-all}" in
raw_command) tags=(--tags @raw-command) ;;
stream_options) tags=(--tags @stream-options) ;;
resource_options_patch) tags=(--tags @resource-options-patch) ;;
+ metadata_changes) tags=(--tags @metadata-changes) ;;
session_contract) tags=(--tags @session-contract) ;;
esac
diff --git a/bdd/docker-compose.coverage.yml b/bdd/docker-compose.coverage.yml
index 1bdf7614c..68421a4df 100644
--- a/bdd/docker-compose.coverage.yml
+++ b/bdd/docker-compose.coverage.yml
@@ -39,7 +39,7 @@ services:
trap 'cargo llvm-cov report --lcov --output-path /reports/rust-bdd-coverage.lcov' EXIT
TEST_ARGS=()
case "$$BDD_FEATURE" in
- basic_messaging|leader_redirection|raw_command|session_contract|stream_crud|stream_options|resource_options_patch) TEST_ARGS=(--test "$$BDD_FEATURE") ;;
+ basic_messaging|leader_redirection|raw_command|session_contract|stream_crud|stream_options|resource_options_patch|metadata_changes) TEST_ARGS=(--test "$$BDD_FEATURE") ;;
esac
cargo test --locked -p bdd --features "$$BDD_RUST_FEATURES" "$${TEST_ARGS[@]}"
@@ -105,6 +105,7 @@ services:
leader_redirection) FILTER='--filter-trait Category=requires-leader-awareness' ;;
stream_options) FILTER='--filter-trait Category=stream-options' ;;
resource_options_patch) FILTER='--filter-trait Category=resource-options-patch' ;;
+ metadata_changes) FILTER='--filter-trait Category=metadata-changes' ;;
session_contract) FILTER='--filter-trait Category=session-contract' ;;
esac
dotnet test $$FILTER --coverage --coverage-output-format cobertura --coverage-output csharp-bdd-coverage.cobertura.xml --results-directory /reports
@@ -124,5 +125,6 @@ services:
stream_crud) export CUCUMBER_FILTER_TAGS='@stream-crud' ;;
stream_options) export CUCUMBER_FILTER_TAGS='@stream-options' ;;
resource_options_patch) export CUCUMBER_FILTER_TAGS='@resource-options-patch' ;;
+ metadata_changes) export CUCUMBER_FILTER_TAGS='@metadata-changes' ;;
esac
gradle --no-daemon test jacocoTestReport && cp build/reports/jacoco/test/jacocoTestReport.xml /reports/java-bdd-coverage.xml
diff --git a/bdd/docker-compose.server.yml b/bdd/docker-compose.server.yml
index 0f4c30093..b9a87337f 100644
--- a/bdd/docker-compose.server.yml
+++ b/bdd/docker-compose.server.yml
@@ -73,6 +73,8 @@ services:
# Every feature shares this server, and stream_options asserts server-generated stream keys.
- IGGY_ENCRYPTION_ENABLED=true
- IGGY_ENCRYPTION_KEY=/rvT1xP4V8u1EAhk4xDdqzqM2UOPXyy9XYkl4uRShgE=
+ # Small enough for metadata_changes to expire a cursor with a few dozen streams.
+ - IGGY_METADATA_CHANGE_LOG_RESOURCE_CAPACITY=64
volumes:
- iggy_data:/app/local_data
- ../target:/profiles
diff --git a/bdd/docker-compose.yml b/bdd/docker-compose.yml
index 681f689c4..3e3e582ad 100644
--- a/bdd/docker-compose.yml
+++ b/bdd/docker-compose.yml
@@ -38,6 +38,7 @@ services:
- ./scenarios/stream_crud.feature:/app/features/stream_crud.feature
- ./scenarios/stream_options.feature:/app/features/stream_options.feature
- ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature
+ - ./scenarios/metadata_changes.feature:/app/features/metadata_changes.feature
command:
- sh
- -c
@@ -50,6 +51,7 @@ services:
stream_crud) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test stream_crud ;;
stream_options) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test stream_options ;;
resource_options_patch) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test resource_options_patch ;;
+ metadata_changes) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test metadata_changes ;;
*) cargo test -p bdd --features "$$BDD_RUST_FEATURES" ;;
esac
networks:
@@ -69,6 +71,7 @@ services:
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
- ./scenarios/stream_options.feature:/app/features/stream_options.feature
- ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature
+ - ./scenarios/metadata_changes.feature:/app/features/metadata_changes.feature
- ./scenarios/session_contract.feature:/app/features/session_contract.feature
working_dir: /app
command:
@@ -81,6 +84,7 @@ services:
session_contract) TEST_PATH='tests/test_session_contract.py' ;;
stream_options) TEST_PATH='tests/test_stream_options.py' ;;
resource_options_patch) TEST_PATH='tests/test_resource_options_patch.py' ;;
+ metadata_changes) TEST_PATH='tests/test_metadata_changes.py' ;;
*) TEST_PATH='tests/' ;;
esac
uv run --project /workspace/bdd/python pytest $$TEST_PATH -v
@@ -100,6 +104,7 @@ services:
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
- ./scenarios/stream_options.feature:/app/features/stream_options.feature
- ./scenarios/session_contract.feature:/app/features/session_contract.feature
+ - ./scenarios/metadata_changes.feature:/app/features/metadata_changes.feature
command:
- sh
- -c
@@ -109,6 +114,7 @@ services:
raw_command) GROUP='--group raw-command' ;;
session_contract) GROUP='--group session-contract' ;;
stream_options) GROUP='--group stream-options' ;;
+ metadata_changes) GROUP='--group metadata-changes' ;;
*) GROUP='' ;;
esac
./scripts/test.sh --configuration /workspace/bdd/php/phpunit.xml.dist $$GROUP
@@ -148,6 +154,7 @@ services:
session_contract) TAGS='--tags @session-contract' ;;
stream_options) TAGS='--tags @stream-options' ;;
resource_options_patch) TAGS='--tags @resource-options-patch' ;;
+ metadata_changes) TAGS='--tags @metadata-changes' ;;
*) TAGS='' ;;
esac
npm run test:bdd -- $$TAGS
@@ -174,6 +181,7 @@ services:
raw_command) FILTER='--filter-trait Category=raw-command' ;;
stream_options) FILTER='--filter-trait Category=stream-options' ;;
resource_options_patch) FILTER='--filter-trait Category=resource-options-patch' ;;
+ metadata_changes) FILTER='--filter-trait Category=metadata-changes' ;;
session_contract) FILTER='--filter-trait Category=session-contract' ;;
esac
dotnet test $$FILTER
@@ -197,6 +205,7 @@ services:
- ./scenarios/stream_crud.feature:/app/features/stream_crud.feature
- ./scenarios/stream_options.feature:/app/features/stream_options.feature
- ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature
+ - ./scenarios/metadata_changes.feature:/app/features/metadata_changes.feature
command:
- sh
- -c
@@ -209,6 +218,7 @@ services:
stream_crud) export CUCUMBER_FILTER_TAGS='@stream-crud' ;;
stream_options) export CUCUMBER_FILTER_TAGS='@stream-options' ;;
resource_options_patch) export CUCUMBER_FILTER_TAGS='@resource-options-patch' ;;
+ metadata_changes) export CUCUMBER_FILTER_TAGS='@metadata-changes' ;;
esac
gradle --no-daemon test
networks:
@@ -228,6 +238,7 @@ services:
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
- ./scenarios/stream_options.feature:/app/features/stream_options.feature
- ./scenarios/resource_options_patch.feature:/app/features/resource_options_patch.feature
+ - ./scenarios/metadata_changes.feature:/app/features/metadata_changes.feature
- ./scenarios/session_contract.feature:/app/features/session_contract.feature
networks:
- iggy-bdd-network
diff --git a/bdd/go/tests/metadata_changes.go b/bdd/go/tests/metadata_changes.go
new file mode 100644
index 000000000..7a35c6d41
--- /dev/null
+++ b/bdd/go/tests/metadata_changes.go
@@ -0,0 +1,471 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package tests
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "maps"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/apache/iggy/bdd/go/tests/env"
+ iggcon "github.com/apache/iggy/foreign/go/contracts"
+ ierror "github.com/apache/iggy/foreign/go/errors"
+ "github.com/cucumber/godog"
+)
+
+const (
+ // A two phase delete commits its record after the call returns.
+ metadataChangesFollowAttempts = 100
+ metadataChangesFollowInterval = 100 * time.Millisecond
+ rootUserID = 0
+)
+
+type metadataChangesCtxKey struct{}
+
+type metadataChangesCtx struct {
+ groups []iggcon.MetadataChangeGroup
+ cursor *iggcon.MetadataChangeCursor
+ followed []iggcon.MetadataChangeRecord
+ currentItems []iggcon.MetadataCurrentItem
+ signedInAsNew bool
+}
+
+func getMetadataChangesCtx(ctx context.Context) *metadataChangesCtx {
+ return ctx.Value(metadataChangesCtxKey{}).(*metadataChangesCtx)
+}
+
+type metadataChangesSteps struct{}
+
+func (s metadataChangesSteps) givenLatestCursor(ctx context.Context, groups string) error {
+ state := getMetadataChangesCtx(ctx)
+ parsed, err := parseMetadataChangeGroups(groups)
+ if err != nil {
+ return err
+ }
+ state.groups = parsed
+ changes, err := s.read(ctx, iggcon.LatestMetadataChangeStart())
+ if err != nil {
+ return fmt.Errorf("read the latest cursor: %w", err)
+ }
+ if changes.Status != iggcon.MetadataChangeStatusOk || changes.Next == nil {
+ return fmt.Errorf("latest answered %s with cursor %v", changes.Status, changes.Next)
+ }
+ state.cursor = changes.Next
+ return nil
+}
+
+func (metadataChangesSteps) whenDeleteStream(ctx context.Context) error {
+ options := getResourceOptionsCtx(ctx)
+ if err := options.client.DeleteStream(ctx, streamIdentifier(options)); err != nil {
+ return fmt.Errorf("delete stream: %w", err)
+ }
+ options.createdStreams = slices.DeleteFunc(options.createdStreams, func(id uint32) bool { return id == *options.streamId })
+ return nil
+}
+
+func (metadataChangesSteps) whenChurnStreams(ctx context.Context, count int, prefix string) error {
+ client := getResourceOptionsCtx(ctx).client
+ for index := range count {
+ stream, err := client.CreateStream(ctx, fmt.Sprintf("%s-%d", prefix, index), nil)
+ if err != nil {
+ return fmt.Errorf("create stream %d: %w", index, err)
+ }
+ streamId, _ := iggcon.NewIdentifier(stream.Id)
+ if err := client.DeleteStream(ctx, streamId); err != nil {
+ return fmt.Errorf("delete stream %d: %w", index, err)
+ }
+ }
+ return nil
+}
+
+func (s metadataChangesSteps) thenFollowedKinds(ctx context.Context, kinds string) error {
+ state := getMetadataChangesCtx(ctx)
+ streamKey, err := s.createdStreamKey(ctx)
+ if err != nil {
+ return err
+ }
+ var want []iggcon.MetadataChangeKind
+ for _, name := range strings.Split(kinds, ",") {
+ kind, err := iggcon.ParseMetadataChangeKind(strings.TrimSpace(name))
+ if err != nil {
+ return err
+ }
+ want = append(want, kind)
+ }
+ cursor := *state.cursor
+ var records []iggcon.MetadataChangeRecord
+ var mine []iggcon.MetadataChangeRecord
+ for range metadataChangesFollowAttempts {
+ changes, err := s.read(ctx, iggcon.AfterMetadataChangeStart(cursor))
+ if err != nil {
+ return fmt.Errorf("follow changes: %w", err)
+ }
+ if changes.Status != iggcon.MetadataChangeStatusOk || changes.Next == nil {
+ return fmt.Errorf("follow answered %s with cursor %v", changes.Status, changes.Next)
+ }
+ cursor = *changes.Next
+ records = append(records, changes.Records...)
+ if changes.HasMore {
+ continue
+ }
+ mine = slices.DeleteFunc(slices.Clone(records), func(record iggcon.MetadataChangeRecord) bool {
+ key, ok := changeStreamKey(record)
+ return !ok || key != streamKey
+ })
+ if len(mine) >= len(want) {
+ break
+ }
+ time.Sleep(metadataChangesFollowInterval)
+ }
+ for _, record := range records {
+ if len(state.groups) > 0 && !slices.Contains(state.groups, record.Group) {
+ return fmt.Errorf("record of group %s outside the requested %v", record.Group, state.groups)
+ }
+ }
+ got := make([]iggcon.MetadataChangeKind, 0, len(mine))
+ for _, record := range mine {
+ got = append(got, record.Kind)
+ }
+ if !slices.Equal(got, want) {
+ return fmt.Errorf("followed kinds %v, want %v", got, want)
+ }
+ state.followed = mine
+ return nil
+}
+
+func (metadataChangesSteps) thenRootActor(ctx context.Context) error {
+ for _, record := range getMetadataChangesCtx(ctx).followed {
+ if record.ActorUserID == nil || *record.ActorUserID != rootUserID {
+ return fmt.Errorf("op %d names actor %v, want the root user", record.Op, record.ActorUserID)
+ }
+ }
+ return nil
+}
+
+func (s metadataChangesSteps) thenRequestedOpBeforeOwnOp(ctx context.Context, kind string) error {
+ record, err := s.followedRecord(ctx, kind)
+ if err != nil {
+ return err
+ }
+ deleted, ok := record.Body.(*iggcon.MetadataStreamDeleted)
+ if !ok {
+ return fmt.Errorf("expected a stream deletion, got %T", record.Body)
+ }
+ if deleted.RequestedOp == nil || *deleted.RequestedOp >= record.Op {
+ return fmt.Errorf("requested op %v is not before op %d", deleted.RequestedOp, record.Op)
+ }
+ return nil
+}
+
+func (s metadataChangesSteps) thenChangeKeySource(ctx context.Context, kind, source string) error {
+ stream, err := s.followedStreamOptions(ctx, kind)
+ if err != nil {
+ return err
+ }
+ if got, ok := stream.EncryptionKeySource(); !ok || got != iggcon.StreamEncryptionSource(source) {
+ return fmt.Errorf("encryption key source is %q (reported %t), want %q", got, ok, source)
+ }
+ return nil
+}
+
+func (s metadataChangesSteps) thenChangeFingerprint(ctx context.Context, kind string) error {
+ stream, err := s.followedStreamOptions(ctx, kind)
+ if err != nil {
+ return err
+ }
+ want, err := iggcon.EncryptionKeyFingerprint(getResourceOptionsCtx(ctx).streamKey)
+ if err != nil {
+ return fmt.Errorf("fingerprint the stream key: %w", err)
+ }
+ if got, ok := stream.EncryptionKeyFingerprint(); !ok || got != want {
+ return fmt.Errorf("encryption key fingerprint is %q (reported %t), want %q", got, ok, want)
+ }
+ return nil
+}
+
+func (metadataChangesSteps) thenChangesHideKeys(ctx context.Context) error {
+ key := getResourceOptionsCtx(ctx).streamKey
+ for _, record := range getMetadataChangesCtx(ctx).followed {
+ stream, ok := record.Body.(*iggcon.MetadataStreamChange)
+ if !ok {
+ continue
+ }
+ if stream.Options.Unreadable {
+ return fmt.Errorf("op %d options must stay readable", record.Op)
+ }
+ for _, options := range []map[string]iggcon.HeaderValue{stream.Options.Options, stream.Options.DerivedOptions} {
+ for _, hidden := range []string{string(iggcon.StreamOptionEncryptionKey), "encryption_key_wrapped", "critical_keys"} {
+ if _, found := options[hidden]; found {
+ return fmt.Errorf("op %d returns %s", record.Op, hidden)
+ }
+ }
+ for name, value := range options {
+ if len(key) > 0 && (bytes.Contains(value.Value, key) ||
+ bytes.Contains(value.Value, []byte(base64.StdEncoding.EncodeToString(key))) ||
+ bytes.Contains(value.Value, []byte(hex.EncodeToString(key)))) {
+ return fmt.Errorf("op %d option %s carries the stream key", record.Op, name)
+ }
+ }
+ }
+ }
+ return nil
+}
+
+func (s metadataChangesSteps) thenFollowStatus(ctx context.Context, status string) error {
+ state := getMetadataChangesCtx(ctx)
+ changes, err := s.read(ctx, iggcon.AfterMetadataChangeStart(*state.cursor))
+ if err != nil {
+ return fmt.Errorf("follow changes: %w", err)
+ }
+ if changes.Status.String() != status {
+ return fmt.Errorf("follow answered %s, want %s", changes.Status, status)
+ }
+ if changes.Status != iggcon.MetadataChangeStatusOk && (len(changes.Records) != 0 || changes.Next != nil) {
+ return fmt.Errorf("a %s answer carries %d records and cursor %v", changes.Status, len(changes.Records), changes.Next)
+ }
+ return nil
+}
+
+func (s metadataChangesSteps) whenReadCurrentState(ctx context.Context, groups string) error {
+ state := getMetadataChangesCtx(ctx)
+ parsed, err := parseMetadataChangeGroups(groups)
+ if err != nil {
+ return err
+ }
+ state.groups = parsed
+ state.currentItems = nil
+ var token *iggcon.MetadataCurrentPageToken
+ for {
+ changes, err := s.read(ctx, iggcon.CurrentMetadataChangeStart(token))
+ if err != nil {
+ return fmt.Errorf("read current state: %w", err)
+ }
+ if changes.Status != iggcon.MetadataChangeStatusOk {
+ return fmt.Errorf("current state answered %s", changes.Status)
+ }
+ state.currentItems = append(state.currentItems, changes.Items...)
+ if changes.CurrentPageToken == nil {
+ if changes.Next == nil {
+ return errors.New("the last current state page returned no anchor")
+ }
+ state.cursor = changes.Next
+ return nil
+ }
+ token = changes.CurrentPageToken
+ }
+}
+
+func (metadataChangesSteps) thenCurrentMatchesList(ctx context.Context) error {
+ listed, err := getResourceOptionsCtx(ctx).client.GetStreams(ctx)
+ if err != nil {
+ return fmt.Errorf("list streams: %w", err)
+ }
+ want := make(map[uint32]string, len(listed))
+ for _, stream := range listed {
+ want[stream.Id] = stream.Name
+ }
+ got := make(map[uint32]string)
+ for _, item := range getMetadataChangesCtx(ctx).currentItems {
+ if stream, ok := item.Body.(*iggcon.MetadataStreamChange); ok {
+ got[stream.StreamID] = stream.Name
+ }
+ }
+ if !maps.Equal(got, want) {
+ return fmt.Errorf("current state lists %v, the stream list %v", got, want)
+ }
+ return nil
+}
+
+func (metadataChangesSteps) givenUserWithoutServerPermissions(ctx context.Context, username, password string) error {
+ options := getResourceOptionsCtx(ctx)
+ permissions := &iggcon.Permissions{Global: iggcon.GlobalPermissions{ReadServers: true}}
+ user, err := options.client.CreateUser(ctx, username, password, iggcon.Active, permissions, nil)
+ if err != nil {
+ return fmt.Errorf("create user %s: %w", username, err)
+ }
+ options.userId = &user.Id
+ options.createdUsers = append(options.createdUsers, user.Id)
+ return nil
+}
+
+func (metadataChangesSteps) whenAuthenticateAs(ctx context.Context, username, password string) error {
+ client := getResourceOptionsCtx(ctx).client
+ if err := client.LogoutUser(ctx); err != nil {
+ return fmt.Errorf("logout: %w", err)
+ }
+ getMetadataChangesCtx(ctx).signedInAsNew = true
+ if _, err := client.LoginUser(ctx, username, password); err != nil {
+ return fmt.Errorf("login as %s: %w", username, err)
+ }
+ return nil
+}
+
+func (s metadataChangesSteps) thenChangesRefused(ctx context.Context) error {
+ getMetadataChangesCtx(ctx).groups = nil
+ _, err := s.read(ctx, iggcon.EarliestMetadataChangeStart())
+ if !errors.Is(err, ierror.ErrUnauthorized) {
+ return fmt.Errorf("reading changes returned %v, want unauthorized", err)
+ }
+ return nil
+}
+
+func (metadataChangesSteps) read(ctx context.Context, start iggcon.MetadataChangeStart) (*iggcon.MetadataChanges, error) {
+ query := iggcon.NewMetadataChangesQuery(start, getMetadataChangesCtx(ctx).groups...)
+ query.Limit = iggcon.MaxMetadataChangesLimit
+ return getResourceOptionsCtx(ctx).client.GetMetadataChanges(ctx, query)
+}
+
+// createdStreamKey reads the key of the scenario's stream from its creation
+// record. Ids are reused after a delete, so a stream id alone can match a
+// stream another test created and deleted meanwhile.
+func (metadataChangesSteps) createdStreamKey(ctx context.Context) (iggcon.MetadataStreamKey, error) {
+ options := getResourceOptionsCtx(ctx)
+ query := iggcon.NewMetadataChangesQuery(
+ iggcon.AfterMetadataChangeStart(*getMetadataChangesCtx(ctx).cursor),
+ iggcon.MetadataChangeGroupStreams,
+ )
+ query.Limit = iggcon.MaxMetadataChangesLimit
+ for {
+ changes, err := options.client.GetMetadataChanges(ctx, query)
+ if err != nil {
+ return iggcon.MetadataStreamKey{}, fmt.Errorf("read stream changes: %w", err)
+ }
+ if changes.Status != iggcon.MetadataChangeStatusOk || changes.Next == nil {
+ return iggcon.MetadataStreamKey{}, fmt.Errorf("stream changes answered %s with cursor %v", changes.Status, changes.Next)
+ }
+ for _, record := range changes.Records {
+ created, ok := record.Body.(*iggcon.MetadataStreamChange)
+ if ok && record.Kind == iggcon.MetadataChangeKindStreamCreated &&
+ created.StreamID == *options.streamId && created.Name == options.streamName {
+ return created.MetadataStreamKey, nil
+ }
+ }
+ if !changes.HasMore {
+ return iggcon.MetadataStreamKey{}, fmt.Errorf("no stream_created change for stream %d %q", *options.streamId, options.streamName)
+ }
+ query.Start = iggcon.AfterMetadataChangeStart(*changes.Next)
+ }
+}
+
+func (metadataChangesSteps) followedRecord(ctx context.Context, kind string) (iggcon.MetadataChangeRecord, error) {
+ want, err := iggcon.ParseMetadataChangeKind(kind)
+ if err != nil {
+ return iggcon.MetadataChangeRecord{}, err
+ }
+ for _, record := range getMetadataChangesCtx(ctx).followed {
+ if record.Kind == want {
+ return record, nil
+ }
+ }
+ return iggcon.MetadataChangeRecord{}, fmt.Errorf("no followed %s change", kind)
+}
+
+// followedStreamOptions views a followed stream change's options as a stream,
+// so the stream's projection helpers read them.
+func (s metadataChangesSteps) followedStreamOptions(ctx context.Context, kind string) (iggcon.Stream, error) {
+ record, err := s.followedRecord(ctx, kind)
+ if err != nil {
+ return iggcon.Stream{}, err
+ }
+ stream, ok := record.Body.(*iggcon.MetadataStreamChange)
+ if !ok {
+ return iggcon.Stream{}, fmt.Errorf("expected a stream change, got %T", record.Body)
+ }
+ if stream.Options.Unreadable {
+ return iggcon.Stream{}, errors.New("stream options must stay readable")
+ }
+ return iggcon.Stream{Options: stream.Options.Options, DerivedOptions: stream.Options.DerivedOptions}, nil
+}
+
+func initMetadataChangesScenario(sc *godog.ScenarioContext) {
+ sc.Before(func(context.Context, *godog.Scenario) (context.Context, error) {
+ ctx := context.WithValue(context.Background(), resourceOptionsCtxKey{}, &resourceOptionsCtx{})
+ return context.WithValue(ctx, metadataChangesCtxKey{}, &metadataChangesCtx{}), nil
+ })
+ options := resourceOptionsSteps{}
+ s := metadataChangesSteps{}
+ sc.Step(`^I have a running Iggy server$`, options.givenRunningServer)
+ sc.Step(`^I am authenticated as the root user$`, options.givenAuthenticationAsRoot)
+ sc.Step(`^I create a stream named "([^"]+)" without an encryption key$`, options.whenCreateKeylessStream)
+ sc.Step(`^I create a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+)$`, options.whenCreateKeyedStream)
+ sc.Step(`^I rename the stream to "([^"]+)"$`, options.whenRenameStream)
+ sc.Step(`^I create a topic named "([^"]+)" with (\d+) partitions? in the stream$`, options.whenCreateTopic)
+ sc.Step(`^I hold the latest metadata change cursor for the groups "([^"]+)"$`, s.givenLatestCursor)
+ sc.Step(`^I delete the stream$`, s.whenDeleteStream)
+ sc.Step(`^I create and delete (\d+) streams named "([^"]+)"$`, s.whenChurnStreams)
+ sc.Step(`^following the metadata changes should return the kinds "([^"]+)" for the stream$`, s.thenFollowedKinds)
+ sc.Step(`^each followed change should name the root user as its actor$`, s.thenRootActor)
+ sc.Step(`^the followed "([^"]+)" change should carry a requested op before its own op$`, s.thenRequestedOpBeforeOwnOp)
+ sc.Step(`^the followed "([^"]+)" change should report encryption key source "([^"]+)"$`, s.thenChangeKeySource)
+ sc.Step(`^the followed "([^"]+)" change should carry the stream key fingerprint$`, s.thenChangeFingerprint)
+ sc.Step(`^the followed changes should expose no encryption key material$`, s.thenChangesHideKeys)
+ sc.Step(`^following the metadata changes should answer "([^"]+)"$`, s.thenFollowStatus)
+ sc.Step(`^I read the current state for the groups "([^"]+)"$`, s.whenReadCurrentState)
+ sc.Step(`^the current state should list the same streams as the stream list$`, s.thenCurrentMatchesList)
+ sc.Step(`^a user named "([^"]+)" with password "([^"]+)" and no server permissions exists$`, s.givenUserWithoutServerPermissions)
+ sc.Step(`^I authenticate as "([^"]+)" with password "([^"]+)"$`, s.whenAuthenticateAs)
+ sc.Step(`^reading the metadata changes should be refused as unauthorized$`, s.thenChangesRefused)
+ sc.After(func(ctx context.Context, scenario *godog.Scenario, scenarioError error) (context.Context, error) {
+ options := getResourceOptionsCtx(ctx)
+ if options.client != nil && getMetadataChangesCtx(ctx).signedInAsNew {
+ username, password := env.RootCredentials()
+ logoutError := options.client.LogoutUser(ctx)
+ _, loginError := options.client.LoginUser(ctx, username, password)
+ scenarioError = errors.Join(scenarioError, logoutError, loginError)
+ }
+ return resourceOptionsCleanup(ctx, scenario, scenarioError)
+ })
+}
+
+func parseMetadataChangeGroups(names string) ([]iggcon.MetadataChangeGroup, error) {
+ var groups []iggcon.MetadataChangeGroup
+ for _, name := range strings.Split(names, ",") {
+ group, err := iggcon.ParseMetadataChangeGroup(strings.TrimSpace(name))
+ if err != nil {
+ return nil, err
+ }
+ groups = append(groups, group)
+ }
+ return groups, nil
+}
+
+// changeStreamKey names the stream a stream, topic or partitions record
+// belongs to.
+func changeStreamKey(record iggcon.MetadataChangeRecord) (iggcon.MetadataStreamKey, bool) {
+ switch body := record.Body.(type) {
+ case *iggcon.MetadataStreamChange:
+ return body.MetadataStreamKey, true
+ case *iggcon.MetadataStreamDeleted:
+ return body.MetadataStreamKey, true
+ case *iggcon.MetadataTopicChange:
+ return body.MetadataStreamKey, true
+ case *iggcon.MetadataTopicDeleted:
+ return body.MetadataStreamKey, true
+ case *iggcon.MetadataPartitionsChanged:
+ return body.MetadataStreamKey, true
+ default:
+ return iggcon.MetadataStreamKey{}, false
+ }
+}
diff --git a/bdd/go/tests/resource_options.go b/bdd/go/tests/resource_options.go
index 9a250ba22..db0c62fcc 100644
--- a/bdd/go/tests/resource_options.go
+++ b/bdd/go/tests/resource_options.go
@@ -42,6 +42,7 @@ type resourceOptionsCtxKey struct{}
type resourceOptionsCtx struct {
client iggcon.Client
streamId *uint32
+ streamName string
streamKey []byte
topicId *uint32
userId *uint32
@@ -100,6 +101,7 @@ func (resourceOptionsSteps) createStream(ctx context.Context, name string, optio
return fmt.Errorf("create stream %s: %w", name, err)
}
state.streamId = &stream.Id
+ state.streamName = name
state.createdStreams = append(state.createdStreams, stream.Id)
return nil
}
@@ -563,21 +565,25 @@ func initResourceOptionsScenario(sc *godog.ScenarioContext) {
sc.Step(`^I update the user username to "([^"]+)" and status to "([^"]+)"$`, s.whenUpdateUser)
sc.Step(`^the user should have username "([^"]+)"$`, s.thenUserUsername)
sc.Step(`^the user should have status "([^"]+)"$`, s.thenUserStatus)
- sc.After(func(ctx context.Context, _ *godog.Scenario, scenarioError error) (context.Context, error) {
- state := getResourceOptionsCtx(ctx)
- if state.client == nil {
- return ctx, scenarioError
- }
- for _, id := range state.createdStreams {
- streamId, err := iggcon.NewIdentifier(id)
- scenarioError = errors.Join(scenarioError, err, state.client.DeleteStream(ctx, streamId))
- }
- for _, id := range state.createdUsers {
- userId, err := iggcon.NewIdentifier(id)
- scenarioError = errors.Join(scenarioError, err, state.client.DeleteUser(ctx, userId))
- }
- return ctx, errors.Join(scenarioError, state.client.Close())
- })
+ sc.After(resourceOptionsCleanup)
+}
+
+// resourceOptionsCleanup deletes the streams and users a scenario created and
+// closes its client.
+func resourceOptionsCleanup(ctx context.Context, _ *godog.Scenario, scenarioError error) (context.Context, error) {
+ state := getResourceOptionsCtx(ctx)
+ if state.client == nil {
+ return ctx, scenarioError
+ }
+ for _, id := range state.createdStreams {
+ streamId, err := iggcon.NewIdentifier(id)
+ scenarioError = errors.Join(scenarioError, err, state.client.DeleteStream(ctx, streamId))
+ }
+ for _, id := range state.createdUsers {
+ userId, err := iggcon.NewIdentifier(id)
+ scenarioError = errors.Join(scenarioError, err, state.client.DeleteUser(ctx, userId))
+ }
+ return ctx, errors.Join(scenarioError, state.client.Close())
}
func streamIdentifier(state *resourceOptionsCtx) iggcon.Identifier {
diff --git a/bdd/go/tests/suite_test.go b/bdd/go/tests/suite_test.go
index 3adf9c1e8..ddbff0039 100644
--- a/bdd/go/tests/suite_test.go
+++ b/bdd/go/tests/suite_test.go
@@ -99,6 +99,18 @@ func TestFeatures(t *testing.T) {
})
}
+ if feature == "all" || feature == "metadata_changes" {
+ suites = append(suites, godog.TestSuite{
+ ScenarioInitializer: initMetadataChangesScenario,
+ Options: &godog.Options{
+ Format: "pretty",
+ Paths: []string{"../../scenarios/metadata_changes.feature"},
+ Strict: true,
+ TestingT: t,
+ },
+ })
+ }
+
if len(suites) == 0 {
t.Fatalf("unknown BDD_FEATURE=%q", feature)
}
diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/MetadataChangesSteps.java b/bdd/java/src/test/java/org/apache/iggy/bdd/MetadataChangesSteps.java
new file mode 100644
index 000000000..6c2db1138
--- /dev/null
+++ b/bdd/java/src/test/java/org/apache/iggy/bdd/MetadataChangesSteps.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iggy.bdd;
+
+import io.cucumber.java.After;
+import io.cucumber.java.en.Given;
+import io.cucumber.java.en.Then;
+import io.cucumber.java.en.When;
+import org.apache.iggy.client.blocking.tcp.IggyTcpClient;
+import org.apache.iggy.exception.IggyErrorCode;
+import org.apache.iggy.exception.IggyServerException;
+import org.apache.iggy.stream.EncryptionKeySource;
+import org.apache.iggy.stream.EncryptionKeys;
+import org.apache.iggy.stream.StreamBase;
+import org.apache.iggy.system.MetadataChangeBody;
+import org.apache.iggy.system.MetadataChangeCursor;
+import org.apache.iggy.system.MetadataChangeGroup;
+import org.apache.iggy.system.MetadataChangeKind;
+import org.apache.iggy.system.MetadataChangeRecord;
+import org.apache.iggy.system.MetadataChangeStart;
+import org.apache.iggy.system.MetadataChangeStatus;
+import org.apache.iggy.system.MetadataChanges;
+import org.apache.iggy.system.MetadataChangesQuery;
+import org.apache.iggy.system.MetadataCurrentItem;
+import org.apache.iggy.user.GlobalPermissions;
+import org.apache.iggy.user.Permissions;
+import org.apache.iggy.user.UserOptions;
+import org.apache.iggy.user.UserStatus;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public final class MetadataChangesSteps {
+
+ private static final long ROOT_USER_ID = 0;
+ private static final int FOLLOW_ATTEMPTS = 100;
+ private static final long FOLLOW_INTERVAL_MILLIS = 100;
+ private static final List<String> HIDDEN_OPTIONS =
+ List.of("encryption_key", "encryption_key_wrapped", "critical_keys");
+
+ private IggyTcpClient client;
+ private List<MetadataChangeGroup> groups = List.of();
+ private MetadataChangeCursor cursor;
+ private List<MetadataChangeRecord> followed = List.of();
+ private List<MetadataCurrentItem> items = List.of();
+ private Long userId;
+
+ // The client may be signed in as a user without manage users by now, so root signs in again to remove it.
+ @After
+ public void close() {
+ if (client == null) {
+ return;
+ }
+ if (userId != null) {
+ client.users().logout();
+ client.users().login(TestEnvironment.rootUsername(), TestEnvironment.rootPassword());
+ client.users().deleteUser(userId);
+ }
+ client.close();
+ }
+
+ @Given("I hold the latest metadata change cursor for the groups {string}")
+ public void holdLatestCursor(String names) {
+ groups = parseGroups(names);
+ var page = read(MetadataChangeStart.latest());
+ assertEquals(MetadataChangeStatus.OK, page.status());
+ cursor = page.next().orElseThrow();
+ }
+
+ @When("I create and delete {int} streams named {string}")
+ public void churnStreams(int count, String prefix) {
+ for (int index = 0; index < count; index++) {
+ var stream = client().streams().createStream(prefix + "-" + index);
+ client().streams().deleteStream(stream.id());
+ }
+ }
+
+ @Then("following the metadata changes should return the kinds {string} for the stream")
+ public void followedKinds(String kinds) throws InterruptedException {
+ List<MetadataChangeKind> expected = Arrays.stream(kinds.split(","))
+ .map(name -> MetadataChangeKind.fromValue(name.trim()).orElseThrow())
+ .toList();
+ long streamId = ResourceOptionsSteps.current().createdStreamId();
+ List<MetadataChangeRecord> records = new ArrayList<>();
+ var position = cursor;
+ // A two phase delete commits its record after the call returns.
+ for (int attempt = 0; attempt < FOLLOW_ATTEMPTS; attempt++) {
+ var page = read(MetadataChangeStart.after(position));
+ assertEquals(MetadataChangeStatus.OK, page.status());
+ position = page.next().orElseThrow();
+ records.addAll(page.records());
+ if (page.hasMore()) {
+ continue;
+ }
+ if (records.stream().filter(record -> namesStream(record, streamId)).count() >= expected.size()) {
+ break;
+ }
+ Thread.sleep(FOLLOW_INTERVAL_MILLIS);
+ }
+ for (MetadataChangeRecord record : records) {
+ assertTrue(groups.contains(record.group().orElseThrow()), record.toString());
+ }
+ followed =
+ records.stream().filter(record -> namesStream(record, streamId)).toList();
+ assertEquals(
+ expected,
+ followed.stream().map(record -> record.kind().orElseThrow()).toList());
+ }
+
+ @Then("each followed change should name the root user as its actor")
+ public void rootActor() {
+ for (MetadataChangeRecord record : followed) {
+ assertEquals(Optional.of(ROOT_USER_ID), record.actorUserId(), record.toString());
+ }
+ }
+
+ @Then("the followed {string} change should carry a requested op before its own op")
+ public void requestedOpBeforeOwnOp(String kind) {
+ var record = followed(kind);
+ var deleted = assertInstanceOf(MetadataChangeBody.StreamDeleted.class, record.body());
+ assertTrue(deleted.requestedOp().orElseThrow().compareTo(record.op()) < 0, record.toString());
+ }
+
+ @Then("the followed {string} change should report encryption key source {string}")
+ public void keySource(String kind, String source) {
+ assertEquals(
+ EncryptionKeySource.fromValue(source).orElseThrow(),
+ followedStream(kind).encryptionKeySource().orElseThrow());
+ }
+
+ @Then("the followed {string} change should carry the stream key fingerprint")
+ public void keyFingerprint(String kind) {
+ assertEquals(
+ EncryptionKeys.fingerprint(ResourceOptionsSteps.current().streamKey()),
+ followedStream(kind).encryptionKeyFingerprint().orElseThrow());
+ }
+
+ @Then("the followed changes should expose no encryption key material")
+ public void noKeyMaterial() {
+ for (MetadataChangeRecord record : followed) {
+ if (record.body() instanceof MetadataChangeBody.StreamChange stream) {
+ assertFalse(stream.optionsUnreadable(), record.toString());
+ for (String hidden : HIDDEN_OPTIONS) {
+ assertFalse(stream.options().containsKey(hidden), hidden);
+ assertFalse(stream.derivedOptions().containsKey(hidden), hidden);
+ }
+ }
+ }
+ }
+
+ @Then("following the metadata changes should answer {string}")
+ public void followStatus(String status) {
+ var expected = MetadataChangeStatus.fromValue(status).orElseThrow();
+ var page = read(MetadataChangeStart.after(cursor));
+ assertEquals(expected, page.status());
+ if (expected != MetadataChangeStatus.OK) {
+ assertTrue(page.records().isEmpty());
+ assertTrue(page.next().isEmpty());
+ }
+ }
+
+ @When("I read the current state for the groups {string}")
+ public void readCurrentState(String names) {
+ groups = parseGroups(names);
+ List<MetadataCurrentItem> scanned = new ArrayList<>();
+ var start = MetadataChangeStart.current();
+ while (true) {
+ var page = read(start);
+ assertEquals(MetadataChangeStatus.OK, page.status());
+ scanned.addAll(page.items());
+ if (page.currentPageToken().isEmpty()) {
+ cursor = page.next().orElseThrow();
+ break;
+ }
+ start = MetadataChangeStart.current(page.currentPageToken().orElseThrow());
+ }
+ items = scanned;
+ }
+
+ @Then("the current state should list the same streams as the stream list")
+ public void currentStateMatchesList() {
+ Map<Long, String> listed =
+ client().streams().getStreams().stream().collect(Collectors.toMap(StreamBase::id, StreamBase::name));
+ Map<Long, String> projected = items.stream()
+ .map(MetadataCurrentItem::body)
+ .filter(MetadataChangeBody.StreamChange.class::isInstance)
+ .map(MetadataChangeBody.StreamChange.class::cast)
+ .collect(Collectors.toMap(
+ MetadataChangeBody.StreamChange::streamId, MetadataChangeBody.StreamChange::name));
+ assertEquals(listed, projected);
+ }
+
+ @Given("a user named {string} with password {string} and no server permissions exists")
+ public void userWithoutServerPermissions(String username, String password) {
+ var permissions = new Permissions(
+ new GlobalPermissions(false, true, false, false, false, false, false, false, false, false), Map.of());
+ var user = client().users()
+ .createUser(username, password, UserStatus.Active, Optional.of(permissions), UserOptions.empty());
+ userId = user.id();
+ }
+
+ @When("I authenticate as {string} with password {string}")
+ public void authenticateAs(String username, String password) {
+ client().users().logout();
+ assertNotNull(client().users().login(username, password));
+ }
+
+ @Then("reading the metadata changes should be refused as unauthorized")
+ public void readingRefused() {
+ groups = List.of();
+ var refusal = assertThrows(IggyServerException.class, () -> read(MetadataChangeStart.earliest()));
+ assertEquals(IggyErrorCode.UNAUTHORIZED, refusal.getErrorCode());
+ }
+
+ private MetadataChanges read(MetadataChangeStart start) {
+ return client().system()
+ .getMetadataChanges(MetadataChangesQuery.builder()
+ .start(start)
+ .groups(groups)
+ .limit(MetadataChangesQuery.MAX_LIMIT)
+ .build());
+ }
+
+ private MetadataChangeRecord followed(String kind) {
+ var parsed = MetadataChangeKind.fromValue(kind).orElseThrow();
+ return followed.stream()
+ .filter(record -> record.kind().equals(Optional.of(parsed)))
+ .findFirst()
+ .orElseThrow();
+ }
+
+ private MetadataChangeBody.StreamChange followedStream(String kind) {
+ var stream = assertInstanceOf(
+ MetadataChangeBody.StreamChange.class, followed(kind).body());
+ assertFalse(stream.optionsUnreadable());
+ return stream;
+ }
+
+ private IggyTcpClient client() {
+ if (client == null) {
+ String[] address = TestEnvironment.serverAddress().split(":", 2);
+ client = IggyTcpClient.builder()
+ .host(address[0])
+ .port(Integer.parseInt(address[1]))
+ .build();
+ client.connect();
+ client.users().login(TestEnvironment.rootUsername(), TestEnvironment.rootPassword());
+ }
+ return client;
+ }
+
+ private static List<MetadataChangeGroup> parseGroups(String names) {
+ return Arrays.stream(names.split(","))
+ .map(name -> MetadataChangeGroup.fromValue(name.trim()).orElseThrow())
+ .toList();
+ }
+
+ private static boolean namesStream(MetadataChangeRecord record, long streamId) {
+ var body = record.body();
+ if (body instanceof MetadataChangeBody.StreamChange stream) {
+ return stream.streamId() == streamId;
+ }
+ if (body instanceof MetadataChangeBody.StreamDeleted stream) {
+ return stream.streamId() == streamId;
+ }
+ if (body instanceof MetadataChangeBody.TopicChange topic) {
+ return topic.streamId() == streamId;
+ }
+ if (body instanceof MetadataChangeBody.TopicDeleted topic) {
+ return topic.streamId() == streamId;
+ }
+ return body instanceof MetadataChangeBody.PartitionsChanged partitions && partitions.streamId() == streamId;
+ }
+}
diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/ResourceOptionsSteps.java b/bdd/java/src/test/java/org/apache/iggy/bdd/ResourceOptionsSteps.java
index c7501d926..de9e70ed1 100644
--- a/bdd/java/src/test/java/org/apache/iggy/bdd/ResourceOptionsSteps.java
+++ b/bdd/java/src/test/java/org/apache/iggy/bdd/ResourceOptionsSteps.java
@@ -20,6 +20,7 @@
package org.apache.iggy.bdd;
import io.cucumber.java.After;
+import io.cucumber.java.Before;
import io.cucumber.java.en.Given;
import io.cucumber.java.en.Then;
import io.cucumber.java.en.When;
@@ -63,14 +64,23 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public final class ResourceOptionsSteps {
+ // Cucumber builds glue classes without injection, so other steps read this scenario's stream through it.
+ private static ResourceOptionsSteps current;
+
private IggyTcpClient client;
private Long streamId;
+ private Long createdStreamId;
private Long topicId;
private Long userId;
private String streamName;
private byte[] streamKey;
private boolean updateRejected;
+ @Before
+ public void begin() {
+ current = this;
+ }
+
@After
public void close() {
if (client == null) {
@@ -96,6 +106,7 @@ public final class ResourceOptionsSteps {
.createStream(
name, StreamOptions.builder().encryptionKey(streamKey).build());
streamId = created.id();
+ createdStreamId = streamId;
streamName = name;
}
@@ -108,6 +119,7 @@ public final class ResourceOptionsSteps {
public void createKeylessStream(String name) {
var created = client().streams().createStream(name);
streamId = created.id();
+ createdStreamId = streamId;
streamName = name;
}
@@ -195,6 +207,12 @@ public final class ResourceOptionsSteps {
HeaderValue.fromString(value), messages.get(0).userHeaders().get(HeaderKey.fromString(key)));
}
+ @When("I delete the stream")
+ public void deleteStream() {
+ client().streams().deleteStream(requireStreamId());
+ streamId = null;
+ }
+
@When("I rename the stream to {string}")
public void renameStream(String name) {
client().streams()
@@ -355,6 +373,21 @@ public final class ResourceOptionsSteps {
.toLowerCase());
}
+ static ResourceOptionsSteps current() {
+ return current;
+ }
+
+ /** The stream this scenario created, still known after a step deleted it. */
+ Long createdStreamId() {
+ assertNotNull(createdStreamId);
+ return createdStreamId;
+ }
+
+ byte[] streamKey() {
+ assertNotNull(streamKey);
+ return streamKey;
+ }
+
private IggyTcpClient client() {
if (client == null) {
String[] address = TestEnvironment.serverAddress().split(":", 2);
diff --git a/bdd/node/Dockerfile b/bdd/node/Dockerfile
index a76c12b7f..7f209a746 100644
--- a/bdd/node/Dockerfile
+++ b/bdd/node/Dockerfile
@@ -23,6 +23,7 @@ COPY ./bdd/scenarios/raw_command.feature ./bdd/
COPY ./bdd/scenarios/stream_options.feature ./bdd/
COPY ./bdd/scenarios/resource_options_patch.feature ./bdd/
COPY ./bdd/scenarios/session_contract.feature ./bdd/
+COPY ./bdd/scenarios/metadata_changes.feature ./bdd/
RUN npm ci
diff --git a/bdd/php/tests/MetadataChangesFeatureTest.php b/bdd/php/tests/MetadataChangesFeatureTest.php
new file mode 100644
index 000000000..6c3438904
--- /dev/null
+++ b/bdd/php/tests/MetadataChangesFeatureTest.php
@@ -0,0 +1,329 @@
+<?php
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+declare(strict_types=1);
+
+require_once __DIR__ . '/SharedFeatureParser.php';
+require_once __DIR__ . '/StreamSteps.php';
+require_once __DIR__ . '/TestEnvironment.php';
+
+use Iggy\Client as IggyClient;
+use Iggy\Exception\AuthenticationException;
+use Iggy\MetadataChangeCursor;
+use Iggy\MetadataChangeGroup;
+use Iggy\MetadataChangeKind;
+use Iggy\MetadataChangeRecord;
+use Iggy\MetadataChanges;
+use Iggy\MetadataChangeStart;
+use Iggy\MetadataChangeStatus;
+use Iggy\MetadataStreamChange;
+use Iggy\MetadataStreamDeleted;
+use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\Attributes\Group;
+use PHPUnit\Framework\TestCase;
+
+final class MetadataChangesFeatureTest extends TestCase
+{
+ use StreamSteps;
+
+ private const ROOT_USER_ID = 0;
+ private const UNAUTHORIZED = 41;
+ private const PAGE_LIMIT = 1000;
+ // A two phase delete commits its record after the call returns.
+ private const FOLLOW_ATTEMPTS = 100;
+ private const FOLLOW_INTERVAL_MICROSECONDS = 100_000;
+ private const NEVER_RETURNED_KEYS = ['encryption_key', 'encryption_key_wrapped', 'critical_keys'];
+
+ /** @var list<MetadataChangeGroup> */
+ private array $groups = [];
+ private ?MetadataChangeCursor $cursor = null;
+ /** @var list<MetadataChangeRecord> */
+ private array $followed = [];
+ /** @var array<int, string> */
+ private array $currentStreams = [];
+ private ?string $username = null;
+
+ public static function scenarioCases(): array
+ {
+ return SharedFeatureParser::load(__DIR__ . '/../../scenarios/metadata_changes.feature');
+ }
+
+ #[DataProvider('scenarioCases')]
+ #[Group('metadata-changes')]
+ public function testMetadataChangesScenario(string $scenarioName, array $steps): void
+ {
+ assert_true($scenarioName !== '', 'scenario name must not be empty');
+ foreach ($steps as $step) {
+ $this->runStep($step);
+ }
+ }
+
+ protected function tearDown(): void
+ {
+ if ($this->streamName !== null || $this->username !== null) {
+ $root = new IggyClient(TestEnvironment::serverAddress());
+ $root->connect();
+ $root->loginUser(TestEnvironment::rootUsername(), TestEnvironment::rootPassword());
+ if ($this->streamName !== null) {
+ cleanup_stream($root, $this->streamName);
+ }
+ if ($this->username !== null) {
+ delete_user_by_name($root, $this->username);
+ }
+ }
+
+ parent::tearDown();
+ }
+
+ private function runStep(string $step): void
+ {
+ if ($this->runStreamStep($step)) {
+ return;
+ }
+
+ if (preg_match('/^I hold the latest metadata change cursor for the groups "([^"]+)"$/', $step, $matches) === 1) {
+ $this->groups = self::parseGroups($matches[1]);
+ $page = $this->read(MetadataChangeStart::latest());
+ assert_same(MetadataChangeStatus::Ok, $page->status);
+ $this->cursor = $page->next;
+ assert_not_null($this->cursor);
+ return;
+ }
+
+ if ($step === 'I delete the stream') {
+ $this->requireClient()->deleteStream($this->requireStreamId());
+ return;
+ }
+
+ if (preg_match('/^I create and delete (\d+) streams named "([^"]+)"$/', $step, $matches) === 1) {
+ $client = $this->requireClient();
+ for ($index = 0; $index < (int) $matches[1]; $index++) {
+ $name = "{$matches[2]}-{$index}";
+ $client->createStream($name);
+ $client->deleteStream($name);
+ }
+ return;
+ }
+
+ if (preg_match('/^following the metadata changes should return the kinds "([^"]+)" for the stream$/', $step, $matches) === 1) {
+ $expected = array_map(
+ static fn (string $name): MetadataChangeKind => MetadataChangeKind::from(trim($name)),
+ explode(',', $matches[1]),
+ );
+ $records = $this->follow(count($expected));
+ foreach ($records as $record) {
+ assert_true(in_array($record->group, $this->groups, true), 'a record of an unrequested group');
+ }
+ $this->followed = $this->streamRecords($records);
+ assert_same(
+ $expected,
+ array_map(static fn (MetadataChangeRecord $record): ?MetadataChangeKind => $record->kind, $this->followed),
+ );
+ return;
+ }
+
+ if ($step === 'each followed change should name the root user as its actor') {
+ assert_true($this->followed !== [], 'changes were followed');
+ foreach ($this->followed as $record) {
+ assert_same(self::ROOT_USER_ID, $record->actor_user_id);
+ }
+ return;
+ }
+
+ if (preg_match('/^the followed "([^"]+)" change should carry a requested op before its own op$/', $step, $matches) === 1) {
+ $record = $this->followedRecord($matches[1]);
+ $deleted = $record->change;
+ assert_instance_of(MetadataStreamDeleted::class, $deleted);
+ assert_not_null($deleted->requested_op);
+ assert_true($deleted->requested_op < $record->op, 'the request precedes the completion');
+ return;
+ }
+
+ if (preg_match('/^the followed "([^"]+)" change should report encryption key source "([^"]+)"$/', $step, $matches) === 1) {
+ assert_same($matches[2], $this->followedStream($matches[1])->derived_options['encryption_key_source'] ?? null);
+ return;
+ }
+
+ if (preg_match('/^the followed "([^"]+)" change should carry the stream key fingerprint$/', $step, $matches) === 1) {
+ assert_not_null($this->keyBase64);
+ assert_same(
+ IggyClient::encryptionKeyFingerprint($this->keyBase64),
+ $this->followedStream($matches[1])->derived_options['encryption_key_fingerprint'] ?? null,
+ );
+ return;
+ }
+
+ if ($step === 'the followed changes should expose no encryption key material') {
+ assert_true($this->followed !== [], 'changes were followed');
+ foreach ($this->followed as $record) {
+ $stream = $record->change;
+ if (!$stream instanceof MetadataStreamChange) {
+ continue;
+ }
+ $explicit = $stream->options;
+ $derived = $stream->derived_options;
+ assert_not_null($explicit);
+ assert_not_null($derived);
+ foreach (self::NEVER_RETURNED_KEYS as $secret) {
+ self::assertArrayNotHasKey($secret, $explicit);
+ self::assertArrayNotHasKey($secret, $derived);
+ }
+ if ($this->keyBase64 !== null) {
+ $values = [...array_values($explicit), ...array_values($derived)];
+ self::assertNotContains($this->keyBase64, $values);
+ self::assertNotContains(base64_decode($this->keyBase64, true), $values);
+ }
+ }
+ return;
+ }
+
+ if (preg_match('/^following the metadata changes should answer "([^"]+)"$/', $step, $matches) === 1) {
+ assert_not_null($this->cursor);
+ $page = $this->read(MetadataChangeStart::after($this->cursor));
+ assert_same(MetadataChangeStatus::from($matches[1]), $page->status);
+ if ($page->status !== MetadataChangeStatus::Ok) {
+ assert_same([], $page->records);
+ assert_null($page->next);
+ }
+ return;
+ }
+
+ if (preg_match('/^I read the current state for the groups "([^"]+)"$/', $step, $matches) === 1) {
+ $this->groups = self::parseGroups($matches[1]);
+ $this->currentStreams = [];
+ $token = null;
+ do {
+ $page = $this->read(MetadataChangeStart::current($token));
+ assert_same(MetadataChangeStatus::Ok, $page->status);
+ foreach ($page->items as $item) {
+ $body = $item->body;
+ if ($body instanceof MetadataStreamChange) {
+ $this->currentStreams[$body->stream_id] = $body->name;
+ }
+ }
+ $token = $page->current_page_token;
+ } while ($token !== null);
+ $this->cursor = $page->next;
+ assert_not_null($this->cursor);
+ ksort($this->currentStreams);
+ return;
+ }
+
+ if ($step === 'the current state should list the same streams as the stream list') {
+ assert_same(stream_list($this->requireClient()), $this->currentStreams);
+ return;
+ }
+
+ if (preg_match('/^a user named "([^"]+)" with password "([^"]+)" and no server permissions exists$/', $step, $matches) === 1) {
+ create_user_without_permissions($this->requireClient(), $matches[1], $matches[2]);
+ $this->username = $matches[1];
+ return;
+ }
+
+ if (preg_match('/^I authenticate as "([^"]+)" with password "([^"]+)"$/', $step, $matches) === 1) {
+ $this->client = new IggyClient(TestEnvironment::serverAddress());
+ $this->client->connect();
+ $this->client->loginUser($matches[1], $matches[2]);
+ return;
+ }
+
+ if ($step === 'reading the metadata changes should be refused as unauthorized') {
+ $this->groups = [];
+ $error = assert_throws(fn () => $this->read(MetadataChangeStart::earliest()));
+ assert_instance_of(AuthenticationException::class, $error);
+ assert_same(self::UNAUTHORIZED, $error->getCode());
+ return;
+ }
+
+ self::fail("Unsupported BDD step: {$step}");
+ }
+
+ private function read(MetadataChangeStart $start): MetadataChanges
+ {
+ return $this->requireClient()->getMetadataChanges($start, $this->groups, limit: self::PAGE_LIMIT);
+ }
+
+ /** @return list<MetadataChangeRecord> */
+ private function follow(int $count): array
+ {
+ assert_not_null($this->cursor);
+ $cursor = $this->cursor;
+ $records = [];
+ for ($attempt = 0; $attempt < self::FOLLOW_ATTEMPTS; $attempt++) {
+ $page = $this->read(MetadataChangeStart::after($cursor));
+ assert_same(MetadataChangeStatus::Ok, $page->status);
+ $cursor = $page->next;
+ assert_not_null($cursor);
+ array_push($records, ...$page->records);
+ if ($page->has_more) {
+ continue;
+ }
+ if (count($this->streamRecords($records)) >= $count) {
+ break;
+ }
+ usleep(self::FOLLOW_INTERVAL_MICROSECONDS);
+ }
+
+ return $records;
+ }
+
+ /** @return list<MetadataChangeRecord> */
+ private function streamRecords(array $records): array
+ {
+ $streamId = $this->requireStreamId();
+
+ return array_values(array_filter(
+ $records,
+ static function (MetadataChangeRecord $record) use ($streamId): bool {
+ $change = $record->change;
+
+ return property_exists($change, 'stream_id') && $change->stream_id === $streamId;
+ },
+ ));
+ }
+
+ private function followedRecord(string $kind): MetadataChangeRecord
+ {
+ $wanted = MetadataChangeKind::from($kind);
+ foreach ($this->followed as $record) {
+ if ($record->kind === $wanted) {
+ return $record;
+ }
+ }
+
+ self::fail("no followed {$kind} change");
+ }
+
+ private function followedStream(string $kind): MetadataStreamChange
+ {
+ $stream = $this->followedRecord($kind)->change;
+ assert_instance_of(MetadataStreamChange::class, $stream);
+ assert_true(!$stream->options_unreadable, 'stream options stay readable');
+
+ return $stream;
+ }
+
+ /** @return list<MetadataChangeGroup> */
+ private static function parseGroups(string $names): array
+ {
+ return array_map(
+ static fn (string $name): MetadataChangeGroup => MetadataChangeGroup::from(trim($name)),
+ explode(',', $names),
+ );
+ }
+}
diff --git a/bdd/php/tests/StreamOptionsFeatureTest.php b/bdd/php/tests/StreamOptionsFeatureTest.php
index 52d1e2d05..fe0335cb5 100644
--- a/bdd/php/tests/StreamOptionsFeatureTest.php
+++ b/bdd/php/tests/StreamOptionsFeatureTest.php
@@ -19,6 +19,7 @@
declare(strict_types=1);
require_once __DIR__ . '/SharedFeatureParser.php';
+require_once __DIR__ . '/StreamSteps.php';
require_once __DIR__ . '/TestEnvironment.php';
use Iggy\Client as IggyClient;
@@ -34,11 +35,8 @@ use PHPUnit\Framework\TestCase;
final class StreamOptionsFeatureTest extends TestCase
{
- private ?IggyClient $client = null;
- private ?int $streamId = null;
- private ?string $streamName = null;
- private ?string $topicName = null;
- private ?string $keyBase64 = null;
+ use StreamSteps;
+
private bool $updateRejected = false;
/** @var array<string, OptionSpec> */
private array $streamOptionSpecs = [];
@@ -76,27 +74,7 @@ final class StreamOptionsFeatureTest extends TestCase
private function runStep(string $step): void
{
- if ($step === 'I have a running Iggy server') {
- $this->client = new IggyClient(TestEnvironment::serverAddress());
- $this->client->connect();
- $this->client->ping();
- return;
- }
-
- if ($step === 'I am authenticated as the root user') {
- $this->requireClient()->loginUser(TestEnvironment::rootUsername(), TestEnvironment::rootPassword());
- return;
- }
-
- if (preg_match('/^I create a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+)$/', $step, $matches) === 1
- || preg_match('/^a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+) exists$/', $step, $matches) === 1) {
- $this->keyBase64 = base64_encode(str_repeat(chr((int) $matches[2]), 32));
- $this->createStream($matches[1], $this->keyBase64);
- return;
- }
-
- if (preg_match('/^I create a stream named "([^"]+)" without an encryption key$/', $step, $matches) === 1) {
- $this->createStream($matches[1], null);
+ if ($this->runStreamStep($step)) {
return;
}
@@ -137,12 +115,6 @@ final class StreamOptionsFeatureTest extends TestCase
return;
}
- if (preg_match('/^I create a topic named "([^"]+)" with (\d+) partition in the stream$/', $step, $matches) === 1) {
- $this->topicName = $matches[1];
- $this->requireClient()->createTopic($this->requireStreamId(), $this->topicName, (int) $matches[2], null, null, null, null);
- return;
- }
-
if (preg_match('/^I send a message with payload "([^"]+)" to the topic$/', $step, $matches) === 1) {
assert_not_null($this->topicName);
$this->requireClient()->sendMessages($this->requireStreamId(), $this->topicName, 0, [new SendMessage($matches[1])]);
@@ -158,12 +130,6 @@ final class StreamOptionsFeatureTest extends TestCase
return;
}
- if (preg_match('/^I rename the stream to "([^"]+)"$/', $step, $matches) === 1) {
- $this->requireClient()->updateStream($this->requireStreamId(), name: $matches[1]);
- $this->streamName = $matches[1];
- return;
- }
-
if (preg_match('/^the stream name should be "([^"]+)"$/', $step, $matches) === 1) {
assert_same($matches[1], $this->requireStream()->name);
return;
@@ -219,27 +185,6 @@ final class StreamOptionsFeatureTest extends TestCase
return $this->streamOptionSpecs[$key];
}
- private function createStream(string $name, ?string $key): void
- {
- $this->requireClient()->createStream($name, $key);
- $stream = $this->requireClient()->getStream($name);
- assert_not_null($stream);
- $this->streamId = $stream->id;
- $this->streamName = $name;
- }
-
- private function requireClient(): IggyClient
- {
- assert_not_null($this->client);
- return $this->client;
- }
-
- private function requireStreamId(): int
- {
- assert_not_null($this->streamId);
- return $this->streamId;
- }
-
private function requireStream(): StreamDetails
{
$stream = $this->requireClient()->getStream($this->requireStreamId());
diff --git a/bdd/php/tests/StreamSteps.php b/bdd/php/tests/StreamSteps.php
new file mode 100644
index 000000000..c5b8e932f
--- /dev/null
+++ b/bdd/php/tests/StreamSteps.php
@@ -0,0 +1,99 @@
+<?php
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+declare(strict_types=1);
+
+require_once __DIR__ . '/TestEnvironment.php';
+
+use Iggy\Client as IggyClient;
+
+/**
+ * The server, authentication, stream, topic and rename steps the stream options and metadata
+ * changes features share.
+ */
+trait StreamSteps
+{
+ private ?IggyClient $client = null;
+ private ?int $streamId = null;
+ private ?string $streamName = null;
+ private ?string $topicName = null;
+ private ?string $keyBase64 = null;
+
+ /** Runs a shared step, and tells whether the step was one of them. */
+ private function runStreamStep(string $step): bool
+ {
+ if ($step === 'I have a running Iggy server') {
+ $this->client = new IggyClient(TestEnvironment::serverAddress());
+ $this->client->connect();
+ $this->client->ping();
+ return true;
+ }
+
+ if ($step === 'I am authenticated as the root user') {
+ $this->requireClient()->loginUser(TestEnvironment::rootUsername(), TestEnvironment::rootPassword());
+ return true;
+ }
+
+ if (preg_match('/^I create a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+)$/', $step, $matches) === 1
+ || preg_match('/^a stream named "([^"]+)" with an encryption key of 32 bytes valued (\d+) exists$/', $step, $matches) === 1) {
+ $this->keyBase64 = base64_encode(str_repeat(chr((int) $matches[2]), 32));
+ $this->createStream($matches[1], $this->keyBase64);
+ return true;
+ }
+
+ if (preg_match('/^I create a stream named "([^"]+)" without an encryption key$/', $step, $matches) === 1) {
+ $this->createStream($matches[1], null);
+ return true;
+ }
+
+ if (preg_match('/^I create a topic named "([^"]+)" with (\d+) partition in the stream$/', $step, $matches) === 1) {
+ $this->topicName = $matches[1];
+ $this->requireClient()->createTopic($this->requireStreamId(), $this->topicName, (int) $matches[2], null, null, null, null);
+ return true;
+ }
+
+ if (preg_match('/^I rename the stream to "([^"]+)"$/', $step, $matches) === 1) {
+ $this->requireClient()->updateStream($this->requireStreamId(), name: $matches[1]);
+ $this->streamName = $matches[1];
+ return true;
+ }
+
+ return false;
+ }
+
+ private function createStream(string $name, ?string $key): void
+ {
+ $this->requireClient()->createStream($name, $key);
+ $stream = $this->requireClient()->getStream($name);
+ assert_not_null($stream);
+ $this->streamId = $stream->id;
+ $this->streamName = $name;
+ }
+
+ private function requireClient(): IggyClient
+ {
+ assert_not_null($this->client);
+ return $this->client;
+ }
+
+ private function requireStreamId(): int
+ {
+ assert_not_null($this->streamId);
+ return $this->streamId;
+ }
+}
diff --git a/bdd/python/tests/conftest.py b/bdd/python/tests/conftest.py
index 9930e66a7..4924f2337 100644
--- a/bdd/python/tests/conftest.py
+++ b/bdd/python/tests/conftest.py
@@ -21,10 +21,25 @@ BDD test configuration and fixtures for Python SDK tests.
import asyncio
import os
+import socket
from dataclasses import dataclass
import pytest
-from apache_iggy import IggyClient, OptionSpec, ReceiveMessage
+from apache_iggy import (
+ IggyClient,
+ IggyExpiry,
+ MetadataChangeCursor,
+ MetadataChangeGroup,
+ MetadataChangeRecord,
+ MetadataCurrentItem,
+ OptionSpec,
+ ReceiveMessage,
+ StreamOptions,
+ TopicOptions,
+)
+from pytest_bdd import given, parsers, when
+
+STREAM_KEY_LENGTH = 32
@dataclass
@@ -47,6 +62,10 @@ class GlobalContext:
last_option_specs: list[OptionSpec] | None = None
last_user_id: int | None = None
last_call_error: RuntimeError | None = None
+ last_change_groups: list[MetadataChangeGroup] | None = None
+ last_change_cursor: MetadataChangeCursor | None = None
+ last_changes: list[MetadataChangeRecord] | None = None
+ last_current_items: list[MetadataCurrentItem] | None = None
def pytest_configure(config: pytest.Config) -> None:
@@ -61,6 +80,8 @@ def pytest_configure(config: pytest.Config) -> None:
"session-contract: shared session contract scenarios",
"stream-options: shared stream option scenarios",
"resource-options-patch: shared resource patch scenarios",
+ "metadata-changes: shared metadata change log scenarios",
+ "serial: scenarios that must not share the server with other runs",
"message-headers: scenarios that assert message headers",
"describe-options: scenarios that assert the option catalog",
):
@@ -110,3 +131,114 @@ def context():
ctx.server_addr = required_env("IGGY_TCP_ADDRESS")
yield ctx
+
+
+# Steps shared by the stream options and metadata changes features. A test
+# module that defines the same step text keeps its own definition.
+
+
+@given("I have a running Iggy server")
+def running_server(context):
+ async def connect():
+ host, port = context.server_addr.split(":")
+ try:
+ address = f"{socket.gethostbyname(host)}:{port}"
+ except socket.gaierror:
+ address = context.server_addr
+
+ context.client = IggyClient(address)
+ await context.client.connect()
+ await context.client.ping()
+
+ asyncio.run(connect())
+
+
+@given("I am authenticated as the root user")
+def authenticated_root_user(context, root_credentials):
+ async def login():
+ await context.client.login_user(*root_credentials)
+
+ asyncio.run(login())
+
+
+@when(
+ parsers.re(
+ r'I create a stream named "(?P<name>[^"]+)" with an encryption key '
+ r"of 32 bytes valued (?P<byte>\d+)"
+ ),
+ converters={"byte": int},
+)
+def create_stream_with_key(context, name, byte):
+ create_keyed_stream(context, name, byte)
+
+
+@given(
+ parsers.re(
+ r'a stream named "(?P<name>[^"]+)" with an encryption key '
+ r"of 32 bytes valued (?P<byte>\d+) exists"
+ ),
+ converters={"byte": int},
+)
+def keyed_stream_exists(context, name, byte):
+ create_keyed_stream(context, name, byte)
+
+
+@when(
+ parsers.re(r'I create a stream named "(?P<name>[^"]+)" without an encryption key')
+)
+def create_stream_without_key(context, name):
+ async def create():
+ stream = await context.client.create_stream(name)
+ context.last_stream_id = stream.id
+ context.last_stream_name = stream.name
+ context.last_stream_key = None
+
+ asyncio.run(create())
+
+
+@when(
+ parsers.re(
+ r'I create a topic named "(?P<name>[^"]+)" with (?P<partitions>\d+) '
+ r"partitions? in the stream"
+ ),
+ converters={"partitions": int},
+)
+def create_topic_in_stream(context, name, partitions):
+ async def create():
+ await context.client.create_topic(
+ context.last_stream_id,
+ name,
+ TopicOptions(
+ partitions_count=partitions,
+ message_expiry=IggyExpiry.NeverExpire(),
+ ),
+ )
+ topic = await context.client.get_topic(context.last_stream_id, name)
+ assert topic is not None
+ context.last_topic_id = topic.id
+ context.last_topic_name = topic.name
+ context.last_topic_partitions = partitions
+
+ asyncio.run(create())
+
+
+@when(parsers.re(r'I rename the stream to "(?P<name>[^"]+)"'))
+def rename_stream(context, name):
+ async def rename():
+ await context.client.update_stream(context.last_stream_id, name=name)
+
+ asyncio.run(rename())
+
+
+def create_keyed_stream(context, name: str, byte: int) -> None:
+ key = bytes([byte]) * STREAM_KEY_LENGTH
+
+ async def create():
+ stream = await context.client.create_stream(
+ name, StreamOptions(encryption_key=key)
+ )
+ context.last_stream_id = stream.id
+ context.last_stream_name = stream.name
+ context.last_stream_key = key
+
+ asyncio.run(create())
diff --git a/bdd/python/tests/test_metadata_changes.py b/bdd/python/tests/test_metadata_changes.py
new file mode 100644
index 000000000..a880ae501
--- /dev/null
+++ b/bdd/python/tests/test_metadata_changes.py
@@ -0,0 +1,315 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Metadata changes BDD steps for the Python SDK.
+
+The server, authentication, stream, topic and rename steps are shared through
+conftest.py.
+"""
+
+import asyncio
+import base64
+from pathlib import Path
+
+from apache_iggy import (
+ GlobalPermissions,
+ IggyError,
+ MetadataChangeGroup,
+ MetadataChangeKind,
+ MetadataChangeRecord,
+ MetadataChanges,
+ MetadataChangeStart,
+ MetadataChangeStatus,
+ MetadataStreamChange,
+ MetadataStreamDeleted,
+ Permissions,
+ encryption_key_fingerprint,
+)
+from pytest_bdd import given, parsers, scenarios, then, when
+
+FEATURE_FILE = Path("/app/features/metadata_changes.feature")
+if not FEATURE_FILE.exists():
+ FEATURE_FILE = (
+ Path(__file__).resolve().parents[2] / "scenarios/metadata_changes.feature"
+ )
+scenarios(str(FEATURE_FILE))
+
+ROOT_USER_ID = 0
+UNAUTHORIZED = 41
+PAGE_LIMIT = 1000
+# A two phase delete commits its record after the call returns.
+FOLLOW_ATTEMPTS = 100
+FOLLOW_INTERVAL_SECONDS = 0.1
+NEVER_RETURNED_KEYS = {"encryption_key", "encryption_key_wrapped", "critical_keys"}
+
+
+@given(
+ parsers.re(
+ r'I hold the latest metadata change cursor for the groups "(?P<groups>[^"]+)"'
+ )
+)
+def hold_latest_cursor(context, groups):
+ context.last_change_groups = parse_groups(groups)
+ page = asyncio.run(read_changes(context, MetadataChangeStart.Latest()))
+ assert page.status == MetadataChangeStatus.Ok
+ assert page.next is not None
+ context.last_change_cursor = page.next
+
+
+@when("I delete the stream")
+def delete_stream(context):
+ async def delete():
+ await context.client.delete_stream(context.last_stream_id)
+
+ asyncio.run(delete())
+
+
+@when(
+ parsers.re(r'I create and delete (?P<count>\d+) streams named "(?P<prefix>[^"]+)"'),
+ converters={"count": int},
+)
+def churn_streams(context, count, prefix):
+ async def churn():
+ for index in range(count):
+ stream = await context.client.create_stream(f"{prefix}-{index}")
+ await context.client.delete_stream(stream.id)
+
+ asyncio.run(churn())
+
+
+@then(
+ parsers.re(
+ r"following the metadata changes should return the kinds "
+ r'"(?P<kinds>[^"]+)" for the stream'
+ )
+)
+def followed_kinds(context, kinds):
+ expected = [parse_member(MetadataChangeKind, name) for name in kinds.split(",")]
+
+ async def follow() -> list[MetadataChangeRecord]:
+ records: list[MetadataChangeRecord] = []
+ cursor = context.last_change_cursor
+ for _ in range(FOLLOW_ATTEMPTS):
+ page = await read_changes(context, MetadataChangeStart.After(cursor))
+ assert page.status == MetadataChangeStatus.Ok
+ assert page.next is not None
+ cursor = page.next
+ records.extend(page.records)
+ if page.has_more:
+ continue
+ if len(stream_records(context, records)) >= len(expected):
+ break
+ await asyncio.sleep(FOLLOW_INTERVAL_SECONDS)
+ return records
+
+ records = asyncio.run(follow())
+ assert all(record.group in context.last_change_groups for record in records)
+ mine = stream_records(context, records)
+ assert [record.kind for record in mine] == expected
+ context.last_changes = mine
+
+
+@then("each followed change should name the root user as its actor")
+def root_actor(context):
+ assert context.last_changes
+ for record in context.last_changes:
+ assert record.actor_user_id == ROOT_USER_ID, record
+
+
+@then(
+ parsers.re(
+ r'the followed "(?P<kind>[^"]+)" change should carry a requested op '
+ r"before its own op"
+ )
+)
+def requested_op_before_own(context, kind):
+ record = followed(context, kind)
+ deleted = record.change
+ assert isinstance(deleted, MetadataStreamDeleted)
+ assert deleted.requested_op is not None
+ assert deleted.requested_op < record.op
+
+
+@then(
+ parsers.re(
+ r'the followed "(?P<kind>[^"]+)" change should report encryption key '
+ r'source "(?P<source>[^"]+)"'
+ )
+)
+def change_key_source(context, kind, source):
+ derived = followed_stream(context, kind).derived_options
+ assert derived is not None
+ assert derived.to_scalar_dict()["encryption_key_source"] == source
+
+
+@then(
+ parsers.re(
+ r'the followed "(?P<kind>[^"]+)" change should carry the stream key '
+ r"fingerprint"
+ )
+)
+def change_key_fingerprint(context, kind):
+ assert context.last_stream_key is not None
+ derived = followed_stream(context, kind).derived_options
+ assert derived is not None
+ assert derived.to_scalar_dict()[
+ "encryption_key_fingerprint"
+ ] == encryption_key_fingerprint(context.last_stream_key)
+
+
+@then("the followed changes should expose no encryption key material")
+def changes_hide_keys(context):
+ assert context.last_changes
+ for record in context.last_changes:
+ stream = record.change
+ if not isinstance(stream, MetadataStreamChange):
+ continue
+ assert stream.options is not None
+ assert stream.derived_options is not None
+ explicit = stream.options.to_scalar_dict()
+ derived = stream.derived_options.to_scalar_dict()
+ assert not NEVER_RETURNED_KEYS & (set(explicit) | set(derived))
+ if context.last_stream_key is not None:
+ values = [*explicit.values(), *derived.values()]
+ assert context.last_stream_key not in values
+ assert base64.b64encode(context.last_stream_key).decode() not in values
+
+
+@then(parsers.re(r'following the metadata changes should answer "(?P<status>[^"]+)"'))
+def follow_status(context, status):
+ page = asyncio.run(
+ read_changes(context, MetadataChangeStart.After(context.last_change_cursor))
+ )
+ assert page.status == parse_member(MetadataChangeStatus, status)
+ if page.status != MetadataChangeStatus.Ok:
+ assert page.records == []
+ assert page.next is None
+
+
+@when(parsers.re(r'I read the current state for the groups "(?P<groups>[^"]+)"'))
+def read_current_state(context, groups):
+ context.last_change_groups = parse_groups(groups)
+
+ async def scan():
+ start = MetadataChangeStart.Current()
+ items = []
+ while True:
+ page = await read_changes(context, start)
+ assert page.status == MetadataChangeStatus.Ok
+ items.extend(page.items)
+ if page.current_page_token is None:
+ assert page.next is not None
+ context.last_change_cursor = page.next
+ return items
+ start = MetadataChangeStart.Current(page.current_page_token)
+
+ context.last_current_items = asyncio.run(scan())
+
+
+@then("the current state should list the same streams as the stream list")
+def current_matches_list(context):
+ async def list_streams():
+ return await context.client.get_streams()
+
+ listed = {(stream.id, stream.name) for stream in asyncio.run(list_streams())}
+ projected = {
+ (item.body.stream_id, item.body.name)
+ for item in context.last_current_items
+ if isinstance(item.body, MetadataStreamChange)
+ }
+ assert projected == listed
+
+
+@given(
+ parsers.re(
+ r'a user named "(?P<username>[^"]+)" with password "(?P<password>[^"]+)" '
+ r"and no server permissions exists"
+ )
+)
+def user_without_server_permissions(context, username, password):
+ async def create():
+ user = await context.client.create_user(
+ username,
+ password,
+ permissions=Permissions(
+ global_permissions=GlobalPermissions(read_servers=True)
+ ),
+ )
+ context.last_user_id = user.id
+
+ asyncio.run(create())
+
+
+@when(
+ parsers.re(
+ r'I authenticate as "(?P<username>[^"]+)" with password "(?P<password>[^"]+)"'
+ )
+)
+def authenticate_as(context, username, password):
+ async def login():
+ await context.client.logout_user()
+ await context.client.login_user(username, password)
+
+ asyncio.run(login())
+
+
+@then("reading the metadata changes should be refused as unauthorized")
+def changes_refused(context):
+ context.last_change_groups = []
+ try:
+ asyncio.run(read_changes(context, MetadataChangeStart.Earliest()))
+ except IggyError as error:
+ assert error.code == UNAUTHORIZED
+ else:
+ raise AssertionError("manage_servers is required")
+
+
+async def read_changes(context, start: MetadataChangeStart) -> MetadataChanges:
+ return await context.client.get_metadata_changes(
+ start, groups=context.last_change_groups, limit=PAGE_LIMIT
+ )
+
+
+def stream_records(
+ context, records: list[MetadataChangeRecord]
+) -> list[MetadataChangeRecord]:
+ return [
+ record
+ for record in records
+ if getattr(record.change, "stream_id", None) == context.last_stream_id
+ ]
+
+
+def followed(context, kind: str) -> MetadataChangeRecord:
+ wanted = parse_member(MetadataChangeKind, kind)
+ return next(record for record in context.last_changes if record.kind == wanted)
+
+
+def followed_stream(context, kind: str) -> MetadataStreamChange:
+ stream = followed(context, kind).change
+ assert isinstance(stream, MetadataStreamChange)
+ assert not stream.options_unreadable
+ return stream
+
+
+def parse_groups(names: str) -> list[MetadataChangeGroup]:
+ return [parse_member(MetadataChangeGroup, name) for name in names.split(",")]
+
+
+def parse_member(enum_type, name: str):
+ return getattr(enum_type, "".join(part.title() for part in name.strip().split("_")))
diff --git a/bdd/python/tests/test_stream_options.py b/bdd/python/tests/test_stream_options.py
index 2a4b9e005..f83e5ce03 100644
--- a/bdd/python/tests/test_stream_options.py
+++ b/bdd/python/tests/test_stream_options.py
@@ -21,22 +21,17 @@ Stream options BDD steps for the Python SDK.
import asyncio
import base64
-import socket
from pathlib import Path
from apache_iggy import (
Consumer,
- IggyClient,
- IggyExpiry,
OptionKey,
PollingStrategy,
SendMessage,
StreamDetails,
- StreamOptions,
- TopicOptions,
encryption_key_fingerprint,
)
-from pytest_bdd import given, parsers, scenarios, then, when
+from pytest_bdd import parsers, scenarios, then, when
FEATURE_FILE = Path("/app/features/stream_options.feature")
if not FEATURE_FILE.exists():
@@ -45,71 +40,11 @@ if not FEATURE_FILE.exists():
)
scenarios(str(FEATURE_FILE))
-KEY_LENGTH = 32
FINGERPRINT_LENGTH = 16
PARTITION_ID = 0
NEVER_RETURNED_KEYS = {"encryption_key", "encryption_key_wrapped", "critical_keys"}
-@given("I have a running Iggy server")
-def running_server(context):
- async def connect():
- host, port = context.server_addr.split(":")
- try:
- address = f"{socket.gethostbyname(host)}:{port}"
- except socket.gaierror:
- address = context.server_addr
-
- context.client = IggyClient(address)
- await context.client.connect()
- await context.client.ping()
-
- asyncio.run(connect())
-
-
-@given("I am authenticated as the root user")
-def authenticated_root_user(context, root_credentials):
- async def login():
- await context.client.login_user(*root_credentials)
-
- asyncio.run(login())
-
-
-@when(
- parsers.re(
- r'I create a stream named "(?P<name>[^"]+)" with an encryption key '
- r"of 32 bytes valued (?P<byte>\d+)"
- ),
- converters={"byte": int},
-)
-def create_stream_with_key(context, name, byte):
- create_keyed_stream(context, name, byte)
-
-
-@given(
- parsers.re(
- r'a stream named "(?P<name>[^"]+)" with an encryption key '
- r"of 32 bytes valued (?P<byte>\d+) exists"
- ),
- converters={"byte": int},
-)
-def keyed_stream_exists(context, name, byte):
- create_keyed_stream(context, name, byte)
-
-
-@when(
- parsers.re(r'I create a stream named "(?P<name>[^"]+)" without an encryption key')
-)
-def create_stream_without_key(context, name):
- async def create():
- stream = await context.client.create_stream(name)
- context.last_stream_id = stream.id
- context.last_stream_name = stream.name
- context.last_stream_key = None
-
- asyncio.run(create())
-
-
@then(parsers.re(r'the stream should report encryption key source "(?P<source>[^"]+)"'))
def stream_reports_source(context, source):
assert fetch_stream(context).encryption_key_source == source
@@ -146,32 +81,6 @@ def stream_exposes_no_key_material(context):
assert base64.b64encode(context.last_stream_key).decode() not in values
-@when(
- parsers.re(
- r'I create a topic named "(?P<name>[^"]+)" with (?P<partitions>\d+) '
- r"partitions? in the stream"
- ),
- converters={"partitions": int},
-)
-def create_topic_in_stream(context, name, partitions):
- async def create():
- await context.client.create_topic(
- context.last_stream_id,
- name,
- TopicOptions(
- partitions_count=partitions,
- message_expiry=IggyExpiry.NeverExpire(),
- ),
- )
- topic = await context.client.get_topic(context.last_stream_id, name)
- assert topic is not None
- context.last_topic_id = topic.id
- context.last_topic_name = topic.name
- context.last_topic_partitions = partitions
-
- asyncio.run(create())
-
-
@when(parsers.re(r'I send a message with payload "(?P<payload>[^"]+)" to the topic'))
def send_payload(context, payload):
send_message(context, SendMessage(payload))
@@ -207,14 +116,6 @@ def poll_returns_payload_and_header(context, payload, header, value):
assert headers.to_scalar_dict()[header] == value
-@when(parsers.re(r'I rename the stream to "(?P<name>[^"]+)"'))
-def rename_stream(context, name):
- async def rename():
- await context.client.update_stream(context.last_stream_id, name=name)
-
- asyncio.run(rename())
-
-
@then(parsers.re(r'the stream name should be "(?P<name>[^"]+)"'))
def stream_name_is(context, name):
assert fetch_stream(context).name == name
@@ -270,20 +171,6 @@ def option_read_only(context, key):
assert spec.read_only and not spec.updatable and not spec.secret
-def create_keyed_stream(context, name: str, byte: int) -> None:
- key = bytes([byte]) * KEY_LENGTH
-
- async def create():
- stream = await context.client.create_stream(
- name, StreamOptions(encryption_key=key)
- )
- context.last_stream_id = stream.id
- context.last_stream_name = stream.name
- context.last_stream_key = key
-
- asyncio.run(create())
-
-
def fetch_stream(context) -> StreamDetails:
async def fetch():
return await context.client.get_stream(context.last_stream_id)
diff --git a/bdd/python/uv.lock b/bdd/python/uv.lock
index 8de7e910f..3f1a79c38 100644
--- a/bdd/python/uv.lock
+++ b/bdd/python/uv.lock
@@ -8,7 +8,7 @@ exclude-newer-span = "P7D"
[[package]]
name = "apache-iggy"
-version = "0.9.1.dev2"
+version = "0.9.1.dev3"
source = { directory = "../../foreign/python" }
[package.metadata]
diff --git a/bdd/rust/Cargo.toml b/bdd/rust/Cargo.toml
index cb7249d56..c6b83c921 100644
--- a/bdd/rust/Cargo.toml
+++ b/bdd/rust/Cargo.toml
@@ -70,5 +70,10 @@ name = "resource_options_patch"
harness = false
required-features = ["bdd"]
+[[test]]
+name = "metadata_changes"
+harness = false
+required-features = ["bdd"]
+
[lints]
workspace = true
diff --git a/bdd/rust/tests/common/global_context.rs b/bdd/rust/tests/common/global_context.rs
index b7d36b86f..d0565a02b 100644
--- a/bdd/rust/tests/common/global_context.rs
+++ b/bdd/rust/tests/common/global_context.rs
@@ -18,7 +18,10 @@
use bytes::Bytes;
use cucumber::World;
use iggy::clients::client::IggyClient;
-use iggy::prelude::{IggyError, IggyMessage, OptionSpec, PolledMessages, StreamEncryptionKey};
+use iggy::prelude::{
+ IggyError, IggyMessage, MetadataChangeCursor, MetadataChangeGroup, MetadataChangeRecord,
+ MetadataCurrentItem, OptionSpec, PolledMessages, StreamEncryptionKey,
+};
#[derive(Debug, World, Default)]
pub struct GlobalContext {
@@ -38,4 +41,8 @@ pub struct GlobalContext {
pub last_option_specs: Vec<OptionSpec>,
pub last_user_id: Option<u32>,
pub last_call_result: Option<Result<(), IggyError>>,
+ pub last_change_cursor: Option<MetadataChangeCursor>,
+ pub last_change_groups: Vec<MetadataChangeGroup>,
+ pub last_changes: Vec<MetadataChangeRecord>,
+ pub last_current_items: Vec<MetadataCurrentItem>,
}
diff --git a/foreign/go/contracts/version.go b/bdd/rust/tests/metadata_changes.rs
similarity index 71%
copy from foreign/go/contracts/version.go
copy to bdd/rust/tests/metadata_changes.rs
index ab6b6fff3..e3a979a65 100644
--- a/foreign/go/contracts/version.go
+++ b/bdd/rust/tests/metadata_changes.rs
@@ -15,6 +15,17 @@
// specific language governing permissions and limitations
// under the License.
-package iggcon
+pub(crate) mod common;
+pub(crate) mod helpers;
+pub(crate) mod steps;
-const Version = "0.9.1-edge.2"
+use crate::common::global_context::GlobalContext;
+use cucumber::World;
+
+#[tokio::main]
+async fn main() {
+ GlobalContext::cucumber()
+ .fail_on_skipped()
+ .run_and_exit("../../bdd/scenarios/metadata_changes.feature")
+ .await;
+}
diff --git a/bdd/rust/tests/steps/metadata_changes.rs b/bdd/rust/tests/steps/metadata_changes.rs
new file mode 100644
index 000000000..9b1604f3f
--- /dev/null
+++ b/bdd/rust/tests/steps/metadata_changes.rs
@@ -0,0 +1,339 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::common::global_context::GlobalContext;
+use cucumber::{given, then, when};
+use iggy::prelude::{
+ GlobalPermissions, HeaderKey, Identifier, IggyError, MAX_METADATA_CHANGES_LIMIT,
+ MetadataChange, MetadataChangeGroup, MetadataChangeKind, MetadataChangeRecord,
+ MetadataChangeStart, MetadataChangeStatus, MetadataChanges, MetadataChangesQuery,
+ MetadataCurrentItem, MetadataOptions, OptionKey, Permissions, ResourceOptions, StreamClient,
+ StreamCreateOptions, StreamEncryptionSource, SystemClient, UserClient, UserCreateOptions,
+ UserStatus,
+};
+use std::collections::BTreeMap;
+use std::str::FromStr;
+use std::time::Duration;
+use tokio::time::sleep;
+
+const ROOT_USER_ID: u32 = 0;
+const FOLLOW_ATTEMPTS: usize = 100;
+const FOLLOW_INTERVAL: Duration = Duration::from_millis(100);
+const HIDDEN_OPTIONS: [OptionKey; 3] = [
+ OptionKey::EncryptionKey,
+ OptionKey::EncryptionKeyWrapped,
+ OptionKey::CriticalKeys,
+];
+
+#[given(regex = r#"^I hold the latest metadata change cursor for the groups "([^"]+)"$"#)]
+pub async fn given_latest_cursor(world: &mut GlobalContext, groups: String) {
+ world.last_change_groups = groups
+ .split(',')
+ .map(|name| MetadataChangeGroup::from_str(name.trim()).expect("known group"))
+ .collect();
+ let page = read_changes(world, MetadataChangeStart::Latest)
+ .await
+ .expect("latest");
+ assert_eq!(page.status, MetadataChangeStatus::Ok);
+ world.last_change_cursor = Some(page.next.expect("latest cursor"));
+}
+
+#[when("I delete the stream")]
+pub async fn when_delete_stream(world: &mut GlobalContext) {
+ let client = world.client.as_ref().expect("client");
+ let stream_id = world.last_stream_id.expect("stream created");
+ client
+ .delete_stream(&Identifier::numeric(stream_id).expect("stream id"))
+ .await
+ .expect("delete stream");
+}
+
+#[when(regex = r#"^I create and delete (\d+) streams named "([^"]+)"$"#)]
+pub async fn when_churn_streams(world: &mut GlobalContext, count: u32, prefix: String) {
+ let client = world.client.as_ref().expect("client");
+ for index in 0..count {
+ let stream = client
+ .create_stream(
+ &format!("{prefix}-{index}"),
+ &StreamCreateOptions::default(),
+ )
+ .await
+ .expect("create stream");
+ client
+ .delete_stream(&Identifier::numeric(stream.id).expect("stream id"))
+ .await
+ .expect("delete stream");
+ }
+}
+
+#[then(
+ regex = r#"^following the metadata changes should return the kinds "([^"]+)" for the stream$"#
+)]
+pub async fn then_followed_kinds(world: &mut GlobalContext, kinds: String) {
+ let expected: Vec<MetadataChangeKind> = kinds
+ .split(',')
+ .map(|name| MetadataChangeKind::from_str(name.trim()).expect("known kind"))
+ .collect();
+ let stream_id = world.last_stream_id.expect("stream created");
+ let mut records = Vec::new();
+ let mut cursor = world.last_change_cursor.expect("held cursor");
+ // A two phase delete commits its record after the call returns.
+ for _ in 0..FOLLOW_ATTEMPTS {
+ let page = read_changes(world, MetadataChangeStart::After(cursor))
+ .await
+ .expect("change page");
+ assert_eq!(page.status, MetadataChangeStatus::Ok);
+ cursor = page.next.expect("continuation");
+ records.extend(page.records);
+ if page.has_more {
+ continue;
+ }
+ let names_stream =
+ |record: &&MetadataChangeRecord| change_stream_id(record) == Some(stream_id);
+ if records.iter().filter(names_stream).count() >= expected.len() {
+ break;
+ }
+ sleep(FOLLOW_INTERVAL).await;
+ }
+ for record in &records {
+ let group = record.change.group().expect("known group");
+ assert!(world.last_change_groups.contains(&group), "{record:?}");
+ }
+ let mine: Vec<MetadataChangeRecord> = records
+ .into_iter()
+ .filter(|record| change_stream_id(record) == Some(stream_id))
+ .collect();
+ let kinds: Vec<MetadataChangeKind> = mine
+ .iter()
+ .filter_map(|record| record.change.kind())
+ .collect();
+ assert_eq!(kinds, expected);
+ world.last_changes = mine;
+}
+
+#[then("each followed change should name the root user as its actor")]
+pub async fn then_root_actor(world: &mut GlobalContext) {
+ for record in &world.last_changes {
+ assert_eq!(record.actor_user_id, Some(ROOT_USER_ID), "{record:?}");
+ }
+}
+
+#[then(regex = r#"^the followed "([^"]+)" change should carry a requested op before its own op$"#)]
+pub async fn then_requested_op(world: &mut GlobalContext, kind: String) {
+ let record = followed(world, &kind);
+ let MetadataChange::StreamDeleted(deleted) = &record.change else {
+ panic!("expected a stream deletion, got {record:?}");
+ };
+ let requested = deleted.requested_op.expect("requested op");
+ assert!(requested < record.op, "{record:?}");
+}
+
+#[then(regex = r#"^the followed "([^"]+)" change should report encryption key source "([^"]+)"$"#)]
+pub async fn then_change_key_source(world: &mut GlobalContext, kind: String, source: String) {
+ let options = followed_stream_options(world, &kind);
+ let expected: &'static str = StreamEncryptionSource::from_str(&source)
+ .expect("known source")
+ .into();
+ let reported = options
+ .get(&HeaderKey::from(OptionKey::EncryptionKeySource))
+ .and_then(|option| option.value.as_str().ok().map(str::to_owned));
+ assert_eq!(reported.as_deref(), Some(expected));
+}
+
+#[then(regex = r#"^the followed "([^"]+)" change should carry the stream key fingerprint$"#)]
+pub async fn then_change_fingerprint(world: &mut GlobalContext, kind: String) {
+ let options = followed_stream_options(world, &kind);
+ let key = world.last_stream_key.as_ref().expect("stream key");
+ let reported = options
+ .get(&HeaderKey::from(OptionKey::EncryptionKeyFingerprint))
+ .and_then(|option| option.value.as_str().ok().map(str::to_owned));
+ assert_eq!(reported, Some(key.fingerprint()));
+}
+
+#[then("the followed changes should expose no encryption key material")]
+pub async fn then_changes_hide_keys(world: &mut GlobalContext) {
+ for record in &world.last_changes {
+ let (MetadataChange::StreamCreated(stream) | MetadataChange::StreamUpdated(stream)) =
+ &record.change
+ else {
+ continue;
+ };
+ let MetadataOptions::Readable(options) = &stream.options else {
+ panic!("stream options must stay readable: {record:?}");
+ };
+ for hidden in HIDDEN_OPTIONS {
+ assert!(!options.contains_key(&HeaderKey::from(hidden)), "{hidden}");
+ }
+ }
+}
+
+#[then(regex = r#"^following the metadata changes should answer "([^"]+)"$"#)]
+pub async fn then_follow_status(world: &mut GlobalContext, status: String) {
+ let expected = MetadataChangeStatus::from_str(&status).expect("known status");
+ let cursor = world.last_change_cursor.expect("held cursor");
+ let page = read_changes(world, MetadataChangeStart::After(cursor))
+ .await
+ .expect("change page");
+ assert_eq!(page.status, expected);
+ if expected != MetadataChangeStatus::Ok {
+ assert!(page.records.is_empty());
+ assert_eq!(page.next, None);
+ }
+}
+
+#[when(regex = r#"^I read the current state for the groups "([^"]+)"$"#)]
+pub async fn when_read_current(world: &mut GlobalContext, groups: String) {
+ world.last_change_groups = groups
+ .split(',')
+ .map(|name| MetadataChangeGroup::from_str(name.trim()).expect("known group"))
+ .collect();
+ let mut start = MetadataChangeStart::Current(None);
+ let mut items = Vec::new();
+ loop {
+ let page = read_changes(world, start).await.expect("current page");
+ assert_eq!(page.status, MetadataChangeStatus::Ok);
+ items.extend(page.items);
+ match page.current_page_token {
+ Some(token) => start = MetadataChangeStart::Current(Some(token)),
+ None => {
+ world.last_change_cursor = Some(page.next.expect("anchor"));
+ break;
+ }
+ }
+ }
+ world.last_current_items = items;
+}
+
+#[then("the current state should list the same streams as the stream list")]
+pub async fn then_current_matches_list(world: &mut GlobalContext) {
+ let client = world.client.as_ref().expect("client");
+ let listed: BTreeMap<u32, String> = client
+ .get_streams()
+ .await
+ .expect("streams")
+ .into_iter()
+ .map(|stream| (stream.id, stream.name))
+ .collect();
+ let projected: BTreeMap<u32, String> = world
+ .last_current_items
+ .iter()
+ .filter_map(|item| match item {
+ MetadataCurrentItem::Stream(stream) => Some((stream.stream_id, stream.name.clone())),
+ _ => None,
+ })
+ .collect();
+ assert_eq!(projected, listed);
+}
+
+#[given(
+ regex = r#"^a user named "([^"]+)" with password "([^"]+)" and no server permissions exists$"#
+)]
+pub async fn given_user_without_server_permissions(
+ world: &mut GlobalContext,
+ username: String,
+ password: String,
+) {
+ let client = world.client.as_ref().expect("client");
+ let user = client
+ .create_user(
+ &username,
+ &password,
+ UserStatus::Active,
+ Some(Permissions {
+ global: GlobalPermissions {
+ read_servers: true,
+ ..GlobalPermissions::default()
+ },
+ streams: None,
+ }),
+ &UserCreateOptions::default(),
+ )
+ .await
+ .expect("create user");
+ world.last_user_id = Some(user.id);
+}
+
+#[when(regex = r#"^I authenticate as "([^"]+)" with password "([^"]+)"$"#)]
+pub async fn when_authenticate_as(world: &mut GlobalContext, username: String, password: String) {
+ let client = world.client.as_ref().expect("client");
+ client.logout_user().await.expect("logout");
+ client
+ .login_user(&username, &password)
+ .await
+ .expect("login");
+}
+
+#[then("reading the metadata changes should be refused as unauthorized")]
+pub async fn then_changes_refused(world: &mut GlobalContext) {
+ world.last_change_groups = Vec::new();
+ let error = read_changes(world, MetadataChangeStart::Earliest)
+ .await
+ .expect_err("manage_servers is required");
+ assert_eq!(error.as_code(), IggyError::Unauthorized.as_code());
+}
+
+async fn read_changes(
+ world: &GlobalContext,
+ start: MetadataChangeStart,
+) -> Result<MetadataChanges, IggyError> {
+ let client = world.client.as_ref().expect("client");
+ client
+ .get_metadata_changes(&MetadataChangesQuery {
+ start,
+ groups: world.last_change_groups.clone(),
+ limit: MAX_METADATA_CHANGES_LIMIT,
+ ..MetadataChangesQuery::default()
+ })
+ .await
+}
+
+fn followed<'a>(world: &'a GlobalContext, kind: &str) -> &'a MetadataChangeRecord {
+ let kind = MetadataChangeKind::from_str(kind).expect("known kind");
+ world
+ .last_changes
+ .iter()
+ .find(|record| record.change.kind() == Some(kind))
+ .expect("followed change of that kind")
+}
+
+fn followed_stream_options(world: &GlobalContext, kind: &str) -> ResourceOptions {
+ let record = followed(world, kind);
+ let (MetadataChange::StreamCreated(stream) | MetadataChange::StreamUpdated(stream)) =
+ &record.change
+ else {
+ panic!("expected a stream change, got {record:?}");
+ };
+ let MetadataOptions::Readable(options) = &stream.options else {
+ panic!("stream options must stay readable: {record:?}");
+ };
+ options.clone()
+}
+
+const fn change_stream_id(record: &MetadataChangeRecord) -> Option<u32> {
+ match &record.change {
+ MetadataChange::StreamCreated(stream) | MetadataChange::StreamUpdated(stream) => {
+ Some(stream.stream_id)
+ }
+ MetadataChange::StreamDeleted(stream) => Some(stream.stream_id),
+ MetadataChange::TopicCreated(topic) | MetadataChange::TopicUpdated(topic) => {
+ Some(topic.stream_id)
+ }
+ MetadataChange::TopicDeleted(topic) => Some(topic.stream_id),
+ MetadataChange::PartitionsChanged(partitions) => Some(partitions.stream_id),
+ _ => None,
+ }
+}
diff --git a/bdd/rust/tests/steps/mod.rs b/bdd/rust/tests/steps/mod.rs
index a091ca2cc..96355c82b 100644
--- a/bdd/rust/tests/steps/mod.rs
+++ b/bdd/rust/tests/steps/mod.rs
@@ -18,6 +18,7 @@
pub mod auth;
pub mod leader_redirection;
pub mod messages;
+pub mod metadata_changes;
pub mod raw_command;
pub mod resource_options;
pub mod server;
diff --git a/bdd/scenarios/metadata_changes.feature b/bdd/scenarios/metadata_changes.feature
new file mode 100644
index 000000000..7fe270b17
--- /dev/null
+++ b/bdd/scenarios/metadata_changes.feature
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+@metadata-changes
+Feature: Metadata changes
+ As an operator of Apache Iggy
+ I want to read the metadata changes the server committed
+ So that a mirror of streams, topics, users and sessions stays current without listing everything again
+
+ Background:
+ Given I have a running Iggy server
+ And I am authenticated as the root user
+
+ Scenario: Follow stream changes from the latest position
+ Given I hold the latest metadata change cursor for the groups "streams"
+ When I create a stream named "changes-follow" without an encryption key
+ And I rename the stream to "changes-follow-renamed"
+ And I delete the stream
+ Then following the metadata changes should return the kinds "stream_created,stream_updated,stream_deleted" for the stream
+ And each followed change should name the root user as its actor
+
+ Scenario: Filter changes by group
+ Given I hold the latest metadata change cursor for the groups "topics"
+ When I create a stream named "changes-filter" without an encryption key
+ And I create a topic named "events" with 1 partition in the stream
+ Then following the metadata changes should return the kinds "topic_created" for the stream
+
+ Scenario: Record a two phase delete at its completion
+ Given I hold the latest metadata change cursor for the groups "streams"
+ When I create a stream named "changes-two-phase" without an encryption key
+ And I create a topic named "events" with 1 partition in the stream
+ And I delete the stream
+ Then following the metadata changes should return the kinds "stream_created,stream_deleted" for the stream
+ And the followed "stream_deleted" change should carry a requested op before its own op
+
+ Scenario: Keyed stream changes carry fingerprints and no key
+ Given I hold the latest metadata change cursor for the groups "streams"
+ When I create a stream named "changes-keyed" with an encryption key of 32 bytes valued 21
+ Then following the metadata changes should return the kinds "stream_created" for the stream
+ And the followed "stream_created" change should report encryption key source "stream"
+ And the followed "stream_created" change should carry the stream key fingerprint
+ And the followed changes should expose no encryption key material
+
+ @serial
+ Scenario: Rebuild from current state after the cursor expired
+ Given I hold the latest metadata change cursor for the groups "streams"
+ When I create and delete 40 streams named "changes-churn"
+ Then following the metadata changes should answer "expired"
+ When I read the current state for the groups "streams"
+ Then the current state should list the same streams as the stream list
+ And following the metadata changes should answer "ok"
+
+ Scenario: Reading changes needs the manage servers permission
+ Given a user named "changes-reader" with password "changes-secret" and no server permissions exists
+ When I authenticate as "changes-reader" with password "changes-secret"
+ Then reading the metadata changes should be refused as unauthorized
diff --git a/core/ai/mcp/Cargo.toml b/core/ai/mcp/Cargo.toml
index 1fc411cbe..763484a7f 100644
--- a/core/ai/mcp/Cargo.toml
+++ b/core/ai/mcp/Cargo.toml
@@ -17,7 +17,7 @@
[package]
name = "iggy-mcp"
-version = "0.5.1-edge.2"
+version = "0.5.1-edge.3"
description = "MCP Server for Iggy message streaming platform"
edition = "2024"
license = "Apache-2.0"
diff --git a/core/bench/Cargo.toml b/core/bench/Cargo.toml
index f05e136d7..577084e8f 100644
--- a/core/bench/Cargo.toml
+++ b/core/bench/Cargo.toml
@@ -17,7 +17,7 @@
[package]
name = "iggy-bench"
-version = "0.6.1-edge.2"
+version = "0.6.1-edge.3"
edition = "2024"
license = "Apache-2.0"
repository = "https://github.com/apache/iggy"
diff --git a/core/binary_protocol/Cargo.toml b/core/binary_protocol/Cargo.toml
index 531d4784d..ee64d4711 100644
--- a/core/binary_protocol/Cargo.toml
+++ b/core/binary_protocol/Cargo.toml
@@ -17,7 +17,7 @@
[package]
name = "iggy_binary_protocol"
-version = "0.11.1-edge.2"
+version = "0.11.1-edge.3"
description = "Wire protocol types and codec for the Iggy binary protocol. Shared between server and SDK."
edition = "2024"
rust-version.workspace = true
diff --git a/core/binary_protocol/src/codes.rs b/core/binary_protocol/src/codes.rs
index ab8e8f9f4..ac84481da 100644
--- a/core/binary_protocol/src/codes.rs
+++ b/core/binary_protocol/src/codes.rs
@@ -29,6 +29,7 @@ pub const DESCRIBE_OPTIONS_CODE: u32 = 13;
// Code 14 is retired.
pub const BIND_SESSION_CODE: u32 = 15;
pub const UPDATE_RESOURCE_CODE: u32 = 16;
+pub const GET_METADATA_CHANGES_CODE: u32 = 17;
pub const GET_ME_CODE: u32 = 20;
pub const GET_CLIENT_CODE: u32 = 21;
pub const GET_CLIENTS_CODE: u32 = 22;
@@ -123,6 +124,7 @@ mod tests {
DESCRIBE_OPTIONS_CODE,
BIND_SESSION_CODE,
UPDATE_RESOURCE_CODE,
+ GET_METADATA_CHANGES_CODE,
GET_ME_CODE,
GET_CLIENT_CODE,
GET_CLIENTS_CODE,
diff --git a/core/binary_protocol/src/dispatch.rs b/core/binary_protocol/src/dispatch.rs
index 0e809b47d..b5106cb4e 100644
--- a/core/binary_protocol/src/dispatch.rs
+++ b/core/binary_protocol/src/dispatch.rs
@@ -187,6 +187,8 @@ pub const COMMAND_TABLE: &[CommandMeta] = &[
"resource.update",
Operation::UpdateResource,
),
+ // Change log of committed metadata operations
+ CommandMeta::non_replicated(GET_METADATA_CHANGES_CODE, "metadata.changes"),
];
/// Lookup command metadata by command code.
@@ -246,6 +248,7 @@ pub const fn lookup_command(code: u32) -> Option<&'static CommandMeta> {
BIND_SESSION_CODE => 45,
GET_PARTITION_ROUTE_CODE => 46,
UPDATE_RESOURCE_CODE => 47,
+ GET_METADATA_CHANGES_CODE => 48,
_ => return None,
};
Some(&COMMAND_TABLE[idx])
diff --git a/core/binary_protocol/src/primitives/metadata_changes/body.rs b/core/binary_protocol/src/primitives/metadata_changes/body.rs
new file mode 100644
index 000000000..ba55e953c
--- /dev/null
+++ b/core/binary_protocol/src/primitives/metadata_changes/body.rs
@@ -0,0 +1,1771 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::{
+ WireMemberLeftReason, WireMetadataChangeGroup, WireMetadataChangeKind,
+ WirePermissionsUpdatedReason, WireSessionClosedReason, WireTokenDeletedReason,
+ optional_u32_size, optional_u64_size, partition_ids_size, put_flag, put_optional_u32,
+ put_optional_u64, put_partition_ids, read_flag, read_optional_u32, read_optional_u64,
+ read_partition_ids, read_wire_name,
+};
+use crate::WireError;
+use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le, read_u128_le};
+use crate::primitives::identifier::WireName;
+use crate::primitives::options::{
+ WireOptions, decode_options_prefixed, encode_options_prefixed, options_prefixed_size,
+};
+use crate::primitives::permissions::WirePermissions;
+use bytes::{BufMut, Bytes, BytesMut};
+use std::borrow::Cow;
+
+/// A change record body, typed by its kind.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum WireMetadataChange {
+ StreamCreated(WireStreamBody),
+ StreamUpdated(WireStreamBody),
+ StreamDeleted(WireStreamDeletedBody),
+ TopicCreated(WireTopicBody),
+ TopicUpdated(WireTopicBody),
+ TopicDeleted(WireTopicDeletedBody),
+ PartitionsChanged(WirePartitionsChangedBody),
+ ConsumerGroupCreated(WireConsumerGroupCreatedBody),
+ ConsumerGroupDeleted(WireConsumerGroupDeletedBody),
+ ConsumerGroupMemberJoined(WireMemberJoinedBody),
+ ConsumerGroupMemberLeft(WireMemberLeftBody),
+ ConsumerGroupPartitionHandedOver(WirePartitionHandedOverBody),
+ UserCreated(WireUserBody),
+ UserUpdated(WireUserBody),
+ UserDeleted(WireUserIdentityBody),
+ UserPasswordChanged(WireUserIdentityBody),
+ UserPermissionsUpdated(WireUserPermissionsBody),
+ PersonalAccessTokenCreated(WireTokenBody),
+ PersonalAccessTokenDeleted(WireTokenDeletedBody),
+ SessionRegistered(WireSessionBody),
+ SessionClosed(WireSessionClosedBody),
+}
+
+/// A current state item body, typed by its group.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum WireMetadataCurrentItem {
+ Stream(WireStreamBody),
+ Topic(WireTopicItemBody),
+ Partitions(WirePartitionsItemBody),
+ ConsumerGroup(WireConsumerGroupItemBody),
+ ConsumerGroupMember(WireMemberItemBody),
+ User(WireUserBody),
+ PersonalAccessToken(WireTokenBody),
+ Session(WireSessionBody),
+}
+
+/// Stream identity: ids are reused, the creating op is not.
+///
+/// Wire format: `[stream_id:u32][stream_created_op:u64]`
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct WireStreamKey {
+ pub stream_id: u32,
+ pub stream_created_op: u64,
+}
+
+/// Topic identity: the stream key plus the topic id and creating op.
+///
+/// Wire format: `[stream key:12][topic_id:u32][topic_created_op:u64]`
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct WireTopicKey {
+ pub stream: WireStreamKey,
+ pub topic_id: u32,
+ pub topic_created_op: u64,
+}
+
+/// User identity, the one sessions and tokens bind to.
+///
+/// Wire format: `[user_id:u32][user_created_at:u64]`
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct WireUserKey {
+ pub user_id: u32,
+ pub user_created_at: u64,
+}
+
+/// Public options of a resource, or the marker for stored critical state this
+/// build cannot read.
+///
+/// Wire format: `[state:u8]`, then for state 0 the explicit and derived blocks
+/// as the read responses encode them, nothing for state 1.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum WireChangeOptions {
+ Readable {
+ explicit: WireOptions,
+ derived: WireOptions,
+ },
+ Unreadable,
+}
+
+/// Body of `stream_created`, `stream_updated` and of a `streams` item.
+///
+/// Wire format: `[stream key][created_at:u64][name][options]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireStreamBody {
+ pub key: WireStreamKey,
+ pub created_at: u64,
+ pub name: WireName,
+ pub options: WireChangeOptions,
+}
+
+/// Body of `stream_deleted`. `requested_op` is present on a deferred
+/// completion.
+///
+/// Wire format: `[stream key][created_at:u64][name][has:u8][requested_op:u64]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireStreamDeletedBody {
+ pub key: WireStreamKey,
+ pub created_at: u64,
+ pub name: WireName,
+ pub requested_op: Option<u64>,
+}
+
+/// Body of `topic_created` and `topic_updated`.
+///
+/// Wire format: `[topic key][created_at:u64][name][partitions_count:u32][options]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireTopicBody {
+ pub key: WireTopicKey,
+ pub created_at: u64,
+ pub name: WireName,
+ pub partitions_count: u32,
+ pub options: WireChangeOptions,
+}
+
+/// Body of a `topics` item. Partition counts travel as `partitions` items.
+///
+/// Wire format: `[topic key][created_at:u64][name][options]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireTopicItemBody {
+ pub key: WireTopicKey,
+ pub created_at: u64,
+ pub name: WireName,
+ pub options: WireChangeOptions,
+}
+
+/// Body of `topic_deleted`.
+///
+/// Wire format: `[topic key][created_at:u64][name][has:u8][requested_op:u64]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireTopicDeletedBody {
+ pub key: WireTopicKey,
+ pub created_at: u64,
+ pub name: WireName,
+ pub requested_op: Option<u64>,
+}
+
+/// Body of `partitions_changed`. Removed ids are the actual deleted tail,
+/// ascending, empty on create.
+///
+/// Wire format: `[topic key][partitions_count:u32][ids][has:u8][requested_op:u64]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WirePartitionsChangedBody {
+ pub topic: WireTopicKey,
+ pub partitions_count: u32,
+ pub removed_partition_ids: Vec<u32>,
+ pub requested_op: Option<u64>,
+}
+
+/// Body of a `partitions` item.
+///
+/// Wire format: `[topic key][partitions_count:u32]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WirePartitionsItemBody {
+ pub topic: WireTopicKey,
+ pub partitions_count: u32,
+}
+
+/// Body of `consumer_group_created`.
+///
+/// Wire format: `[topic key][group_id:u32][name][consumer_groups_count:u32]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireConsumerGroupCreatedBody {
+ pub topic: WireTopicKey,
+ pub group_id: u32,
+ pub name: WireName,
+ pub consumer_groups_count: u32,
+}
+
+/// Body of `consumer_group_deleted`.
+///
+/// Wire format: `[topic key][group_id:u32][name][consumer_groups_count:u32][has:u8][requested_op:u64]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireConsumerGroupDeletedBody {
+ pub topic: WireTopicKey,
+ pub group_id: u32,
+ pub name: WireName,
+ pub consumer_groups_count: u32,
+ pub requested_op: Option<u64>,
+}
+
+/// Body of a `consumer_groups` item.
+///
+/// Wire format: `[topic key][group_id:u32][name]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireConsumerGroupItemBody {
+ pub topic: WireTopicKey,
+ pub group_id: u32,
+ pub name: WireName,
+}
+
+/// Body of `consumer_group_member_joined`.
+///
+/// Wire format: `[topic key][group_id:u32][member_id:u32][client_id:u128][members_count:u32][ids]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireMemberJoinedBody {
+ pub topic: WireTopicKey,
+ pub group_id: u32,
+ pub member_id: u32,
+ pub client_id: u128,
+ pub members_count: u32,
+ pub partition_ids: Vec<u32>,
+}
+
+/// Body of `consumer_group_member_left`.
+///
+/// Wire format: `[topic key][group_id:u32][member_id:u32][client_id:u128][members_count:u32][reason:u8]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireMemberLeftBody {
+ pub topic: WireTopicKey,
+ pub group_id: u32,
+ pub member_id: u32,
+ pub client_id: u128,
+ pub members_count: u32,
+ pub reason: WireMemberLeftReason,
+}
+
+/// Body of `consumer_group_partition_handed_over`. The owner is absent when
+/// the last member left.
+///
+/// Wire format: `[topic key][group_id:u32][partition_id:u32][has:u8][member_id:u32]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WirePartitionHandedOverBody {
+ pub topic: WireTopicKey,
+ pub group_id: u32,
+ pub partition_id: u32,
+ pub member_id: Option<u32>,
+}
+
+/// Body of a `consumer_group_members` item.
+///
+/// Wire format: `[topic key][group_id:u32][member_id:u32][client_id:u128][ids]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireMemberItemBody {
+ pub topic: WireTopicKey,
+ pub group_id: u32,
+ pub member_id: u32,
+ pub client_id: u128,
+ pub partition_ids: Vec<u32>,
+}
+
+/// Body of `user_created`, `user_updated` and of a `users` item.
+///
+/// Wire format: `[user key][username][status:u8][has:u8][permissions][options]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireUserBody {
+ pub key: WireUserKey,
+ pub username: WireName,
+ pub status: u8,
+ pub permissions: Option<WirePermissions>,
+ pub options: WireChangeOptions,
+}
+
+/// Body of `user_deleted` and `user_password_changed`.
+///
+/// Wire format: `[user key][username]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireUserIdentityBody {
+ pub key: WireUserKey,
+ pub username: WireName,
+}
+
+/// Body of `user_permissions_updated`.
+///
+/// Wire format: `[user key][username][has:u8][permissions][reason:u8]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireUserPermissionsBody {
+ pub key: WireUserKey,
+ pub username: WireName,
+ pub permissions: Option<WirePermissions>,
+ pub reason: WirePermissionsUpdatedReason,
+}
+
+/// Body of `personal_access_token_created` and of a `personal_access_tokens`
+/// item. Expiry 0 means the token never expires.
+///
+/// Wire format: `[user key][name][expiry:u64]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireTokenBody {
+ pub user: WireUserKey,
+ pub name: WireName,
+ pub expiry: u64,
+}
+
+/// Body of `personal_access_token_deleted`.
+///
+/// Wire format: `[user key][name][reason:u8]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireTokenDeletedBody {
+ pub user: WireUserKey,
+ pub name: WireName,
+ pub reason: WireTokenDeletedReason,
+}
+
+/// Body of `session_registered` and of a `sessions` item.
+///
+/// Wire format: `[user key][client_id:u128][session:u64]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireSessionBody {
+ pub user: WireUserKey,
+ pub client_id: u128,
+ pub session: u64,
+}
+
+/// Body of `session_closed`.
+///
+/// Wire format: `[user key][client_id:u128][session:u64][reason:u8]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireSessionClosedBody {
+ pub user: WireUserKey,
+ pub client_id: u128,
+ pub session: u64,
+ pub reason: WireSessionClosedReason,
+}
+
+impl WireMetadataChange {
+ #[must_use]
+ pub const fn kind(&self) -> WireMetadataChangeKind {
+ match self {
+ Self::StreamCreated(_) => WireMetadataChangeKind::StreamCreated,
+ Self::StreamUpdated(_) => WireMetadataChangeKind::StreamUpdated,
+ Self::StreamDeleted(_) => WireMetadataChangeKind::StreamDeleted,
+ Self::TopicCreated(_) => WireMetadataChangeKind::TopicCreated,
+ Self::TopicUpdated(_) => WireMetadataChangeKind::TopicUpdated,
+ Self::TopicDeleted(_) => WireMetadataChangeKind::TopicDeleted,
+ Self::PartitionsChanged(_) => WireMetadataChangeKind::PartitionsChanged,
+ Self::ConsumerGroupCreated(_) => WireMetadataChangeKind::ConsumerGroupCreated,
+ Self::ConsumerGroupDeleted(_) => WireMetadataChangeKind::ConsumerGroupDeleted,
+ Self::ConsumerGroupMemberJoined(_) => WireMetadataChangeKind::ConsumerGroupMemberJoined,
+ Self::ConsumerGroupMemberLeft(_) => WireMetadataChangeKind::ConsumerGroupMemberLeft,
+ Self::ConsumerGroupPartitionHandedOver(_) => {
+ WireMetadataChangeKind::ConsumerGroupPartitionHandedOver
+ }
+ Self::UserCreated(_) => WireMetadataChangeKind::UserCreated,
+ Self::UserUpdated(_) => WireMetadataChangeKind::UserUpdated,
+ Self::UserDeleted(_) => WireMetadataChangeKind::UserDeleted,
+ Self::UserPasswordChanged(_) => WireMetadataChangeKind::UserPasswordChanged,
+ Self::UserPermissionsUpdated(_) => WireMetadataChangeKind::UserPermissionsUpdated,
+ Self::PersonalAccessTokenCreated(_) => {
+ WireMetadataChangeKind::PersonalAccessTokenCreated
+ }
+ Self::PersonalAccessTokenDeleted(_) => {
+ WireMetadataChangeKind::PersonalAccessTokenDeleted
+ }
+ Self::SessionRegistered(_) => WireMetadataChangeKind::SessionRegistered,
+ Self::SessionClosed(_) => WireMetadataChangeKind::SessionClosed,
+ }
+ }
+
+ #[must_use]
+ pub const fn group(&self) -> WireMetadataChangeGroup {
+ self.kind().group()
+ }
+
+ /// Encoded body bytes, without the record envelope.
+ #[must_use]
+ pub fn body_bytes(&self) -> Bytes {
+ match self {
+ Self::StreamCreated(body) | Self::StreamUpdated(body) => body.to_bytes(),
+ Self::StreamDeleted(body) => body.to_bytes(),
+ Self::TopicCreated(body) | Self::TopicUpdated(body) => body.to_bytes(),
+ Self::TopicDeleted(body) => body.to_bytes(),
+ Self::PartitionsChanged(body) => body.to_bytes(),
+ Self::ConsumerGroupCreated(body) => body.to_bytes(),
+ Self::ConsumerGroupDeleted(body) => body.to_bytes(),
+ Self::ConsumerGroupMemberJoined(body) => body.to_bytes(),
+ Self::ConsumerGroupMemberLeft(body) => body.to_bytes(),
+ Self::ConsumerGroupPartitionHandedOver(body) => body.to_bytes(),
+ Self::UserCreated(body) | Self::UserUpdated(body) => body.to_bytes(),
+ Self::UserDeleted(body) | Self::UserPasswordChanged(body) => body.to_bytes(),
+ Self::UserPermissionsUpdated(body) => body.to_bytes(),
+ Self::PersonalAccessTokenCreated(body) => body.to_bytes(),
+ Self::PersonalAccessTokenDeleted(body) => body.to_bytes(),
+ Self::SessionRegistered(body) => body.to_bytes(),
+ Self::SessionClosed(body) => body.to_bytes(),
+ }
+ }
+
+ /// Decode a record body by its group and kind codes.
+ ///
+ /// Returns `Ok(None)` for a group or kind this build does not know, so a
+ /// caller can keep the record with its raw body.
+ ///
+ /// # Errors
+ /// Returns `WireError` for a known kind under another group, a malformed
+ /// body, an unknown reason code or trailing bytes.
+ pub fn decode(group: u8, kind: u8, body: &[u8]) -> Result<Option<Self>, WireError> {
+ let Ok(group) = WireMetadataChangeGroup::from_code(group) else {
+ return Ok(None);
+ };
+ let Ok(kind) = WireMetadataChangeKind::from_code(kind) else {
+ return Ok(None);
+ };
+ if kind.group() != group {
+ return Err(WireError::Validation(Cow::Owned(format!(
+ "change kind {} does not belong to group {}",
+ kind.as_u8(),
+ group.as_u8()
+ ))));
+ }
+ let change = match kind {
+ WireMetadataChangeKind::StreamCreated => Self::StreamCreated(decode_exact(body)?),
+ WireMetadataChangeKind::StreamUpdated => Self::StreamUpdated(decode_exact(body)?),
+ WireMetadataChangeKind::StreamDeleted => Self::StreamDeleted(decode_exact(body)?),
+ WireMetadataChangeKind::TopicCreated => Self::TopicCreated(decode_exact(body)?),
+ WireMetadataChangeKind::TopicUpdated => Self::TopicUpdated(decode_exact(body)?),
+ WireMetadataChangeKind::TopicDeleted => Self::TopicDeleted(decode_exact(body)?),
+ WireMetadataChangeKind::PartitionsChanged => {
+ Self::PartitionsChanged(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::ConsumerGroupCreated => {
+ Self::ConsumerGroupCreated(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::ConsumerGroupDeleted => {
+ Self::ConsumerGroupDeleted(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::ConsumerGroupMemberJoined => {
+ Self::ConsumerGroupMemberJoined(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::ConsumerGroupMemberLeft => {
+ Self::ConsumerGroupMemberLeft(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::ConsumerGroupPartitionHandedOver => {
+ Self::ConsumerGroupPartitionHandedOver(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::UserCreated => Self::UserCreated(decode_exact(body)?),
+ WireMetadataChangeKind::UserUpdated => Self::UserUpdated(decode_exact(body)?),
+ WireMetadataChangeKind::UserDeleted => Self::UserDeleted(decode_exact(body)?),
+ WireMetadataChangeKind::UserPasswordChanged => {
+ Self::UserPasswordChanged(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::UserPermissionsUpdated => {
+ Self::UserPermissionsUpdated(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::PersonalAccessTokenCreated => {
+ Self::PersonalAccessTokenCreated(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::PersonalAccessTokenDeleted => {
+ Self::PersonalAccessTokenDeleted(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::SessionRegistered => {
+ Self::SessionRegistered(decode_exact(body)?)
+ }
+ WireMetadataChangeKind::SessionClosed => Self::SessionClosed(decode_exact(body)?),
+ };
+ Ok(Some(change))
+ }
+}
+
+impl WireMetadataCurrentItem {
+ #[must_use]
+ pub const fn group(&self) -> WireMetadataChangeGroup {
+ match self {
+ Self::Stream(_) => WireMetadataChangeGroup::Streams,
+ Self::Topic(_) => WireMetadataChangeGroup::Topics,
+ Self::Partitions(_) => WireMetadataChangeGroup::Partitions,
+ Self::ConsumerGroup(_) => WireMetadataChangeGroup::ConsumerGroups,
+ Self::ConsumerGroupMember(_) => WireMetadataChangeGroup::ConsumerGroupMembers,
+ Self::User(_) => WireMetadataChangeGroup::Users,
+ Self::PersonalAccessToken(_) => WireMetadataChangeGroup::PersonalAccessTokens,
+ Self::Session(_) => WireMetadataChangeGroup::Sessions,
+ }
+ }
+
+ /// Encoded body bytes, without the item envelope.
+ #[must_use]
+ pub fn body_bytes(&self) -> Bytes {
+ match self {
+ Self::Stream(body) => body.to_bytes(),
+ Self::Topic(body) => body.to_bytes(),
+ Self::Partitions(body) => body.to_bytes(),
+ Self::ConsumerGroup(body) => body.to_bytes(),
+ Self::ConsumerGroupMember(body) => body.to_bytes(),
+ Self::User(body) => body.to_bytes(),
+ Self::PersonalAccessToken(body) => body.to_bytes(),
+ Self::Session(body) => body.to_bytes(),
+ }
+ }
+
+ /// Decode an item body by its group code.
+ ///
+ /// Returns `Ok(None)` for a group this build does not know.
+ ///
+ /// # Errors
+ /// Returns `WireError` for a malformed body or trailing bytes.
+ pub fn decode(group: u8, body: &[u8]) -> Result<Option<Self>, WireError> {
+ let Ok(group) = WireMetadataChangeGroup::from_code(group) else {
+ return Ok(None);
+ };
+ let item = match group {
+ WireMetadataChangeGroup::Streams => Self::Stream(decode_exact(body)?),
+ WireMetadataChangeGroup::Topics => Self::Topic(decode_exact(body)?),
+ WireMetadataChangeGroup::Partitions => Self::Partitions(decode_exact(body)?),
+ WireMetadataChangeGroup::ConsumerGroups => Self::ConsumerGroup(decode_exact(body)?),
+ WireMetadataChangeGroup::ConsumerGroupMembers => {
+ Self::ConsumerGroupMember(decode_exact(body)?)
+ }
+ WireMetadataChangeGroup::Users => Self::User(decode_exact(body)?),
+ WireMetadataChangeGroup::PersonalAccessTokens => {
+ Self::PersonalAccessToken(decode_exact(body)?)
+ }
+ WireMetadataChangeGroup::Sessions => Self::Session(decode_exact(body)?),
+ };
+ Ok(Some(item))
+ }
+}
+
+/// Decode a body that must fill `body` exactly.
+fn decode_exact<T: WireDecode>(body: &[u8]) -> Result<T, WireError> {
+ let (value, consumed) = T::decode(body)?;
+ if consumed != body.len() {
+ return Err(WireError::Validation(Cow::Owned(format!(
+ "{} trailing bytes after a change body",
+ body.len() - consumed
+ ))));
+ }
+ Ok(value)
+}
+
+/// Sequential reader over one body. Every read checks bounds.
+struct BodyReader<'a> {
+ buf: &'a [u8],
+ position: usize,
+}
+
+impl<'a> BodyReader<'a> {
+ const fn new(buf: &'a [u8]) -> Self {
+ Self { buf, position: 0 }
+ }
+
+ const fn consumed(&self) -> usize {
+ self.position
+ }
+
+ fn u8(&mut self) -> Result<u8, WireError> {
+ let value = read_u8(self.buf, self.position)?;
+ self.position += 1;
+ Ok(value)
+ }
+
+ fn u32(&mut self) -> Result<u32, WireError> {
+ let value = read_u32_le(self.buf, self.position)?;
+ self.position += 4;
+ Ok(value)
+ }
+
+ fn u64(&mut self) -> Result<u64, WireError> {
+ let value = read_u64_le(self.buf, self.position)?;
+ self.position += 8;
+ Ok(value)
+ }
+
+ fn u128(&mut self) -> Result<u128, WireError> {
+ let value = read_u128_le(self.buf, self.position)?;
+ self.position += 16;
+ Ok(value)
+ }
+
+ fn flag(&mut self) -> Result<bool, WireError> {
+ let value = read_flag(self.buf, self.position)?;
+ self.position += 1;
+ Ok(value)
+ }
+
+ fn name(&mut self) -> Result<WireName, WireError> {
+ let (name, consumed) = read_wire_name(self.buf, self.position)?;
+ self.position += consumed;
+ Ok(name)
+ }
+
+ fn optional_u64(&mut self) -> Result<Option<u64>, WireError> {
+ let (value, consumed) = read_optional_u64(self.buf, self.position)?;
+ self.position += consumed;
+ Ok(value)
+ }
+
+ fn optional_u32(&mut self) -> Result<Option<u32>, WireError> {
+ let (value, consumed) = read_optional_u32(self.buf, self.position)?;
+ self.position += consumed;
+ Ok(value)
+ }
+
+ fn partition_ids(&mut self) -> Result<Vec<u32>, WireError> {
+ let (ids, consumed) = read_partition_ids(self.buf, self.position)?;
+ self.position += consumed;
+ Ok(ids)
+ }
+
+ fn decode<T: WireDecode>(&mut self) -> Result<T, WireError> {
+ let rest = self
+ .buf
+ .get(self.position..)
+ .ok_or(WireError::UnexpectedEof {
+ offset: self.position,
+ need: 1,
+ have: 0,
+ })?;
+ let (value, consumed) = T::decode(rest)?;
+ self.position += consumed;
+ Ok(value)
+ }
+
+ fn options(&mut self) -> Result<WireChangeOptions, WireError> {
+ let offset = self.position;
+ match self.u8()? {
+ 0 => {
+ let (explicit, explicit_consumed) =
+ decode_options_prefixed(self.buf, self.position)?;
+ self.position += explicit_consumed;
+ let (derived, derived_consumed) = decode_options_prefixed(self.buf, self.position)?;
+ self.position += derived_consumed;
+ Ok(WireChangeOptions::Readable { explicit, derived })
+ }
+ 1 => Ok(WireChangeOptions::Unreadable),
+ value => Err(WireError::UnknownDiscriminant {
+ type_name: "WireChangeOptions",
+ value,
+ offset,
+ }),
+ }
+ }
+
+ fn permissions(&mut self) -> Result<Option<WirePermissions>, WireError> {
+ if self.flag()? {
+ Ok(Some(self.decode()?))
+ } else {
+ Ok(None)
+ }
+ }
+
+ fn reason<T>(&mut self, from_code: fn(u8) -> Result<T, WireError>) -> Result<T, WireError> {
+ let offset = self.position;
+ let code = self.u8()?;
+ from_code(code).map_err(|error| match error {
+ WireError::UnknownDiscriminant {
+ type_name, value, ..
+ } => WireError::UnknownDiscriminant {
+ type_name,
+ value,
+ offset,
+ },
+ other => other,
+ })
+ }
+}
+
+fn options_size(options: &WireChangeOptions) -> usize {
+ match options {
+ WireChangeOptions::Readable { explicit, derived } => {
+ 1 + options_prefixed_size(explicit) + options_prefixed_size(derived)
+ }
+ WireChangeOptions::Unreadable => 1,
+ }
+}
+
+fn put_options(buf: &mut BytesMut, options: &WireChangeOptions) {
+ match options {
+ WireChangeOptions::Readable { explicit, derived } => {
+ buf.put_u8(0);
+ encode_options_prefixed(explicit, buf);
+ encode_options_prefixed(derived, buf);
+ }
+ WireChangeOptions::Unreadable => buf.put_u8(1),
+ }
+}
+
+fn permissions_size(permissions: Option<&WirePermissions>) -> usize {
+ 1 + permissions.map_or(0, WireEncode::encoded_size)
+}
+
+fn put_permissions(buf: &mut BytesMut, permissions: Option<&WirePermissions>) {
+ put_flag(buf, permissions.is_some());
+ if let Some(permissions) = permissions {
+ permissions.encode(buf);
+ }
+}
+
+impl WireStreamKey {
+ const SIZE: usize = 12;
+}
+
+impl WireEncode for WireStreamKey {
+ fn encoded_size(&self) -> usize {
+ Self::SIZE
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ buf.put_u32_le(self.stream_id);
+ buf.put_u64_le(self.stream_created_op);
+ }
+}
+
+impl WireDecode for WireStreamKey {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let key = Self {
+ stream_id: reader.u32()?,
+ stream_created_op: reader.u64()?,
+ };
+ Ok((key, reader.consumed()))
+ }
+}
+
+impl WireTopicKey {
+ const SIZE: usize = WireStreamKey::SIZE + 12;
+}
+
+impl WireEncode for WireTopicKey {
+ fn encoded_size(&self) -> usize {
+ Self::SIZE
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.stream.encode(buf);
+ buf.put_u32_le(self.topic_id);
+ buf.put_u64_le(self.topic_created_op);
+ }
+}
+
+impl WireDecode for WireTopicKey {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let key = Self {
+ stream: reader.decode()?,
+ topic_id: reader.u32()?,
+ topic_created_op: reader.u64()?,
+ };
+ Ok((key, reader.consumed()))
+ }
+}
+
+impl WireUserKey {
+ const SIZE: usize = 12;
+}
+
+impl WireEncode for WireUserKey {
+ fn encoded_size(&self) -> usize {
+ Self::SIZE
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ buf.put_u32_le(self.user_id);
+ buf.put_u64_le(self.user_created_at);
+ }
+}
+
+impl WireDecode for WireUserKey {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let key = Self {
+ user_id: reader.u32()?,
+ user_created_at: reader.u64()?,
+ };
+ Ok((key, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireStreamBody {
+ fn encoded_size(&self) -> usize {
+ WireStreamKey::SIZE + 8 + self.name.encoded_size() + options_size(&self.options)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.key.encode(buf);
+ buf.put_u64_le(self.created_at);
+ self.name.encode(buf);
+ put_options(buf, &self.options);
+ }
+}
+
+impl WireDecode for WireStreamBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ key: reader.decode()?,
+ created_at: reader.u64()?,
+ name: reader.name()?,
+ options: reader.options()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireStreamDeletedBody {
+ fn encoded_size(&self) -> usize {
+ WireStreamKey::SIZE + 8 + self.name.encoded_size() + optional_u64_size(self.requested_op)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.key.encode(buf);
+ buf.put_u64_le(self.created_at);
+ self.name.encode(buf);
+ put_optional_u64(buf, self.requested_op);
+ }
+}
+
+impl WireDecode for WireStreamDeletedBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ key: reader.decode()?,
+ created_at: reader.u64()?,
+ name: reader.name()?,
+ requested_op: reader.optional_u64()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireTopicBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 8 + self.name.encoded_size() + 4 + options_size(&self.options)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.key.encode(buf);
+ buf.put_u64_le(self.created_at);
+ self.name.encode(buf);
+ buf.put_u32_le(self.partitions_count);
+ put_options(buf, &self.options);
+ }
+}
+
+impl WireDecode for WireTopicBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ key: reader.decode()?,
+ created_at: reader.u64()?,
+ name: reader.name()?,
+ partitions_count: reader.u32()?,
+ options: reader.options()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireTopicItemBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 8 + self.name.encoded_size() + options_size(&self.options)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.key.encode(buf);
+ buf.put_u64_le(self.created_at);
+ self.name.encode(buf);
+ put_options(buf, &self.options);
+ }
+}
+
+impl WireDecode for WireTopicItemBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ key: reader.decode()?,
+ created_at: reader.u64()?,
+ name: reader.name()?,
+ options: reader.options()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireTopicDeletedBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 8 + self.name.encoded_size() + optional_u64_size(self.requested_op)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.key.encode(buf);
+ buf.put_u64_le(self.created_at);
+ self.name.encode(buf);
+ put_optional_u64(buf, self.requested_op);
+ }
+}
+
+impl WireDecode for WireTopicDeletedBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ key: reader.decode()?,
+ created_at: reader.u64()?,
+ name: reader.name()?,
+ requested_op: reader.optional_u64()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WirePartitionsChangedBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE
+ + 4
+ + partition_ids_size(&self.removed_partition_ids)
+ + optional_u64_size(self.requested_op)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.topic.encode(buf);
+ buf.put_u32_le(self.partitions_count);
+ put_partition_ids(buf, &self.removed_partition_ids);
+ put_optional_u64(buf, self.requested_op);
+ }
+}
+
+impl WireDecode for WirePartitionsChangedBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ topic: reader.decode()?,
+ partitions_count: reader.u32()?,
+ removed_partition_ids: reader.partition_ids()?,
+ requested_op: reader.optional_u64()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WirePartitionsItemBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 4
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.topic.encode(buf);
+ buf.put_u32_le(self.partitions_count);
+ }
+}
+
+impl WireDecode for WirePartitionsItemBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ topic: reader.decode()?,
+ partitions_count: reader.u32()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireConsumerGroupCreatedBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 4 + self.name.encoded_size() + 4
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.topic.encode(buf);
+ buf.put_u32_le(self.group_id);
+ self.name.encode(buf);
+ buf.put_u32_le(self.consumer_groups_count);
+ }
+}
+
+impl WireDecode for WireConsumerGroupCreatedBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ topic: reader.decode()?,
+ group_id: reader.u32()?,
+ name: reader.name()?,
+ consumer_groups_count: reader.u32()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireConsumerGroupDeletedBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 4 + self.name.encoded_size() + 4 + optional_u64_size(self.requested_op)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.topic.encode(buf);
+ buf.put_u32_le(self.group_id);
+ self.name.encode(buf);
+ buf.put_u32_le(self.consumer_groups_count);
+ put_optional_u64(buf, self.requested_op);
+ }
+}
+
+impl WireDecode for WireConsumerGroupDeletedBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ topic: reader.decode()?,
+ group_id: reader.u32()?,
+ name: reader.name()?,
+ consumer_groups_count: reader.u32()?,
+ requested_op: reader.optional_u64()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireConsumerGroupItemBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 4 + self.name.encoded_size()
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.topic.encode(buf);
+ buf.put_u32_le(self.group_id);
+ self.name.encode(buf);
+ }
+}
+
+impl WireDecode for WireConsumerGroupItemBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ topic: reader.decode()?,
+ group_id: reader.u32()?,
+ name: reader.name()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireMemberJoinedBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 4 + 4 + 16 + 4 + partition_ids_size(&self.partition_ids)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.topic.encode(buf);
+ buf.put_u32_le(self.group_id);
+ buf.put_u32_le(self.member_id);
+ buf.put_u128_le(self.client_id);
+ buf.put_u32_le(self.members_count);
+ put_partition_ids(buf, &self.partition_ids);
+ }
+}
+
+impl WireDecode for WireMemberJoinedBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ topic: reader.decode()?,
+ group_id: reader.u32()?,
+ member_id: reader.u32()?,
+ client_id: reader.u128()?,
+ members_count: reader.u32()?,
+ partition_ids: reader.partition_ids()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireMemberLeftBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 4 + 4 + 16 + 4 + 1
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.topic.encode(buf);
+ buf.put_u32_le(self.group_id);
+ buf.put_u32_le(self.member_id);
+ buf.put_u128_le(self.client_id);
+ buf.put_u32_le(self.members_count);
+ buf.put_u8(self.reason.as_u8());
+ }
+}
+
+impl WireDecode for WireMemberLeftBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ topic: reader.decode()?,
+ group_id: reader.u32()?,
+ member_id: reader.u32()?,
+ client_id: reader.u128()?,
+ members_count: reader.u32()?,
+ reason: reader.reason(WireMemberLeftReason::from_code)?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WirePartitionHandedOverBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 4 + 4 + optional_u32_size(self.member_id)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.topic.encode(buf);
+ buf.put_u32_le(self.group_id);
+ buf.put_u32_le(self.partition_id);
+ put_optional_u32(buf, self.member_id);
+ }
+}
+
+impl WireDecode for WirePartitionHandedOverBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ topic: reader.decode()?,
+ group_id: reader.u32()?,
+ partition_id: reader.u32()?,
+ member_id: reader.optional_u32()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireMemberItemBody {
+ fn encoded_size(&self) -> usize {
+ WireTopicKey::SIZE + 4 + 4 + 16 + partition_ids_size(&self.partition_ids)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.topic.encode(buf);
+ buf.put_u32_le(self.group_id);
+ buf.put_u32_le(self.member_id);
+ buf.put_u128_le(self.client_id);
+ put_partition_ids(buf, &self.partition_ids);
+ }
+}
+
+impl WireDecode for WireMemberItemBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ topic: reader.decode()?,
+ group_id: reader.u32()?,
+ member_id: reader.u32()?,
+ client_id: reader.u128()?,
+ partition_ids: reader.partition_ids()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireUserBody {
+ fn encoded_size(&self) -> usize {
+ WireUserKey::SIZE
+ + self.username.encoded_size()
+ + 1
+ + permissions_size(self.permissions.as_ref())
+ + options_size(&self.options)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.key.encode(buf);
+ self.username.encode(buf);
+ buf.put_u8(self.status);
+ put_permissions(buf, self.permissions.as_ref());
+ put_options(buf, &self.options);
+ }
+}
+
+impl WireDecode for WireUserBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ key: reader.decode()?,
+ username: reader.name()?,
+ status: reader.u8()?,
+ permissions: reader.permissions()?,
+ options: reader.options()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireUserIdentityBody {
+ fn encoded_size(&self) -> usize {
+ WireUserKey::SIZE + self.username.encoded_size()
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.key.encode(buf);
+ self.username.encode(buf);
+ }
+}
+
+impl WireDecode for WireUserIdentityBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ key: reader.decode()?,
+ username: reader.name()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireUserPermissionsBody {
+ fn encoded_size(&self) -> usize {
+ WireUserKey::SIZE
+ + self.username.encoded_size()
+ + permissions_size(self.permissions.as_ref())
+ + 1
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.key.encode(buf);
+ self.username.encode(buf);
+ put_permissions(buf, self.permissions.as_ref());
+ buf.put_u8(self.reason.as_u8());
+ }
+}
+
+impl WireDecode for WireUserPermissionsBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ key: reader.decode()?,
+ username: reader.name()?,
+ permissions: reader.permissions()?,
+ reason: reader.reason(WirePermissionsUpdatedReason::from_code)?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireTokenBody {
+ fn encoded_size(&self) -> usize {
+ WireUserKey::SIZE + self.name.encoded_size() + 8
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.user.encode(buf);
+ self.name.encode(buf);
+ buf.put_u64_le(self.expiry);
+ }
+}
+
+impl WireDecode for WireTokenBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ user: reader.decode()?,
+ name: reader.name()?,
+ expiry: reader.u64()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireTokenDeletedBody {
+ fn encoded_size(&self) -> usize {
+ WireUserKey::SIZE + self.name.encoded_size() + 1
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.user.encode(buf);
+ self.name.encode(buf);
+ buf.put_u8(self.reason.as_u8());
+ }
+}
+
+impl WireDecode for WireTokenDeletedBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ user: reader.decode()?,
+ name: reader.name()?,
+ reason: reader.reason(WireTokenDeletedReason::from_code)?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireSessionBody {
+ fn encoded_size(&self) -> usize {
+ WireUserKey::SIZE + 16 + 8
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.user.encode(buf);
+ buf.put_u128_le(self.client_id);
+ buf.put_u64_le(self.session);
+ }
+}
+
+impl WireDecode for WireSessionBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ user: reader.decode()?,
+ client_id: reader.u128()?,
+ session: reader.u64()?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+impl WireEncode for WireSessionClosedBody {
+ fn encoded_size(&self) -> usize {
+ WireUserKey::SIZE + 16 + 8 + 1
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.user.encode(buf);
+ buf.put_u128_le(self.client_id);
+ buf.put_u64_le(self.session);
+ buf.put_u8(self.reason.as_u8());
+ }
+}
+
+impl WireDecode for WireSessionClosedBody {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mut reader = BodyReader::new(buf);
+ let body = Self {
+ user: reader.decode()?,
+ client_id: reader.u128()?,
+ session: reader.u64()?,
+ reason: reader.reason(WireSessionClosedReason::from_code)?,
+ };
+ Ok((body, reader.consumed()))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::primitives::permissions::{
+ WireGlobalPermissions, WireStreamPermissions, WireTopicPermissions,
+ };
+
+ #[test]
+ fn given_every_change_kind_when_round_tripping_should_preserve_every_field() {
+ for change in sample_changes() {
+ let body = change.body_bytes();
+ let decoded =
+ WireMetadataChange::decode(change.group().as_u8(), change.kind().as_u8(), &body)
+ .unwrap()
+ .unwrap();
+ assert_eq!(decoded, change);
+ }
+ }
+
+ #[test]
+ fn given_every_item_group_when_round_tripping_should_preserve_every_field() {
+ for item in sample_items() {
+ let body = item.body_bytes();
+ let decoded = WireMetadataCurrentItem::decode(item.group().as_u8(), &body)
+ .unwrap()
+ .unwrap();
+ assert_eq!(decoded, item);
+ }
+ }
+
+ #[test]
+ fn given_every_change_kind_when_truncating_the_body_should_refuse_every_prefix() {
+ for change in sample_changes() {
+ let body = change.body_bytes();
+ for length in 0..body.len() {
+ assert!(
+ WireMetadataChange::decode(
+ change.group().as_u8(),
+ change.kind().as_u8(),
+ &body[..length]
+ )
+ .is_err(),
+ "{:?} accepted a body truncated to {length} bytes",
+ change.kind()
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn given_trailing_bytes_when_decoding_a_change_body_should_refuse() {
+ for change in sample_changes() {
+ let mut body = change.body_bytes().to_vec();
+ body.push(0);
+ assert!(
+ WireMetadataChange::decode(change.group().as_u8(), change.kind().as_u8(), &body)
+ .is_err(),
+ "{:?} accepted a trailing byte",
+ change.kind()
+ );
+ }
+ for item in sample_items() {
+ let mut body = item.body_bytes().to_vec();
+ body.push(0);
+ assert!(WireMetadataCurrentItem::decode(item.group().as_u8(), &body).is_err());
+ }
+ }
+
+ #[test]
+ fn given_known_kind_under_wrong_group_when_decoding_should_refuse() {
+ let change = WireMetadataChange::StreamCreated(stream_body());
+ let body = change.body_bytes();
+ let error = WireMetadataChange::decode(
+ WireMetadataChangeGroup::Topics.as_u8(),
+ WireMetadataChangeKind::StreamCreated.as_u8(),
+ &body,
+ );
+ assert!(error.is_err());
+ }
+
+ #[test]
+ fn given_unknown_kind_or_group_when_decoding_should_return_none() {
+ assert_eq!(WireMetadataChange::decode(1, 9, &[1, 2, 3]), Ok(None));
+ assert_eq!(WireMetadataChange::decode(200, 1, &[]), Ok(None));
+ assert_eq!(WireMetadataChange::decode(0, 1, &[]), Ok(None));
+ assert_eq!(WireMetadataCurrentItem::decode(9, &[7]), Ok(None));
+ }
+
+ #[test]
+ fn given_unknown_reason_code_when_decoding_member_left_should_refuse() {
+ let change = WireMetadataChange::ConsumerGroupMemberLeft(WireMemberLeftBody {
+ topic: topic_key(),
+ group_id: 1,
+ member_id: 2,
+ client_id: 3,
+ members_count: 0,
+ reason: WireMemberLeftReason::Left,
+ });
+ let mut body = change.body_bytes().to_vec();
+ *body.last_mut().unwrap() = 2;
+ assert!(
+ WireMetadataChange::decode(change.group().as_u8(), change.kind().as_u8(), &body)
+ .is_err()
+ );
+ }
+
+ #[test]
+ fn given_unknown_reasons_when_decoding_token_session_and_permission_bodies_should_refuse() {
+ let changes = [
+ WireMetadataChange::PersonalAccessTokenDeleted(WireTokenDeletedBody {
+ user: user_key(),
+ name: WireName::new("token").unwrap(),
+ reason: WireTokenDeletedReason::Expired,
+ }),
+ WireMetadataChange::SessionClosed(WireSessionClosedBody {
+ user: user_key(),
+ client_id: 9,
+ session: 4,
+ reason: WireSessionClosedReason::Expired,
+ }),
+ WireMetadataChange::UserPermissionsUpdated(WireUserPermissionsBody {
+ key: user_key(),
+ username: WireName::new("alice").unwrap(),
+ permissions: None,
+ reason: WirePermissionsUpdatedReason::ResourceDeleted,
+ }),
+ ];
+ for change in changes {
+ let mut body = change.body_bytes().to_vec();
+ *body.last_mut().unwrap() = 7;
+ assert!(
+ WireMetadataChange::decode(change.group().as_u8(), change.kind().as_u8(), &body)
+ .is_err(),
+ "{:?} accepted reason code 7",
+ change.kind()
+ );
+ }
+ }
+
+ #[test]
+ fn given_options_state_byte_above_one_when_decoding_should_refuse() {
+ let change = WireMetadataChange::StreamCreated(stream_body());
+ let mut body = change.body_bytes().to_vec();
+ let state_offset = WireStreamKey::SIZE + 8 + 1 + "orders".len();
+ body[state_offset] = 2;
+ assert!(
+ WireMetadataChange::decode(change.group().as_u8(), change.kind().as_u8(), &body)
+ .is_err()
+ );
+ }
+
+ #[test]
+ fn given_presence_byte_above_one_when_decoding_requested_op_should_refuse() {
+ let change = WireMetadataChange::StreamDeleted(WireStreamDeletedBody {
+ key: stream_key(),
+ created_at: 1,
+ name: WireName::new("orders").unwrap(),
+ requested_op: None,
+ });
+ let mut body = change.body_bytes().to_vec();
+ *body.last_mut().unwrap() = 2;
+ assert!(
+ WireMetadataChange::decode(change.group().as_u8(), change.kind().as_u8(), &body)
+ .is_err()
+ );
+ }
+
+ #[test]
+ fn given_unreadable_options_when_round_tripping_should_keep_the_marker() {
+ let change = WireMetadataChange::TopicUpdated(WireTopicBody {
+ options: WireChangeOptions::Unreadable,
+ ..topic_body()
+ });
+ let body = change.body_bytes();
+ let decoded =
+ WireMetadataChange::decode(change.group().as_u8(), change.kind().as_u8(), &body)
+ .unwrap()
+ .unwrap();
+ assert_eq!(decoded, change);
+ }
+
+ fn stream_key() -> WireStreamKey {
+ WireStreamKey {
+ stream_id: 7,
+ stream_created_op: 4121,
+ }
+ }
+
+ fn topic_key() -> WireTopicKey {
+ WireTopicKey {
+ stream: stream_key(),
+ topic_id: 3,
+ topic_created_op: 4125,
+ }
+ }
+
+ fn user_key() -> WireUserKey {
+ WireUserKey {
+ user_id: 9,
+ user_created_at: 1_760_300_000_000_000,
+ }
+ }
+
+ fn options() -> WireChangeOptions {
+ let mut buf = BytesMut::new();
+ let value = 1_000u64.to_le_bytes();
+ crate::primitives::user_headers::encode_user_headers(
+ &[(2u8, b"message_expiry".as_slice(), 8u8, value.as_slice())],
+ &mut buf,
+ );
+ WireChangeOptions::Readable {
+ explicit: WireOptions::from_slice(&buf).unwrap(),
+ derived: WireOptions::empty(),
+ }
+ }
+
+ fn permissions() -> WirePermissions {
+ WirePermissions {
+ global: WireGlobalPermissions {
+ manage_servers: false,
+ read_servers: true,
+ manage_users: false,
+ read_users: true,
+ manage_streams: false,
+ read_streams: true,
+ manage_topics: false,
+ read_topics: true,
+ poll_messages: true,
+ send_messages: false,
+ },
+ streams: vec![WireStreamPermissions {
+ stream_id: 7,
+ manage_stream: false,
+ read_stream: true,
+ manage_topics: false,
+ read_topics: true,
+ poll_messages: true,
+ send_messages: false,
+ topics: vec![WireTopicPermissions {
+ topic_id: 3,
+ manage_topic: false,
+ read_topic: true,
+ poll_messages: true,
+ send_messages: false,
+ }],
+ }],
+ }
+ }
+
+ fn stream_body() -> WireStreamBody {
+ WireStreamBody {
+ key: stream_key(),
+ created_at: 1_760_356_800_123_456,
+ name: WireName::new("orders").unwrap(),
+ options: options(),
+ }
+ }
+
+ fn topic_body() -> WireTopicBody {
+ WireTopicBody {
+ key: topic_key(),
+ created_at: 1_760_356_801_000_000,
+ name: WireName::new("events").unwrap(),
+ partitions_count: 8,
+ options: options(),
+ }
+ }
+
+ fn user_body(permissions: Option<WirePermissions>) -> WireUserBody {
+ WireUserBody {
+ key: user_key(),
+ username: WireName::new("alice").unwrap(),
+ status: 1,
+ permissions,
+ options: WireChangeOptions::Readable {
+ explicit: WireOptions::empty(),
+ derived: WireOptions::empty(),
+ },
+ }
+ }
+
+ fn sample_changes() -> Vec<WireMetadataChange> {
+ let name = || WireName::new("audit").unwrap();
+ vec![
+ WireMetadataChange::StreamCreated(stream_body()),
+ WireMetadataChange::StreamUpdated(WireStreamBody {
+ options: WireChangeOptions::Unreadable,
+ ..stream_body()
+ }),
+ WireMetadataChange::StreamDeleted(WireStreamDeletedBody {
+ key: stream_key(),
+ created_at: 1,
+ name: name(),
+ requested_op: Some(4140),
+ }),
+ WireMetadataChange::TopicCreated(topic_body()),
+ WireMetadataChange::TopicUpdated(topic_body()),
+ WireMetadataChange::TopicDeleted(WireTopicDeletedBody {
+ key: topic_key(),
+ created_at: 2,
+ name: name(),
+ requested_op: None,
+ }),
+ WireMetadataChange::PartitionsChanged(WirePartitionsChangedBody {
+ topic: topic_key(),
+ partitions_count: 5,
+ removed_partition_ids: vec![5, 6, 7],
+ requested_op: Some(4200),
+ }),
+ WireMetadataChange::ConsumerGroupCreated(WireConsumerGroupCreatedBody {
+ topic: topic_key(),
+ group_id: 0,
+ name: name(),
+ consumer_groups_count: 1,
+ }),
+ WireMetadataChange::ConsumerGroupDeleted(WireConsumerGroupDeletedBody {
+ topic: topic_key(),
+ group_id: 0,
+ name: name(),
+ consumer_groups_count: 0,
+ requested_op: Some(4300),
+ }),
+ WireMetadataChange::ConsumerGroupMemberJoined(WireMemberJoinedBody {
+ topic: topic_key(),
+ group_id: 1,
+ member_id: 2,
+ client_id: u128::MAX - 1,
+ members_count: 2,
+ partition_ids: vec![],
+ }),
+ WireMetadataChange::ConsumerGroupMemberLeft(WireMemberLeftBody {
+ topic: topic_key(),
+ group_id: 1,
+ member_id: 2,
+ client_id: 77,
+ members_count: 1,
+ reason: WireMemberLeftReason::SessionEnded,
+ }),
+ WireMetadataChange::ConsumerGroupPartitionHandedOver(WirePartitionHandedOverBody {
+ topic: topic_key(),
+ group_id: 1,
+ partition_id: 4,
+ member_id: None,
+ }),
+ WireMetadataChange::UserCreated(user_body(Some(permissions()))),
+ WireMetadataChange::UserUpdated(user_body(None)),
+ WireMetadataChange::UserDeleted(WireUserIdentityBody {
+ key: user_key(),
+ username: WireName::new("alice").unwrap(),
+ }),
+ WireMetadataChange::UserPasswordChanged(WireUserIdentityBody {
+ key: user_key(),
+ username: WireName::new("alice").unwrap(),
+ }),
+ WireMetadataChange::UserPermissionsUpdated(WireUserPermissionsBody {
+ key: user_key(),
+ username: WireName::new("alice").unwrap(),
+ permissions: Some(permissions()),
+ reason: WirePermissionsUpdatedReason::Updated,
+ }),
+ WireMetadataChange::PersonalAccessTokenCreated(WireTokenBody {
+ user: user_key(),
+ name: WireName::new("ci-token").unwrap(),
+ expiry: 0,
+ }),
+ WireMetadataChange::PersonalAccessTokenDeleted(WireTokenDeletedBody {
+ user: user_key(),
+ name: WireName::new("ci-token").unwrap(),
+ reason: WireTokenDeletedReason::Deleted,
+ }),
+ WireMetadataChange::SessionRegistered(WireSessionBody {
+ user: user_key(),
+ client_id: 1 << 100,
+ session: 4,
+ }),
+ WireMetadataChange::SessionClosed(WireSessionClosedBody {
+ user: user_key(),
+ client_id: 1 << 100,
+ session: 4,
+ reason: WireSessionClosedReason::Logout,
+ }),
+ ]
+ }
+
+ fn sample_items() -> Vec<WireMetadataCurrentItem> {
+ vec![
+ WireMetadataCurrentItem::Stream(stream_body()),
+ WireMetadataCurrentItem::Topic(WireTopicItemBody {
+ key: topic_key(),
+ created_at: 3,
+ name: WireName::new("events").unwrap(),
+ options: options(),
+ }),
+ WireMetadataCurrentItem::Partitions(WirePartitionsItemBody {
+ topic: topic_key(),
+ partitions_count: 12,
+ }),
+ WireMetadataCurrentItem::ConsumerGroup(WireConsumerGroupItemBody {
+ topic: topic_key(),
+ group_id: 4,
+ name: WireName::new("readers").unwrap(),
+ }),
+ WireMetadataCurrentItem::ConsumerGroupMember(WireMemberItemBody {
+ topic: topic_key(),
+ group_id: 4,
+ member_id: 1,
+ client_id: 12,
+ partition_ids: vec![0, 2],
+ }),
+ WireMetadataCurrentItem::User(user_body(Some(permissions()))),
+ WireMetadataCurrentItem::PersonalAccessToken(WireTokenBody {
+ user: user_key(),
+ name: WireName::new("ci-token").unwrap(),
+ expiry: 1_760_400_000_000_000,
+ }),
+ WireMetadataCurrentItem::Session(WireSessionBody {
+ user: user_key(),
+ client_id: 5,
+ session: 6,
+ }),
+ ]
+ }
+}
diff --git a/core/binary_protocol/src/primitives/metadata_changes/mod.rs b/core/binary_protocol/src/primitives/metadata_changes/mod.rs
new file mode 100644
index 000000000..ce508f154
--- /dev/null
+++ b/core/binary_protocol/src/primitives/metadata_changes/mod.rs
@@ -0,0 +1,454 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Wire codes, record and item bodies, the change cursor and the current
+//! state page token of the metadata change log.
+//!
+//! A record body is encoded once on the server and copied verbatim into every
+//! answer, so decoding is strict: unknown reason codes, a known kind under the
+//! wrong group, presence bytes other than 0 and 1 and trailing bytes are all
+//! refused. A kind or group this build does not know is not an error, the
+//! caller keeps it with its raw body.
+
+mod body;
+mod position;
+
+pub use body::{
+ WireChangeOptions, WireConsumerGroupCreatedBody, WireConsumerGroupDeletedBody,
+ WireConsumerGroupItemBody, WireMemberItemBody, WireMemberJoinedBody, WireMemberLeftBody,
+ WireMetadataChange, WireMetadataCurrentItem, WirePartitionHandedOverBody,
+ WirePartitionsChangedBody, WirePartitionsItemBody, WireSessionBody, WireSessionClosedBody,
+ WireStreamBody, WireStreamDeletedBody, WireStreamKey, WireTokenBody, WireTokenDeletedBody,
+ WireTopicBody, WireTopicDeletedBody, WireTopicItemBody, WireTopicKey, WireUserBody,
+ WireUserIdentityBody, WireUserKey, WireUserPermissionsBody,
+};
+pub use position::{
+ MAX_PAGE_TOKEN_KEY_SIZE, MAX_PAGE_TOKEN_SIZE, METADATA_CHANGE_CURSOR_SIZE,
+ WireMetadataChangeCursor, WireMetadataCurrentPageToken,
+};
+
+use crate::WireDecode;
+use crate::WireError;
+use crate::codec::{bounded_capacity, read_u8, read_u32_le, read_u64_le};
+use crate::primitives::identifier::WireName;
+use bytes::{BufMut, BytesMut};
+use std::borrow::Cow;
+
+macro_rules! wire_code_enum {
+ (
+ $(#[$meta:meta])*
+ $name:ident {
+ $($(#[$variant_meta:meta])* $variant:ident = $code:literal),+ $(,)?
+ }
+ ) => {
+ $(#[$meta])*
+ #[repr(u8)]
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+ pub enum $name {
+ $($(#[$variant_meta])* $variant = $code),+
+ }
+
+ impl $name {
+ /// Decode from the wire code.
+ ///
+ /// # Errors
+ /// Returns `WireError::UnknownDiscriminant` for a code this build
+ /// does not know.
+ pub const fn from_code(code: u8) -> Result<Self, WireError> {
+ match code {
+ $($code => Ok(Self::$variant),)+
+ value => Err(WireError::UnknownDiscriminant {
+ type_name: stringify!($name),
+ value,
+ offset: 0,
+ }),
+ }
+ }
+
+ /// The wire code.
+ #[must_use]
+ pub const fn as_u8(self) -> u8 {
+ self as u8
+ }
+ }
+ };
+}
+
+wire_code_enum! {
+ /// Family of change records and the unit of filtering. Codes 9 to 127 are
+ /// reserved for future groups, 128 to 255 for downstream extensions.
+ ///
+ /// A page token holds the groups in an 8 bit mask, so a ninth group needs
+ /// a new token format.
+ WireMetadataChangeGroup {
+ Streams = 1,
+ Topics = 2,
+ Partitions = 3,
+ ConsumerGroups = 4,
+ ConsumerGroupMembers = 5,
+ Users = 6,
+ PersonalAccessTokens = 7,
+ Sessions = 8,
+ }
+}
+
+impl WireMetadataChangeGroup {
+ /// Bit of this group in a page token's group mask.
+ #[must_use]
+ pub const fn mask_bit(self) -> u8 {
+ 1 << (self as u8 - 1)
+ }
+}
+
+wire_code_enum! {
+ /// Kind of a change record. Existing kind bodies never change in place, a
+ /// new layout is a new kind.
+ WireMetadataChangeKind {
+ StreamCreated = 1,
+ StreamUpdated = 2,
+ StreamDeleted = 3,
+ TopicCreated = 10,
+ TopicUpdated = 11,
+ TopicDeleted = 12,
+ PartitionsChanged = 20,
+ ConsumerGroupCreated = 30,
+ ConsumerGroupDeleted = 31,
+ ConsumerGroupMemberJoined = 40,
+ ConsumerGroupMemberLeft = 41,
+ ConsumerGroupPartitionHandedOver = 42,
+ UserCreated = 50,
+ UserUpdated = 51,
+ UserDeleted = 52,
+ UserPasswordChanged = 53,
+ UserPermissionsUpdated = 54,
+ PersonalAccessTokenCreated = 60,
+ PersonalAccessTokenDeleted = 61,
+ SessionRegistered = 70,
+ SessionClosed = 71,
+ }
+}
+
+impl WireMetadataChangeKind {
+ /// The group every record of this kind belongs to.
+ #[must_use]
+ pub const fn group(self) -> WireMetadataChangeGroup {
+ match self {
+ Self::StreamCreated | Self::StreamUpdated | Self::StreamDeleted => {
+ WireMetadataChangeGroup::Streams
+ }
+ Self::TopicCreated | Self::TopicUpdated | Self::TopicDeleted => {
+ WireMetadataChangeGroup::Topics
+ }
+ Self::PartitionsChanged => WireMetadataChangeGroup::Partitions,
+ Self::ConsumerGroupCreated | Self::ConsumerGroupDeleted => {
+ WireMetadataChangeGroup::ConsumerGroups
+ }
+ Self::ConsumerGroupMemberJoined
+ | Self::ConsumerGroupMemberLeft
+ | Self::ConsumerGroupPartitionHandedOver => {
+ WireMetadataChangeGroup::ConsumerGroupMembers
+ }
+ Self::UserCreated
+ | Self::UserUpdated
+ | Self::UserDeleted
+ | Self::UserPasswordChanged
+ | Self::UserPermissionsUpdated => WireMetadataChangeGroup::Users,
+ Self::PersonalAccessTokenCreated | Self::PersonalAccessTokenDeleted => {
+ WireMetadataChangeGroup::PersonalAccessTokens
+ }
+ Self::SessionRegistered | Self::SessionClosed => WireMetadataChangeGroup::Sessions,
+ }
+ }
+}
+
+wire_code_enum! {
+ /// Why a consumer group member was removed.
+ WireMemberLeftReason {
+ Left = 0,
+ SessionEnded = 1,
+ }
+}
+
+wire_code_enum! {
+ /// Why a personal access token was removed.
+ WireTokenDeletedReason {
+ Deleted = 0,
+ Expired = 1,
+ }
+}
+
+wire_code_enum! {
+ /// Why a session was closed.
+ WireSessionClosedReason {
+ Logout = 0,
+ Expired = 1,
+ /// The user was deactivated or deleted.
+ Revoked = 2,
+ }
+}
+
+wire_code_enum! {
+ /// Why a user's public permission set changed.
+ WirePermissionsUpdatedReason {
+ Updated = 0,
+ ResourceDeleted = 1,
+ }
+}
+
+wire_code_enum! {
+ /// Outcome of a change log query.
+ WireMetadataChangeStatus {
+ Ok = 0,
+ Behind = 1,
+ Expired = 2,
+ Reset = 3,
+ }
+}
+
+wire_code_enum! {
+ /// Whether an answer holds change records or current state items.
+ WireMetadataChangePhase {
+ Changes = 0,
+ Current = 1,
+ }
+}
+
+wire_code_enum! {
+ /// Where a change log query starts.
+ WireMetadataChangeStartMode {
+ Earliest = 0,
+ Latest = 1,
+ After = 2,
+ Current = 3,
+ }
+}
+
+/// Read a presence or boolean byte, refusing anything but 0 and 1.
+pub(crate) fn read_flag(buf: &[u8], offset: usize) -> Result<bool, WireError> {
+ match read_u8(buf, offset)? {
+ 0 => Ok(false),
+ 1 => Ok(true),
+ value => Err(WireError::UnknownDiscriminant {
+ type_name: "flag",
+ value,
+ offset,
+ }),
+ }
+}
+
+pub(crate) fn put_flag(buf: &mut BytesMut, value: bool) {
+ buf.put_u8(u8::from(value));
+}
+
+pub(crate) fn read_wire_name(buf: &[u8], offset: usize) -> Result<(WireName, usize), WireError> {
+ let rest = buf.get(offset..).ok_or(WireError::UnexpectedEof {
+ offset,
+ need: 1,
+ have: 0,
+ })?;
+ WireName::decode(rest)
+}
+
+pub(crate) const fn optional_u64_size(value: Option<u64>) -> usize {
+ if value.is_some() { 9 } else { 1 }
+}
+
+pub(crate) fn put_optional_u64(buf: &mut BytesMut, value: Option<u64>) {
+ put_flag(buf, value.is_some());
+ if let Some(value) = value {
+ buf.put_u64_le(value);
+ }
+}
+
+pub(crate) fn read_optional_u64(
+ buf: &[u8],
+ offset: usize,
+) -> Result<(Option<u64>, usize), WireError> {
+ if read_flag(buf, offset)? {
+ Ok((Some(read_u64_le(buf, offset + 1)?), 9))
+ } else {
+ Ok((None, 1))
+ }
+}
+
+pub(crate) const fn optional_u32_size(value: Option<u32>) -> usize {
+ if value.is_some() { 5 } else { 1 }
+}
+
+pub(crate) fn put_optional_u32(buf: &mut BytesMut, value: Option<u32>) {
+ put_flag(buf, value.is_some());
+ if let Some(value) = value {
+ buf.put_u32_le(value);
+ }
+}
+
+pub(crate) fn read_optional_u32(
+ buf: &[u8],
+ offset: usize,
+) -> Result<(Option<u32>, usize), WireError> {
+ if read_flag(buf, offset)? {
+ Ok((Some(read_u32_le(buf, offset + 1)?), 5))
+ } else {
+ Ok((None, 1))
+ }
+}
+
+pub(crate) const fn partition_ids_size(ids: &[u32]) -> usize {
+ 4 + ids.len() * 4
+}
+
+/// Encode `[count:u32][id:u32]*`. Callers hold ascending distinct ids.
+pub(crate) fn put_partition_ids(buf: &mut BytesMut, ids: &[u32]) {
+ // A topic holds at most `u32::MAX` partitions, so the count fits.
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(ids.len() as u32);
+ for &id in ids {
+ buf.put_u32_le(id);
+ }
+}
+
+/// Decode `[count:u32][id:u32]*`, refusing ids that are not strictly
+/// ascending.
+pub(crate) fn read_partition_ids(
+ buf: &[u8],
+ offset: usize,
+) -> Result<(Vec<u32>, usize), WireError> {
+ let count = read_u32_le(buf, offset)? as usize;
+ let mut position = offset + 4;
+ let remaining = buf.len().saturating_sub(position);
+ let mut ids = Vec::with_capacity(bounded_capacity(count, remaining, 4));
+ for _ in 0..count {
+ let id = read_u32_le(buf, position)?;
+ if ids.last().is_some_and(|&previous| previous >= id) {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "partition ids must be strictly ascending",
+ )));
+ }
+ ids.push(id);
+ position += 4;
+ }
+ Ok((ids, position - offset))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn given_every_group_when_decoding_its_code_should_round_trip_and_set_one_mask_bit() {
+ let mut mask = 0u8;
+ for code in 1..=8u8 {
+ let group = WireMetadataChangeGroup::from_code(code).unwrap();
+ assert_eq!(group.as_u8(), code);
+ assert_eq!(mask & group.mask_bit(), 0);
+ mask |= group.mask_bit();
+ }
+ assert_eq!(mask, u8::MAX);
+ }
+
+ #[test]
+ fn given_reserved_group_codes_when_decoding_should_refuse() {
+ for code in [0u8, 9, 127, 128, 255] {
+ assert!(WireMetadataChangeGroup::from_code(code).is_err());
+ }
+ }
+
+ #[test]
+ fn given_every_kind_when_decoding_its_code_should_round_trip() {
+ let codes = [
+ 1u8, 2, 3, 10, 11, 12, 20, 30, 31, 40, 41, 42, 50, 51, 52, 53, 54, 60, 61, 70, 71,
+ ];
+ for code in codes {
+ let kind = WireMetadataChangeKind::from_code(code).unwrap();
+ assert_eq!(kind.as_u8(), code);
+ }
+ for code in 0..=u8::MAX {
+ assert_eq!(
+ WireMetadataChangeKind::from_code(code).is_ok(),
+ codes.contains(&code)
+ );
+ }
+ }
+
+ #[test]
+ fn given_kind_codes_when_mapping_groups_should_follow_the_catalog_decades() {
+ for code in 0..=u8::MAX {
+ if let Ok(kind) = WireMetadataChangeKind::from_code(code) {
+ let expected = match code {
+ 1..=3 => 1,
+ 10..=12 => 2,
+ 20 => 3,
+ 30 | 31 => 4,
+ 40..=42 => 5,
+ 50..=54 => 6,
+ 60 | 61 => 7,
+ _ => 8,
+ };
+ assert_eq!(kind.group().as_u8(), expected, "kind {code}");
+ }
+ }
+ }
+
+ #[test]
+ fn given_unknown_reason_codes_when_decoding_should_refuse() {
+ assert!(WireMemberLeftReason::from_code(2).is_err());
+ assert!(WireTokenDeletedReason::from_code(2).is_err());
+ assert!(WireSessionClosedReason::from_code(3).is_err());
+ assert!(WirePermissionsUpdatedReason::from_code(2).is_err());
+ assert!(WireMetadataChangeStatus::from_code(4).is_err());
+ assert!(WireMetadataChangePhase::from_code(2).is_err());
+ assert!(WireMetadataChangeStartMode::from_code(4).is_err());
+ }
+
+ #[test]
+ fn given_flag_byte_above_one_when_reading_should_refuse() {
+ assert!(!read_flag(&[0], 0).unwrap());
+ assert!(read_flag(&[1], 0).unwrap());
+ assert!(read_flag(&[2], 0).is_err());
+ assert!(read_flag(&[], 0).is_err());
+ }
+
+ #[test]
+ fn given_partition_ids_when_round_tripping_should_preserve_order() {
+ let ids = [0u32, 3, 7, u32::MAX];
+ let mut buf = BytesMut::new();
+ put_partition_ids(&mut buf, &ids);
+ assert_eq!(buf.len(), partition_ids_size(&ids));
+ let (decoded, consumed) = read_partition_ids(&buf, 0).unwrap();
+ assert_eq!(decoded, ids);
+ assert_eq!(consumed, buf.len());
+ }
+
+ #[test]
+ fn given_unsorted_or_duplicate_partition_ids_when_decoding_should_refuse() {
+ for ids in [[3u32, 1], [2, 2]] {
+ let mut buf = BytesMut::new();
+ buf.put_u32_le(2);
+ for id in ids {
+ buf.put_u32_le(id);
+ }
+ assert!(read_partition_ids(&buf, 0).is_err());
+ }
+ }
+
+ #[test]
+ fn given_huge_partition_count_when_decoding_should_fail_without_allocating_it() {
+ let mut buf = BytesMut::new();
+ buf.put_u32_le(u32::MAX);
+ assert!(read_partition_ids(&buf, 0).is_err());
+ }
+}
diff --git a/core/binary_protocol/src/primitives/metadata_changes/position.rs b/core/binary_protocol/src/primitives/metadata_changes/position.rs
new file mode 100644
index 000000000..879007f2e
--- /dev/null
+++ b/core/binary_protocol/src/primitives/metadata_changes/position.rs
@@ -0,0 +1,384 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::{WireMetadataChangeGroup, read_wire_name};
+use crate::WireError;
+use crate::codec::{WireDecode, WireEncode, read_bytes, read_u8, read_u16_le, read_u32_le};
+use crate::codec::{read_u64_le, read_u128_le};
+use bytes::{BufMut, Bytes, BytesMut};
+use std::borrow::Cow;
+
+/// Encoded size of a change cursor.
+pub const METADATA_CHANGE_CURSOR_SIZE: usize = 44;
+
+/// Longest page token key: a user id and a personal access token name.
+pub const MAX_PAGE_TOKEN_KEY_SIZE: usize = 4 + 1 + 255;
+
+/// Longest decoded page token payload.
+pub const MAX_PAGE_TOKEN_SIZE: usize =
+ METADATA_CHANGE_CURSOR_SIZE + 8 + 1 + 1 + 2 + MAX_PAGE_TOKEN_KEY_SIZE;
+
+/// Position of the last record a caller consumed or scanned.
+///
+/// `(op, checksum, ordinal) = (0, 0, 0)` names the beginning of the history.
+///
+/// Wire format: `[log_id:u128][op:u64][checksum:u128][ordinal:u32]`
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+pub struct WireMetadataChangeCursor {
+ pub log_id: u128,
+ pub op: u64,
+ pub checksum: u128,
+ pub ordinal: u32,
+}
+
+impl WireMetadataChangeCursor {
+ /// The cursor before the first record of `log_id`.
+ #[must_use]
+ pub const fn zero(log_id: u128) -> Self {
+ Self {
+ log_id,
+ op: 0,
+ checksum: 0,
+ ordinal: 0,
+ }
+ }
+
+ /// Whether the cursor names the beginning of its history.
+ #[must_use]
+ pub const fn is_zero(&self) -> bool {
+ self.op == 0
+ }
+
+ /// Op 0 carries no checksum and no ordinal.
+ #[must_use]
+ pub const fn is_canonical(&self) -> bool {
+ self.op != 0 || (self.checksum == 0 && self.ordinal == 0)
+ }
+}
+
+impl WireEncode for WireMetadataChangeCursor {
+ fn encoded_size(&self) -> usize {
+ METADATA_CHANGE_CURSOR_SIZE
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ buf.put_u128_le(self.log_id);
+ buf.put_u64_le(self.op);
+ buf.put_u128_le(self.checksum);
+ buf.put_u32_le(self.ordinal);
+ }
+}
+
+impl WireDecode for WireMetadataChangeCursor {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let cursor = Self {
+ log_id: read_u128_le(buf, 0)?,
+ op: read_u64_le(buf, 16)?,
+ checksum: read_u128_le(buf, 24)?,
+ ordinal: read_u32_le(buf, 40)?,
+ };
+ if !cursor.is_canonical() {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "a cursor at op 0 carries no checksum and no ordinal",
+ )));
+ }
+ Ok((cursor, METADATA_CHANGE_CURSOR_SIZE))
+ }
+}
+
+/// Continuation of a current state scan.
+///
+/// The page token names the anchor captured before the first page, the
+/// highest head any previous page reported, the requested groups and the key
+/// of the last item returned, so any node that covers `required_head` can
+/// serve the next page.
+///
+/// Wire format:
+/// ```text
+/// [anchor:44][required_head:u64][groups_mask:u8][group:u8][key_len:u16][last_key]
+/// ```
+///
+/// Keys are little-endian ids: streams `[stream_id:u32]`, topics and
+/// partitions `[stream_id:u32][topic_id:u32]`, consumer groups add
+/// `[group_id:u32]`, members add `[member_id:u32]`, users `[user_id:u32]`,
+/// personal access tokens `[user_id:u32][name]`, sessions `[epoch:u64]`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireMetadataCurrentPageToken {
+ pub anchor: WireMetadataChangeCursor,
+ pub required_head: u64,
+ pub groups_mask: u8,
+ pub group: WireMetadataChangeGroup,
+ pub last_key: Bytes,
+}
+
+impl WireMetadataCurrentPageToken {
+ const FIXED_SIZE: usize = METADATA_CHANGE_CURSOR_SIZE + 8 + 1 + 1 + 2;
+
+ /// Check the token's own invariants: the group is selected, the key has
+ /// the exact width of that group and the head covers the anchor.
+ ///
+ /// # Errors
+ /// Returns `WireError::Validation` naming the first broken invariant.
+ pub fn validate(&self) -> Result<(), WireError> {
+ if self.groups_mask & self.group.mask_bit() == 0 {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "page token group is not in its group mask",
+ )));
+ }
+ if self.required_head < self.anchor.op {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "page token head is below its anchor",
+ )));
+ }
+ if !self.anchor.is_canonical() {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "page token anchor is not canonical",
+ )));
+ }
+ if self.last_key.len() > MAX_PAGE_TOKEN_KEY_SIZE {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "page token key is too long",
+ )));
+ }
+ if !key_fits_group(self.group, &self.last_key) {
+ return Err(WireError::Validation(Cow::Owned(format!(
+ "page token key of {} bytes does not fit group {}",
+ self.last_key.len(),
+ self.group.as_u8()
+ ))));
+ }
+ Ok(())
+ }
+}
+
+impl WireEncode for WireMetadataCurrentPageToken {
+ fn encoded_size(&self) -> usize {
+ Self::FIXED_SIZE + self.last_key.len()
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ self.anchor.encode(buf);
+ buf.put_u64_le(self.required_head);
+ buf.put_u8(self.groups_mask);
+ buf.put_u8(self.group.as_u8());
+ // Keys are at most `MAX_PAGE_TOKEN_KEY_SIZE` bytes, see `validate`.
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u16_le(self.last_key.len() as u16);
+ buf.put_slice(&self.last_key);
+ }
+}
+
+impl WireDecode for WireMetadataCurrentPageToken {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let (anchor, mut position) = WireMetadataChangeCursor::decode(buf)?;
+ let required_head = read_u64_le(buf, position)?;
+ position += 8;
+ let groups_mask = read_u8(buf, position)?;
+ position += 1;
+ let group_offset = position;
+ let group = WireMetadataChangeGroup::from_code(read_u8(buf, position)?).map_err(|_| {
+ WireError::Validation(Cow::Borrowed("page token names an unknown group"))
+ })?;
+ position += 1;
+ let key_length = usize::from(read_u16_le(buf, position)?);
+ position += 2;
+ if key_length > MAX_PAGE_TOKEN_KEY_SIZE {
+ return Err(WireError::Validation(Cow::Owned(format!(
+ "page token key of {key_length} bytes at offset {group_offset} is too long"
+ ))));
+ }
+ let last_key = Bytes::copy_from_slice(read_bytes(buf, position, key_length)?);
+ position += key_length;
+ let token = Self {
+ anchor,
+ required_head,
+ groups_mask,
+ group,
+ last_key,
+ };
+ token.validate()?;
+ Ok((token, position))
+ }
+}
+
+fn key_fits_group(group: WireMetadataChangeGroup, key: &[u8]) -> bool {
+ match group {
+ WireMetadataChangeGroup::Streams | WireMetadataChangeGroup::Users => key.len() == 4,
+ WireMetadataChangeGroup::Topics
+ | WireMetadataChangeGroup::Partitions
+ | WireMetadataChangeGroup::Sessions => key.len() == 8,
+ WireMetadataChangeGroup::ConsumerGroups => key.len() == 12,
+ WireMetadataChangeGroup::ConsumerGroupMembers => key.len() == 16,
+ WireMetadataChangeGroup::PersonalAccessTokens => {
+ key.len() > 4
+ && read_wire_name(key, 4).is_ok_and(|(_, consumed)| 4 + consumed == key.len())
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn cursor() -> WireMetadataChangeCursor {
+ WireMetadataChangeCursor {
+ log_id: 0x0d4b_8f2a_6e1c_93d7,
+ op: 4187,
+ checksum: 0xf2a4_c6e8_b0d1_f359,
+ ordinal: 1,
+ }
+ }
+
+ fn token(group: WireMetadataChangeGroup, last_key: &[u8]) -> WireMetadataCurrentPageToken {
+ WireMetadataCurrentPageToken {
+ anchor: cursor(),
+ required_head: 4200,
+ groups_mask: u8::MAX,
+ group,
+ last_key: Bytes::copy_from_slice(last_key),
+ }
+ }
+
+ fn token_key(user_id: u32, name: &str) -> Vec<u8> {
+ let mut key = user_id.to_le_bytes().to_vec();
+ key.push(u8::try_from(name.len()).unwrap());
+ key.extend_from_slice(name.as_bytes());
+ key
+ }
+
+ #[test]
+ fn given_cursor_when_round_tripping_should_use_44_little_endian_bytes() {
+ let bytes = cursor().to_bytes();
+ assert_eq!(bytes.len(), METADATA_CHANGE_CURSOR_SIZE);
+ assert_eq!(&bytes[16..24], &4187u64.to_le_bytes());
+ let (decoded, consumed) = WireMetadataChangeCursor::decode(&bytes).unwrap();
+ assert_eq!(decoded, cursor());
+ assert_eq!(consumed, METADATA_CHANGE_CURSOR_SIZE);
+ }
+
+ #[test]
+ fn given_zero_cursor_when_round_tripping_should_keep_its_log_id() {
+ let zero = WireMetadataChangeCursor::zero(42);
+ assert!(zero.is_zero());
+ let (decoded, _) = WireMetadataChangeCursor::decode(&zero.to_bytes()).unwrap();
+ assert_eq!(decoded, zero);
+ }
+
+ #[test]
+ fn given_op_zero_with_checksum_or_ordinal_when_decoding_should_refuse() {
+ for (checksum, ordinal) in [(1u128, 0u32), (0, 1)] {
+ let cursor = WireMetadataChangeCursor {
+ log_id: 1,
+ op: 0,
+ checksum,
+ ordinal,
+ };
+ assert!(!cursor.is_canonical());
+ assert!(WireMetadataChangeCursor::decode(&cursor.to_bytes()).is_err());
+ }
+ }
+
+ #[test]
+ fn given_truncated_cursor_when_decoding_should_refuse() {
+ let bytes = cursor().to_bytes();
+ assert!(WireMetadataChangeCursor::decode(&bytes[..43]).is_err());
+ }
+
+ #[test]
+ fn given_every_group_key_width_when_round_tripping_page_tokens_should_preserve_them() {
+ let cases = [
+ (WireMetadataChangeGroup::Streams, vec![0u8; 4]),
+ (WireMetadataChangeGroup::Topics, vec![1u8; 8]),
+ (WireMetadataChangeGroup::Partitions, vec![2u8; 8]),
+ (WireMetadataChangeGroup::ConsumerGroups, vec![3u8; 12]),
+ (WireMetadataChangeGroup::ConsumerGroupMembers, vec![4u8; 16]),
+ (WireMetadataChangeGroup::Users, vec![5u8; 4]),
+ (
+ WireMetadataChangeGroup::PersonalAccessTokens,
+ token_key(9, "ci"),
+ ),
+ (WireMetadataChangeGroup::Sessions, vec![6u8; 8]),
+ ];
+ for (group, key) in cases {
+ let original = token(group, &key);
+ let bytes = original.to_bytes();
+ let (decoded, consumed) = WireMetadataCurrentPageToken::decode(&bytes).unwrap();
+ assert_eq!(decoded, original);
+ assert_eq!(consumed, bytes.len());
+ }
+ }
+
+ #[test]
+ fn given_longest_token_name_when_encoding_should_fit_the_maximum_payload() {
+ let longest = "n".repeat(255);
+ let original = token(
+ WireMetadataChangeGroup::PersonalAccessTokens,
+ &token_key(1, &longest),
+ );
+ assert_eq!(original.to_bytes().len(), MAX_PAGE_TOKEN_SIZE);
+ assert!(WireMetadataCurrentPageToken::decode(&original.to_bytes()).is_ok());
+ }
+
+ #[test]
+ fn given_wrong_key_width_when_decoding_page_token_should_refuse() {
+ for (group, width) in [
+ (WireMetadataChangeGroup::Streams, 8),
+ (WireMetadataChangeGroup::Topics, 4),
+ (WireMetadataChangeGroup::ConsumerGroups, 16),
+ (WireMetadataChangeGroup::Sessions, 4),
+ (WireMetadataChangeGroup::PersonalAccessTokens, 4),
+ ] {
+ let bytes = token(group, &vec![0u8; width]).to_bytes();
+ assert!(WireMetadataCurrentPageToken::decode(&bytes).is_err());
+ }
+ let mut bad_name = token_key(1, "ci");
+ bad_name.push(0);
+ let bytes = token(WireMetadataChangeGroup::PersonalAccessTokens, &bad_name).to_bytes();
+ assert!(WireMetadataCurrentPageToken::decode(&bytes).is_err());
+ }
+
+ #[test]
+ fn given_group_outside_mask_when_decoding_page_token_should_refuse() {
+ let mut original = token(WireMetadataChangeGroup::Topics, &[0u8; 8]);
+ original.groups_mask = WireMetadataChangeGroup::Streams.mask_bit();
+ assert!(WireMetadataCurrentPageToken::decode(&original.to_bytes()).is_err());
+ }
+
+ #[test]
+ fn given_head_below_anchor_when_decoding_page_token_should_refuse() {
+ let mut original = token(WireMetadataChangeGroup::Streams, &[0u8; 4]);
+ original.required_head = original.anchor.op - 1;
+ assert!(WireMetadataCurrentPageToken::decode(&original.to_bytes()).is_err());
+ }
+
+ #[test]
+ fn given_unknown_group_code_when_decoding_page_token_should_refuse() {
+ let mut bytes = token(WireMetadataChangeGroup::Streams, &[0u8; 4])
+ .to_bytes()
+ .to_vec();
+ bytes[METADATA_CHANGE_CURSOR_SIZE + 9] = 9;
+ assert!(WireMetadataCurrentPageToken::decode(&bytes).is_err());
+ }
+
+ #[test]
+ fn given_truncated_page_token_when_decoding_should_refuse_every_prefix() {
+ let bytes = token(WireMetadataChangeGroup::Users, &[1u8; 4]).to_bytes();
+ for length in 0..bytes.len() {
+ assert!(WireMetadataCurrentPageToken::decode(&bytes[..length]).is_err());
+ }
+ }
+}
diff --git a/core/binary_protocol/src/primitives/mod.rs b/core/binary_protocol/src/primitives/mod.rs
index 07f107578..7a031d95d 100644
--- a/core/binary_protocol/src/primitives/mod.rs
+++ b/core/binary_protocol/src/primitives/mod.rs
@@ -20,6 +20,7 @@
pub mod ack_level;
pub mod consumer;
pub mod identifier;
+pub mod metadata_changes;
pub mod options;
pub mod partition_assignment;
pub mod partition_history;
diff --git a/core/binary_protocol/src/requests/system/get_metadata_changes.rs b/core/binary_protocol/src/requests/system/get_metadata_changes.rs
new file mode 100644
index 000000000..c35360238
--- /dev/null
+++ b/core/binary_protocol/src/requests/system/get_metadata_changes.rs
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::WireError;
+use crate::codec::{WireDecode, WireEncode, read_bytes, read_u8, read_u16_le, read_u32_le};
+use crate::primitives::metadata_changes::{
+ METADATA_CHANGE_CURSOR_SIZE, WireMetadataChangeCursor, WireMetadataChangeStartMode,
+ WireMetadataCurrentPageToken,
+};
+use crate::primitives::metadata_changes::{put_optional_u64, read_optional_u64};
+use bytes::{BufMut, BytesMut};
+use std::borrow::Cow;
+
+/// Where a change log query starts.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum WireMetadataChangeStart {
+ /// The oldest operation the requested classes still hold completely.
+ Earliest,
+ /// No records, a cursor at the newest retained record.
+ Latest,
+ /// Records after a cursor.
+ After(WireMetadataChangeCursor),
+ /// Current state pages, the first one without a token.
+ Current(Option<WireMetadataCurrentPageToken>),
+}
+
+/// `GetMetadataChanges` request.
+///
+/// Wire format:
+/// ```text
+/// [start:u8]
+/// [log_id:u128][op:u64][checksum:u128][ordinal:u32] cursor, all zero unless start = 2
+/// [has_since:u8][since:u64 if 1]
+/// [has_until:u8][until:u64 if 1]
+/// [limit:u32]
+/// [groups_count:u8][group:u8]* 0 entries = all groups
+/// [token_len:u16][token] 0 unless start = 3 with a token
+/// ```
+///
+/// Structural rules are enforced here. Which groups exist, the limit range
+/// and the time bounds are checked where the request is served.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct GetMetadataChangesRequest {
+ pub start: WireMetadataChangeStart,
+ pub since: Option<u64>,
+ pub until: Option<u64>,
+ pub limit: u32,
+ pub groups: Vec<u8>,
+}
+
+impl GetMetadataChangesRequest {
+ const fn start_mode(&self) -> WireMetadataChangeStartMode {
+ match self.start {
+ WireMetadataChangeStart::Earliest => WireMetadataChangeStartMode::Earliest,
+ WireMetadataChangeStart::Latest => WireMetadataChangeStartMode::Latest,
+ WireMetadataChangeStart::After(_) => WireMetadataChangeStartMode::After,
+ WireMetadataChangeStart::Current(_) => WireMetadataChangeStartMode::Current,
+ }
+ }
+
+ /// Group codes the request can carry, at most `u8::MAX` of them.
+ fn encoded_groups(&self) -> &[u8] {
+ &self.groups[..self.groups.len().min(usize::from(u8::MAX))]
+ }
+
+ const fn token(&self) -> Option<&WireMetadataCurrentPageToken> {
+ match &self.start {
+ WireMetadataChangeStart::Current(Some(token)) => Some(token),
+ _ => None,
+ }
+ }
+}
+
+impl WireEncode for GetMetadataChangesRequest {
+ fn encoded_size(&self) -> usize {
+ 1 + METADATA_CHANGE_CURSOR_SIZE
+ + if self.since.is_some() { 9 } else { 1 }
+ + if self.until.is_some() { 9 } else { 1 }
+ + 4
+ + 1
+ + self.encoded_groups().len()
+ + 2
+ + self.token().map_or(0, WireEncode::encoded_size)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ buf.put_u8(self.start_mode().as_u8());
+ let cursor = match &self.start {
+ WireMetadataChangeStart::After(cursor) => *cursor,
+ _ => WireMetadataChangeCursor::default(),
+ };
+ cursor.encode(buf);
+ put_optional_u64(buf, self.since);
+ put_optional_u64(buf, self.until);
+ buf.put_u32_le(self.limit);
+ let groups = self.encoded_groups();
+ // `encoded_groups` holds at most `u8::MAX` entries.
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u8(groups.len() as u8);
+ buf.put_slice(groups);
+ match self.token() {
+ Some(token) => {
+ // A valid token is at most `MAX_PAGE_TOKEN_SIZE` bytes.
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u16_le(token.encoded_size() as u16);
+ token.encode(buf);
+ }
+ None => buf.put_u16_le(0),
+ }
+ }
+}
+
+impl WireDecode for GetMetadataChangesRequest {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let mode = WireMetadataChangeStartMode::from_code(read_u8(buf, 0)?)?;
+ let cursor_bytes = read_bytes(buf, 1, METADATA_CHANGE_CURSOR_SIZE)?;
+ let mut position = 1 + METADATA_CHANGE_CURSOR_SIZE;
+ let (since, consumed) = read_optional_u64(buf, position)?;
+ position += consumed;
+ let (until, consumed) = read_optional_u64(buf, position)?;
+ position += consumed;
+ let limit = read_u32_le(buf, position)?;
+ position += 4;
+ let groups_count = usize::from(read_u8(buf, position)?);
+ position += 1;
+ let groups = read_bytes(buf, position, groups_count)?.to_vec();
+ position += groups_count;
+ let token_length = usize::from(read_u16_le(buf, position)?);
+ position += 2;
+ let token_bytes = read_bytes(buf, position, token_length)?;
+ position += token_length;
+ if position != buf.len() {
+ return Err(WireError::Validation(Cow::Owned(format!(
+ "{} trailing bytes after a metadata changes request",
+ buf.len() - position
+ ))));
+ }
+
+ if mode != WireMetadataChangeStartMode::After && cursor_bytes.iter().any(|&byte| byte != 0)
+ {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "a cursor is allowed only when starting after it",
+ )));
+ }
+ if mode != WireMetadataChangeStartMode::Current && token_length != 0 {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "a page token is allowed only in current mode",
+ )));
+ }
+ let start = match mode {
+ WireMetadataChangeStartMode::Earliest => WireMetadataChangeStart::Earliest,
+ WireMetadataChangeStartMode::Latest => WireMetadataChangeStart::Latest,
+ WireMetadataChangeStartMode::After => {
+ WireMetadataChangeStart::After(WireMetadataChangeCursor::decode(cursor_bytes)?.0)
+ }
+ WireMetadataChangeStartMode::Current if token_length == 0 => {
+ WireMetadataChangeStart::Current(None)
+ }
+ WireMetadataChangeStartMode::Current => {
+ let (token, consumed) = WireMetadataCurrentPageToken::decode(token_bytes)?;
+ if consumed != token_length {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "trailing bytes after a page token",
+ )));
+ }
+ WireMetadataChangeStart::Current(Some(token))
+ }
+ };
+ Ok((
+ Self {
+ start,
+ since,
+ until,
+ limit,
+ groups,
+ },
+ position,
+ ))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::primitives::metadata_changes::WireMetadataChangeGroup;
+ use bytes::Bytes;
+
+ fn cursor() -> WireMetadataChangeCursor {
+ WireMetadataChangeCursor {
+ log_id: 7,
+ op: 9,
+ checksum: 11,
+ ordinal: 2,
+ }
+ }
+
+ fn token() -> WireMetadataCurrentPageToken {
+ WireMetadataCurrentPageToken {
+ anchor: cursor(),
+ required_head: 12,
+ groups_mask: 0b11,
+ group: WireMetadataChangeGroup::Topics,
+ last_key: Bytes::from_static(&[1, 0, 0, 0, 2, 0, 0, 0]),
+ }
+ }
+
+ fn request(start: WireMetadataChangeStart) -> GetMetadataChangesRequest {
+ GetMetadataChangesRequest {
+ start,
+ since: Some(1_760_353_200_000_000),
+ until: None,
+ limit: 500,
+ groups: vec![1, 2],
+ }
+ }
+
+ #[test]
+ fn given_every_start_mode_when_round_tripping_should_preserve_every_field() {
+ for start in [
+ WireMetadataChangeStart::Earliest,
+ WireMetadataChangeStart::Latest,
+ WireMetadataChangeStart::After(cursor()),
+ WireMetadataChangeStart::Current(None),
+ WireMetadataChangeStart::Current(Some(token())),
+ ] {
+ let original = request(start);
+ let bytes = original.to_bytes();
+ assert_eq!(bytes.len(), original.encoded_size());
+ let (decoded, consumed) = GetMetadataChangesRequest::decode(&bytes).unwrap();
+ assert_eq!(decoded, original);
+ assert_eq!(consumed, bytes.len());
+ }
+ }
+
+ #[test]
+ fn given_no_groups_and_no_bounds_when_round_tripping_should_preserve_them() {
+ let original = GetMetadataChangesRequest {
+ start: WireMetadataChangeStart::Latest,
+ since: None,
+ until: None,
+ limit: 1,
+ groups: vec![],
+ };
+ let bytes = original.to_bytes();
+ assert_eq!(bytes.len(), 1 + 44 + 1 + 1 + 4 + 1 + 2);
+ assert_eq!(
+ GetMetadataChangesRequest::decode(&bytes).unwrap().0,
+ original
+ );
+ }
+
+ #[test]
+ fn given_trailing_bytes_when_decoding_request_should_refuse() {
+ let mut bytes = request(WireMetadataChangeStart::Earliest)
+ .to_bytes()
+ .to_vec();
+ bytes.push(0);
+ assert!(GetMetadataChangesRequest::decode(&bytes).is_err());
+ }
+
+ #[test]
+ fn given_cursor_bytes_without_after_when_decoding_request_should_refuse() {
+ let mut bytes = request(WireMetadataChangeStart::Latest).to_bytes().to_vec();
+ bytes[5] = 1;
+ assert!(GetMetadataChangesRequest::decode(&bytes).is_err());
+ }
+
+ #[test]
+ fn given_non_canonical_cursor_when_decoding_request_should_refuse() {
+ let original = request(WireMetadataChangeStart::After(WireMetadataChangeCursor {
+ log_id: 1,
+ op: 0,
+ checksum: 3,
+ ordinal: 0,
+ }));
+ assert!(GetMetadataChangesRequest::decode(&original.to_bytes()).is_err());
+ }
+
+ #[test]
+ fn given_page_token_outside_current_mode_when_decoding_request_should_refuse() {
+ let mut bytes = request(WireMetadataChangeStart::Current(Some(token())))
+ .to_bytes()
+ .to_vec();
+ bytes[0] = WireMetadataChangeStartMode::Earliest.as_u8();
+ assert!(GetMetadataChangesRequest::decode(&bytes).is_err());
+ }
+
+ #[test]
+ fn given_unknown_start_or_presence_byte_when_decoding_request_should_refuse() {
+ let bytes = request(WireMetadataChangeStart::Earliest)
+ .to_bytes()
+ .to_vec();
+ let mut unknown_start = bytes.clone();
+ unknown_start[0] = 4;
+ assert!(GetMetadataChangesRequest::decode(&unknown_start).is_err());
+ let mut bad_presence = bytes;
+ bad_presence[1 + 44] = 2;
+ assert!(GetMetadataChangesRequest::decode(&bad_presence).is_err());
+ }
+
+ #[test]
+ fn given_truncated_request_when_decoding_should_refuse_every_prefix() {
+ let bytes = request(WireMetadataChangeStart::Current(Some(token()))).to_bytes();
+ for length in 0..bytes.len() {
+ assert!(GetMetadataChangesRequest::decode(&bytes[..length]).is_err());
+ }
+ }
+}
diff --git a/core/binary_protocol/src/requests/system/mod.rs b/core/binary_protocol/src/requests/system/mod.rs
index 9d5c568c8..e59a75d2d 100644
--- a/core/binary_protocol/src/requests/system/mod.rs
+++ b/core/binary_protocol/src/requests/system/mod.rs
@@ -22,6 +22,7 @@ pub mod get_client;
pub mod get_clients;
pub mod get_cluster_metadata;
pub mod get_me;
+pub mod get_metadata_changes;
pub mod get_snapshot;
pub mod get_stats;
pub mod ping;
@@ -37,6 +38,7 @@ pub use get_client::GetClientRequest;
pub use get_clients::GetClientsRequest;
pub use get_cluster_metadata::GetClusterMetadataRequest;
pub use get_me::GetMeRequest;
+pub use get_metadata_changes::{GetMetadataChangesRequest, WireMetadataChangeStart};
pub use get_snapshot::GetSnapshotRequest;
pub use get_stats::GetStatsRequest;
pub use ping::PingRequest;
diff --git a/core/binary_protocol/src/responses/system/get_metadata_changes.rs b/core/binary_protocol/src/responses/system/get_metadata_changes.rs
new file mode 100644
index 000000000..b5a12ec06
--- /dev/null
+++ b/core/binary_protocol/src/responses/system/get_metadata_changes.rs
@@ -0,0 +1,488 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::WireError;
+use crate::codec::{
+ WireDecode, WireEncode, bounded_capacity, read_bytes, read_u8, read_u16_le, read_u32_le,
+ read_u64_le, read_u128_le,
+};
+use crate::primitives::metadata_changes::{
+ METADATA_CHANGE_CURSOR_SIZE, WireMetadataChangeCursor, WireMetadataChangePhase,
+ WireMetadataChangeStatus, WireMetadataCurrentPageToken,
+};
+use crate::primitives::metadata_changes::{
+ put_flag, put_optional_u32, read_flag, read_optional_u32,
+};
+use bytes::{BufMut, Bytes, BytesMut};
+use std::borrow::Cow;
+
+/// One change record: the envelope stamped from the prepare header and the
+/// body the handler encoded at apply, kept as raw bytes so a client can hold a
+/// kind it does not know.
+///
+/// Wire format:
+/// ```text
+/// [op:u64][ordinal:u32][checksum:u128][timestamp:u64]
+/// [has_actor:u8][actor_user_id:u32 if 1]
+/// [group:u8][kind:u8][body_len:u32][body]
+/// ```
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireMetadataChangeRecord {
+ pub op: u64,
+ pub ordinal: u32,
+ pub checksum: u128,
+ pub timestamp: u64,
+ pub actor_user_id: Option<u32>,
+ pub group: u8,
+ pub kind: u8,
+ pub body: Bytes,
+}
+
+/// One current state item.
+///
+/// Wire format: `[group:u8][body_len:u32][body]`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WireMetadataItem {
+ pub group: u8,
+ pub body: Bytes,
+}
+
+/// `GetMetadataChanges` response.
+///
+/// Wire format:
+/// ```text
+/// [status:u8][phase:u8]
+/// [has_log_id:u8][log_id:u128 if 1]
+/// [oldest_op:u64][retained_since:u64][head_op:u64]
+/// [has_next:u8][cursor:44 if 1]
+/// [has_more:u8]
+/// [records_count:u32][record]*
+/// [items_count:u32][item]*
+/// [token_len:u16][token]
+/// ```
+///
+/// A changes page carries records and no items or token, a current page
+/// carries items and a token and no records.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct GetMetadataChangesResponse {
+ pub status: WireMetadataChangeStatus,
+ pub phase: WireMetadataChangePhase,
+ pub log_id: Option<u128>,
+ pub oldest_op: u64,
+ pub retained_since: u64,
+ pub head_op: u64,
+ pub next: Option<WireMetadataChangeCursor>,
+ pub has_more: bool,
+ pub records: Vec<WireMetadataChangeRecord>,
+ pub items: Vec<WireMetadataItem>,
+ pub current_page_token: Option<WireMetadataCurrentPageToken>,
+}
+
+impl WireMetadataChangeRecord {
+ const MIN_SIZE: usize = 8 + 4 + 16 + 8 + 1 + 1 + 1 + 4;
+}
+
+impl WireEncode for WireMetadataChangeRecord {
+ fn encoded_size(&self) -> usize {
+ Self::MIN_SIZE + if self.actor_user_id.is_some() { 4 } else { 0 } + self.body.len()
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ buf.put_u64_le(self.op);
+ buf.put_u32_le(self.ordinal);
+ buf.put_u128_le(self.checksum);
+ buf.put_u64_le(self.timestamp);
+ put_optional_u32(buf, self.actor_user_id);
+ buf.put_u8(self.group);
+ buf.put_u8(self.kind);
+ // Bodies are bounded by the options budget, far below `u32::MAX`.
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(self.body.len() as u32);
+ buf.put_slice(&self.body);
+ }
+}
+
+impl WireDecode for WireMetadataChangeRecord {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let op = read_u64_le(buf, 0)?;
+ let ordinal = read_u32_le(buf, 8)?;
+ let checksum = read_u128_le(buf, 12)?;
+ let timestamp = read_u64_le(buf, 28)?;
+ let (actor_user_id, consumed) = read_optional_u32(buf, 36)?;
+ let mut position = 36 + consumed;
+ let group = read_u8(buf, position)?;
+ let kind = read_u8(buf, position + 1)?;
+ let body_length = read_u32_le(buf, position + 2)? as usize;
+ position += 6;
+ let body = Bytes::copy_from_slice(read_bytes(buf, position, body_length)?);
+ position += body_length;
+ Ok((
+ Self {
+ op,
+ ordinal,
+ checksum,
+ timestamp,
+ actor_user_id,
+ group,
+ kind,
+ body,
+ },
+ position,
+ ))
+ }
+}
+
+impl WireEncode for WireMetadataItem {
+ fn encoded_size(&self) -> usize {
+ 1 + 4 + self.body.len()
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ buf.put_u8(self.group);
+ // Item bodies are bounded by the options budget.
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(self.body.len() as u32);
+ buf.put_slice(&self.body);
+ }
+}
+
+impl WireDecode for WireMetadataItem {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let group = read_u8(buf, 0)?;
+ let body_length = read_u32_le(buf, 1)? as usize;
+ let body = Bytes::copy_from_slice(read_bytes(buf, 5, body_length)?);
+ Ok((Self { group, body }, 5 + body_length))
+ }
+}
+
+impl WireEncode for GetMetadataChangesResponse {
+ fn encoded_size(&self) -> usize {
+ 2 + if self.log_id.is_some() { 17 } else { 1 }
+ + 24
+ + if self.next.is_some() {
+ 1 + METADATA_CHANGE_CURSOR_SIZE
+ } else {
+ 1
+ }
+ + 1
+ + 4
+ + self
+ .records
+ .iter()
+ .map(WireEncode::encoded_size)
+ .sum::<usize>()
+ + 4
+ + self
+ .items
+ .iter()
+ .map(WireEncode::encoded_size)
+ .sum::<usize>()
+ + 2
+ + self
+ .current_page_token
+ .as_ref()
+ .map_or(0, WireEncode::encoded_size)
+ }
+
+ fn encode(&self, buf: &mut BytesMut) {
+ buf.put_u8(self.status.as_u8());
+ buf.put_u8(self.phase.as_u8());
+ put_flag(buf, self.log_id.is_some());
+ if let Some(log_id) = self.log_id {
+ buf.put_u128_le(log_id);
+ }
+ buf.put_u64_le(self.oldest_op);
+ buf.put_u64_le(self.retained_since);
+ buf.put_u64_le(self.head_op);
+ put_flag(buf, self.next.is_some());
+ if let Some(next) = &self.next {
+ next.encode(buf);
+ }
+ put_flag(buf, self.has_more);
+ // A page holds at most 1000 records or items.
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(self.records.len() as u32);
+ for record in &self.records {
+ record.encode(buf);
+ }
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u32_le(self.items.len() as u32);
+ for item in &self.items {
+ item.encode(buf);
+ }
+ match &self.current_page_token {
+ Some(token) => {
+ // A valid token is at most `MAX_PAGE_TOKEN_SIZE` bytes.
+ #[allow(clippy::cast_possible_truncation)]
+ buf.put_u16_le(token.encoded_size() as u16);
+ token.encode(buf);
+ }
+ None => buf.put_u16_le(0),
+ }
+ }
+}
+
+impl WireDecode for GetMetadataChangesResponse {
+ fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+ let status = WireMetadataChangeStatus::from_code(read_u8(buf, 0)?)?;
+ let phase = WireMetadataChangePhase::from_code(read_u8(buf, 1)?)?;
+ let mut position = 2;
+ let log_id = if read_flag(buf, position)? {
+ position += 1;
+ let log_id = read_u128_le(buf, position)?;
+ position += 16;
+ Some(log_id)
+ } else {
+ position += 1;
+ None
+ };
+ let oldest_op = read_u64_le(buf, position)?;
+ let retained_since = read_u64_le(buf, position + 8)?;
+ let head_op = read_u64_le(buf, position + 16)?;
+ position += 24;
+ let next = if read_flag(buf, position)? {
+ position += 1;
+ let rest = buf.get(position..).unwrap_or_default();
+ let (cursor, consumed) = WireMetadataChangeCursor::decode(rest)?;
+ position += consumed;
+ Some(cursor)
+ } else {
+ position += 1;
+ None
+ };
+ let has_more = read_flag(buf, position)?;
+ position += 1;
+
+ let records_count = read_u32_le(buf, position)? as usize;
+ position += 4;
+ let mut records = Vec::with_capacity(bounded_capacity(
+ records_count,
+ buf.len().saturating_sub(position),
+ WireMetadataChangeRecord::MIN_SIZE,
+ ));
+ for _ in 0..records_count {
+ let rest = buf.get(position..).unwrap_or_default();
+ let (record, consumed) = WireMetadataChangeRecord::decode(rest)?;
+ records.push(record);
+ position += consumed;
+ }
+
+ let items_count = read_u32_le(buf, position)? as usize;
+ position += 4;
+ let mut items = Vec::with_capacity(bounded_capacity(
+ items_count,
+ buf.len().saturating_sub(position),
+ 5,
+ ));
+ for _ in 0..items_count {
+ let rest = buf.get(position..).unwrap_or_default();
+ let (item, consumed) = WireMetadataItem::decode(rest)?;
+ items.push(item);
+ position += consumed;
+ }
+
+ let token_length = usize::from(read_u16_le(buf, position)?);
+ position += 2;
+ let current_page_token = if token_length == 0 {
+ None
+ } else {
+ let token_bytes = read_bytes(buf, position, token_length)?;
+ let (token, consumed) = WireMetadataCurrentPageToken::decode(token_bytes)?;
+ if consumed != token_length {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "trailing bytes after a page token",
+ )));
+ }
+ position += token_length;
+ Some(token)
+ };
+
+ let phase_holds_other_content = match phase {
+ WireMetadataChangePhase::Changes => !items.is_empty() || current_page_token.is_some(),
+ WireMetadataChangePhase::Current => !records.is_empty(),
+ };
+ if phase_holds_other_content {
+ return Err(WireError::Validation(Cow::Borrowed(
+ "a changes page carries only records and a current page only items",
+ )));
+ }
+ Ok((
+ Self {
+ status,
+ phase,
+ log_id,
+ oldest_op,
+ retained_since,
+ head_op,
+ next,
+ has_more,
+ records,
+ items,
+ current_page_token,
+ },
+ position,
+ ))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::primitives::metadata_changes::WireMetadataChangeGroup;
+
+ fn cursor(op: u64, ordinal: u32) -> WireMetadataChangeCursor {
+ WireMetadataChangeCursor {
+ log_id: 0x0d4b_8f2a_6e1c_93d7,
+ op,
+ checksum: 0xa5c1_f703_8b9e_2d4c,
+ ordinal,
+ }
+ }
+
+ fn record(actor_user_id: Option<u32>) -> WireMetadataChangeRecord {
+ WireMetadataChangeRecord {
+ op: 4121,
+ ordinal: 0,
+ checksum: 0xa5c1_f703_8b9e_2d4c,
+ timestamp: 1_760_356_800_123_456,
+ actor_user_id,
+ group: 1,
+ kind: 1,
+ body: Bytes::from_static(&[1, 2, 3, 4]),
+ }
+ }
+
+ fn changes_page() -> GetMetadataChangesResponse {
+ GetMetadataChangesResponse {
+ status: WireMetadataChangeStatus::Ok,
+ phase: WireMetadataChangePhase::Changes,
+ log_id: Some(0x0d4b_8f2a_6e1c_93d7),
+ oldest_op: 3871,
+ retained_since: 1_760_352_000_000_000,
+ head_op: 4187,
+ next: Some(cursor(4187, 1)),
+ has_more: false,
+ records: vec![record(Some(4)), record(None)],
+ items: vec![],
+ current_page_token: None,
+ }
+ }
+
+ fn current_page() -> GetMetadataChangesResponse {
+ GetMetadataChangesResponse {
+ status: WireMetadataChangeStatus::Ok,
+ phase: WireMetadataChangePhase::Current,
+ log_id: Some(1),
+ oldest_op: 1,
+ retained_since: 0,
+ head_op: 40,
+ next: Some(cursor(30, 0)),
+ has_more: true,
+ records: vec![],
+ items: vec![WireMetadataItem {
+ group: 1,
+ body: Bytes::from_static(&[9; 12]),
+ }],
+ current_page_token: Some(WireMetadataCurrentPageToken {
+ anchor: cursor(30, 0),
+ required_head: 40,
+ groups_mask: WireMetadataChangeGroup::Streams.mask_bit(),
+ group: WireMetadataChangeGroup::Streams,
+ last_key: Bytes::from_static(&[3, 0, 0, 0]),
+ }),
+ }
+ }
+
+ #[test]
+ fn given_changes_page_when_round_tripping_should_preserve_every_field() {
+ let original = changes_page();
+ let bytes = original.to_bytes();
+ assert_eq!(bytes.len(), original.encoded_size());
+ let (decoded, consumed) = GetMetadataChangesResponse::decode(&bytes).unwrap();
+ assert_eq!(decoded, original);
+ assert_eq!(consumed, bytes.len());
+ }
+
+ #[test]
+ fn given_current_page_when_round_tripping_should_preserve_every_field() {
+ let original = current_page();
+ let bytes = original.to_bytes();
+ let (decoded, consumed) = GetMetadataChangesResponse::decode(&bytes).unwrap();
+ assert_eq!(decoded, original);
+ assert_eq!(consumed, bytes.len());
+ }
+
+ #[test]
+ fn given_behind_without_log_id_or_next_when_round_tripping_should_preserve_absence() {
+ let original = GetMetadataChangesResponse {
+ status: WireMetadataChangeStatus::Behind,
+ log_id: None,
+ next: None,
+ records: vec![],
+ ..changes_page()
+ };
+ let bytes = original.to_bytes();
+ assert_eq!(
+ GetMetadataChangesResponse::decode(&bytes).unwrap().0,
+ original
+ );
+ }
+
+ #[test]
+ fn given_items_in_changes_phase_when_decoding_should_refuse() {
+ let mut page = current_page();
+ page.phase = WireMetadataChangePhase::Changes;
+ assert!(GetMetadataChangesResponse::decode(&page.to_bytes()).is_err());
+ }
+
+ #[test]
+ fn given_records_in_current_phase_when_decoding_should_refuse() {
+ let mut page = changes_page();
+ page.phase = WireMetadataChangePhase::Current;
+ assert!(GetMetadataChangesResponse::decode(&page.to_bytes()).is_err());
+ }
+
+ #[test]
+ fn given_unknown_status_when_decoding_should_refuse() {
+ let mut bytes = changes_page().to_bytes().to_vec();
+ bytes[0] = 4;
+ assert!(GetMetadataChangesResponse::decode(&bytes).is_err());
+ }
+
+ #[test]
+ fn given_truncated_response_when_decoding_should_refuse_every_prefix() {
+ for page in [changes_page(), current_page()] {
+ let bytes = page.to_bytes();
+ for length in 0..bytes.len() {
+ assert!(
+ GetMetadataChangesResponse::decode(&bytes[..length]).is_err(),
+ "accepted a response truncated to {length} bytes"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn given_record_count_beyond_the_frame_when_decoding_should_fail_without_allocating_it() {
+ let mut page = changes_page();
+ page.records.clear();
+ let mut bytes = page.to_bytes().to_vec();
+ let count_offset = 2 + 17 + 24 + 1 + METADATA_CHANGE_CURSOR_SIZE + 1;
+ bytes[count_offset..count_offset + 4].copy_from_slice(&u32::MAX.to_le_bytes());
+ assert!(GetMetadataChangesResponse::decode(&bytes).is_err());
+ }
+}
diff --git a/core/binary_protocol/src/responses/system/mod.rs b/core/binary_protocol/src/responses/system/mod.rs
index a8e21d3b1..a43a53cfd 100644
--- a/core/binary_protocol/src/responses/system/mod.rs
+++ b/core/binary_protocol/src/responses/system/mod.rs
@@ -18,6 +18,7 @@
pub mod describe_options;
pub mod get_cluster_metadata;
pub mod get_me;
+pub mod get_metadata_changes;
pub mod get_snapshot;
pub mod get_stats;
mod ping;
@@ -27,6 +28,9 @@ pub use super::EmptyResponse;
pub use describe_options::{DescribeOptionsResponse, OptionDescriptor};
pub use get_cluster_metadata::{ClusterMetadataResponse, ClusterNodeResponse};
pub use get_me::GetMeResponse;
+pub use get_metadata_changes::{
+ GetMetadataChangesResponse, WireMetadataChangeRecord, WireMetadataItem,
+};
pub use get_snapshot::GetSnapshotResponse;
pub use get_stats::StatsResponse;
pub use ping::PingResponse;
diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml
index f8d433742..c7d365c73 100644
--- a/core/cli/Cargo.toml
+++ b/core/cli/Cargo.toml
@@ -17,7 +17,7 @@
[package]
name = "iggy-cli"
-version = "0.14.1-edge.2"
+version = "0.14.1-edge.3"
edition = "2024"
rust-version.workspace = true
authors = ["bartosz.ciesla@gmail.com"]
diff --git a/core/cli/src/args/metadata.rs b/core/cli/src/args/metadata.rs
new file mode 100644
index 000000000..3f9ddf905
--- /dev/null
+++ b/core/cli/src/args/metadata.rs
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use clap::{Args, Subcommand, ValueEnum};
+use iggy::prelude::{
+ IggyDuration, IggyTimestamp, MetadataChangeCursor, MetadataChangeGroup, MetadataChangeStart,
+ MetadataChangesQuery,
+};
+
+#[derive(Debug, Clone, Subcommand)]
+pub(crate) enum MetadataAction {
+ /// read the metadata change log
+ ///
+ /// Prints every answered page as one JSON object per line: the change
+ /// records or current state items, the status and the cursor to continue
+ /// from. Requires root or the global manage_servers permission.
+ ///
+ /// Examples
+ /// iggy metadata changes
+ /// iggy metadata changes --from latest
+ /// iggy metadata changes --after mc1_... --groups streams,topics
+ /// iggy metadata changes --from current --groups streams,topics --follow
+ #[clap(verbatim_doc_comment, visible_alias = "c")]
+ Changes(MetadataChangesArgs),
+}
+
+#[derive(Debug, Clone, Args)]
+pub(crate) struct MetadataChangesArgs {
+ /// Where to start reading
+ ///
+ /// earliest: the oldest operation the log still holds completely,
+ /// latest: no records, only the cursor at the newest record,
+ /// current: the current state, page by page, then its catch-up cursor.
+ #[arg(verbatim_doc_comment, long, value_enum, default_value_t = MetadataChangesFrom::Earliest, conflicts_with = "after")]
+ pub(crate) from: MetadataChangesFrom,
+
+ /// Continue after this cursor, the `next` value of an earlier page
+ #[arg(long)]
+ pub(crate) after: Option<MetadataChangeCursor>,
+
+ /// Groups to read, comma separated, every group when omitted
+ ///
+ /// streams, topics, partitions, consumer_groups, consumer_group_members,
+ /// users, personal_access_tokens, sessions
+ #[arg(verbatim_doc_comment, long, value_delimiter = ',')]
+ pub(crate) groups: Vec<MetadataChangeGroup>,
+
+ /// Oldest prepare timestamp to include, in microseconds since the epoch
+ #[arg(long, value_parser = parse_micros)]
+ pub(crate) since: Option<IggyTimestamp>,
+
+ /// Newest prepare timestamp to include, in microseconds since the epoch
+ #[arg(long, value_parser = parse_micros)]
+ pub(crate) until: Option<IggyTimestamp>,
+
+ /// Records or items per page, 1 to 1000
+ #[arg(short, long, default_value_t = 100)]
+ pub(crate) limit: u32,
+
+ /// Keep reading new changes once the log has no more
+ ///
+ /// When the cursor's history is no longer retained, the command rebuilds
+ /// from the current state and goes on following.
+ #[arg(verbatim_doc_comment, short, long, default_value_t = false)]
+ pub(crate) follow: bool,
+
+ /// Pause between pages while following
+ #[arg(long, default_value = "1s")]
+ pub(crate) interval: IggyDuration,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
+pub(crate) enum MetadataChangesFrom {
+ Earliest,
+ Latest,
+ Current,
+}
+
+impl From<&MetadataChangesArgs> for MetadataChangesQuery {
+ fn from(args: &MetadataChangesArgs) -> Self {
+ let start = match (args.after, args.from) {
+ (Some(cursor), _) => MetadataChangeStart::After(cursor),
+ (None, MetadataChangesFrom::Earliest) => MetadataChangeStart::Earliest,
+ (None, MetadataChangesFrom::Latest) => MetadataChangeStart::Latest,
+ (None, MetadataChangesFrom::Current) => MetadataChangeStart::Current(None),
+ };
+ Self {
+ start,
+ groups: args.groups.clone(),
+ since: args.since,
+ until: args.until,
+ limit: args.limit,
+ }
+ }
+}
+
+fn parse_micros(raw: &str) -> Result<IggyTimestamp, String> {
+ let micros: u64 = raw
+ .parse()
+ .map_err(|error| format!("expected microseconds since the epoch, got {raw}: {error}"))?;
+ IggyTimestamp::checked_from_micros(micros)
+ .ok_or_else(|| format!("{micros} microseconds is past what this platform's clock holds"))
+}
diff --git a/core/cli/src/args/mod.rs b/core/cli/src/args/mod.rs
index ffe1145d4..24a3ca669 100644
--- a/core/cli/src/args/mod.rs
+++ b/core/cli/src/args/mod.rs
@@ -34,6 +34,7 @@ use crate::args::{
consumer_offset::ConsumerOffsetAction,
context::ContextAction,
message::MessageAction,
+ metadata::MetadataAction,
partition::PartitionAction,
personal_access_token::PersonalAccessTokenAction,
stream::StreamAction,
@@ -56,6 +57,7 @@ pub(crate) mod consumer_group;
pub(crate) mod consumer_offset;
pub(crate) mod context;
pub(crate) mod message;
+pub(crate) mod metadata;
pub(crate) mod partition;
pub(crate) mod permissions;
pub(crate) mod personal_access_token;
@@ -191,6 +193,9 @@ pub(crate) enum Command {
/// collect iggy server troubleshooting data
#[clap(verbatim_doc_comment)]
Snapshot(SnapshotArgs),
+ /// metadata change log operations
+ #[command(subcommand, visible_alias = "md")]
+ Metadata(MetadataAction),
/// personal access token operations
#[command(subcommand)]
Pat(PersonalAccessTokenAction),
diff --git a/core/cli/src/commands/binary_system/metadata_changes.rs b/core/cli/src/commands/binary_system/metadata_changes.rs
new file mode 100644
index 000000000..64156fa7c
--- /dev/null
+++ b/core/cli/src/commands/binary_system/metadata_changes.rs
@@ -0,0 +1,337 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::commands::cli_command::CliCommand;
+use anyhow::{Context, bail};
+use async_trait::async_trait;
+use iggy_common::{
+ Client, IggyDuration, IggyError, MetadataChangeGroup, MetadataChangeStart,
+ MetadataChangeStatus, MetadataChanges, MetadataChangesQuery,
+};
+use std::future::Future;
+use std::io::{Write, stdout};
+
+/// Reads metadata change pages and prints each as one JSON line.
+pub struct GetMetadataChangesCmd {
+ query: MetadataChangesQuery,
+ follow: bool,
+ interval: IggyDuration,
+}
+
+impl GetMetadataChangesCmd {
+ pub fn new(query: MetadataChangesQuery, follow: bool, interval: IggyDuration) -> Self {
+ Self {
+ query,
+ follow,
+ interval,
+ }
+ }
+
+ /// Reads pages from `read` and writes each to `output` until the query ends.
+ async fn read_pages<R, F>(
+ &self,
+ mut read: R,
+ output: &mut (impl Write + Send),
+ ) -> anyhow::Result<()>
+ where
+ R: FnMut(MetadataChangesQuery) -> F + Send,
+ F: Future<Output = Result<MetadataChanges, IggyError>> + Send,
+ {
+ let mut query = self.first_query();
+ loop {
+ let page = match read(query.clone()).await {
+ Ok(page) => page,
+ Err(IggyError::TransientNotAccepted) if self.follow => {
+ tokio::time::sleep(self.interval.get_duration()).await;
+ continue;
+ }
+ Err(error) => {
+ return Err(error)
+ .with_context(|| "Problem reading metadata changes".to_owned());
+ }
+ };
+ let quiet_follow_page = self.follow
+ && page.status == MetadataChangeStatus::Ok
+ && page.records.is_empty()
+ && page.items.is_empty();
+ if !quiet_follow_page {
+ writeln!(output, "{}", serde_json::to_string(&page)?)
+ .and_then(|()| output.flush())
+ .with_context(|| "Problem writing metadata changes page".to_owned())?;
+ }
+ query = match self.step(query, page)? {
+ Step::Read(next) => next,
+ Step::Wait(next) => {
+ tokio::time::sleep(self.interval.get_duration()).await;
+ next
+ }
+ Step::Stop => return Ok(()),
+ };
+ }
+ }
+
+ /// The query that starts the read, with the parent groups a current scan needs.
+ fn first_query(&self) -> MetadataChangesQuery {
+ match self.query.start {
+ MetadataChangeStart::Current(_) => MetadataChangesQuery {
+ groups: self.scan_groups(),
+ ..self.query.clone()
+ },
+ _ => self.query.clone(),
+ }
+ }
+
+ /// What to read after `page` answered `query`.
+ fn step(&self, mut query: MetadataChangesQuery, page: MetadataChanges) -> anyhow::Result<Step> {
+ match page.status {
+ MetadataChangeStatus::Ok => {}
+ MetadataChangeStatus::Behind if self.follow => return Ok(Step::Wait(query)),
+ MetadataChangeStatus::Behind => return Ok(Step::Stop),
+ MetadataChangeStatus::Expired | MetadataChangeStatus::Reset if self.follow => {
+ return Ok(Step::Read(self.current_state_query()));
+ }
+ MetadataChangeStatus::Expired | MetadataChangeStatus::Reset => {
+ bail!(
+ "The history after the cursor is not available ({}), read it again with --from current",
+ serde_json::to_string(&page.status)?
+ );
+ }
+ }
+ if let MetadataChangeStart::Current(_) = query.start {
+ return Ok(match (page.current_page_token, page.next) {
+ (Some(token), _) => {
+ query.start = MetadataChangeStart::Current(Some(token));
+ Step::Read(query)
+ }
+ (None, Some(anchor)) if self.follow => Step::Read(MetadataChangesQuery {
+ start: MetadataChangeStart::After(anchor),
+ ..self.query.clone()
+ }),
+ _ => Step::Stop,
+ });
+ }
+ if let Some(next) = page.next {
+ query.start = MetadataChangeStart::After(next);
+ }
+ Ok(if page.has_more {
+ Step::Read(query)
+ } else if self.follow {
+ Step::Wait(query)
+ } else {
+ Step::Stop
+ })
+ }
+
+ /// The rebuild scan, with the parent groups and no time bounds the server requires.
+ fn current_state_query(&self) -> MetadataChangesQuery {
+ MetadataChangesQuery {
+ start: MetadataChangeStart::Current(None),
+ groups: self.scan_groups(),
+ since: None,
+ until: None,
+ limit: self.query.limit,
+ }
+ }
+
+ /// The requested groups and their parents.
+ fn scan_groups(&self) -> Vec<MetadataChangeGroup> {
+ let mut groups = self.query.groups.clone();
+ groups.extend(self.query.groups.iter().flat_map(|group| group.parents()));
+ groups
+ }
+}
+
+#[async_trait]
+impl CliCommand for GetMetadataChangesCmd {
+ fn explain(&self) -> String {
+ "metadata changes command".to_owned()
+ }
+
+ fn stdout_is_data(&self) -> bool {
+ true
+ }
+
+ async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> {
+ self.read_pages(
+ |query| async move { client.get_metadata_changes(&query).await },
+ &mut stdout(),
+ )
+ .await
+ }
+}
+
+#[derive(Debug, PartialEq)]
+enum Step {
+ Read(MetadataChangesQuery),
+ Wait(MetadataChangesQuery),
+ Stop,
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use iggy_common::{
+ IggyTimestamp, MetadataChangeCursor, MetadataChangeGroup, MetadataChangeGroups,
+ MetadataChangePhase,
+ };
+ use std::collections::VecDeque;
+ use std::future::ready;
+
+ #[test]
+ fn given_current_start_for_topics_when_building_first_query_should_add_their_parent_groups() {
+ let command = GetMetadataChangesCmd::new(
+ MetadataChangesQuery {
+ start: MetadataChangeStart::Current(None),
+ groups: vec![MetadataChangeGroup::Topics],
+ ..MetadataChangesQuery::default()
+ },
+ false,
+ IggyDuration::ONE_SECOND,
+ );
+
+ assert_eq!(
+ command.first_query().validate(),
+ Ok(MetadataChangeGroups::requested(&[
+ MetadataChangeGroup::Streams,
+ MetadataChangeGroup::Topics
+ ]))
+ );
+ }
+
+ #[tokio::test]
+ async fn given_transient_refusal_when_following_should_retry_the_same_query() {
+ let command = GetMetadataChangesCmd::new(
+ MetadataChangesQuery::default(),
+ true,
+ IggyDuration::from(1),
+ );
+ let mut answers = VecDeque::from([
+ Err(IggyError::TransientNotAccepted),
+ Ok(page(MetadataChangeStatus::Behind, None)),
+ Err(IggyError::Unauthenticated),
+ ]);
+ let mut queries = Vec::new();
+ let mut output = Vec::new();
+
+ let result = command
+ .read_pages(
+ |query| {
+ queries.push(query);
+ ready(answers.pop_front().expect("scripted answer"))
+ },
+ &mut output,
+ )
+ .await;
+
+ assert!(result.is_err(), "the scripted refusal ends the read");
+ assert_eq!(queries.len(), 3);
+ assert!(
+ queries
+ .iter()
+ .all(|query| *query == MetadataChangesQuery::default())
+ );
+ assert_eq!(String::from_utf8(output).expect("utf-8").lines().count(), 1);
+ }
+
+ #[test]
+ fn given_expired_cursor_when_following_topics_should_rebuild_with_their_parent_groups() {
+ let command = following(vec![MetadataChangeGroup::Topics]);
+ let step = command
+ .step(
+ command.query.clone(),
+ page(MetadataChangeStatus::Expired, None),
+ )
+ .expect("recovery step");
+ let Step::Read(recovery) = step else {
+ panic!("expected an immediate read, got {step:?}");
+ };
+ assert_eq!(recovery.start, MetadataChangeStart::Current(None));
+ assert_eq!(
+ recovery.validate(),
+ Ok(MetadataChangeGroups::requested(&[
+ MetadataChangeGroup::Streams,
+ MetadataChangeGroup::Topics
+ ]))
+ );
+ }
+
+ #[test]
+ fn given_finished_recovery_scan_when_following_should_restore_requested_groups_and_bounds() {
+ let command = following(vec![MetadataChangeGroup::Topics]);
+ let Step::Read(recovery) = command
+ .step(
+ command.query.clone(),
+ page(MetadataChangeStatus::Reset, None),
+ )
+ .expect("recovery step")
+ else {
+ panic!("expected an immediate read");
+ };
+ let anchor = MetadataChangeCursor {
+ log_id: 7,
+ op: 42,
+ checksum: 9,
+ ordinal: 0,
+ };
+ let step = command
+ .step(
+ recovery,
+ MetadataChanges {
+ phase: MetadataChangePhase::Current,
+ ..page(MetadataChangeStatus::Ok, Some(anchor))
+ },
+ )
+ .expect("hand over step");
+ assert_eq!(
+ step,
+ Step::Read(MetadataChangesQuery {
+ start: MetadataChangeStart::After(anchor),
+ ..command.query.clone()
+ })
+ );
+ }
+
+ fn following(groups: Vec<MetadataChangeGroup>) -> GetMetadataChangesCmd {
+ GetMetadataChangesCmd::new(
+ MetadataChangesQuery {
+ start: MetadataChangeStart::Earliest,
+ groups,
+ since: Some(IggyTimestamp::from(1_000)),
+ until: Some(IggyTimestamp::from(9_000)),
+ limit: 50,
+ },
+ true,
+ IggyDuration::ONE_SECOND,
+ )
+ }
+
+ fn page(status: MetadataChangeStatus, next: Option<MetadataChangeCursor>) -> MetadataChanges {
+ MetadataChanges {
+ status,
+ phase: MetadataChangePhase::Changes,
+ log_id: Some(7),
+ oldest_op: 1,
+ retained_since: 0,
+ head_op: 42,
+ next,
+ has_more: false,
+ records: vec![],
+ items: vec![],
+ current_page_token: None,
+ }
+ }
+}
diff --git a/core/cli/src/commands/binary_system/mod.rs b/core/cli/src/commands/binary_system/mod.rs
index 2bbc45a3b..69223a41d 100644
--- a/core/cli/src/commands/binary_system/mod.rs
+++ b/core/cli/src/commands/binary_system/mod.rs
@@ -18,6 +18,7 @@
pub mod login;
pub mod logout;
pub mod me;
+pub mod metadata_changes;
pub mod options;
pub mod ping;
pub mod session;
diff --git a/core/cli/src/commands/cli_command.rs b/core/cli/src/commands/cli_command.rs
index eb2520a02..02bef7b51 100644
--- a/core/cli/src/commands/cli_command.rs
+++ b/core/cli/src/commands/cli_command.rs
@@ -33,6 +33,10 @@ pub trait CliCommand {
fn use_tracing(&self) -> bool {
true
}
+ /// Whether stdout carries only the command's data, so the banner goes to stderr.
+ fn stdout_is_data(&self) -> bool {
+ false
+ }
fn login_required(&self) -> bool {
true
}
diff --git a/core/cli/src/main.rs b/core/cli/src/main.rs
index a3ccad4dc..b5ee3ebf5 100644
--- a/core/cli/src/main.rs
+++ b/core/cli/src/main.rs
@@ -31,6 +31,7 @@ use crate::error::{CmdToolError, IggyCmdError};
use crate::logging::Logging;
use args::context::ContextAction;
use args::message::MessageAction;
+use args::metadata::MetadataAction;
use args::partition::PartitionAction;
use args::segment::SegmentAction;
use args::user::UserAction;
@@ -39,8 +40,8 @@ use clap::Parser;
use iggy::client_provider::{self, ClientProviderConfig};
use iggy::clients::client::IggyClient;
use iggy::prelude::{
- Aes256GcmEncryptor, ClientError, EncryptorKind, PersonalAccessTokenExpiry, StreamUpdateOptions,
- TopicUpdateOptions,
+ Aes256GcmEncryptor, ClientError, EncryptorKind, MetadataChangesQuery,
+ PersonalAccessTokenExpiry, StreamUpdateOptions, TopicUpdateOptions,
};
use iggy_cli::commands::binary_context::common::{ContextManager, validate_transport};
use iggy_cli::commands::binary_context::create_context::CreateContextCmd;
@@ -48,6 +49,7 @@ use iggy_cli::commands::binary_context::delete_context::DeleteContextCmd;
use iggy_cli::commands::binary_context::show_context::ShowContextCmd;
use iggy_cli::commands::binary_context::use_context::UseContextCmd;
use iggy_cli::commands::binary_segments::delete_segments::DeleteSegmentsCmd;
+use iggy_cli::commands::binary_system::metadata_changes::GetMetadataChangesCmd;
use iggy_cli::commands::binary_system::snapshot::GetSnapshotCmd;
#[cfg(feature = "login-session")]
use iggy_cli::commands::cli_command::DIAGNOSTIC_TARGET;
@@ -187,6 +189,13 @@ fn get_command(
args.snapshot_types,
args.out_dir,
)),
+ Command::Metadata(command) => match command {
+ MetadataAction::Changes(args) => Box::new(GetMetadataChangesCmd::new(
+ MetadataChangesQuery::from(&args),
+ args.follow,
+ args.interval,
+ )),
+ },
Command::Pat(command) => match command {
PersonalAccessTokenAction::Create(pat_create_args) => {
Box::new(CreatePersonalAccessTokenCmd::new(
@@ -439,7 +448,11 @@ async fn run() -> Result<(), IggyCmdError> {
credentials.set_iggy_client(&client);
credentials.login_user().await?;
- if command.use_tracing() {
+ if command.stdout_is_data() {
+ if !cli_options.quiet {
+ eprintln!("Executing {}", command.explain());
+ }
+ } else if command.use_tracing() {
event!(target: PRINT_TARGET, Level::INFO, "Executing {}", command.explain());
} else {
println!("Executing {}", command.explain());
diff --git a/core/common/Cargo.toml b/core/common/Cargo.toml
index a48d622d6..09c5008df 100644
--- a/core/common/Cargo.toml
+++ b/core/common/Cargo.toml
@@ -17,7 +17,7 @@
[package]
name = "iggy_common"
-version = "0.11.1-edge.2"
+version = "0.11.1-edge.3"
description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second."
edition = "2024"
rust-version.workspace = true
@@ -49,7 +49,7 @@ papaya = { workspace = true }
ring = { workspace = true }
secrecy = { workspace = true }
serde = { workspace = true }
-serde_json = { workspace = true }
+serde_json = { workspace = true, features = ["raw_value"] }
serde_with = { workspace = true, features = ["base64", "json"] }
strum = { workspace = true }
thiserror = { workspace = true }
diff --git a/foreign/swift/Sources/Iggy/Iggy.swift b/core/common/examples/metadata_changes_fixtures.rs
similarity index 58%
copy from foreign/swift/Sources/Iggy/Iggy.swift
copy to core/common/examples/metadata_changes_fixtures.rs
index 932138513..8f0aaaaeb 100644
--- a/foreign/swift/Sources/Iggy/Iggy.swift
+++ b/core/common/examples/metadata_changes_fixtures.rs
@@ -15,11 +15,23 @@
// specific language governing permissions and limitations
// under the License.
-/// Version identity the SDK sends to the server during the login handshake.
-public enum IggyVersion {
- /// Name of this SDK as reported to the server.
- public static let sdkName = "swift-sdk"
- /// Version of this SDK as reported to the server. Provisional until the
- /// release wiring lands; the release tooling reads this constant.
- public static let sdkVersion = "0.1.1"
+#[path = "metadata_changes_fixtures/schema.rs"]
+mod schema;
+
+use schema::{CANONICAL_PATH, FixtureResult, LANGUAGE_COPIES, render};
+use std::fs;
+use std::iter;
+use std::path::Path;
+
+fn main() -> FixtureResult<()> {
+ let json = render()?;
+ let manifest = Path::new(env!("CARGO_MANIFEST_DIR"));
+ for relative in iter::once(CANONICAL_PATH).chain(LANGUAGE_COPIES) {
+ let path = manifest.join(relative);
+ if let Some(parent) = path.parent() {
+ fs::create_dir_all(parent)?;
+ }
+ fs::write(&path, &json)?;
+ }
+ Ok(())
}
diff --git a/core/common/examples/metadata_changes_fixtures/schema.rs b/core/common/examples/metadata_changes_fixtures/schema.rs
new file mode 100644
index 000000000..619765536
--- /dev/null
+++ b/core/common/examples/metadata_changes_fixtures/schema.rs
@@ -0,0 +1,1692 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Shape and builder of the metadata change log golden vectors.
+//!
+//! Shared by the generator example and the drift test, so the test rebuilds
+//! the fixture in process and compares it byte for byte with the canonical
+//! file and every SDK copy.
+
+use base64::Engine;
+use base64::engine::general_purpose::URL_SAFE_NO_PAD;
+use bytes::Bytes;
+use iggy_binary_protocol::codes::GET_METADATA_CHANGES_CODE;
+use iggy_binary_protocol::lookup_command;
+use iggy_binary_protocol::primitives::metadata_changes::{
+ MAX_PAGE_TOKEN_KEY_SIZE, MAX_PAGE_TOKEN_SIZE, METADATA_CHANGE_CURSOR_SIZE, WireChangeOptions,
+ WireConsumerGroupCreatedBody, WireConsumerGroupDeletedBody, WireConsumerGroupItemBody,
+ WireMemberItemBody, WireMemberJoinedBody, WireMemberLeftBody, WireMemberLeftReason,
+ WireMetadataChange, WireMetadataChangeCursor, WireMetadataChangeGroup, WireMetadataChangeKind,
+ WireMetadataChangePhase, WireMetadataChangeStartMode, WireMetadataChangeStatus,
+ WireMetadataCurrentItem, WireMetadataCurrentPageToken, WirePartitionHandedOverBody,
+ WirePartitionsChangedBody, WirePartitionsItemBody, WirePermissionsUpdatedReason,
+ WireSessionBody, WireSessionClosedBody, WireSessionClosedReason, WireStreamBody,
+ WireStreamDeletedBody, WireStreamKey, WireTokenBody, WireTokenDeletedBody,
+ WireTokenDeletedReason, WireTopicBody, WireTopicDeletedBody, WireTopicItemBody, WireTopicKey,
+ WireUserBody, WireUserIdentityBody, WireUserKey, WireUserPermissionsBody,
+};
+use iggy_binary_protocol::requests::system::GetMetadataChangesRequest;
+use iggy_binary_protocol::responses::system::{
+ GetMetadataChangesResponse, WireMetadataChangeRecord, WireMetadataItem,
+};
+use iggy_binary_protocol::{WireEncode, WireName, WireOptions};
+use iggy_common::wire_conversions::{permissions_to_wire, resource_options_to_wire_split};
+use iggy_common::{
+ DEFAULT_METADATA_CHANGES_LIMIT, GlobalPermissions, HeaderKey, HeaderValue, IggyError,
+ IggyTimestamp, MAX_METADATA_CHANGES_LIMIT, METADATA_CHANGE_CURSOR_PREFIX,
+ METADATA_CURRENT_PAGE_TOKEN_PREFIX, MetadataChangeClass, MetadataChangeCursor,
+ MetadataChangeGroup, MetadataChangeKind, MetadataChangePhase, MetadataChangeRecord,
+ MetadataChangeStart, MetadataChangeStatus, MetadataChanges, MetadataChangesQuery,
+ MetadataCurrentItem, MetadataCurrentPageToken, MetadataMemberLeftReason,
+ MetadataPermissionsUpdatedReason, MetadataSessionClosedReason, MetadataTokenDeletedReason,
+ OptionKey, OptionValue, Permissions, ResourceOptions, StreamPermissions, TopicPermissions,
+};
+use serde::Serialize;
+use std::collections::BTreeMap;
+use std::error::Error;
+use std::str::FromStr;
+use strum::IntoEnumIterator;
+
+pub const SCHEMA_VERSION: u32 = 1;
+pub const GENERATOR: &str = "core/common/examples/metadata_changes_fixtures.rs";
+/// Relative to the `iggy_common` manifest directory.
+pub const CANONICAL_PATH: &str = "fixtures/metadata_changes.json";
+/// Byte-identical copies each foreign SDK loads from its own test resources.
+pub const LANGUAGE_COPIES: [&str; 8] = [
+ "../../foreign/python/tests/fixtures/metadata_changes.json",
+ "../../foreign/cpp/tests/fixtures/metadata_changes.json",
+ "../../foreign/php/tests/fixtures/metadata_changes.json",
+ "../../foreign/node/src/wire/fixtures/metadata_changes.json",
+ "../../foreign/go/contracts/testdata/metadata_changes.json",
+ "../../foreign/java/java-sdk/src/test/resources/metadata_changes.json",
+ "../../foreign/csharp/Iggy_SDK_Tests/Fixtures/metadata_changes.json",
+ "../../foreign/swift/Tests/IggyTests/Fixtures/metadata_changes.json",
+];
+
+const LOG_ID: u128 = 0x0d4b_8f2a_6e1c_93d7;
+const ABOVE_2_POW_53: u64 = (1 << 53) + 1;
+
+pub type FixtureResult<T> = Result<T, Box<dyn Error>>;
+
+#[derive(Debug, Serialize)]
+pub struct MetadataChangesFixture {
+ pub schema_version: u32,
+ pub generated_by: String,
+ pub conventions: BTreeMap<String, String>,
+ pub constants: Constants,
+ pub records: Vec<RecordVector>,
+ pub items: Vec<ItemVector>,
+ pub cursors: Vec<CursorVector>,
+ pub page_tokens: Vec<PageTokenVector>,
+ pub requests: Vec<RequestVector>,
+ pub responses: Vec<ResponseVector>,
+ pub large_values: LargeValuesVector,
+ pub refused: RefusedVectors,
+}
+
+#[derive(Debug, Serialize)]
+pub struct Constants {
+ pub command_code: u32,
+ pub command_name: String,
+ pub cursor_prefix: String,
+ pub page_token_prefix: String,
+ pub cursor_size: usize,
+ pub cursor_text_length: usize,
+ pub max_page_token_size: usize,
+ pub max_page_token_key_size: usize,
+ pub default_limit: u32,
+ pub max_limit: u32,
+ pub groups: BTreeMap<String, GroupConstant>,
+ pub kinds: BTreeMap<String, KindConstant>,
+ pub statuses: BTreeMap<String, u8>,
+ pub phases: BTreeMap<String, u8>,
+ pub start_modes: BTreeMap<String, u8>,
+ pub member_left_reasons: BTreeMap<String, u8>,
+ pub token_deleted_reasons: BTreeMap<String, u8>,
+ pub session_closed_reasons: BTreeMap<String, u8>,
+ pub permissions_updated_reasons: BTreeMap<String, u8>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct GroupConstant {
+ pub code: u8,
+ pub class: String,
+ pub parents: Vec<String>,
+ pub page_token_key: String,
+}
+
+#[derive(Debug, Serialize)]
+pub struct KindConstant {
+ pub code: u8,
+ pub group: String,
+}
+
+/// A record's binary forms and the JSON the HTTP route renders for it.
+#[derive(Debug, Serialize)]
+pub struct RecordVector {
+ pub case: String,
+ pub group: u8,
+ pub kind: u8,
+ pub body_hex: String,
+ pub record_hex: String,
+ pub expected: MetadataChangeRecord,
+}
+
+#[derive(Debug, Serialize)]
+pub struct ItemVector {
+ pub case: String,
+ pub group: u8,
+ pub body_hex: String,
+ pub item_hex: String,
+ pub expected: MetadataCurrentItem,
+}
+
+#[derive(Debug, Serialize)]
+pub struct CursorVector {
+ pub case: String,
+ pub text: String,
+ pub hex: String,
+ pub log_id: String,
+ pub op: u64,
+ pub checksum: String,
+ pub ordinal: u32,
+}
+
+#[derive(Debug, Serialize)]
+pub struct PageTokenVector {
+ pub case: String,
+ pub text: String,
+ pub hex: String,
+ pub anchor: String,
+ pub required_head: u64,
+ pub groups_mask: u8,
+ pub group: String,
+ pub last_key_hex: String,
+}
+
+/// A query in the shape every SDK can build: start mode, positions as text.
+#[derive(Debug, Clone, Serialize)]
+pub struct QueryInput {
+ pub start: String,
+ pub cursor: Option<String>,
+ pub page_token: Option<String>,
+ pub groups: Vec<String>,
+ pub since: Option<u64>,
+ pub until: Option<u64>,
+ pub limit: u32,
+}
+
+#[derive(Debug, Serialize)]
+pub struct RequestVector {
+ pub case: String,
+ pub input: QueryInput,
+ pub request_hex: String,
+ pub http_query: String,
+}
+
+#[derive(Debug, Serialize)]
+pub struct ResponseVector {
+ pub case: String,
+ pub response_hex: String,
+ pub expected: MetadataChanges,
+}
+
+/// Unsigned values above 2^53 as decimal strings, so SDKs can prove they
+/// decode them losslessly from the binary form.
+#[derive(Debug, Serialize)]
+pub struct LargeValuesVector {
+ pub record_hex: String,
+ pub op: String,
+ pub ordinal: u32,
+ pub timestamp: String,
+ pub checksum: String,
+ pub user_created_at: String,
+ pub session: String,
+ pub client_id: String,
+}
+
+#[derive(Debug, Serialize)]
+pub struct RefusedVectors {
+ pub cursors: Vec<RefusedText>,
+ pub page_tokens: Vec<RefusedText>,
+ pub record_bodies: Vec<RefusedBody>,
+ pub requests: Vec<RefusedHex>,
+ pub responses: Vec<RefusedHex>,
+ pub queries: Vec<RefusedQuery>,
+ pub http_queries: Vec<RefusedHttpQuery>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct RefusedText {
+ pub case: String,
+ pub text: String,
+}
+
+#[derive(Debug, Serialize)]
+pub struct RefusedBody {
+ pub case: String,
+ pub group: u8,
+ pub kind: u8,
+ pub body_hex: String,
+}
+
+#[derive(Debug, Serialize)]
+pub struct RefusedHex {
+ pub case: String,
+ pub hex: String,
+}
+
+#[derive(Debug, Serialize)]
+pub struct RefusedQuery {
+ pub case: String,
+ pub input: QueryInput,
+}
+
+#[derive(Debug, Serialize)]
+pub struct RefusedHttpQuery {
+ pub case: String,
+ pub query: String,
+}
+
+impl QueryInput {
+ fn new(start: &str) -> Self {
+ Self {
+ start: start.to_owned(),
+ cursor: None,
+ page_token: None,
+ groups: Vec::new(),
+ since: None,
+ until: None,
+ limit: DEFAULT_METADATA_CHANGES_LIMIT,
+ }
+ }
+
+ /// The domain query this input describes.
+ ///
+ /// # Errors
+ ///
+ /// `InvalidCommand` for an unknown start mode, group or position text.
+ pub fn query(&self) -> Result<MetadataChangesQuery, IggyError> {
+ let start = match (self.start.as_str(), &self.cursor, &self.page_token) {
+ ("earliest", None, None) => MetadataChangeStart::Earliest,
+ ("latest", None, None) => MetadataChangeStart::Latest,
+ ("after", Some(cursor), None) => {
+ MetadataChangeStart::After(MetadataChangeCursor::from_str(cursor)?)
+ }
+ ("current", None, token) => MetadataChangeStart::Current(
+ token
+ .as_deref()
+ .map(MetadataCurrentPageToken::from_str)
+ .transpose()?,
+ ),
+ _ => return Err(IggyError::InvalidCommand),
+ };
+ let groups = self
+ .groups
+ .iter()
+ .map(|name| MetadataChangeGroup::from_str(name).map_err(|_| IggyError::InvalidCommand))
+ .collect::<Result<_, _>>()?;
+ Ok(MetadataChangesQuery {
+ start,
+ groups,
+ since: self.since.map(IggyTimestamp::from),
+ until: self.until.map(IggyTimestamp::from),
+ limit: self.limit,
+ })
+ }
+}
+
+/// Build the whole fixture from the live codecs.
+///
+/// # Errors
+///
+/// Propagates any codec or conversion failure, which means the fixture cannot
+/// describe the current code.
+pub fn build() -> FixtureResult<MetadataChangesFixture> {
+ let records = record_vectors()?;
+ let items = item_vectors()?;
+ let cursors = cursor_vectors();
+ let page_tokens = page_token_vectors();
+ let requests = request_vectors()?;
+ let responses = response_vectors(&records, &items)?;
+ Ok(MetadataChangesFixture {
+ schema_version: SCHEMA_VERSION,
+ generated_by: GENERATOR.to_owned(),
+ conventions: conventions(),
+ constants: constants()?,
+ records,
+ items,
+ cursors,
+ page_tokens,
+ requests,
+ responses,
+ large_values: large_values_vector()?,
+ refused: refused_vectors()?,
+ })
+}
+
+/// The fixture as the canonical file text.
+///
+/// # Errors
+///
+/// Propagates build and serialization failures.
+pub fn render() -> FixtureResult<String> {
+ let mut json = serde_json::to_string_pretty(&build()?)?;
+ json.push('\n');
+ Ok(json)
+}
+
+fn conventions() -> BTreeMap<String, String> {
+ [
+ (
+ "byte_strings",
+ "Every *_hex field is lowercase hex without separators.",
+ ),
+ (
+ "u128_values",
+ "128-bit values (log ids, checksums, client ids) are 32 lowercase hex characters in JSON.",
+ ),
+ (
+ "u64_values",
+ "Ops and timestamps are JSON numbers below 2^53 in every vector except large_values, which carries them as decimal strings.",
+ ),
+ (
+ "record_hex",
+ "record_hex is one record envelope as a response carries it. body_hex is the body alone.",
+ ),
+ (
+ "expected",
+ "expected is the JSON the HTTP route renders. Binary decoders must reach the same values.",
+ ),
+ (
+ "request_hex",
+ "request_hex is the command payload without framing. http_query is the query string of GET /metadata/changes.",
+ ),
+ (
+ "refused",
+ "Every refused vector must fail to decode or validate. Query and HTTP refusals are invalid_command.",
+ ),
+ (
+ "unknown_kinds",
+ "A record of an unknown group or kind is kept with its codes and raw body, never refused.",
+ ),
+ ]
+ .into_iter()
+ .map(|(name, text)| (name.to_owned(), text.to_owned()))
+ .collect()
+}
+
+fn constants() -> FixtureResult<Constants> {
+ let command = lookup_command(GET_METADATA_CHANGES_CODE).ok_or("code 17 is not registered")?;
+ let groups = MetadataChangeGroup::iter()
+ .map(|group| {
+ let class = match group.class() {
+ MetadataChangeClass::Resource => "resource",
+ MetadataChangeClass::Activity => "activity",
+ };
+ let key = match group {
+ MetadataChangeGroup::Streams => "[stream_id:u32]",
+ MetadataChangeGroup::Topics | MetadataChangeGroup::Partitions => {
+ "[stream_id:u32][topic_id:u32]"
+ }
+ MetadataChangeGroup::ConsumerGroups => {
+ "[stream_id:u32][topic_id:u32][group_id:u32]"
+ }
+ MetadataChangeGroup::ConsumerGroupMembers => {
+ "[stream_id:u32][topic_id:u32][group_id:u32][member_id:u32]"
+ }
+ MetadataChangeGroup::Users => "[user_id:u32]",
+ MetadataChangeGroup::PersonalAccessTokens => "[user_id:u32][name_len:u8][name]",
+ MetadataChangeGroup::Sessions => "[epoch:u64]",
+ };
+ (
+ group.to_string(),
+ GroupConstant {
+ code: WireMetadataChangeGroup::from(group).as_u8(),
+ class: class.to_owned(),
+ parents: group.parents().iter().map(ToString::to_string).collect(),
+ page_token_key: key.to_owned(),
+ },
+ )
+ })
+ .collect();
+ let kinds = (0..=u8::MAX)
+ .filter_map(|code| WireMetadataChangeKind::from_code(code).ok())
+ .map(|kind| {
+ let domain = MetadataChangeKind::from(kind);
+ (
+ domain.to_string(),
+ KindConstant {
+ code: kind.as_u8(),
+ group: domain.group().to_string(),
+ },
+ )
+ })
+ .collect();
+ Ok(Constants {
+ command_code: GET_METADATA_CHANGES_CODE,
+ command_name: command.name.to_owned(),
+ cursor_prefix: METADATA_CHANGE_CURSOR_PREFIX.to_owned(),
+ page_token_prefix: METADATA_CURRENT_PAGE_TOKEN_PREFIX.to_owned(),
+ cursor_size: METADATA_CHANGE_CURSOR_SIZE,
+ cursor_text_length: MetadataChangeCursor::default().to_string().len(),
+ max_page_token_size: MAX_PAGE_TOKEN_SIZE,
+ max_page_token_key_size: MAX_PAGE_TOKEN_KEY_SIZE,
+ default_limit: DEFAULT_METADATA_CHANGES_LIMIT,
+ max_limit: MAX_METADATA_CHANGES_LIMIT,
+ groups,
+ kinds,
+ statuses: codes(
+ (0..=u8::MAX).filter_map(|code| WireMetadataChangeStatus::from_code(code).ok()),
+ |status| MetadataChangeStatus::from(status).to_string(),
+ WireMetadataChangeStatus::as_u8,
+ ),
+ phases: codes(
+ (0..=u8::MAX).filter_map(|code| WireMetadataChangePhase::from_code(code).ok()),
+ |phase| MetadataChangePhase::from(phase).to_string(),
+ WireMetadataChangePhase::as_u8,
+ ),
+ start_modes: codes(
+ (0..=u8::MAX).filter_map(|code| WireMetadataChangeStartMode::from_code(code).ok()),
+ |mode| format!("{mode:?}").to_lowercase(),
+ WireMetadataChangeStartMode::as_u8,
+ ),
+ member_left_reasons: codes(
+ (0..=u8::MAX).filter_map(|code| WireMemberLeftReason::from_code(code).ok()),
+ |reason| MetadataMemberLeftReason::from(reason).to_string(),
+ WireMemberLeftReason::as_u8,
+ ),
+ token_deleted_reasons: codes(
+ (0..=u8::MAX).filter_map(|code| WireTokenDeletedReason::from_code(code).ok()),
+ |reason| MetadataTokenDeletedReason::from(reason).to_string(),
+ WireTokenDeletedReason::as_u8,
+ ),
+ session_closed_reasons: codes(
+ (0..=u8::MAX).filter_map(|code| WireSessionClosedReason::from_code(code).ok()),
+ |reason| MetadataSessionClosedReason::from(reason).to_string(),
+ WireSessionClosedReason::as_u8,
+ ),
+ permissions_updated_reasons: codes(
+ (0..=u8::MAX).filter_map(|code| WirePermissionsUpdatedReason::from_code(code).ok()),
+ |reason| MetadataPermissionsUpdatedReason::from(reason).to_string(),
+ WirePermissionsUpdatedReason::as_u8,
+ ),
+ })
+}
+
+fn codes<T: Copy>(
+ values: impl Iterator<Item = T>,
+ name: impl Fn(T) -> String,
+ code: impl Fn(T) -> u8,
+) -> BTreeMap<String, u8> {
+ values.map(|value| (name(value), code(value))).collect()
+}
+
+fn stream_key() -> WireStreamKey {
+ WireStreamKey {
+ stream_id: 3,
+ stream_created_op: 4121,
+ }
+}
+
+fn topic_key() -> WireTopicKey {
+ WireTopicKey {
+ stream: stream_key(),
+ topic_id: 0,
+ topic_created_op: 4125,
+ }
+}
+
+fn user_key() -> WireUserKey {
+ WireUserKey {
+ user_id: 7,
+ user_created_at: 1_760_300_000_000_000,
+ }
+}
+
+fn name(text: &str) -> FixtureResult<WireName> {
+ Ok(WireName::new(text)?)
+}
+
+fn readable(options: &ResourceOptions) -> FixtureResult<WireChangeOptions> {
+ let (explicit, derived) = resource_options_to_wire_split(options)?;
+ Ok(WireChangeOptions::Readable { explicit, derived })
+}
+
+fn stream_options() -> FixtureResult<WireChangeOptions> {
+ readable(&ResourceOptions::from([
+ (
+ HeaderKey::from(OptionKey::EncryptionKeySource),
+ OptionValue::derived(HeaderValue::try_from("generated")?),
+ ),
+ (
+ HeaderKey::from(OptionKey::EncryptionKeyFingerprint),
+ OptionValue::derived(HeaderValue::try_from("9f2c1a7e44b0d816")?),
+ ),
+ (
+ HeaderKey::from(OptionKey::EncryptionKekFingerprint),
+ OptionValue::derived(HeaderValue::try_from("51d0e3a9c27b6f48")?),
+ ),
+ ]))
+}
+
+fn topic_options() -> FixtureResult<WireChangeOptions> {
+ readable(&ResourceOptions::from([
+ (
+ HeaderKey::from(OptionKey::MessageExpiry),
+ OptionValue::explicit(HeaderValue::from(604_800_000_000u64)),
+ ),
+ (
+ HeaderKey::from(OptionKey::SegmentSize),
+ OptionValue::derived(HeaderValue::from(1_073_741_824u64)),
+ ),
+ ]))
+}
+
+fn empty_options() -> WireChangeOptions {
+ WireChangeOptions::Readable {
+ explicit: WireOptions::empty(),
+ derived: WireOptions::empty(),
+ }
+}
+
+fn permissions() -> Permissions {
+ Permissions {
+ global: GlobalPermissions {
+ read_streams: true,
+ ..GlobalPermissions::default()
+ },
+ streams: Some(BTreeMap::from([(
+ 3,
+ StreamPermissions {
+ read_stream: true,
+ topics: Some(BTreeMap::from([(
+ 0,
+ TopicPermissions {
+ poll_messages: true,
+ ..TopicPermissions::default()
+ },
+ )])),
+ ..StreamPermissions::default()
+ },
+ )])),
+ }
+}
+
+fn record_hex(record: &WireMetadataChangeRecord) -> String {
+ hex::encode(record.to_bytes())
+}
+
+fn wire_record(
+ change: &WireMetadataChange,
+ op: u64,
+ ordinal: u32,
+ actor_user_id: Option<u32>,
+) -> WireMetadataChangeRecord {
+ WireMetadataChangeRecord {
+ op,
+ ordinal,
+ checksum: 0x0000_0000_0000_0000_a5c1_f703_8b9e_2d4c + u128::from(op),
+ timestamp: 1_760_356_800_000_000 + op,
+ actor_user_id,
+ group: change.group().as_u8(),
+ kind: change.kind().as_u8(),
+ body: change.body_bytes(),
+ }
+}
+
+fn change_cases() -> FixtureResult<Vec<(&'static str, WireMetadataChange, Option<u32>)>> {
+ let token_name = || name("ci-token");
+ Ok(vec![
+ (
+ "stream_created",
+ WireMetadataChange::StreamCreated(WireStreamBody {
+ key: stream_key(),
+ created_at: 1_760_356_800_123_456,
+ name: name("telemetry")?,
+ options: stream_options()?,
+ }),
+ Some(4),
+ ),
+ (
+ "stream_updated_unreadable_options",
+ WireMetadataChange::StreamUpdated(WireStreamBody {
+ key: stream_key(),
+ created_at: 1_760_356_800_123_456,
+ name: name("telemetry-v2")?,
+ options: WireChangeOptions::Unreadable,
+ }),
+ None,
+ ),
+ (
+ "stream_deleted_deferred",
+ WireMetadataChange::StreamDeleted(WireStreamDeletedBody {
+ key: stream_key(),
+ created_at: 1_760_356_800_123_456,
+ name: name("telemetry")?,
+ requested_op: Some(4140),
+ }),
+ Some(4),
+ ),
+ (
+ "topic_created",
+ WireMetadataChange::TopicCreated(WireTopicBody {
+ key: topic_key(),
+ created_at: 1_760_356_801_000_000,
+ name: name("node-metrics")?,
+ partitions_count: 8,
+ options: topic_options()?,
+ }),
+ Some(4),
+ ),
+ (
+ "topic_updated",
+ WireMetadataChange::TopicUpdated(WireTopicBody {
+ key: topic_key(),
+ created_at: 1_760_356_801_000_000,
+ name: name("node-metrics-v2")?,
+ partitions_count: 8,
+ options: empty_options(),
+ }),
+ Some(4),
+ ),
+ (
+ "topic_deleted_immediate",
+ WireMetadataChange::TopicDeleted(WireTopicDeletedBody {
+ key: topic_key(),
+ created_at: 1_760_356_801_000_000,
+ name: name("node-metrics")?,
+ requested_op: None,
+ }),
+ Some(4),
+ ),
+ (
+ "partitions_changed_deleted_tail",
+ WireMetadataChange::PartitionsChanged(WirePartitionsChangedBody {
+ topic: topic_key(),
+ partitions_count: 5,
+ removed_partition_ids: vec![5, 6, 7],
+ requested_op: Some(4200),
+ }),
+ Some(4),
+ ),
+ (
+ "partitions_changed_created",
+ WireMetadataChange::PartitionsChanged(WirePartitionsChangedBody {
+ topic: topic_key(),
+ partitions_count: 10,
+ removed_partition_ids: vec![],
+ requested_op: None,
+ }),
+ Some(4),
+ ),
+ (
+ "consumer_group_created",
+ WireMetadataChange::ConsumerGroupCreated(WireConsumerGroupCreatedBody {
+ topic: topic_key(),
+ group_id: 0,
+ name: name("readers")?,
+ consumer_groups_count: 1,
+ }),
+ Some(4),
+ ),
+ (
+ "consumer_group_deleted",
+ WireMetadataChange::ConsumerGroupDeleted(WireConsumerGroupDeletedBody {
+ topic: topic_key(),
+ group_id: 0,
+ name: name("readers")?,
+ consumer_groups_count: 0,
+ requested_op: Some(4300),
+ }),
+ Some(4),
+ ),
+ (
+ "consumer_group_member_joined",
+ WireMetadataChange::ConsumerGroupMemberJoined(WireMemberJoinedBody {
+ topic: topic_key(),
+ group_id: 0,
+ member_id: 1,
+ client_id: 0x0102_0304_0506_0708_090a_0b0c_0d0e_0f10,
+ members_count: 2,
+ partition_ids: vec![],
+ }),
+ Some(9),
+ ),
+ (
+ "consumer_group_member_left_session_ended",
+ WireMetadataChange::ConsumerGroupMemberLeft(WireMemberLeftBody {
+ topic: topic_key(),
+ group_id: 0,
+ member_id: 1,
+ client_id: 0x0102_0304_0506_0708_090a_0b0c_0d0e_0f10,
+ members_count: 1,
+ reason: WireMemberLeftReason::SessionEnded,
+ }),
+ Some(9),
+ ),
+ (
+ "consumer_group_partition_handed_over",
+ WireMetadataChange::ConsumerGroupPartitionHandedOver(WirePartitionHandedOverBody {
+ topic: topic_key(),
+ group_id: 0,
+ partition_id: 2,
+ member_id: Some(1),
+ }),
+ None,
+ ),
+ (
+ "consumer_group_partition_released",
+ WireMetadataChange::ConsumerGroupPartitionHandedOver(WirePartitionHandedOverBody {
+ topic: topic_key(),
+ group_id: 0,
+ partition_id: 3,
+ member_id: None,
+ }),
+ None,
+ ),
+ (
+ "user_created_with_permissions",
+ WireMetadataChange::UserCreated(WireUserBody {
+ key: user_key(),
+ username: name("telemetry-writer")?,
+ status: 1,
+ permissions: Some(permissions_to_wire(&permissions())),
+ options: empty_options(),
+ }),
+ Some(0),
+ ),
+ (
+ "user_updated_inactive",
+ WireMetadataChange::UserUpdated(WireUserBody {
+ key: user_key(),
+ username: name("telemetry-writer")?,
+ status: 2,
+ permissions: None,
+ options: empty_options(),
+ }),
+ Some(0),
+ ),
+ (
+ "user_deleted",
+ WireMetadataChange::UserDeleted(WireUserIdentityBody {
+ key: user_key(),
+ username: name("telemetry-writer")?,
+ }),
+ Some(0),
+ ),
+ (
+ "user_password_changed",
+ WireMetadataChange::UserPasswordChanged(WireUserIdentityBody {
+ key: user_key(),
+ username: name("telemetry-writer")?,
+ }),
+ Some(7),
+ ),
+ (
+ "user_permissions_pruned",
+ WireMetadataChange::UserPermissionsUpdated(WireUserPermissionsBody {
+ key: user_key(),
+ username: name("telemetry-writer")?,
+ permissions: Some(permissions_to_wire(&Permissions::default())),
+ reason: WirePermissionsUpdatedReason::ResourceDeleted,
+ }),
+ Some(4),
+ ),
+ (
+ "personal_access_token_created",
+ WireMetadataChange::PersonalAccessTokenCreated(WireTokenBody {
+ user: user_key(),
+ name: token_name()?,
+ expiry: 1_760_400_000_000_000,
+ }),
+ Some(7),
+ ),
+ (
+ "personal_access_token_deleted_expired",
+ WireMetadataChange::PersonalAccessTokenDeleted(WireTokenDeletedBody {
+ user: user_key(),
+ name: token_name()?,
+ reason: WireTokenDeletedReason::Expired,
+ }),
+ None,
+ ),
+ (
+ "session_registered",
+ WireMetadataChange::SessionRegistered(WireSessionBody {
+ user: user_key(),
+ client_id: 0x0102_0304_0506_0708_090a_0b0c_0d0e_0f10,
+ session: 4500,
+ }),
+ Some(7),
+ ),
+ (
+ "session_closed_expired",
+ WireMetadataChange::SessionClosed(WireSessionClosedBody {
+ user: user_key(),
+ client_id: 0x0102_0304_0506_0708_090a_0b0c_0d0e_0f10,
+ session: 4500,
+ reason: WireSessionClosedReason::Expired,
+ }),
+ None,
+ ),
+ (
+ "session_closed_revoked",
+ WireMetadataChange::SessionClosed(WireSessionClosedBody {
+ user: user_key(),
+ client_id: 0x0102_0304_0506_0708_090a_0b0c_0d0e_0f10,
+ session: 4500,
+ reason: WireSessionClosedReason::Revoked,
+ }),
+ Some(7),
+ ),
+ ])
+}
+
+fn record_vectors() -> FixtureResult<Vec<RecordVector>> {
+ change_cases()?
+ .into_iter()
+ .enumerate()
+ .map(|(index, (case, change, actor))| {
+ let op = 4121 + u64::try_from(index)?;
+ let record = wire_record(&change, op, 0, actor);
+ Ok(RecordVector {
+ case: case.to_owned(),
+ group: record.group,
+ kind: record.kind,
+ body_hex: hex::encode(&record.body),
+ record_hex: record_hex(&record),
+ expected: MetadataChangeRecord::try_from(record)?,
+ })
+ })
+ .collect()
+}
+
+fn item_vectors() -> FixtureResult<Vec<ItemVector>> {
+ let items = vec![
+ (
+ "stream",
+ WireMetadataCurrentItem::Stream(WireStreamBody {
+ key: stream_key(),
+ created_at: 1_760_356_800_123_456,
+ name: name("telemetry")?,
+ options: stream_options()?,
+ }),
+ ),
+ (
+ "topic",
+ WireMetadataCurrentItem::Topic(WireTopicItemBody {
+ key: topic_key(),
+ created_at: 1_760_356_801_000_000,
+ name: name("node-metrics")?,
+ options: topic_options()?,
+ }),
+ ),
+ (
+ "partitions",
+ WireMetadataCurrentItem::Partitions(WirePartitionsItemBody {
+ topic: topic_key(),
+ partitions_count: 8,
+ }),
+ ),
+ (
+ "consumer_group",
+ WireMetadataCurrentItem::ConsumerGroup(WireConsumerGroupItemBody {
+ topic: topic_key(),
+ group_id: 0,
+ name: name("readers")?,
+ }),
+ ),
+ (
+ "consumer_group_member",
+ WireMetadataCurrentItem::ConsumerGroupMember(WireMemberItemBody {
+ topic: topic_key(),
+ group_id: 0,
+ member_id: 1,
+ client_id: 0x0102_0304_0506_0708_090a_0b0c_0d0e_0f10,
+ partition_ids: vec![0, 2, 4],
+ }),
+ ),
+ (
+ "user",
+ WireMetadataCurrentItem::User(WireUserBody {
+ key: user_key(),
+ username: name("telemetry-writer")?,
+ status: 1,
+ permissions: Some(permissions_to_wire(&permissions())),
+ options: empty_options(),
+ }),
+ ),
+ (
+ "personal_access_token_without_expiry",
+ WireMetadataCurrentItem::PersonalAccessToken(WireTokenBody {
+ user: user_key(),
+ name: name("ci-token")?,
+ expiry: 0,
+ }),
+ ),
+ (
+ "session",
+ WireMetadataCurrentItem::Session(WireSessionBody {
+ user: user_key(),
+ client_id: 0x0102_0304_0506_0708_090a_0b0c_0d0e_0f10,
+ session: 4500,
+ }),
+ ),
+ ];
+ items
+ .into_iter()
+ .map(|(case, item)| {
+ let envelope = WireMetadataItem {
+ group: item.group().as_u8(),
+ body: item.body_bytes(),
+ };
+ Ok(ItemVector {
+ case: case.to_owned(),
+ group: envelope.group,
+ body_hex: hex::encode(&envelope.body),
+ item_hex: hex::encode(envelope.to_bytes()),
+ expected: MetadataCurrentItem::try_from(envelope)?,
+ })
+ })
+ .collect()
+}
+
+fn cursor_vector(case: &str, cursor: MetadataChangeCursor) -> CursorVector {
+ CursorVector {
+ case: case.to_owned(),
+ text: cursor.to_string(),
+ hex: hex::encode(WireMetadataChangeCursor::from(cursor).to_bytes()),
+ log_id: format!("{:032x}", cursor.log_id),
+ op: cursor.op,
+ checksum: format!("{:032x}", cursor.checksum),
+ ordinal: cursor.ordinal,
+ }
+}
+
+fn cursor_vectors() -> Vec<CursorVector> {
+ vec![
+ cursor_vector("zero", MetadataChangeCursor::zero(LOG_ID)),
+ cursor_vector(
+ "first_record_of_op",
+ MetadataChangeCursor {
+ log_id: LOG_ID,
+ op: 4120,
+ checksum: 0x4c2d_9e8b_03f7_c1a5,
+ ordinal: 0,
+ },
+ ),
+ cursor_vector(
+ "inside_op",
+ MetadataChangeCursor {
+ log_id: LOG_ID,
+ op: 4187,
+ checksum: 0xf2a4_c6e8_b0d1_f359,
+ ordinal: 1,
+ },
+ ),
+ cursor_vector(
+ "every_field_at_maximum",
+ MetadataChangeCursor {
+ log_id: u128::MAX,
+ op: u64::MAX,
+ checksum: u128::MAX,
+ ordinal: u32::MAX,
+ },
+ ),
+ ]
+}
+
+fn page_token(
+ group: WireMetadataChangeGroup,
+ groups_mask: u8,
+ last_key: Vec<u8>,
+) -> MetadataCurrentPageToken {
+ MetadataCurrentPageToken::from(WireMetadataCurrentPageToken {
+ anchor: MetadataChangeCursor {
+ log_id: LOG_ID,
+ op: 4187,
+ checksum: 0xf2a4_c6e8_b0d1_f359,
+ ordinal: 1,
+ }
+ .into(),
+ required_head: 4190,
+ groups_mask,
+ group,
+ last_key: Bytes::from(last_key),
+ })
+}
+
+fn token_name_key(user_id: u32, token_name: &str) -> Vec<u8> {
+ let mut key = user_id.to_le_bytes().to_vec();
+ key.extend(
+ WireName::new(token_name)
+ .map(|name| name.to_bytes().to_vec())
+ .unwrap_or_default(),
+ );
+ key
+}
+
+fn page_token_vectors() -> Vec<PageTokenVector> {
+ let all = u8::MAX;
+ let ids = |values: &[u32]| {
+ values
+ .iter()
+ .flat_map(|value| value.to_le_bytes())
+ .collect::<Vec<u8>>()
+ };
+ let cases = vec![
+ ("streams", WireMetadataChangeGroup::Streams, 0b1, ids(&[3])),
+ (
+ "topics",
+ WireMetadataChangeGroup::Topics,
+ 0b11,
+ ids(&[3, 0]),
+ ),
+ (
+ "partitions",
+ WireMetadataChangeGroup::Partitions,
+ all,
+ ids(&[3, 0]),
+ ),
+ (
+ "consumer_groups",
+ WireMetadataChangeGroup::ConsumerGroups,
+ all,
+ ids(&[3, 0, 2]),
+ ),
+ (
+ "consumer_group_members",
+ WireMetadataChangeGroup::ConsumerGroupMembers,
+ all,
+ ids(&[3, 0, 2, 1]),
+ ),
+ (
+ "users",
+ WireMetadataChangeGroup::Users,
+ 0b10_0000,
+ ids(&[7]),
+ ),
+ (
+ "personal_access_tokens",
+ WireMetadataChangeGroup::PersonalAccessTokens,
+ 0b110_0000,
+ token_name_key(7, "ci-token"),
+ ),
+ (
+ "sessions",
+ WireMetadataChangeGroup::Sessions,
+ 0b1010_0000,
+ 4500u64.to_le_bytes().to_vec(),
+ ),
+ (
+ "longest_token_name",
+ WireMetadataChangeGroup::PersonalAccessTokens,
+ all,
+ token_name_key(u32::MAX, &"n".repeat(255)),
+ ),
+ ];
+ cases
+ .into_iter()
+ .map(|(case, group, mask, key)| {
+ let token = page_token(group, mask, key);
+ let wire: &WireMetadataCurrentPageToken = token.as_ref();
+ PageTokenVector {
+ case: case.to_owned(),
+ text: token.to_string(),
+ hex: hex::encode(wire.to_bytes()),
+ anchor: token.anchor().to_string(),
+ required_head: token.required_head(),
+ groups_mask: token.groups().mask(),
+ group: token.last_group().to_string(),
+ last_key_hex: hex::encode(token.last_key()),
+ }
+ })
+ .collect()
+}
+
+fn request_vectors() -> FixtureResult<Vec<RequestVector>> {
+ let cursor = MetadataChangeCursor {
+ log_id: LOG_ID,
+ op: 4187,
+ checksum: 0xf2a4_c6e8_b0d1_f359,
+ ordinal: 1,
+ }
+ .to_string();
+ let topics_token = page_token(
+ WireMetadataChangeGroup::Topics,
+ 0b11,
+ [3u32, 0]
+ .iter()
+ .flat_map(|value| value.to_le_bytes())
+ .collect(),
+ )
+ .to_string();
+ let inputs = vec![
+ ("earliest_every_group", QueryInput::new("earliest")),
+ (
+ "earliest_many_duplicate_groups",
+ QueryInput {
+ groups: std::iter::repeat_n("streams".to_owned(), 255)
+ .chain(std::iter::once("users".to_owned()))
+ .collect(),
+ ..QueryInput::new("earliest")
+ },
+ ),
+ (
+ "earliest_since_users_and_tokens",
+ QueryInput {
+ groups: vec!["users".to_owned(), "personal_access_tokens".to_owned()],
+ since: Some(1_760_353_200_000_000),
+ ..QueryInput::new("earliest")
+ },
+ ),
+ (
+ "latest_streams",
+ QueryInput {
+ groups: vec!["streams".to_owned()],
+ limit: 1,
+ ..QueryInput::new("latest")
+ },
+ ),
+ (
+ "after_cursor_with_time_range",
+ QueryInput {
+ cursor: Some(cursor),
+ groups: vec!["topics".to_owned(), "streams".to_owned()],
+ since: Some(1_760_356_800_000_000),
+ until: Some(1_760_360_400_000_000),
+ limit: 500,
+ ..QueryInput::new("after")
+ },
+ ),
+ (
+ "current_first_page",
+ QueryInput {
+ groups: vec!["streams".to_owned(), "topics".to_owned()],
+ limit: 1000,
+ ..QueryInput::new("current")
+ },
+ ),
+ (
+ "current_continuation",
+ QueryInput {
+ page_token: Some(topics_token),
+ groups: vec!["streams".to_owned(), "topics".to_owned()],
+ limit: 1000,
+ ..QueryInput::new("current")
+ },
+ ),
+ ];
+ inputs
+ .into_iter()
+ .map(|(case, input)| {
+ let query = input.query()?;
+ query.validate()?;
+ let request = GetMetadataChangesRequest::from(&query);
+ let http_query = query
+ .http_query_pairs()
+ .into_iter()
+ .map(|(key, value)| format!("{key}={value}"))
+ .collect::<Vec<_>>()
+ .join("&");
+ Ok(RequestVector {
+ case: case.to_owned(),
+ input,
+ request_hex: hex::encode(request.to_bytes()),
+ http_query,
+ })
+ })
+ .collect()
+}
+
+fn response_vector(
+ case: &str,
+ response: GetMetadataChangesResponse,
+) -> FixtureResult<ResponseVector> {
+ let response_hex = hex::encode(response.to_bytes());
+ Ok(ResponseVector {
+ case: case.to_owned(),
+ response_hex,
+ expected: MetadataChanges::try_from(response)?,
+ })
+}
+
+fn wire_cursor(op: u64, ordinal: u32) -> WireMetadataChangeCursor {
+ WireMetadataChangeCursor {
+ log_id: LOG_ID,
+ op,
+ checksum: 0x0000_0000_0000_0000_a5c1_f703_8b9e_2d4c + u128::from(op),
+ ordinal,
+ }
+}
+
+fn response_vectors(
+ records: &[RecordVector],
+ items: &[ItemVector],
+) -> FixtureResult<Vec<ResponseVector>> {
+ let decode_record = |vector: &RecordVector| -> FixtureResult<WireMetadataChangeRecord> {
+ let bytes = hex::decode(&vector.record_hex)?;
+ Ok(iggy_binary_protocol::WireDecode::decode_from(&bytes)?)
+ };
+ let decode_item = |vector: &ItemVector| -> FixtureResult<WireMetadataItem> {
+ let bytes = hex::decode(&vector.item_hex)?;
+ Ok(iggy_binary_protocol::WireDecode::decode_from(&bytes)?)
+ };
+ let page_records = records
+ .iter()
+ .take(3)
+ .map(decode_record)
+ .collect::<FixtureResult<Vec<_>>>()?;
+ let last = page_records.last().ok_or("no records")?;
+ let changes = GetMetadataChangesResponse {
+ status: WireMetadataChangeStatus::Ok,
+ phase: WireMetadataChangePhase::Changes,
+ log_id: Some(LOG_ID),
+ oldest_op: 3871,
+ retained_since: 1_760_352_000_000_000,
+ head_op: 4190,
+ next: Some(wire_cursor(last.op, last.ordinal)),
+ has_more: true,
+ records: page_records,
+ items: vec![],
+ current_page_token: None,
+ };
+ let mut unknown_record = decode_record(records.first().ok_or("no records")?)?;
+ unknown_record.kind = 9;
+ unknown_record.body = Bytes::from_static(&[0xde, 0xad, 0xbe, 0xef]);
+ let mut future_group = unknown_record.clone();
+ future_group.group = 200;
+ future_group.kind = 1;
+ future_group.ordinal = 1;
+ let unknown = GetMetadataChangesResponse {
+ next: Some(wire_cursor(future_group.op, future_group.ordinal)),
+ has_more: false,
+ records: vec![unknown_record, future_group],
+ ..changes.clone()
+ };
+ let current_items = items
+ .iter()
+ .take(2)
+ .map(decode_item)
+ .collect::<FixtureResult<Vec<_>>>()?;
+ let current_token = page_token(
+ WireMetadataChangeGroup::Topics,
+ 0b11,
+ [3u32, 0]
+ .iter()
+ .flat_map(|value| value.to_le_bytes())
+ .collect(),
+ );
+ let current = GetMetadataChangesResponse {
+ status: WireMetadataChangeStatus::Ok,
+ phase: WireMetadataChangePhase::Current,
+ log_id: Some(LOG_ID),
+ oldest_op: 1,
+ retained_since: 0,
+ head_op: 4190,
+ next: Some(wire_cursor(4187, 1)),
+ has_more: true,
+ records: vec![],
+ items: current_items,
+ current_page_token: Some(current_token.into()),
+ };
+ let behind = GetMetadataChangesResponse {
+ status: WireMetadataChangeStatus::Behind,
+ phase: WireMetadataChangePhase::Changes,
+ log_id: None,
+ oldest_op: 0,
+ retained_since: 0,
+ head_op: 0,
+ next: None,
+ has_more: false,
+ records: vec![],
+ items: vec![],
+ current_page_token: None,
+ };
+ let expired = GetMetadataChangesResponse {
+ status: WireMetadataChangeStatus::Expired,
+ log_id: Some(LOG_ID),
+ oldest_op: 5000,
+ retained_since: 1_760_360_000_000_000,
+ head_op: 9000,
+ ..behind.clone()
+ };
+ let reset = GetMetadataChangesResponse {
+ status: WireMetadataChangeStatus::Reset,
+ ..expired.clone()
+ };
+ Ok(vec![
+ response_vector("changes_page", changes)?,
+ response_vector("unknown_kind_and_group", unknown)?,
+ response_vector("current_page", current)?,
+ response_vector("behind_before_op_1", behind)?,
+ response_vector("expired", expired)?,
+ response_vector("reset", reset)?,
+ ])
+}
+
+fn large_values_vector() -> FixtureResult<LargeValuesVector> {
+ let client_id = u128::MAX - 1;
+ let change = WireMetadataChange::SessionRegistered(WireSessionBody {
+ user: WireUserKey {
+ user_id: u32::MAX,
+ user_created_at: ABOVE_2_POW_53,
+ },
+ client_id,
+ session: ABOVE_2_POW_53 + 1,
+ });
+ let mut record = wire_record(&change, ABOVE_2_POW_53 + 2, u32::MAX, Some(u32::MAX));
+ record.checksum = u128::MAX;
+ record.timestamp = u64::MAX;
+ Ok(LargeValuesVector {
+ record_hex: record_hex(&record),
+ op: record.op.to_string(),
+ ordinal: record.ordinal,
+ timestamp: record.timestamp.to_string(),
+ checksum: format!("{:032x}", record.checksum),
+ user_created_at: ABOVE_2_POW_53.to_string(),
+ session: (ABOVE_2_POW_53 + 1).to_string(),
+ client_id: format!("{client_id:032x}"),
+ })
+}
+
+fn refused_vectors() -> FixtureResult<RefusedVectors> {
+ let valid_cursor = MetadataChangeCursor {
+ log_id: LOG_ID,
+ op: 4187,
+ checksum: 0xf2a4_c6e8_b0d1_f359,
+ ordinal: 1,
+ }
+ .to_string();
+ let encoded = &valid_cursor[4..];
+ let non_canonical_cursor = WireMetadataChangeCursor {
+ log_id: LOG_ID,
+ op: 0,
+ checksum: 1,
+ ordinal: 0,
+ };
+ let cursors = vec![
+ ("wrong_prefix", format!("mc2_{encoded}")),
+ ("missing_prefix", encoded.to_owned()),
+ ("too_long", format!("{valid_cursor}A")),
+ ("too_short", valid_cursor[..62].to_owned()),
+ ("padded", format!("mc1_{}=", &encoded[..58])),
+ (
+ "standard_alphabet_character",
+ format!("mc1_+{}", &encoded[1..]),
+ ),
+ (
+ "op_zero_with_checksum",
+ format!(
+ "mc1_{}",
+ base64_url(&WireMetadataChangeCursor::to_bytes(&non_canonical_cursor))
+ ),
+ ),
+ ];
+ let valid_token = page_token(
+ WireMetadataChangeGroup::Streams,
+ 0b1,
+ 3u32.to_le_bytes().to_vec(),
+ );
+ let token_text = valid_token.to_string();
+ let mut wrong_width = WireMetadataCurrentPageToken::from(valid_token.clone());
+ wrong_width.last_key = Bytes::from_static(&[3, 0, 0, 0, 0, 0, 0, 0]);
+ let mut outside_mask = WireMetadataCurrentPageToken::from(valid_token.clone());
+ outside_mask.groups_mask = 0b10;
+ let mut head_below_anchor = WireMetadataCurrentPageToken::from(valid_token);
+ head_below_anchor.required_head = 1;
+ let page_tokens = vec![
+ ("wrong_prefix", token_text.replacen("cs1_", "mc1_", 1)),
+ (
+ "trailing_byte",
+ format!(
+ "cs1_{}",
+ base64_url(&[unvalidated_token_bytes(&token_text)?, vec![0]].concat())
+ ),
+ ),
+ (
+ "wrong_key_width",
+ format!("cs1_{}", base64_url(&wrong_width.to_bytes())),
+ ),
+ (
+ "group_outside_mask",
+ format!("cs1_{}", base64_url(&outside_mask.to_bytes())),
+ ),
+ (
+ "head_below_anchor",
+ format!("cs1_{}", base64_url(&head_below_anchor.to_bytes())),
+ ),
+ ("empty", "cs1_".to_owned()),
+ ];
+
+ let records = change_cases()?;
+ let body = |case: &str| -> FixtureResult<(WireMetadataChange, Vec<u8>)> {
+ let (_, change, _) = records
+ .iter()
+ .find(|(name, _, _)| *name == case)
+ .ok_or("missing case")?;
+ Ok((change.clone(), change.body_bytes().to_vec()))
+ };
+ let (member_left, mut bad_reason) = body("consumer_group_member_left_session_ended")?;
+ if let Some(last) = bad_reason.last_mut() {
+ *last = 2;
+ }
+ let (stream_deleted, mut bad_presence) = body("stream_deleted_deferred")?;
+ let presence_offset = bad_presence.len() - 9;
+ bad_presence[presence_offset] = 2;
+ let (partitions, mut unsorted_ids) = body("partitions_changed_deleted_tail")?;
+ let first_id = 24 + 4 + 4;
+ unsorted_ids[first_id..first_id + 4].copy_from_slice(&9u32.to_le_bytes());
+ let (stream_created, stream_body) = body("stream_created")?;
+ let (session, session_body) = body("session_registered")?;
+ let record_bodies = vec![
+ RefusedBody {
+ case: "unknown_member_left_reason".to_owned(),
+ group: member_left.group().as_u8(),
+ kind: member_left.kind().as_u8(),
+ body_hex: hex::encode(&bad_reason),
+ },
+ RefusedBody {
+ case: "presence_byte_two".to_owned(),
+ group: stream_deleted.group().as_u8(),
+ kind: stream_deleted.kind().as_u8(),
+ body_hex: hex::encode(&bad_presence),
+ },
+ RefusedBody {
+ case: "unsorted_removed_partition_ids".to_owned(),
+ group: partitions.group().as_u8(),
+ kind: partitions.kind().as_u8(),
+ body_hex: hex::encode(&unsorted_ids),
+ },
+ RefusedBody {
+ case: "known_kind_under_wrong_group".to_owned(),
+ group: WireMetadataChangeGroup::Topics.as_u8(),
+ kind: stream_created.kind().as_u8(),
+ body_hex: hex::encode(&stream_body),
+ },
+ RefusedBody {
+ case: "trailing_byte".to_owned(),
+ group: session.group().as_u8(),
+ kind: session.kind().as_u8(),
+ body_hex: hex::encode([session_body.as_slice(), &[0]].concat()),
+ },
+ RefusedBody {
+ case: "truncated".to_owned(),
+ group: session.group().as_u8(),
+ kind: session.kind().as_u8(),
+ body_hex: hex::encode(&session_body[..session_body.len() - 1]),
+ },
+ ];
+
+ let earliest = GetMetadataChangesRequest::from(&MetadataChangesQuery::default())
+ .to_bytes()
+ .to_vec();
+ let mut cursor_without_after = earliest.clone();
+ cursor_without_after[5] = 1;
+ let mut trailing = earliest.clone();
+ trailing.push(0);
+ let mut unknown_start = earliest;
+ unknown_start[0] = 4;
+ let mut non_canonical_after = vec![WireMetadataChangeStartMode::After.as_u8()];
+ non_canonical_after.extend(WireMetadataChangeCursor::to_bytes(&non_canonical_cursor));
+ non_canonical_after.extend([0, 0]);
+ non_canonical_after.extend(DEFAULT_METADATA_CHANGES_LIMIT.to_le_bytes());
+ non_canonical_after.extend([0, 0, 0]);
+ let requests = vec![
+ RefusedHex {
+ case: "cursor_bytes_without_after".to_owned(),
+ hex: hex::encode(&cursor_without_after),
+ },
+ RefusedHex {
+ case: "trailing_byte".to_owned(),
+ hex: hex::encode(&trailing),
+ },
+ RefusedHex {
+ case: "unknown_start_mode".to_owned(),
+ hex: hex::encode(&unknown_start),
+ },
+ RefusedHex {
+ case: "non_canonical_cursor".to_owned(),
+ hex: hex::encode(&non_canonical_after),
+ },
+ ];
+
+ let items_in_changes_phase = GetMetadataChangesResponse {
+ status: WireMetadataChangeStatus::Ok,
+ phase: WireMetadataChangePhase::Changes,
+ log_id: Some(LOG_ID),
+ oldest_op: 1,
+ retained_since: 0,
+ head_op: 2,
+ next: Some(wire_cursor(2, 0)),
+ has_more: false,
+ records: vec![],
+ items: vec![WireMetadataItem {
+ group: 1,
+ body: item_body_for_refusal()?,
+ }],
+ current_page_token: None,
+ }
+ .to_bytes()
+ .to_vec();
+ let mut unknown_status = items_in_changes_phase.clone();
+ unknown_status[0] = 4;
+ let responses = vec![
+ RefusedHex {
+ case: "items_in_changes_phase".to_owned(),
+ hex: hex::encode(&items_in_changes_phase),
+ },
+ RefusedHex {
+ case: "unknown_status".to_owned(),
+ hex: hex::encode(&unknown_status),
+ },
+ ];
+
+ let queries = vec![
+ (
+ "limit_zero",
+ QueryInput {
+ limit: 0,
+ ..QueryInput::new("earliest")
+ },
+ ),
+ (
+ "limit_above_maximum",
+ QueryInput {
+ limit: MAX_METADATA_CHANGES_LIMIT + 1,
+ ..QueryInput::new("earliest")
+ },
+ ),
+ (
+ "since_above_until",
+ QueryInput {
+ since: Some(2),
+ until: Some(1),
+ ..QueryInput::new("earliest")
+ },
+ ),
+ (
+ "time_bound_with_latest",
+ QueryInput {
+ since: Some(1),
+ ..QueryInput::new("latest")
+ },
+ ),
+ (
+ "time_bound_with_current",
+ QueryInput {
+ until: Some(1),
+ ..QueryInput::new("current")
+ },
+ ),
+ (
+ "current_missing_parent_group",
+ QueryInput {
+ groups: vec!["topics".to_owned()],
+ ..QueryInput::new("current")
+ },
+ ),
+ (
+ "page_token_for_other_groups",
+ QueryInput {
+ page_token: Some(
+ page_token(
+ WireMetadataChangeGroup::Streams,
+ 0b1,
+ 3u32.to_le_bytes().to_vec(),
+ )
+ .to_string(),
+ ),
+ groups: vec!["streams".to_owned(), "topics".to_owned()],
+ ..QueryInput::new("current")
+ },
+ ),
+ ];
+ let http_queries = vec![
+ ("no_start", "groups=streams"),
+ ("from_and_after", "from=earliest&after=mc1_x"),
+ ("unknown_from", "from=newest"),
+ ("duplicate_key", "from=earliest&from=latest"),
+ ("unknown_key", "from=earliest&verbose=1"),
+ (
+ "empty_group_element",
+ "from=earliest&groups=streams,,topics",
+ ),
+ ("trailing_group_comma", "from=earliest&groups=streams,"),
+ ("unknown_group", "from=earliest&groups=everything"),
+ ("negative_since", "from=earliest&since=-1"),
+ ("leading_zero_until", "from=earliest&until=01"),
+ ("non_numeric_limit", "from=earliest&limit=ten"),
+ (
+ "token_without_current",
+ "from=latest¤t_after=cs1_AAAA",
+ ),
+ ];
+ Ok(RefusedVectors {
+ cursors: cursors
+ .into_iter()
+ .map(|(case, text)| RefusedText {
+ case: case.to_owned(),
+ text,
+ })
+ .collect(),
+ page_tokens: page_tokens
+ .into_iter()
+ .map(|(case, text)| RefusedText {
+ case: case.to_owned(),
+ text,
+ })
+ .collect(),
+ record_bodies,
+ requests,
+ responses,
+ queries: queries
+ .into_iter()
+ .map(|(case, input)| RefusedQuery {
+ case: case.to_owned(),
+ input,
+ })
+ .collect(),
+ http_queries: http_queries
+ .into_iter()
+ .map(|(case, query)| RefusedHttpQuery {
+ case: case.to_owned(),
+ query: query.to_owned(),
+ })
+ .collect(),
+ })
+}
+
+fn item_body_for_refusal() -> FixtureResult<Bytes> {
+ Ok(WireMetadataCurrentItem::Stream(WireStreamBody {
+ key: stream_key(),
+ created_at: 1,
+ name: name("orders")?,
+ options: empty_options(),
+ })
+ .body_bytes())
+}
+
+fn unvalidated_token_bytes(text: &str) -> FixtureResult<Vec<u8>> {
+ let encoded = text.strip_prefix("cs1_").ok_or("missing prefix")?;
+ Ok(URL_SAFE_NO_PAD.decode(encoded)?)
+}
+
+fn base64_url(bytes: &[u8]) -> String {
+ URL_SAFE_NO_PAD.encode(bytes)
+}
diff --git a/core/common/fixtures/metadata_changes.json b/core/common/fixtures/metadata_changes.json
new file mode 100644
index 000000000..26e7faf22
--- /dev/null
+++ b/core/common/fixtures/metadata_changes.json
@@ -0,0 +1,1997 @@
+{
+ "schema_version": 1,
+ "generated_by": "core/common/examples/metadata_changes_fixtures.rs",
+ "conventions": {
+ "byte_strings": "Every *_hex field is lowercase hex without separators.",
+ "expected": "expected is the JSON the HTTP route renders. Binary decoders must reach the same values.",
+ "record_hex": "record_hex is one record envelope as a response carries it. body_hex is the body alone.",
+ "refused": "Every refused vector must fail to decode or validate. Query and HTTP refusals are invalid_command.",
+ "request_hex": "request_hex is the command payload without framing. http_query is the query string of GET /metadata/changes.",
+ "u128_values": "128-bit values (log ids, checksums, client ids) are 32 lowercase hex characters in JSON.",
+ "u64_values": "Ops and timestamps are JSON numbers below 2^53 in every vector except large_values, which carries them as decimal strings.",
+ "unknown_kinds": "A record of an unknown group or kind is kept with its codes and raw body, never refused."
+ },
+ "constants": {
+ "command_code": 17,
+ "command_name": "metadata.changes",
+ "cursor_prefix": "mc1_",
+ "page_token_prefix": "cs1_",
+ "cursor_size": 44,
+ "cursor_text_length": 63,
+ "max_page_token_size": 316,
+ "max_page_token_key_size": 260,
+ "default_limit": 100,
+ "max_limit": 1000,
+ "groups": {
+ "consumer_group_members": {
+ "code": 5,
+ "class": "activity",
+ "parents": [
+ "consumer_groups",
+ "partitions",
+ "topics",
+ "streams"
+ ],
+ "page_token_key": "[stream_id:u32][topic_id:u32][group_id:u32][member_id:u32]"
+ },
+ "consumer_groups": {
+ "code": 4,
+ "class": "resource",
+ "parents": [
+ "topics",
+ "streams"
+ ],
+ "page_token_key": "[stream_id:u32][topic_id:u32][group_id:u32]"
+ },
+ "partitions": {
+ "code": 3,
+ "class": "resource",
+ "parents": [
+ "topics",
+ "streams"
+ ],
+ "page_token_key": "[stream_id:u32][topic_id:u32]"
+ },
+ "personal_access_tokens": {
+ "code": 7,
+ "class": "resource",
+ "parents": [
+ "users"
+ ],
+ "page_token_key": "[user_id:u32][name_len:u8][name]"
+ },
+ "sessions": {
+ "code": 8,
+ "class": "activity",
+ "parents": [
+ "users"
+ ],
+ "page_token_key": "[epoch:u64]"
+ },
+ "streams": {
+ "code": 1,
+ "class": "resource",
+ "parents": [],
+ "page_token_key": "[stream_id:u32]"
+ },
+ "topics": {
+ "code": 2,
+ "class": "resource",
+ "parents": [
+ "streams"
+ ],
+ "page_token_key": "[stream_id:u32][topic_id:u32]"
+ },
+ "users": {
+ "code": 6,
+ "class": "resource",
+ "parents": [],
+ "page_token_key": "[user_id:u32]"
+ }
+ },
+ "kinds": {
+ "consumer_group_created": {
+ "code": 30,
+ "group": "consumer_groups"
+ },
+ "consumer_group_deleted": {
+ "code": 31,
+ "group": "consumer_groups"
+ },
+ "consumer_group_member_joined": {
+ "code": 40,
+ "group": "consumer_group_members"
+ },
+ "consumer_group_member_left": {
+ "code": 41,
+ "group": "consumer_group_members"
+ },
+ "consumer_group_partition_handed_over": {
+ "code": 42,
+ "group": "consumer_group_members"
+ },
+ "partitions_changed": {
+ "code": 20,
+ "group": "partitions"
+ },
+ "personal_access_token_created": {
+ "code": 60,
+ "group": "personal_access_tokens"
+ },
+ "personal_access_token_deleted": {
+ "code": 61,
+ "group": "personal_access_tokens"
+ },
+ "session_closed": {
+ "code": 71,
+ "group": "sessions"
+ },
+ "session_registered": {
+ "code": 70,
+ "group": "sessions"
+ },
+ "stream_created": {
+ "code": 1,
+ "group": "streams"
+ },
+ "stream_deleted": {
+ "code": 3,
+ "group": "streams"
+ },
+ "stream_updated": {
+ "code": 2,
+ "group": "streams"
+ },
+ "topic_created": {
+ "code": 10,
+ "group": "topics"
+ },
+ "topic_deleted": {
+ "code": 12,
+ "group": "topics"
+ },
+ "topic_updated": {
+ "code": 11,
+ "group": "topics"
+ },
+ "user_created": {
+ "code": 50,
+ "group": "users"
+ },
+ "user_deleted": {
+ "code": 52,
+ "group": "users"
+ },
+ "user_password_changed": {
+ "code": 53,
+ "group": "users"
+ },
+ "user_permissions_updated": {
+ "code": 54,
+ "group": "users"
+ },
+ "user_updated": {
+ "code": 51,
+ "group": "users"
+ }
+ },
+ "statuses": {
+ "behind": 1,
+ "expired": 2,
+ "ok": 0,
+ "reset": 3
+ },
+ "phases": {
+ "changes": 0,
+ "current": 1
+ },
+ "start_modes": {
+ "after": 2,
+ "current": 3,
+ "earliest": 0,
+ "latest": 1
+ },
+ "member_left_reasons": {
+ "left": 0,
+ "session_ended": 1
+ },
+ "token_deleted_reasons": {
+ "deleted": 0,
+ "expired": 1
+ },
+ "session_closed_reasons": {
+ "expired": 1,
+ "logout": 0,
+ "revoked": 2
+ },
+ "permissions_updated_reasons": {
+ "resource_deleted": 1,
+ "updated": 0
... 72482 lines suppressed ...