You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by pi...@apache.org on 2026/10/22 18:50:42 UTC
(iggy) branch feat/spread-placement updated (15b176c16 -> bb9041ec5)
This is an automated email from the ASF dual-hosted git repository.
spetz pushed a change to branch feat/spread-placement
in repository https://gitbox.apache.org/repos/asf/iggy.git
omit 15b176c16 fix
new bb9041ec5 fix
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (15b176c16)
\
N -- N -- N refs/heads/feat/spread-placement (bb9041ec5)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
foreign/go/client/tcp/tcp_connect_test.go | 23 ++++++++++++++++++++++
foreign/go/client/tcp/tcp_session_management.go | 3 ++-
.../swift/Tests/IggyE2ETests/ClientE2ETests.swift | 12 +++++++++--
3 files changed, 35 insertions(+), 3 deletions(-)
(iggy) 01/01: fix
Posted by pi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spetz pushed a commit to branch feat/spread-placement
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit bb9041ec502dabd2c496692501221a51356b1a1c
Author: spetz <pi...@gmail.com>
AuthorDate: Thu Oct 22 19:12:16 2026 +0200
fix
---
.dockerignore | 8 +
.github/config/components.yml | 1 +
bdd/README.md | 5 +-
.../features/step_definitions/background_steps.cpp | 18 +-
.../step_definitions/session_contract_steps.cpp | 93 ++++
bdd/cpp/features/step_definitions/world.hpp | 11 +
bdd/cpp/scripts/entrypoint.sh | 1 +
bdd/docker-compose.coverage.yml | 4 +-
bdd/docker-compose.no-quorum.yml | 134 +++++
bdd/docker-compose.yml | 29 +-
bdd/go/tests/env/env.go | 5 +
bdd/go/tests/session_contract.go | 139 ++++++
bdd/go/tests/suite_test.go | 11 +
bdd/java/build.gradle.kts | 4 +
.../org/apache/iggy/bdd/BasicMessagingSteps.java | 45 ++
.../test/java/org/apache/iggy/bdd/TestContext.java | 1 +
.../java/org/apache/iggy/bdd/TestEnvironment.java | 4 +
bdd/node/Dockerfile | 1 +
bdd/php/tests/SessionContractFeatureTest.php | 118 +++++
bdd/php/tests/SharedFeatureParser.php | 4 +-
bdd/php/tests/TestEnvironment.php | 5 +
bdd/python/tests/conftest.py | 7 +
bdd/python/tests/test_session_contract.py | 101 ++++
bdd/rust/Cargo.toml | 5 +
bdd/rust/tests/common/global_context.rs | 1 +
bdd/rust/tests/helpers/env.rs | 4 +
.../tests/{steps/mod.rs => session_contract.rs} | 21 +-
bdd/rust/tests/steps/mod.rs | 1 +
bdd/rust/tests/steps/session.rs | 65 +++
.../session_contract.feature} | 24 +-
core/configs/src/server_config/defaults.rs | 2 +
core/configs/src/server_config/displays.rs | 7 +-
core/configs/src/server_config/message_bus.rs | 69 +++
core/consensus/src/impls.rs | 13 +-
core/consensus/src/metadata_helpers.rs | 1 -
core/consensus/src/plane_helpers.rs | 2 -
.../tests/cluster/partition_primary_routing.rs | 496 +++++++++++++++++-
core/message_bus/src/config.rs | 5 +
core/message_bus/src/error.rs | 34 +-
core/message_bus/src/installer/mod.rs | 4 +-
core/message_bus/src/lib.rs | 167 ++-----
.../src/lifecycle/connection_registry.rs | 161 +++---
core/message_bus/src/transports/mod.rs | 2 +-
core/metadata/src/impls/metadata.rs | 87 ++--
core/metadata/src/impls/placement.rs | 556 +++++++++++++++++----
core/metadata/src/stm/stream.rs | 118 ++++-
core/partitions/src/iggy_partition.rs | 199 +++++---
core/partitions/src/io.rs | 4 +-
core/partitions/src/lib.rs | 8 +
core/partitions/src/persistence.rs | 47 +-
core/partitions/src/state_transfer.rs | 6 +-
core/partitions/src/wal_storage.rs | 442 ++++++++++++++++
core/sdk/src/links/mux.rs | 72 ++-
core/server/README.md | 16 +-
core/server/config.toml | 14 +
core/server/src/boot/mod.rs | 2 +-
core/server/src/boot/threads.rs | 2 +-
core/server/src/consumer_group/liveness.rs | 22 +-
core/server/src/dispatch/failure.rs | 3 +-
core/server/src/dispatch/mod.rs | 150 +++++-
core/server/src/dispatch/partition.rs | 2 +-
core/server/src/dispatch/test_support.rs | 7 +-
core/server/src/http/error.rs | 3 -
core/server/src/http/extractor.rs | 146 +++++-
core/server/src/http/forward.rs | 261 +++++++---
core/server/src/http/state.rs | 2 +
core/server/src/partition_helpers.rs | 2 +-
core/shard/src/builder.rs | 39 +-
core/shard/src/lib.rs | 122 +++--
core/shard/src/metrics.rs | 64 ++-
core/shard/src/partition_io.rs | 149 +++++-
core/shard/src/poll/timeout_tests.rs | 4 +-
core/shard/src/primary_rebalance/tests.rs | 1 -
core/shard/src/router.rs | 11 +-
core/simulator/src/bus.rs | 6 +-
core/simulator/src/lib.rs | 123 ++++-
core/simulator/src/partition_wal.rs | 119 +++++
core/simulator/src/workload/invariants.rs | 40 +-
core/system_stats/src/lib.rs | 9 +-
foreign/cpp/BUILD.bazel | 1 +
.../Iggy_SDK.Tests.BDD/Context/TestContext.cs | 2 +
.../Iggy_SDK.Tests.BDD/Context/TestEnvironment.cs | 1 +
.../csharp/Iggy_SDK.Tests.BDD/Context/TestHooks.cs | 1 +
.../Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj | 3 +
.../StepDefinitions/SessionContractSteps.cs | 89 ++++
.../IggyConsumerTests.cs | 3 +-
.../IggyTypedConsumerTests.cs | 3 +-
.../csharp/Iggy_SDK/Consumers/AutoCommitMode.cs | 4 +-
.../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 2 +-
foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 12 +-
.../csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs | 2 +-
.../TcpMessageStream.PollRouting.cs | 10 +-
.../PartitionIncarnationTests.Disposal.cs | 39 +-
foreign/csharp/README.md | 2 +
foreign/go/README.md | 13 +-
foreign/go/client/tcp/tcp_connect_test.go | 73 +++
foreign/go/client/tcp/tcp_core.go | 15 +-
foreign/go/client/tcp/tcp_driver.go | 14 +-
foreign/go/client/tcp/tcp_replicas_test.go | 147 ++++--
foreign/go/client/tcp/tcp_session_management.go | 48 +-
foreign/go/errors/errors.yaml | 6 +-
foreign/go/errors/errors_gen.go | 5 +-
foreign/go/errors/errors_test.go | 6 +-
foreign/java/README.md | 4 +
.../iggy/client/async/tcp/AsyncIggyTcpClient.java | 7 +-
.../iggy/client/async/tcp/AsyncTcpConnection.java | 142 ++----
.../iggy/client/async/tcp/ConnectionHeartbeat.java | 132 +++++
.../iggy/client/async/tcp/IggyAuthenticator.java | 15 +-
.../client/async/tcp/vsr/ConsensusSession.java | 20 +-
.../client/async/AsyncConnectionPoolAuthTest.java | 22 +-
.../AsyncIggyTcpClientEndpointFailoverTest.java | 14 +-
.../tcp/AsyncIggyTcpClientFailoverSignInTest.java | 5 +-
.../tcp/AsyncIggyTcpClientLostLinkReplayTest.java | 105 ++++
.../tcp/AsyncIggyTcpClientSessionRulesTest.java | 77 ++-
.../tcp/AsyncTcpConnectionLifecycleBusyTest.java | 102 ++++
foreign/node/README.md | 6 +-
foreign/node/src/bdd/README.md | 1 +
foreign/node/src/bdd/env.ts | 10 +-
foreign/node/src/bdd/session.ts | 63 +++
foreign/node/src/bdd/world.ts | 3 +-
foreign/node/src/client/client.socket.test.ts | 49 +-
foreign/node/src/client/client.socket.ts | 25 +-
foreign/node/src/wire/error.code.ts | 2 +-
foreign/python/README.md | 17 +-
foreign/python/apache_iggy.pyi | 18 +-
foreign/python/src/bin/stub_gen.rs | 2 +
foreign/python/src/client.rs | 65 ++-
foreign/python/src/identifier.rs | 11 +-
foreign/python/src/lib.rs | 3 +-
foreign/python/src/producer.rs | 42 +-
foreign/python/tests/test_raw_command.py | 27 +-
.../swift/Tests/IggyE2ETests/ClientE2ETests.swift | 12 +-
scripts/ci/apt-install.sh | 72 ++-
scripts/run-bdd-tests.sh | 11 +-
134 files changed, 5162 insertions(+), 1114 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index 02100b387..b4405acc2 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -11,6 +11,14 @@
/licenses
/local_data
/performance_results
+/test_logs
+/foreign/swift/.build
+/bdd/java/build
+/examples/java/**/build
+/foreign/java/**/build
+/web/build
+**/.gradle
+**/node_modules
# Selectively exclude scripts/ subdirs not needed in container builds.
# scripts/ci/third-party-licenses.sh + render-node-licenses.mjs are
# required by the convenience-binary Dockerfiles to render LICENSE-binary
diff --git a/.github/config/components.yml b/.github/config/components.yml
index fc66b3d01..c304adb51 100644
--- a/.github/config/components.yml
+++ b/.github/config/components.yml
@@ -354,6 +354,7 @@ components:
- "bdd/docker-compose.yml"
- "bdd/docker-compose.server.yml"
- "bdd/docker-compose.cluster.yml"
+ - "bdd/docker-compose.no-quorum.yml"
- "bdd/docker-compose.coverage.yml"
# Individual BDD tests per SDK - only run when specific SDK changes
diff --git a/bdd/README.md b/bdd/README.md
index b50246b03..a1507ae33 100644
--- a/bdd/README.md
+++ b/bdd/README.md
@@ -38,6 +38,7 @@ bdd/
├── docker-compose.yml # Base: SDK test clients (always included)
├── docker-compose.server.yml # Single iggy-server test setup
├── docker-compose.cluster.yml # Leader + follower test setup
+├── docker-compose.no-quorum.yml # Cluster node that never reaches quorum
├── docker-compose.coverage.yml # Coverage collection overlay
├── Dockerfile # Debug build of Iggy server
└── README.md
@@ -50,7 +51,7 @@ bdd/
```bash
# Usage: ../scripts/run-bdd-tests.sh [--coverage] <sdk> [feature]
# sdk: rust | python | php | go | go-race | node | csharp | java | all | clean (default: all)
-# feature: basic_messaging | leader_redirection | all (default: all)
+# feature: basic_messaging | leader_redirection | raw_command | session_contract | stream_crud | all (default: all)
# Run all features for all SDKs
../scripts/run-bdd-tests.sh all
@@ -84,7 +85,7 @@ bdd/
### How it Works
-1. **Server Containers**: `docker-compose.server.yml` runs a single Iggy server; `docker-compose.cluster.yml` adds a leader + follower pair for cluster scenarios
+1. **Server Containers**: `docker-compose.server.yml` runs a single Iggy server; `docker-compose.cluster.yml` adds a leader + follower pair for cluster scenarios. `docker-compose.no-quorum.yml` adds a cluster node whose only peer never starts, so it refuses every sign-in until the caller's deadline
2. **SDK Containers**: Each SDK has its own container with the appropriate runtime and dependencies
3. **Shared Features**: All SDKs test against the same `.feature` files for consistency
4. **Health Checks**: Containers wait for the server to be healthy before running tests
diff --git a/bdd/cpp/features/step_definitions/background_steps.cpp b/bdd/cpp/features/step_definitions/background_steps.cpp
index 55cfa0935..f4673c2e5 100644
--- a/bdd/cpp/features/step_definitions/background_steps.cpp
+++ b/bdd/cpp/features/step_definitions/background_steps.cpp
@@ -26,29 +26,17 @@
#include <cucumber-cpp/autodetect.hpp>
-#include <cstdlib>
-#include <stdexcept>
#include <string>
#include <utility>
#include "world.hpp"
-namespace {
-std::string required_env(const char *name) {
- const char *value = std::getenv(name);
- if (value == nullptr || *value == '\0') {
- throw std::runtime_error(std::string(name) + " must be set; run the suite via scripts/run-bdd-tests.sh");
- }
- return std::string(value);
-}
-} // namespace
-
GIVEN("^I have a running Iggy server$") {
cucumber::ScenarioScope<bdd::GlobalContext> context;
// Empty address makes the SDK fall back to its default TCP endpoint; in CI the address
// is supplied via IGGY_TCP_ADDRESS (e.g. iggy-server:8090).
- const std::string address = required_env("IGGY_TCP_ADDRESS");
+ const std::string address = bdd::required_env("IGGY_TCP_ADDRESS");
iggy::ffi::IggyClientConfig config{};
config.server_address = address;
iggy::ffi::Client *client = iggy::ffi::new_connection(std::move(config));
@@ -62,7 +50,7 @@ GIVEN("^I am authenticated as the root user$") {
cucumber::ScenarioScope<bdd::GlobalContext> context;
ASSERT_NE(context->client, nullptr);
- const std::string username = required_env("IGGY_ROOT_USERNAME");
- const std::string password = required_env("IGGY_ROOT_PASSWORD");
+ const std::string username = bdd::required_env("IGGY_ROOT_USERNAME");
+ const std::string password = bdd::required_env("IGGY_ROOT_PASSWORD");
context->client->login_user(username, password, error_info);
}
diff --git a/bdd/cpp/features/step_definitions/session_contract_steps.cpp b/bdd/cpp/features/step_definitions/session_contract_steps.cpp
new file mode 100644
index 000000000..d2ca6e986
--- /dev/null
+++ b/bdd/cpp/features/step_definitions/session_contract_steps.cpp
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#define CUKE_OBJECT_PREFIX IggyBddSessionContract
+
+#include <gtest/gtest.h>
+
+#include <cucumber-cpp/autodetect.hpp>
+
+#include <cstdint>
+#include <exception>
+#include <map>
+#include <string>
+#include <utility>
+
+#include "world.hpp"
+
+namespace {
+// The bridge reports no error code for reads, so the check reads the error text.
+const std::map<std::uint32_t, std::string> ERROR_MESSAGES = {
+ {40, "Unauthenticated"},
+ {58, "Request transiently not accepted"},
+};
+} // namespace
+
+GIVEN("^I have a cluster node that cannot reach quorum$") {
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ iggy::ffi::IggyClientConfig config{};
+ config.server_address = bdd::required_env("IGGY_TCP_ADDRESS_NO_QUORUM");
+ iggy::ffi::Client *client = iggy::ffi::new_connection(std::move(config));
+ ASSERT_NE(client, nullptr);
+ context->client = client;
+ context->client->connect();
+}
+
+WHEN("^I sign out$") {
+ iggy::ffi::ErrorInfo error_info{};
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+ ASSERT_NE(context->client, nullptr);
+
+ context->client->logout_user(error_info);
+}
+
+WHEN("^I request the stream with ID ([0-9]+)$") {
+ REGEX_PARAM(std::uint32_t, stream_id);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+ ASSERT_NE(context->client, nullptr);
+
+ context->call_error.clear();
+ try {
+ context->client->get_stream(bdd::make_numeric_identifier(stream_id));
+ } catch (const std::exception &error) {
+ context->call_error = error.what();
+ }
+}
+
+WHEN("^I sign in to that node as the root user$") {
+ iggy::ffi::ErrorInfo error_info{};
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+ ASSERT_NE(context->client, nullptr);
+
+ context->call_error.clear();
+ try {
+ context->client->login_user(bdd::required_env("IGGY_ROOT_USERNAME"), bdd::required_env("IGGY_ROOT_PASSWORD"),
+ error_info);
+ } catch (const std::exception &error) {
+ context->call_error = error.what();
+ }
+}
+
+THEN("^the call should fail with error code ([0-9]+)$") {
+ REGEX_PARAM(std::uint32_t, code);
+ cucumber::ScenarioScope<bdd::GlobalContext> context;
+
+ EXPECT_NE(context->call_error.find(ERROR_MESSAGES.at(code)), std::string::npos) << context->call_error;
+}
diff --git a/bdd/cpp/features/step_definitions/world.hpp b/bdd/cpp/features/step_definitions/world.hpp
index 7b8fdc487..886be4fe4 100644
--- a/bdd/cpp/features/step_definitions/world.hpp
+++ b/bdd/cpp/features/step_definitions/world.hpp
@@ -20,6 +20,8 @@
#pragma once
#include <cstdint>
+#include <cstdlib>
+#include <stdexcept>
#include <string>
#include <vector>
@@ -47,6 +49,7 @@ struct GlobalContext {
PolledData polled;
std::vector<std::uint8_t> raw_response;
std::string raw_error;
+ std::string call_error;
GlobalContext() = default;
GlobalContext(const GlobalContext &) = delete;
@@ -63,6 +66,14 @@ struct GlobalContext {
}
};
+inline std::string required_env(const char *name) {
+ const char *value = std::getenv(name);
+ if (value == nullptr || *value == '\0') {
+ throw std::runtime_error(std::string(name) + " must be set; run the suite via scripts/run-bdd-tests.sh");
+ }
+ return std::string(value);
+}
+
inline iggy::ffi::Identifier make_string_identifier(const std::string &value) {
iggy::ffi::Identifier identifier;
identifier.set_string(value);
diff --git a/bdd/cpp/scripts/entrypoint.sh b/bdd/cpp/scripts/entrypoint.sh
index b9cf1b794..94c4eed4b 100755
--- a/bdd/cpp/scripts/entrypoint.sh
+++ b/bdd/cpp/scripts/entrypoint.sh
@@ -58,6 +58,7 @@ tags=()
case "${BDD_FEATURE:-all}" in
basic_messaging) tags=(--tags @basic-messaging) ;;
raw_command) tags=(--tags @raw-command) ;;
+ session_contract) tags=(--tags @session-contract) ;;
esac
# --strict fails the run on undefined steps; without it a feature step with no matching C++
diff --git a/bdd/docker-compose.coverage.yml b/bdd/docker-compose.coverage.yml
index e1c58fea8..7caa2153b 100644
--- a/bdd/docker-compose.coverage.yml
+++ b/bdd/docker-compose.coverage.yml
@@ -39,7 +39,7 @@ services:
trap 'cargo llvm-cov report --lcov --output-path /reports/rust-bdd-coverage.lcov' EXIT
TEST_ARGS=()
case "$$BDD_FEATURE" in
- basic_messaging|leader_redirection|raw_command|stream_crud) TEST_ARGS=(--test "$$BDD_FEATURE") ;;
+ basic_messaging|leader_redirection|raw_command|session_contract|stream_crud) TEST_ARGS=(--test "$$BDD_FEATURE") ;;
esac
cargo test --locked -p bdd --features "$$BDD_RUST_FEATURES" "$${TEST_ARGS[@]}"
@@ -103,6 +103,7 @@ services:
case "$$BDD_FEATURE" in
basic_messaging) FILTER='--filter-trait Category=basic-messaging' ;;
leader_redirection) FILTER='--filter-trait Category=requires-leader-awareness' ;;
+ session_contract) FILTER='--filter-trait Category=session-contract' ;;
esac
dotnet test $$FILTER --coverage --coverage-output-format cobertura --coverage-output csharp-bdd-coverage.cobertura.xml --results-directory /reports
@@ -117,6 +118,7 @@ services:
basic_messaging) export CUCUMBER_FILTER_TAGS='@basic-messaging' ;;
leader_redirection) export CUCUMBER_FILTER_TAGS='@requires-leader-awareness' ;;
raw_command) export CUCUMBER_FILTER_TAGS='@raw-command' ;;
+ session_contract) export CUCUMBER_FILTER_TAGS='@session-contract' ;;
stream_crud) export CUCUMBER_FILTER_TAGS='@stream-crud' ;;
esac
gradle --no-daemon test jacocoTestReport && cp build/reports/jacoco/test/jacocoTestReport.xml /reports/java-bdd-coverage.xml
diff --git a/bdd/docker-compose.no-quorum.yml b/bdd/docker-compose.no-quorum.yml
new file mode 100644
index 000000000..296d505ad
--- /dev/null
+++ b/bdd/docker-compose.no-quorum.yml
@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# A cluster node that can never reach quorum: replica 0 of a two-node roster
+# whose replica 1 is never started.
+# Activated by: ./scripts/run-bdd-tests.sh <sdk> session_contract
+# ./scripts/run-bdd-tests.sh <sdk> all
+#
+# A two-replica cluster needs both replicas to commit, so this node answers
+# pings but refuses every sign-in as not accepted (58) for as long as it runs.
+# An SDK keeps retrying until the call's deadline, which makes the deadline
+# outcome deterministic. The roster IPs follow the same static-address rules
+# as docker-compose.cluster.yml, and the absent replica's address is never
+# assigned to a container.
+
+x-no-quorum-bdd-deps: &no-quorum-bdd-deps
+ depends_on:
+ iggy-no-quorum:
+ condition: service_healthy
+ environment:
+ - IGGY_TCP_ADDRESS_NO_QUORUM=iggy-no-quorum:8093
+
+services:
+ iggy-no-quorum:
+ image: iggy-bdd-server
+ platform: linux/amd64
+ build:
+ context: ..
+ dockerfile: core/server/Dockerfile
+ target: runtime-prebuilt
+ args:
+ PREBUILT_IGGY_SERVER: ${IGGY_SERVER_PATH:-target/debug/iggy-server}
+ PREBUILT_IGGY_CLI: ${IGGY_CLI_PATH:-target/debug/iggy}
+ LIBC: glibc
+ PROFILE: debug
+ command: [ "--replica-id", "0" ]
+ cap_add:
+ - SYS_NICE
+ security_opt:
+ - seccomp:unconfined
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ healthcheck:
+ test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "172.28.0.111:8093", "ping" ]
+ interval: 5s
+ timeout: 3s
+ retries: 30
+ start_period: 2s
+ environment:
+ LLVM_PROFILE_FILE: ${BDD_SERVER_PROFILE_FILE:-}
+ RUST_LOG: info
+ IGGY_ROOT_USERNAME: iggy
+ IGGY_ROOT_PASSWORD: iggy
+ IGGY_CLUSTER_ENABLED: "true"
+ IGGY_CLUSTER_NAME: no-quorum-cluster
+ IGGY_CLUSTER_NODES_0_NAME: no-quorum-node
+ IGGY_CLUSTER_NODES_0_IP: 172.28.0.111
+ IGGY_CLUSTER_NODES_0_REPLICA_ID: "0"
+ IGGY_CLUSTER_NODES_0_PORTS_TCP: "8093"
+ IGGY_CLUSTER_NODES_0_PORTS_QUIC: "8083"
+ IGGY_CLUSTER_NODES_0_PORTS_HTTP: "3003"
+ IGGY_CLUSTER_NODES_0_PORTS_WEBSOCKET: "8073"
+ IGGY_CLUSTER_NODES_0_PORTS_TCP_REPLICA: "8193"
+ IGGY_CLUSTER_NODES_1_NAME: absent-node
+ IGGY_CLUSTER_NODES_1_IP: 172.28.0.112
+ IGGY_CLUSTER_NODES_1_REPLICA_ID: "1"
+ IGGY_CLUSTER_NODES_1_PORTS_TCP: "8094"
+ IGGY_CLUSTER_NODES_1_PORTS_QUIC: "8084"
+ IGGY_CLUSTER_NODES_1_PORTS_HTTP: "3004"
+ IGGY_CLUSTER_NODES_1_PORTS_WEBSOCKET: "8074"
+ IGGY_CLUSTER_NODES_1_PORTS_TCP_REPLICA: "8194"
+ IGGY_CLUSTER_AUTH_ENABLED: "true"
+ IGGY_CLUSTER_AUTH_SHARED_SECRET: "bdd-vsr-no-quorum-shared-secret-0123456789"
+ IGGY_PATH: local_data_no_quorum
+ IGGY_TCP_ADDRESS: 0.0.0.0:8093
+ IGGY_HTTP_ADDRESS: 0.0.0.0:3003
+ IGGY_QUIC_ADDRESS: 0.0.0.0:8083
+ IGGY_WEBSOCKET_ADDRESS: 0.0.0.0:8073
+ networks:
+ iggy-bdd-network:
+ ipv4_address: 172.28.0.111
+ volumes:
+ - iggy_no_quorum_data:/app/local_data_no_quorum
+ - ../target:/profiles
+
+ rust-bdd:
+ <<: *no-quorum-bdd-deps
+
+ python-bdd:
+ <<: *no-quorum-bdd-deps
+
+ php-bdd:
+ <<: *no-quorum-bdd-deps
+
+ go-bdd:
+ <<: *no-quorum-bdd-deps
+
+ node-bdd:
+ <<: *no-quorum-bdd-deps
+
+ csharp-bdd:
+ <<: *no-quorum-bdd-deps
+
+ java-bdd:
+ <<: *no-quorum-bdd-deps
+
+ cpp-bdd:
+ <<: *no-quorum-bdd-deps
+
+networks:
+ iggy-bdd-network:
+ driver: bridge
+ ipam:
+ config:
+ - subnet: 172.28.0.0/24
+
+volumes:
+ iggy_no_quorum_data:
diff --git a/bdd/docker-compose.yml b/bdd/docker-compose.yml
index 37a460ca8..3dd3e1049 100644
--- a/bdd/docker-compose.yml
+++ b/bdd/docker-compose.yml
@@ -34,6 +34,7 @@ services:
- ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature
- ./scenarios/leader_redirection.feature:/app/features/leader_redirection.feature
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
+ - ./scenarios/session_contract.feature:/app/features/session_contract.feature
- ./scenarios/stream_crud.feature:/app/features/stream_crud.feature
command:
- sh
@@ -43,6 +44,7 @@ services:
basic_messaging) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test basic_messaging ;;
leader_redirection) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test leader_redirection ;;
raw_command) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test raw_command ;;
+ session_contract) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test session_contract ;;
stream_crud) cargo test -p bdd --features "$$BDD_RUST_FEATURES" --test stream_crud ;;
*) cargo test -p bdd --features "$$BDD_RUST_FEATURES" ;;
esac
@@ -61,15 +63,17 @@ services:
volumes:
- ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
+ - ./scenarios/session_contract.feature:/app/features/session_contract.feature
working_dir: /app
command:
- sh
- -c
- |
case "$$BDD_FEATURE" in
- basic_messaging) TEST_PATH='tests/test_basic_messaging.py' ;;
- raw_command) TEST_PATH='tests/test_raw_command.py' ;;
- *) TEST_PATH='tests/' ;;
+ basic_messaging) TEST_PATH='tests/test_basic_messaging.py' ;;
+ raw_command) TEST_PATH='tests/test_raw_command.py' ;;
+ session_contract) TEST_PATH='tests/test_session_contract.py' ;;
+ *) TEST_PATH='tests/' ;;
esac
uv run --project /workspace/bdd/python pytest $$TEST_PATH -v
networks:
@@ -86,14 +90,16 @@ services:
volumes:
- ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
+ - ./scenarios/session_contract.feature:/app/features/session_contract.feature
command:
- sh
- -c
- |
case "$$BDD_FEATURE" in
- basic_messaging) GROUP='--group basic-messaging' ;;
- raw_command) GROUP='--group raw-command' ;;
- *) GROUP='' ;;
+ basic_messaging) GROUP='--group basic-messaging' ;;
+ raw_command) GROUP='--group raw-command' ;;
+ session_contract) GROUP='--group session-contract' ;;
+ *) GROUP='' ;;
esac
./scripts/test.sh --configuration /workspace/bdd/php/phpunit.xml.dist $$GROUP
networks:
@@ -127,9 +133,10 @@ services:
- -c
- |
case "$$BDD_FEATURE" in
- basic_messaging) TAGS='--tags @basic-messaging' ;;
- raw_command) TAGS='--tags @raw-command' ;;
- *) TAGS='' ;;
+ basic_messaging) TAGS='--tags @basic-messaging' ;;
+ raw_command) TAGS='--tags @raw-command' ;;
+ session_contract) TAGS='--tags @session-contract' ;;
+ *) TAGS='' ;;
esac
npm run test:bdd -- $$TAGS
networks:
@@ -153,6 +160,7 @@ services:
basic_messaging) FILTER='--filter-trait Category=basic-messaging' ;;
leader_redirection) FILTER='--filter-trait Category=requires-leader-awareness' ;;
raw_command) FILTER='--filter-trait Category=raw-command' ;;
+ session_contract) FILTER='--filter-trait Category=session-contract' ;;
esac
dotnet test $$FILTER
networks:
@@ -171,6 +179,7 @@ services:
- ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature
- ./scenarios/leader_redirection.feature:/app/features/leader_redirection.feature
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
+ - ./scenarios/session_contract.feature:/app/features/session_contract.feature
- ./scenarios/stream_crud.feature:/app/features/stream_crud.feature
command:
- sh
@@ -180,6 +189,7 @@ services:
basic_messaging) export CUCUMBER_FILTER_TAGS='@basic-messaging' ;;
leader_redirection) export CUCUMBER_FILTER_TAGS='@requires-leader-awareness' ;;
raw_command) export CUCUMBER_FILTER_TAGS='@raw-command' ;;
+ session_contract) export CUCUMBER_FILTER_TAGS='@session-contract' ;;
stream_crud) export CUCUMBER_FILTER_TAGS='@stream-crud' ;;
esac
gradle --no-daemon test
@@ -198,6 +208,7 @@ services:
volumes:
- ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature
- ./scenarios/raw_command.feature:/app/features/raw_command.feature
+ - ./scenarios/session_contract.feature:/app/features/session_contract.feature
networks:
- iggy-bdd-network
diff --git a/bdd/go/tests/env/env.go b/bdd/go/tests/env/env.go
index 002c9bfce..a01aa2ff0 100644
--- a/bdd/go/tests/env/env.go
+++ b/bdd/go/tests/env/env.go
@@ -49,6 +49,11 @@ func FollowerAddress() string {
return required("IGGY_TCP_ADDRESS_FOLLOWER")
}
+// NoQuorumAddress returns the address of the cluster node that cannot reach quorum.
+func NoQuorumAddress() string {
+ return required("IGGY_TCP_ADDRESS_NO_QUORUM")
+}
+
// RootCredentials returns the root username and password.
func RootCredentials() (string, string) {
return required("IGGY_ROOT_USERNAME"), required("IGGY_ROOT_PASSWORD")
diff --git a/bdd/go/tests/session_contract.go b/bdd/go/tests/session_contract.go
new file mode 100644
index 000000000..2746253e9
--- /dev/null
+++ b/bdd/go/tests/session_contract.go
@@ -0,0 +1,139 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package tests
+
+import (
+ "context"
+ "errors"
+ "fmt"
+
+ "github.com/apache/iggy/bdd/go/tests/env"
+ "github.com/apache/iggy/foreign/go/client"
+ "github.com/apache/iggy/foreign/go/client/tcp"
+ iggcon "github.com/apache/iggy/foreign/go/contracts"
+ ierror "github.com/apache/iggy/foreign/go/errors"
+ "github.com/cucumber/godog"
+)
+
+type sessionContractCtxKey struct{}
+
+type sessionContractCtx struct {
+ client iggcon.Client
+ serverAddr string
+ lastError error
+}
+
+func getSessionContractCtx(ctx context.Context) *sessionContractCtx {
+ return ctx.Value(sessionContractCtxKey{}).(*sessionContractCtx)
+}
+
+type sessionContractSteps struct{}
+
+func (sessionContractSteps) givenRunningServer(ctx context.Context) error {
+ getSessionContractCtx(ctx).serverAddr = env.ServerAddress()
+ return nil
+}
+
+func (sessionContractSteps) givenNodeWithoutQuorum(ctx context.Context) error {
+ getSessionContractCtx(ctx).serverAddr = env.NoQuorumAddress()
+ return nil
+}
+
+func (sessionContractSteps) givenAuthenticationAsRoot(ctx context.Context) error {
+ state := getSessionContractCtx(ctx)
+ if err := state.connect(ctx); err != nil {
+ return err
+ }
+ username, password := env.RootCredentials()
+ if _, err := state.client.LoginUser(ctx, username, password); err != nil {
+ return fmt.Errorf("authenticate client: %w", err)
+ }
+ return nil
+}
+
+func (sessionContractSteps) whenSignOut(ctx context.Context) error {
+ if err := getSessionContractCtx(ctx).client.LogoutUser(ctx); err != nil {
+ return fmt.Errorf("sign out: %w", err)
+ }
+ return nil
+}
+
+func (sessionContractSteps) whenRequestStream(ctx context.Context, streamID uint32) error {
+ state := getSessionContractCtx(ctx)
+ identifier, err := iggcon.NewIdentifier(streamID)
+ if err != nil {
+ return fmt.Errorf("stream identifier: %w", err)
+ }
+ _, state.lastError = state.client.GetStream(ctx, identifier)
+ return nil
+}
+
+func (sessionContractSteps) whenSignInToNode(ctx context.Context) error {
+ state := getSessionContractCtx(ctx)
+ if err := state.connect(ctx); err != nil {
+ return err
+ }
+ username, password := env.RootCredentials()
+ _, state.lastError = state.client.LoginUser(ctx, username, password)
+ return nil
+}
+
+func (sessionContractSteps) thenCallFailsWithCode(ctx context.Context, code uint32) error {
+ lastError := getSessionContractCtx(ctx).lastError
+ var iggyError ierror.IggyError
+ if !errors.As(lastError, &iggyError) {
+ return fmt.Errorf("expected error code %d, got %v", code, lastError)
+ }
+ if uint32(iggyError.Code()) != code {
+ return fmt.Errorf("expected error code %d, got %d (%v)", code, iggyError.Code(), lastError)
+ }
+ return nil
+}
+
+func (state *sessionContractCtx) connect(ctx context.Context) error {
+ iggyClient, err := client.NewIggyClient(client.WithTcp(tcp.WithServerAddress(state.serverAddr)))
+ if err != nil {
+ return fmt.Errorf("create client: %w", err)
+ }
+ if err = iggyClient.Connect(ctx); err != nil {
+ return fmt.Errorf("connect client: %w", err)
+ }
+ state.client = iggyClient
+ return nil
+}
+
+func initSessionContractScenario(sc *godog.ScenarioContext) {
+ sc.Before(func(context.Context, *godog.Scenario) (context.Context, error) {
+ return context.WithValue(context.Background(), sessionContractCtxKey{}, &sessionContractCtx{}), nil
+ })
+ steps := sessionContractSteps{}
+ sc.Step(`^I have a running Iggy server$`, steps.givenRunningServer)
+ sc.Step(`^I have a cluster node that cannot reach quorum$`, steps.givenNodeWithoutQuorum)
+ sc.Step(`^I am authenticated as the root user$`, steps.givenAuthenticationAsRoot)
+ sc.Step(`^I sign out$`, steps.whenSignOut)
+ sc.Step(`^I request the stream with ID (\d+)$`, steps.whenRequestStream)
+ sc.Step(`^I sign in to that node as the root user$`, steps.whenSignInToNode)
+ sc.Step(`^the call should fail with error code (\d+)$`, steps.thenCallFailsWithCode)
+ sc.After(func(ctx context.Context, _ *godog.Scenario, scenarioError error) (context.Context, error) {
+ state := getSessionContractCtx(ctx)
+ if state.client != nil {
+ scenarioError = errors.Join(scenarioError, state.client.Close())
+ }
+ return ctx, scenarioError
+ })
+}
diff --git a/bdd/go/tests/suite_test.go b/bdd/go/tests/suite_test.go
index ec81262e6..7841899a7 100644
--- a/bdd/go/tests/suite_test.go
+++ b/bdd/go/tests/suite_test.go
@@ -64,6 +64,17 @@ func TestFeatures(t *testing.T) {
},
})
}
+ if feature == "all" || feature == "session_contract" {
+ suites = append(suites, godog.TestSuite{
+ ScenarioInitializer: initSessionContractScenario,
+ Options: &godog.Options{
+ Format: "pretty",
+ Paths: []string{"../../scenarios/session_contract.feature"},
+ Strict: true,
+ TestingT: t,
+ },
+ })
+ }
if len(suites) == 0 {
t.Fatalf("unknown BDD_FEATURE=%q", feature)
diff --git a/bdd/java/build.gradle.kts b/bdd/java/build.gradle.kts
index 603db6268..3d230a930 100644
--- a/bdd/java/build.gradle.kts
+++ b/bdd/java/build.gradle.kts
@@ -49,6 +49,10 @@ spotless {
tasks.test {
useJUnitPlatform()
+ System.getenv("CUCUMBER_FILTER_TAGS")?.let { systemProperty("cucumber.filter.tags", it) }
+ testLogging {
+ exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL
+ }
}
tasks.jacocoTestReport {
diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java b/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java
index f5b9a8c19..34b800829 100644
--- a/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java
+++ b/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java
@@ -300,6 +300,51 @@ public class BasicMessagingSteps {
assertEquals(IggyErrorCode.INVALID_COMMAND, error.getErrorCode());
}
+ @Given("I have a cluster node that cannot reach quorum")
+ public void clusterNodeWithoutQuorum() {
+ HostPort hostPort = HostPort.parse(TestEnvironment.noQuorumAddress());
+
+ IggyTcpClient client =
+ IggyTcpClient.builder().host(hostPort.host).port(hostPort.port).build();
+
+ client.connect();
+ context.client = client;
+ }
+
+ @When("I sign out")
+ public void signOut() {
+ getClient().users().logout();
+ }
+
+ @When("I request the stream with ID {int}")
+ public void requestStream(int streamId) {
+ recordCallError(() -> getClient().streams().getStream((long) streamId));
+ }
+
+ @When("I sign in to that node as the root user")
+ public void signInToNode() {
+ recordCallError(
+ () -> getClient().users().login(TestEnvironment.rootUsername(), TestEnvironment.rootPassword()));
+ }
+
+ @Then("the call should fail with error code {int}")
+ public void callFailsWithCode(int code) {
+ assertTrue(
+ context.lastCallError instanceof IggyServerException,
+ "Expected an Iggy server error, got " + context.lastCallError);
+ IggyServerException error = (IggyServerException) context.lastCallError;
+ assertEquals(code, error.getRawErrorCode());
+ }
+
+ private void recordCallError(Runnable call) {
+ try {
+ call.run();
+ context.lastCallError = null;
+ } catch (RuntimeException error) {
+ context.lastCallError = error;
+ }
+ }
+
private IggyBaseClient getClient() {
if (context.client == null) {
throw new IllegalStateException("Iggy client not initialized");
diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/TestContext.java b/bdd/java/src/test/java/org/apache/iggy/bdd/TestContext.java
index 62f0e2b89..c39b7fe71 100644
--- a/bdd/java/src/test/java/org/apache/iggy/bdd/TestContext.java
+++ b/bdd/java/src/test/java/org/apache/iggy/bdd/TestContext.java
@@ -35,4 +35,5 @@ class TestContext {
String lastSentMessage;
byte[] lastRawResponse;
RuntimeException lastRawError;
+ RuntimeException lastCallError;
}
diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/TestEnvironment.java b/bdd/java/src/test/java/org/apache/iggy/bdd/TestEnvironment.java
index 0a3cdc9aa..17bdfc483 100644
--- a/bdd/java/src/test/java/org/apache/iggy/bdd/TestEnvironment.java
+++ b/bdd/java/src/test/java/org/apache/iggy/bdd/TestEnvironment.java
@@ -49,6 +49,10 @@ final class TestEnvironment {
return require("IGGY_TCP_ADDRESS_FOLLOWER");
}
+ static String noQuorumAddress() {
+ return require("IGGY_TCP_ADDRESS_NO_QUORUM");
+ }
+
static String rootUsername() {
return require("IGGY_ROOT_USERNAME");
}
diff --git a/bdd/node/Dockerfile b/bdd/node/Dockerfile
index c9d2b25c2..e11d1977d 100644
--- a/bdd/node/Dockerfile
+++ b/bdd/node/Dockerfile
@@ -20,6 +20,7 @@ FROM node:26-slim
COPY ./foreign/node .
COPY ./bdd/scenarios/basic_messaging.feature ./bdd/
COPY ./bdd/scenarios/raw_command.feature ./bdd/
+COPY ./bdd/scenarios/session_contract.feature ./bdd/
RUN npm ci
diff --git a/bdd/php/tests/SessionContractFeatureTest.php b/bdd/php/tests/SessionContractFeatureTest.php
new file mode 100644
index 000000000..b50603fe1
--- /dev/null
+++ b/bdd/php/tests/SessionContractFeatureTest.php
@@ -0,0 +1,118 @@
+<?php
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+declare(strict_types=1);
+require_once __DIR__ . '/SharedFeatureParser.php';
+require_once __DIR__ . '/TestEnvironment.php';
+
+use Iggy\Client as IggyClient;
+use Iggy\Exception\IggyException;
+use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\Attributes\Group;
+use PHPUnit\Framework\Attributes\TestDox;
+use PHPUnit\Framework\TestCase;
+
+final class SessionContractFeatureTest extends TestCase
+{
+ // The PHP SDK raises exceptions with the Rust error text and no code.
+ private const ERROR_MESSAGES = [
+ 40 => 'Unauthenticated',
+ 58 => 'Request transiently not accepted',
+ ];
+
+ private ?IggyClient $client = null;
+ private ?Throwable $lastError = null;
+
+ #[DataProvider('scenarioCases')]
+ #[Group('session-contract')]
+ #[TestDox('Session contract shared BDD scenario passes for the PHP SDK')]
+ public function testSessionContractScenario(string $scenarioName, array $steps): void
+ {
+ assert_true($scenarioName !== '', 'scenario name must not be empty');
+ foreach ($steps as $step) {
+ $this->runStep($step);
+ }
+ }
+
+ public static function scenarioCases(): array
+ {
+ $cases = SharedFeatureParser::load(__DIR__ . '/../../scenarios/session_contract.feature');
+
+ // The PHP SDK has no sign-out.
+ return array_filter($cases, static fn (array $case): bool => !in_array('I sign out', $case[1], true));
+ }
+
+ private function runStep(string $step): void
+ {
+ if ($step === 'I have a running Iggy server') {
+ $this->client = new IggyClient(TestEnvironment::serverAddress());
+ $this->client->connect();
+ $this->client->ping();
+ return;
+ }
+ if ($step === 'I have a cluster node that cannot reach quorum') {
+ $this->client = new IggyClient(TestEnvironment::noQuorumAddress());
+ $this->client->connect();
+ return;
+ }
+ if ($step === 'I am authenticated as the root user') {
+ $this->requireClient()->loginUser(
+ TestEnvironment::rootUsername(),
+ TestEnvironment::rootPassword(),
+ );
+ return;
+ }
+ if (preg_match('/^I request the stream with ID (\d+)$/', $step, $matches) === 1) {
+ $this->recordCallError(fn () => $this->requireClient()->getStream((int) $matches[1]));
+ return;
+ }
+ if ($step === 'I sign in to that node as the root user') {
+ $this->recordCallError(fn () => $this->requireClient()->loginUser(
+ TestEnvironment::rootUsername(),
+ TestEnvironment::rootPassword(),
+ ));
+ return;
+ }
+ if (preg_match('/^the call should fail with error code (\d+)$/', $step, $matches) === 1) {
+ assert_instance_of(IggyException::class, $this->lastError);
+ assert_true(
+ str_contains((string) $this->lastError?->getMessage(), self::ERROR_MESSAGES[(int) $matches[1]]),
+ (string) $this->lastError?->getMessage(),
+ );
+ return;
+ }
+
+ self::fail("Unsupported BDD step: {$step}");
+ }
+
+ private function recordCallError(callable $call): void
+ {
+ try {
+ $call();
+ $this->lastError = null;
+ } catch (Throwable $error) {
+ $this->lastError = $error;
+ }
+ }
+
+ private function requireClient(): IggyClient
+ {
+ assert_not_null($this->client, 'BDD client has not been initialized');
+ return $this->client;
+ }
+}
diff --git a/bdd/php/tests/SharedFeatureParser.php b/bdd/php/tests/SharedFeatureParser.php
index 38a8d8c5e..9a0d07009 100644
--- a/bdd/php/tests/SharedFeatureParser.php
+++ b/bdd/php/tests/SharedFeatureParser.php
@@ -77,8 +77,8 @@ final class SharedFeatureParser
if ($currentScenario !== null) {
$scenarios[$currentScenario] = $currentSteps;
}
- if ($backgroundSteps === [] || $scenarios === []) {
- throw new RuntimeException("feature must contain a background and at least one scenario: {$featureFile}");
+ if ($scenarios === []) {
+ throw new RuntimeException("feature must contain at least one scenario: {$featureFile}");
}
$cases = [];
diff --git a/bdd/php/tests/TestEnvironment.php b/bdd/php/tests/TestEnvironment.php
index 3c999fe2b..87b9ac627 100644
--- a/bdd/php/tests/TestEnvironment.php
+++ b/bdd/php/tests/TestEnvironment.php
@@ -30,6 +30,11 @@ final class TestEnvironment
return self::env('IGGY_TCP_ADDRESS');
}
+ public static function noQuorumAddress(): string
+ {
+ return self::env('IGGY_TCP_ADDRESS_NO_QUORUM');
+ }
+
public static function rootUsername(): string
{
return self::env('IGGY_ROOT_USERNAME');
diff --git a/bdd/python/tests/conftest.py b/bdd/python/tests/conftest.py
index a4a43e892..0b1ddcefa 100644
--- a/bdd/python/tests/conftest.py
+++ b/bdd/python/tests/conftest.py
@@ -42,6 +42,7 @@ class GlobalContext:
last_sent_message: str | None = None # Store message payload as string
last_raw_response: bytes | None = None
last_raw_error: RuntimeError | None = None
+ last_call_error: RuntimeError | None = None
def required_env(name: str) -> str:
@@ -65,6 +66,12 @@ def root_credentials() -> tuple[str, str]:
return required_env("IGGY_ROOT_USERNAME"), required_env("IGGY_ROOT_PASSWORD")
+@pytest.fixture(scope="session")
+def no_quorum_address() -> str:
+ """Address of the cluster node that cannot reach quorum."""
+ return required_env("IGGY_TCP_ADDRESS_NO_QUORUM")
+
+
@pytest.fixture(scope="session")
def event_loop():
"""Create an instance of the default event loop for the test session."""
diff --git a/bdd/python/tests/test_session_contract.py b/bdd/python/tests/test_session_contract.py
new file mode 100644
index 000000000..e81e16ddd
--- /dev/null
+++ b/bdd/python/tests/test_session_contract.py
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import asyncio
+import socket
+
+from apache_iggy import IggyClient
+from pytest_bdd import given, parsers, scenarios, then, when
+
+scenarios("/app/features/session_contract.feature")
+
+# The Python SDK raises RuntimeError with the Rust error text and no code.
+ERROR_MESSAGES = {
+ 40: "Unauthenticated",
+ 58: "Request transiently not accepted",
+}
+
+
+def connect(context):
+ async def open_connection():
+ host, port = context.server_addr.split(":")
+ try:
+ address = f"{socket.gethostbyname(host)}:{port}"
+ except socket.gaierror:
+ address = context.server_addr
+
+ context.client = IggyClient(address)
+ await context.client.connect()
+
+ asyncio.run(open_connection())
+
+
+@given("I have a running Iggy server")
+def running_server(context):
+ connect(context)
+
+
+@given("I have a cluster node that cannot reach quorum")
+def node_without_quorum(context, no_quorum_address):
+ context.server_addr = no_quorum_address
+ connect(context)
+
+
+@given("I am authenticated as the root user")
+def authenticated_root_user(context, root_credentials):
+ async def login():
+ await context.client.login_user(*root_credentials)
+
+ asyncio.run(login())
+
+
+@when("I sign out")
+def sign_out(context):
+ async def logout():
+ await context.client.logout_user()
+
+ asyncio.run(logout())
+
+
+@when(parsers.parse("I request the stream with ID {stream_id:d}"))
+def request_stream(context, stream_id):
+ async def request():
+ try:
+ await context.client.get_stream(stream_id)
+ context.last_call_error = None
+ except RuntimeError as error:
+ context.last_call_error = error
+
+ asyncio.run(request())
+
+
+@when("I sign in to that node as the root user")
+def sign_in_to_node(context, root_credentials):
+ async def sign_in():
+ try:
+ await context.client.login_user(*root_credentials)
+ context.last_call_error = None
+ except RuntimeError as error:
+ context.last_call_error = error
+
+ asyncio.run(sign_in())
+
+
+@then(parsers.parse("the call should fail with error code {code:d}"))
+def call_fails_with_code(context, code):
+ assert context.last_call_error is not None
+ assert ERROR_MESSAGES[code] in str(context.last_call_error)
diff --git a/bdd/rust/Cargo.toml b/bdd/rust/Cargo.toml
index d9e60a518..03207ef69 100644
--- a/bdd/rust/Cargo.toml
+++ b/bdd/rust/Cargo.toml
@@ -50,6 +50,11 @@ name = "raw_command"
harness = false
required-features = ["bdd"]
+[[test]]
+name = "session_contract"
+harness = false
+required-features = ["bdd"]
+
[[test]]
name = "stream_crud"
harness = false
diff --git a/bdd/rust/tests/common/global_context.rs b/bdd/rust/tests/common/global_context.rs
index caed89e6b..3313c003e 100644
--- a/bdd/rust/tests/common/global_context.rs
+++ b/bdd/rust/tests/common/global_context.rs
@@ -33,4 +33,5 @@ pub struct GlobalContext {
pub last_polled_messages: Option<PolledMessages>,
pub last_sent_message: Option<IggyMessage>,
pub last_raw_result: Option<Result<Bytes, IggyError>>,
+ pub last_call_result: Option<Result<(), IggyError>>,
}
diff --git a/bdd/rust/tests/helpers/env.rs b/bdd/rust/tests/helpers/env.rs
index 75c65683a..55e0328c4 100644
--- a/bdd/rust/tests/helpers/env.rs
+++ b/bdd/rust/tests/helpers/env.rs
@@ -41,6 +41,10 @@ pub fn follower_address() -> String {
required_env("IGGY_TCP_ADDRESS_FOLLOWER")
}
+pub fn no_quorum_address() -> String {
+ required_env("IGGY_TCP_ADDRESS_NO_QUORUM")
+}
+
pub fn root_username() -> String {
required_env("IGGY_ROOT_USERNAME")
}
diff --git a/bdd/rust/tests/steps/mod.rs b/bdd/rust/tests/session_contract.rs
similarity index 71%
copy from bdd/rust/tests/steps/mod.rs
copy to bdd/rust/tests/session_contract.rs
index 51b40fe99..84f6aa9ab 100644
--- a/bdd/rust/tests/steps/mod.rs
+++ b/bdd/rust/tests/session_contract.rs
@@ -15,10 +15,17 @@
// specific language governing permissions and limitations
// under the License.
-pub mod auth;
-pub mod leader_redirection;
-pub mod messages;
-pub mod raw_command;
-pub mod server;
-pub mod streams;
-pub mod topics;
+pub(crate) mod common;
+pub(crate) mod helpers;
+pub(crate) mod steps;
+
+use crate::common::global_context::GlobalContext;
+use cucumber::World;
+
+#[tokio::main]
+async fn main() {
+ GlobalContext::cucumber()
+ .fail_on_skipped()
+ .run_and_exit("../../bdd/scenarios/session_contract.feature")
+ .await;
+}
diff --git a/bdd/rust/tests/steps/mod.rs b/bdd/rust/tests/steps/mod.rs
index 51b40fe99..e6c88141d 100644
--- a/bdd/rust/tests/steps/mod.rs
+++ b/bdd/rust/tests/steps/mod.rs
@@ -20,5 +20,6 @@ pub mod leader_redirection;
pub mod messages;
pub mod raw_command;
pub mod server;
+pub mod session;
pub mod streams;
pub mod topics;
diff --git a/bdd/rust/tests/steps/session.rs b/bdd/rust/tests/steps/session.rs
new file mode 100644
index 000000000..84ed5fa55
--- /dev/null
+++ b/bdd/rust/tests/steps/session.rs
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::common::global_context::GlobalContext;
+use crate::helpers::cluster::create_and_connect_client;
+use crate::helpers::env::{no_quorum_address, root_password, root_username};
+use cucumber::{given, then, when};
+use iggy::prelude::{Identifier, StreamClient, UserClient};
+
+#[given("I have a cluster node that cannot reach quorum")]
+pub async fn given_node_without_quorum(world: &mut GlobalContext) {
+ world.server_addr = Some(no_quorum_address());
+}
+
+#[when("I sign out")]
+pub async fn when_sign_out(world: &mut GlobalContext) {
+ let client = world.client.as_ref().expect("Client should be available");
+ client.logout_user().await.expect("Sign-out should succeed");
+}
+
+#[when(regex = r"^I request the stream with ID (\d+)$")]
+pub async fn when_request_stream(world: &mut GlobalContext, stream_id: u32) {
+ let client = world.client.as_ref().expect("Client should be available");
+ let stream_id = Identifier::numeric(stream_id).expect("Stream ID should be valid");
+ world.last_call_result = Some(client.get_stream(&stream_id).await.map(|_| ()));
+}
+
+#[when("I sign in to that node as the root user")]
+pub async fn when_sign_in_to_node(world: &mut GlobalContext) {
+ let server_addr = world
+ .server_addr
+ .as_ref()
+ .expect("Server address should be set");
+ let client = create_and_connect_client(server_addr).await;
+ let result = client
+ .login_user(&root_username(), &root_password())
+ .await
+ .map(|_| ());
+ world.client = Some(client);
+ world.last_call_result = Some(result);
+}
+
+#[then(regex = r"^the call should fail with error code (\d+)$")]
+pub async fn then_call_fails_with_code(world: &mut GlobalContext, code: u32) {
+ let error = world
+ .last_call_result
+ .take()
+ .expect("Should have made a call")
+ .expect_err("Call should have failed");
+ assert_eq!(error.as_code(), code, "Call failed with {error}");
+}
diff --git a/bdd/node/Dockerfile b/bdd/scenarios/session_contract.feature
similarity index 51%
copy from bdd/node/Dockerfile
copy to bdd/scenarios/session_contract.feature
index c9d2b25c2..4262177ac 100644
--- a/bdd/node/Dockerfile
+++ b/bdd/scenarios/session_contract.feature
@@ -15,14 +15,20 @@
# specific language governing permissions and limitations
# under the License.
-FROM node:26-slim
+@session-contract
+Feature: Session contract
+ As a developer using Apache Iggy
+ I want every SDK to end a call with the same error code when it has no session or no primary
+ So that I can handle a sign-out and an unavailable cluster the same way in every language
-COPY ./foreign/node .
-COPY ./bdd/scenarios/basic_messaging.feature ./bdd/
-COPY ./bdd/scenarios/raw_command.feature ./bdd/
+ Scenario: A call after sign-out fails as unauthenticated
+ Given I have a running Iggy server
+ And I am authenticated as the root user
+ When I sign out
+ And I request the stream with ID 1
+ Then the call should fail with error code 40
-RUN npm ci
-
-USER node
-
-CMD ["npm", "run", "test"]
+ Scenario: A sign-in that no primary accepts fails at its deadline
+ Given I have a cluster node that cannot reach quorum
+ When I sign in to that node as the root user
+ Then the call should fail with error code 58
diff --git a/core/configs/src/server_config/defaults.rs b/core/configs/src/server_config/defaults.rs
index 87c192a9c..de4f1cd88 100644
--- a/core/configs/src/server_config/defaults.rs
+++ b/core/configs/src/server_config/defaults.rs
@@ -300,6 +300,8 @@ impl Default for MessageBusConfig {
replica_read_buffer_size: bus.replica_read_buffer_size.parse().unwrap(),
peer_queue_capacity: bus.peer_queue_capacity as usize,
client_queue_capacity: bus.client_queue_capacity as usize,
+ client_write_stall_timeout: bus.client_write_stall_timeout.parse().unwrap(),
+ client_request_queue_bytes_max: bus.client_request_queue_bytes_max.parse().unwrap(),
reconnect_period: bus.reconnect_period.parse().unwrap(),
close_peer_timeout: bus.close_peer_timeout.parse().unwrap(),
close_grace: bus.close_grace.parse().unwrap(),
diff --git a/core/configs/src/server_config/displays.rs b/core/configs/src/server_config/displays.rs
index 6eb1e90cd..413b21ccb 100644
--- a/core/configs/src/server_config/displays.rs
+++ b/core/configs/src/server_config/displays.rs
@@ -100,13 +100,16 @@ impl Display for MessageBusConfig {
write!(
f,
"{{ max_batch: {}, max_message_size: {}, replica_read_buffer_size: {}, \
- peer_queue_capacity: {}, client_queue_capacity: {}, reconnect_period: {}, \
- close_peer_timeout: {}, close_grace: {}, handshake_grace: {}, connections_max: {} }}",
+ peer_queue_capacity: {}, client_queue_capacity: {}, client_write_stall_timeout: {}, \
+ client_request_queue_bytes_max: {}, reconnect_period: {}, close_peer_timeout: {}, \
+ close_grace: {}, handshake_grace: {}, connections_max: {} }}",
self.max_batch,
self.max_message_size,
self.replica_read_buffer_size,
self.peer_queue_capacity,
self.client_queue_capacity,
+ self.client_write_stall_timeout,
+ self.client_request_queue_bytes_max,
self.reconnect_period,
self.close_peer_timeout,
self.close_grace,
diff --git a/core/configs/src/server_config/message_bus.rs b/core/configs/src/server_config/message_bus.rs
index 0c7b6a2a5..7810c587a 100644
--- a/core/configs/src/server_config/message_bus.rs
+++ b/core/configs/src/server_config/message_bus.rs
@@ -120,6 +120,19 @@ pub struct MessageBusConfig {
#[serde(default = "default_client_queue_capacity")]
pub client_queue_capacity: usize,
+ /// How long a reply waits for room in a full client queue before the
+ /// connection is closed.
+ #[serde(default = "default_client_write_stall_timeout")]
+ #[serde_as(as = "DisplayFromStr")]
+ #[config_env(leaf)]
+ pub client_write_stall_timeout: IggyDuration,
+
+ /// Bytes of requests one SDK connection may have queued behind the
+ /// requests the server runs for it. An empty queue takes any frame.
+ #[serde(default = "default_client_request_queue_bytes_max")]
+ #[config_env(leaf)]
+ pub client_request_queue_bytes_max: IggyByteSize,
+
/// Interval between outbound reconnect attempts to peers with
/// `peer_id > self_id`. Also bounds each dial, so a peer that drops
/// SYNs cannot stall the sweep.
@@ -201,6 +214,14 @@ impl Validatable<ConfigurationError> for MessageBusConfig {
eprintln!("{COMPONENT} message_bus.client_queue_capacity must be > 0");
return Err(ConfigurationError::InvalidConfigurationValue);
}
+ if self.client_write_stall_timeout.as_micros() == 0 {
+ eprintln!("{COMPONENT} message_bus.client_write_stall_timeout must be > 0");
+ return Err(ConfigurationError::InvalidConfigurationValue);
+ }
+ if self.client_request_queue_bytes_max.as_bytes_u64() == 0 {
+ eprintln!("{COMPONENT} message_bus.client_request_queue_bytes_max must be > 0");
+ return Err(ConfigurationError::InvalidConfigurationValue);
+ }
if self.max_message_size.as_bytes_u64() == 0 {
eprintln!("{COMPONENT} message_bus.max_message_size must be > 0");
return Err(ConfigurationError::InvalidConfigurationValue);
@@ -240,6 +261,22 @@ fn default_client_queue_capacity() -> usize {
SERVER_CONFIG.message_bus.client_queue_capacity as usize
}
+fn default_client_write_stall_timeout() -> IggyDuration {
+ SERVER_CONFIG
+ .message_bus
+ .client_write_stall_timeout
+ .parse()
+ .expect("message_bus.client_write_stall_timeout is a duration")
+}
+
+fn default_client_request_queue_bytes_max() -> IggyByteSize {
+ SERVER_CONFIG
+ .message_bus
+ .client_request_queue_bytes_max
+ .parse()
+ .expect("message_bus.client_request_queue_bytes_max is a byte size")
+}
+
/// [`IggyByteSize`]'s own `Default` is 0 bytes, which turns the read-ahead
/// buffer off, so the value comes from the embedded `config.toml`.
fn default_replica_read_buffer_size() -> IggyByteSize {
@@ -315,6 +352,38 @@ mod tests {
assert!(config.validate().is_err());
}
+ #[test]
+ fn rejects_zero_client_write_stall_timeout() {
+ let mut config = baseline();
+ config.client_write_stall_timeout = IggyDuration::from(std::time::Duration::ZERO);
+ assert!(config.validate().is_err());
+ }
+
+ #[test]
+ fn rejects_zero_client_request_queue_bytes_max() {
+ let mut config = baseline();
+ config.client_request_queue_bytes_max = IggyByteSize::from(0_u64);
+ assert!(config.validate().is_err());
+ }
+
+ #[test]
+ fn missing_client_stall_and_queue_bytes_keep_config_defaults() {
+ let mut config = serde_json::to_value(baseline()).unwrap();
+ let fields = config.as_object_mut().unwrap();
+ fields.remove("client_write_stall_timeout");
+ fields.remove("client_request_queue_bytes_max");
+ let decoded: MessageBusConfig = serde_json::from_value(config).unwrap();
+ assert_eq!(
+ decoded.client_write_stall_timeout,
+ baseline().client_write_stall_timeout
+ );
+ assert_eq!(
+ decoded.client_request_queue_bytes_max,
+ baseline().client_request_queue_bytes_max
+ );
+ decoded.validate().unwrap();
+ }
+
#[test]
fn default_replica_read_buffer_is_32_kib() {
assert_eq!(
diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs
index bccaa3460..5c2afdfe3 100644
--- a/core/consensus/src/impls.rs
+++ b/core/consensus/src/impls.rs
@@ -1460,6 +1460,8 @@ where
/// Views whose logs were refused for naming a different prepare at an op
/// this replica applied; see [`Self::contradicts_applied_log`].
applied_log_contradictions: Cell<u64>,
+ /// Part of `applied_log_contradictions` already handed to a metric.
+ applied_log_contradictions_taken: Cell<u64>,
/// Newest view counted in `applied_log_contradictions`.
contradiction_reported_view: Cell<Option<u32>>,
@@ -1835,6 +1837,7 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
eager_commit: Cell::new(false),
commit_announcements: RefCell::new(CommitAnnouncements::new(replica_count)),
applied_log_contradictions: Cell::new(0),
+ applied_log_contradictions_taken: Cell::new(0),
contradiction_reported_view: Cell::new(None),
clock,
}
@@ -4074,6 +4077,14 @@ impl<B: MessageBus, P: Pipeline<Entry = PipelineEntry>> VsrConsensus<B, P> {
self.applied_log_contradictions.get()
}
+ /// Views counted by [`Self::applied_log_contradictions`] since the last
+ /// call. The total keeps counting, so invariants still read it whole.
+ #[must_use]
+ pub const fn take_applied_log_contradictions(&self) -> u64 {
+ let total = self.applied_log_contradictions.get();
+ total - self.applied_log_contradictions_taken.replace(total)
+ }
+
/// [`Self::handle_start_view_guarded`] for a caller that cannot read back the
/// prepares it applied.
///
@@ -5930,7 +5941,6 @@ pub mod test_bus {
fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {}
fn track_background(&self, _handle: message_bus::JoinHandle<()>) {}
}
}
@@ -6634,7 +6644,6 @@ mod vsr_consensus_tests {
fn track_background(&self, _handle: message_bus::JoinHandle<()>) {}
fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {}
}
#[test]
diff --git a/core/consensus/src/metadata_helpers.rs b/core/consensus/src/metadata_helpers.rs
index d8c162af7..5d8ab2a8f 100644
--- a/core/consensus/src/metadata_helpers.rs
+++ b/core/consensus/src/metadata_helpers.rs
@@ -542,7 +542,6 @@ mod tests {
fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {}
}
// A registered client's fresh Register dispatches instead of being
diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs
index e9c8dfe87..92ab7366d 100644
--- a/core/consensus/src/plane_helpers.rs
+++ b/core/consensus/src/plane_helpers.rs
@@ -1174,7 +1174,6 @@ mod tests {
fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {}
}
#[allow(clippy::cast_possible_truncation)]
@@ -2689,7 +2688,6 @@ mod tests {
fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {}
}
#[test]
diff --git a/core/integration/tests/cluster/partition_primary_routing.rs b/core/integration/tests/cluster/partition_primary_routing.rs
index 1d2fbe56f..408c4b79b 100644
--- a/core/integration/tests/cluster/partition_primary_routing.rs
+++ b/core/integration/tests/cluster/partition_primary_routing.rs
@@ -21,28 +21,35 @@
//! settle on the metadata leader, so both writes to new partitions and group
//! polls of existing partitions must reach an owner that can commit progress.
+use std::collections::HashSet;
use std::net::SocketAddr;
use std::str::FromStr;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::thread::{self, JoinHandle};
use std::time::Duration;
use super::register_forwarding::connect_without_login;
-use bytes::Bytes;
+use bytes::{Bytes, BytesMut};
use futures::StreamExt;
+use futures::future::join_all;
use iggy::prelude::*;
use iggy_binary_protocol::codes::{
BIND_SESSION_CODE, GET_CONSUMER_GROUP_CODE, GET_PARTITION_ROUTE_CODE, POLL_MESSAGES_CODE,
- STORE_CONSUMER_OFFSET_CODE,
+ SEND_MESSAGES_CODE, STORE_CONSUMER_OFFSET_CODE,
};
use iggy_binary_protocol::consensus::{RequestHeader, result_code, result_section_len};
use iggy_binary_protocol::requests::consumer_groups::GetConsumerGroupRequest;
use iggy_binary_protocol::requests::consumer_offsets::StoreConsumerOffsetRequest;
+use iggy_binary_protocol::requests::messages::send_messages::{RawMessage, SendMessagesEncoder};
use iggy_binary_protocol::requests::messages::{GetPartitionRouteRequest, PollMessagesRequest};
use iggy_binary_protocol::requests::system::BindSessionRequest;
use iggy_binary_protocol::requests::users::login_register::BindSecret;
use iggy_binary_protocol::responses::consumer_groups::ConsumerGroupDetailsResponse;
use iggy_binary_protocol::responses::messages::PartitionRouteResponse;
use iggy_binary_protocol::{
- ClientVersionInfo, IGGY_PROTOCOL_VERSION, WireDecode, WireEncode, WireName,
+ ClientVersionInfo, IGGY_PROTOCOL_VERSION, WireDecode, WireEncode, WireIdentifier, WireName,
+ WirePartitioning,
};
use iggy_common::locking::IggyRwLockFn;
use iggy_common::store_consumer_offset::StoreConsumerOffset;
@@ -63,6 +70,7 @@ use reqwest::StatusCode;
use reqwest::header::RETRY_AFTER;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpStream;
+use tokio::runtime;
use tokio::time::{Instant, sleep, timeout};
use crate::server::http_client::{
@@ -151,6 +159,30 @@ const SEND_BUDGET: Duration = Duration::from_secs(20);
const PRECONDITION_BUDGET: Duration = Duration::from_secs(20);
const PRECONDITION_POLL: Duration = Duration::from_millis(500);
const REBALANCE_BUDGET: Duration = Duration::from_secs(120);
+/// `cluster.heartbeat_timeout` at its default.
+const HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(5);
+/// How long quorum evidence keeps partition admission open past
+/// `HEARTBEAT_TIMEOUT`, as the server sets it.
+const FRESHNESS_MARGIN: Duration = Duration::from_secs(3);
+/// How long a source primary drains a granted move before it gives the grant
+/// up, as the server sets it.
+const GRANT_TIMEOUT: Duration = Duration::from_secs(1);
+/// The longest pause the SDK takes between two attempts of one call.
+const RETRY_BACKOFF_MAX: Duration = Duration::from_secs(1);
+/// A send that meets a move waits out the source's drain and a retry pause. A
+/// handover that stalls ends in an election after a heartbeat timeout.
+const MOVE_STALL_BOUND: Duration = GRANT_TIMEOUT
+ .saturating_add(HEARTBEAT_TIMEOUT)
+ .saturating_add(RETRY_BACKOFF_MAX);
+/// Logged by a source primary when it hands its partition to the move target.
+const TRANSFER_MARKER: &str = "rebalancing partition primary after target readiness confirmation";
+const READBACK_BATCH: u32 = 1_000;
+/// How late an isolated primary may close admission on a loaded machine.
+const FENCE_SLACK: Duration = Duration::from_secs(2);
+const FENCE_BOUND: Duration = HEARTBEAT_TIMEOUT
+ .saturating_add(FRESHNESS_MARGIN)
+ .saturating_add(FENCE_SLACK);
+const FENCE_PROBE_INTERVAL: Duration = Duration::from_millis(100);
fn message(payload: &str) -> IggyMessage {
IggyMessage::from_str(payload).expect("build message")
@@ -566,6 +598,464 @@ async fn append_and_assert_partition_sentinels(
.expect("every partition must preserve and serve its history across rebalancing");
}
+/// Producers write every partition back to back while the rebalancer moves
+/// primaries to a returning replica. Every acknowledged send stays readable in
+/// send order. A send that ends uncertain is never sent again, so no message
+/// may appear twice. Each send ends acknowledged, uncertain or with the refusal
+/// the SDK retries, and none waits longer than a move can fence its partition.
+#[iggy_harness(cluster_nodes = 3, server(sharding.cpu_allocation = "2"))]
+async fn given_producers_on_every_partition_when_primaries_move_should_keep_acknowledged_order_without_stalls(
+ harness: &mut TestHarness,
+) {
+ const TOPIC_COUNT: usize = 12;
+ const RETURNING_MOVES: usize = 4;
+
+ let client = connect_without_login(harness.node(1).tcp_addr().unwrap()).await;
+ client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ let identity = client.session_identity().await.unwrap();
+ client.create_stream(STREAM_NAME).await.unwrap();
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let options = TopicCreateOptions {
+ partitions_count: Some(1),
+ durability: Durability::Persisted,
+ message_expiry: Some(IggyExpiry::NeverExpire),
+ ..TopicCreateOptions::default()
+ };
+ let mut topics = Vec::with_capacity(TOPIC_COUNT);
+ for index in 0..TOPIC_COUNT {
+ let name = format!("moving-{index}");
+ client.create_topic(&stream, &name, &options).await.unwrap();
+ topics.push(Identifier::named(&name).unwrap());
+ }
+ let mut request = RequestHeader {
+ client: identity.client_id,
+ session: identity.session,
+ request: RAW_REQUEST_START,
+ ..Default::default()
+ };
+ wait_for_balanced_routes(harness, &client, &stream, &topics, &[4; 3], &mut request).await;
+ harness.kill_node(0).unwrap();
+ wait_for_balanced_routes(harness, &client, &stream, &topics, &[0, 6, 6], &mut request).await;
+
+ let producing = Arc::new(AtomicUsize::new(0));
+ let stop = Arc::new(AtomicBool::new(false));
+ let producers = spawn_producers(
+ harness.node(1).tcp_addr().unwrap(),
+ topics.clone(),
+ Arc::clone(&producing),
+ Arc::clone(&stop),
+ );
+ timeout(PRECONDITION_BUDGET, async {
+ while producing.load(Ordering::Acquire) < topics.len() {
+ sleep(PRECONDITION_POLL).await;
+ }
+ })
+ .await
+ .expect("every producer must acknowledge a send before the replica returns");
+ let transfers_before = transfers_started(harness);
+ harness.restart_node(0).unwrap();
+ wait_for_balanced_routes(harness, &client, &stream, &topics, &[4; 3], &mut request).await;
+ let transfers = transfers_started(harness) - transfers_before;
+ stop.store(true, Ordering::Release);
+ let logs = producers.join().expect("producer thread");
+
+ let mut latencies: Vec<Duration> = logs
+ .iter()
+ .flat_map(|log| log.latencies.iter().copied())
+ .collect();
+ latencies.sort_unstable();
+ println!(
+ "{} sends across {transfers} primary moves: {} acknowledged, {} uncertain, {} refused, \
+ {} unexpected; latency p50 {:?}, p99 {:?}, max {:?}",
+ latencies.len(),
+ logs.iter().map(|log| log.acknowledged.len()).sum::<usize>(),
+ logs.iter().map(|log| log.uncertain.len()).sum::<usize>(),
+ logs.iter().map(|log| log.refused.len()).sum::<usize>(),
+ logs.iter().map(|log| log.unexpected.len()).sum::<usize>(),
+ latencies[latencies.len() / 2],
+ latencies[latencies.len() * 99 / 100],
+ latencies[latencies.len() - 1],
+ );
+ assert!(
+ transfers >= RETURNING_MOVES,
+ "the producers ran across {transfers} moves, expected at least {RETURNING_MOVES}"
+ );
+ for (topic, log) in topics.iter().zip(&logs) {
+ let sequences = read_sequences(&client, &stream, topic).await;
+ assert_produced_history(topic, log, &sequences);
+ }
+}
+
+/// What one producer saw of its sends. Sequence `n` is the `n`th send.
+#[derive(Default)]
+struct ProducerLog {
+ acknowledged: Vec<u64>,
+ uncertain: Vec<u64>,
+ refused: Vec<u64>,
+ unexpected: Vec<(u64, IggyError)>,
+ latencies: Vec<Duration>,
+}
+
+/// The producers run on a thread of their own, so a blocking harness call on
+/// the test thread does not stretch their latencies.
+fn spawn_producers(
+ address: SocketAddr,
+ topics: Vec<Identifier>,
+ producing: Arc<AtomicUsize>,
+ stop: Arc<AtomicBool>,
+) -> JoinHandle<Vec<ProducerLog>> {
+ thread::spawn(move || {
+ runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .expect("producer runtime")
+ .block_on(join_all(topics.iter().map(|topic| {
+ produce_until_stopped(address, topic, &producing, &stop)
+ })))
+ })
+}
+
+async fn produce_until_stopped(
+ address: SocketAddr,
+ topic: &Identifier,
+ producing: &AtomicUsize,
+ stop: &AtomicBool,
+) -> ProducerLog {
+ let client = connect_without_login(address).await;
+ client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let partitioning = Partitioning::partition_id(PARTITION_ID);
+ let mut log = ProducerLog::default();
+ let mut sequence = 0;
+ while !stop.load(Ordering::Acquire) {
+ let payload = format!("{topic}:{sequence}");
+ let started = Instant::now();
+ let outcome = client
+ .send_messages(&stream, topic, &partitioning, &mut [message(&payload)])
+ .await;
+ log.latencies.push(started.elapsed());
+ match outcome {
+ Ok(_) => {
+ if log.acknowledged.is_empty() {
+ producing.fetch_add(1, Ordering::Release);
+ }
+ log.acknowledged.push(sequence);
+ }
+ Err(IggyError::OperationUncertain(operation)) => {
+ client
+ .abandon_operation(operation)
+ .await
+ .expect("free the lane of an uncertain send");
+ log.uncertain.push(sequence);
+ }
+ Err(IggyError::TransientNotAccepted) => log.refused.push(sequence),
+ Err(error) => log.unexpected.push((sequence, error)),
+ }
+ sequence += 1;
+ }
+ log
+}
+
+fn transfers_started(harness: &TestHarness) -> usize {
+ // Node 0 restarts, which truncates its log, and it never moves a primary away.
+ (1..harness.cluster_size())
+ .map(|node| harness.node(node).stdout_occurrences(TRANSFER_MARKER))
+ .sum()
+}
+
+async fn read_sequences(client: &TcpClient, stream: &Identifier, topic: &Identifier) -> Vec<u64> {
+ let mut sequences = Vec::new();
+ let mut offset = 0;
+ loop {
+ let polled = client
+ .poll_messages(
+ stream,
+ topic,
+ Some(PARTITION_ID),
+ &Consumer::default(),
+ &PollingStrategy::offset(offset),
+ READBACK_BATCH,
+ false,
+ )
+ .await
+ .unwrap();
+ let Some(last) = polled.messages.last() else {
+ return sequences;
+ };
+ offset = last.header.offset + 1;
+ for message in &polled.messages {
+ let payload = std::str::from_utf8(&message.payload).expect("a produced payload");
+ let (name, sequence) = payload.rsplit_once(':').expect("a produced payload");
+ assert_eq!(name, topic.to_string());
+ sequences.push(sequence.parse().expect("a produced sequence"));
+ }
+ }
+}
+
+fn assert_produced_history(topic: &Identifier, log: &ProducerLog, sequences: &[u64]) {
+ assert!(
+ log.unexpected.is_empty(),
+ "{topic}: sends ended with errors the SDK does not retry: {:?}",
+ log.unexpected
+ );
+ let stalled: Vec<_> = log
+ .latencies
+ .iter()
+ .enumerate()
+ .filter(|(_, latency)| **latency > MOVE_STALL_BOUND)
+ .collect();
+ assert!(
+ stalled.is_empty(),
+ "{topic}: sends stalled past {MOVE_STALL_BOUND:?}: {stalled:?}"
+ );
+ if let Some(pair) = sequences.windows(2).find(|pair| pair[0] >= pair[1]) {
+ panic!(
+ "{topic}: sequence {} follows {} in the log",
+ pair[1], pair[0]
+ );
+ }
+ let read: HashSet<u64> = sequences.iter().copied().collect();
+ let lost: Vec<_> = log
+ .acknowledged
+ .iter()
+ .filter(|sequence| !read.contains(sequence))
+ .collect();
+ assert!(
+ lost.is_empty(),
+ "{topic}: acknowledged sends lost: {lost:?}"
+ );
+ let sent: HashSet<u64> = log
+ .acknowledged
+ .iter()
+ .chain(&log.uncertain)
+ .copied()
+ .collect();
+ let refused: Vec<_> = sequences
+ .iter()
+ .filter(|sequence| !sent.contains(sequence))
+ .collect();
+ assert!(
+ refused.is_empty(),
+ "{topic}: refused sends reached the log: {refused:?}"
+ );
+}
+
+#[iggy_harness(cluster_nodes = 3)]
+async fn given_a_partition_primary_leading_metadata_when_its_quorum_freezes_should_refuse_writes_until_it_resumes(
+ harness: &mut TestHarness,
+) {
+ assert_isolated_primary_fence(harness, true).await;
+}
+
+#[iggy_harness(cluster_nodes = 3)]
+async fn given_a_partition_primary_backing_up_metadata_when_its_quorum_freezes_should_refuse_writes_until_it_resumes(
+ harness: &mut TestHarness,
+) {
+ assert_isolated_primary_fence(harness, false).await;
+}
+
+/// A partition primary whose quorum peers are frozen gets no fresh metadata
+/// quorum evidence, whether it credits answered probes as the metadata primary
+/// or probes as a backup. It must refuse partition writes with 58 once that
+/// evidence is `heartbeat_timeout + 3s` old, admit nothing it refused, and
+/// serve the partition again after the peers resume.
+async fn assert_isolated_primary_fence(harness: &TestHarness, leads_metadata: bool) {
+ const ADMITTED: [&str; 2] = ["before-isolation", "after-resume"];
+ const REFUSED: &str = "refused-while-isolated";
+
+ let client = connect_without_login(harness.node(0).tcp_addr().unwrap()).await;
+ client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ client.create_stream(STREAM_NAME).await.unwrap();
+ let stream = Identifier::named(STREAM_NAME).unwrap();
+ let topic = Identifier::named(TOPIC_NAME).unwrap();
+ let partitions = u32::try_from(harness.cluster_size()).unwrap();
+ let options = TopicCreateOptions {
+ partitions_count: Some(partitions),
+ ..TopicCreateOptions::default()
+ };
+ client
+ .create_topic(&stream, TOPIC_NAME, &options)
+ .await
+ .unwrap();
+ let leader = leader_node_index_via(harness, 0).await;
+ let mut routes = Vec::with_capacity(harness.cluster_size());
+ for partition_id in 0..partitions {
+ routes.push(partition_route(&client, &stream, &topic, partition_id).await);
+ }
+ let (partition_id, route) = (0..partitions)
+ .zip(routes)
+ .find(|(_, route)| (usize::from(route.primary_replica) == leader) == leads_metadata)
+ .expect("one partition primary on every node, the metadata primary among them");
+ let primary = usize::from(route.primary_replica);
+ let address = harness.node(primary).tcp_addr().unwrap();
+ let partitioning = Partitioning::partition_id(partition_id);
+ client
+ .send_messages(&stream, &topic, &partitioning, &mut [message(ADMITTED[0])])
+ .await
+ .unwrap();
+
+ let probe = connect_without_login(address).await;
+ probe
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ let identity = probe.session_identity().await.unwrap();
+ let binding = BindSessionRequest {
+ version_info: ClientVersionInfo {
+ protocol_version: IGGY_PROTOCOL_VERSION,
+ sdk_name: WireName::new("routing-test").unwrap(),
+ sdk_version: WireName::new("1.0.0").unwrap(),
+ },
+ identity,
+ bind_secret: probe.session_bind_secret().await.unwrap(),
+ }
+ .to_bytes();
+ let mut request = RequestHeader {
+ client: identity.client_id,
+ session: identity.session,
+ request: RAW_REQUEST_START,
+ ..Default::default()
+ };
+ let mut socket = raw_tcp::connect_to(address).await;
+ data_request(&mut socket, &mut request, BIND_SESSION_CODE, &binding, None)
+ .await
+ .expect("bind the probe session on the partition primary");
+ let poll = PollMessagesRequest {
+ consumer: consumer_to_wire(&Consumer::default()).unwrap(),
+ stream_id: identifier_to_wire(&stream).unwrap(),
+ topic_id: identifier_to_wire(&topic).unwrap(),
+ partition_id: Some(partition_id),
+ strategy: polling_strategy_to_wire(&PollingStrategy::offset(0)),
+ count: 1,
+ auto_commit: false,
+ }
+ .to_bytes();
+ data_request(
+ &mut socket,
+ &mut request,
+ POLL_MESSAGES_CODE,
+ &poll,
+ Some(route.context()),
+ )
+ .await
+ .expect("a primary with its quorum must serve the partition");
+
+ let peers: Vec<usize> = (0..harness.cluster_size())
+ .filter(|node| *node != primary)
+ .collect();
+ let isolated_at = Instant::now();
+ for &peer in &peers {
+ pause_node(harness, peer, true);
+ }
+ let fenced = timeout(FENCE_BOUND, async {
+ loop {
+ match data_request(
+ &mut socket,
+ &mut request,
+ POLL_MESSAGES_CODE,
+ &poll,
+ Some(route.context()),
+ )
+ .await
+ {
+ Ok(_) => sleep(FENCE_PROBE_INTERVAL).await,
+ Err(error) => break error,
+ }
+ }
+ })
+ .await;
+ let write = data_request(
+ &mut socket,
+ &mut request,
+ SEND_MESSAGES_CODE,
+ &raw_send_body(partition_id, REFUSED),
+ Some(route.context()),
+ )
+ .await;
+ let refused_after = isolated_at.elapsed();
+ for &peer in &peers {
+ pause_node(harness, peer, false);
+ }
+ println!(
+ "node {primary}, metadata leader {leader}, refused a write to partition {partition_id} \
+ {refused_after:?} after isolation"
+ );
+ assert_eq!(
+ fenced,
+ Ok(IggyError::TransientNotAccepted),
+ "the isolated primary must close partition admission within {FENCE_BOUND:?}"
+ );
+ assert_eq!(
+ write.err(),
+ Some(IggyError::TransientNotAccepted),
+ "the isolated primary must refuse a write with 58"
+ );
+ assert!(
+ refused_after <= FENCE_BOUND,
+ "the write was refused {refused_after:?} after isolation, past {FENCE_BOUND:?}"
+ );
+
+ timeout(
+ DATA_FAILOVER_BUDGET,
+ client.send_messages(&stream, &topic, &partitioning, &mut [message(ADMITTED[1])]),
+ )
+ .await
+ .expect("the cluster must answer a write after the peers resume")
+ .expect("the cluster must accept a write after the peers resume");
+ let polled = client
+ .poll_messages(
+ &stream,
+ &topic,
+ Some(partition_id),
+ &Consumer::default(),
+ &PollingStrategy::offset(0),
+ u32::try_from(ADMITTED.len() + 1).unwrap(),
+ false,
+ )
+ .await
+ .unwrap();
+ let payloads: Vec<&[u8]> = polled
+ .messages
+ .iter()
+ .map(|message| message.payload.as_ref())
+ .collect();
+ assert_eq!(
+ payloads,
+ ADMITTED.map(str::as_bytes),
+ "a write refused with 58 must not reach the log"
+ );
+}
+
+fn raw_send_body(partition_id: u32, payload: &str) -> Bytes {
+ let stream_id = WireIdentifier::named(STREAM_NAME).unwrap();
+ let topic_id = WireIdentifier::named(TOPIC_NAME).unwrap();
+ let partitioning = WirePartitioning::PartitionId(partition_id);
+ let messages = [RawMessage {
+ id: 0,
+ origin_timestamp: 0,
+ headers: None,
+ payload: payload.as_bytes(),
+ }];
+ let mut body = BytesMut::with_capacity(SendMessagesEncoder::encoded_size(
+ &stream_id,
+ &topic_id,
+ &partitioning,
+ &messages,
+ ));
+ SendMessagesEncoder::encode(&mut body, &stream_id, &topic_id, &partitioning, &messages)
+ .unwrap();
+ body.freeze()
+}
+
#[iggy_harness(cluster_nodes = 3)]
async fn given_metadata_view_moved_when_creating_topic_should_route_to_its_assigned_primary(
harness: &mut TestHarness,
diff --git a/core/message_bus/src/config.rs b/core/message_bus/src/config.rs
index 991a21cd2..7164c5344 100644
--- a/core/message_bus/src/config.rs
+++ b/core/message_bus/src/config.rs
@@ -153,6 +153,10 @@ pub struct MessageBusConfig {
/// Bound on each SDK connection's inbound and outbound queues.
pub client_queue_capacity: usize,
+ /// How long a reply waits for room in a full client queue before the
+ /// connection is closed.
+ pub client_write_stall_timeout: Duration,
+
/// Interval between outbound reconnect attempts to peers with
/// `peer_id > self_id`.
pub reconnect_period: Duration,
@@ -227,6 +231,7 @@ impl From<&ServerConfig> for MessageBusConfig {
.expect("message_bus.replica_read_buffer_size fits usize on supported targets"),
peer_queue_capacity: bus.peer_queue_capacity,
client_queue_capacity: bus.client_queue_capacity,
+ client_write_stall_timeout: bus.client_write_stall_timeout.get_duration(),
reconnect_period: bus.reconnect_period.get_duration(),
mesh_expected_peers: if cfg.cluster.enabled {
cfg.cluster.nodes.len().saturating_sub(1)
diff --git a/core/message_bus/src/error.rs b/core/message_bus/src/error.rs
index 7d5fdd7df..92f188eb7 100644
--- a/core/message_bus/src/error.rs
+++ b/core/message_bus/src/error.rs
@@ -23,9 +23,9 @@ use thiserror::Error;
/// All variants are non-fatal from the consensus perspective.
/// VSR handles message loss via timeout-driven retransmission from the WAL.
///
-/// The client- and replica-keyed variants separate three physically
-/// distinct failure modes so operators can tell apart routing bugs from
-/// transient disconnects:
+/// The replica-keyed variants separate three physically distinct failure
+/// modes so operators can tell apart routing bugs from transient
+/// disconnects:
/// - `*NotFound` / `*NotConnected` - the key is unknown to this shard's
/// local registry (genuine peer state, often recoverable by reconnect).
/// - `*RouteMissing` - the bus cannot forward to the owning shard
@@ -33,36 +33,16 @@ use thiserror::Error;
/// - `*ForwardFailed` - the forward fn rejected the frame (inter-shard
/// queue full, shutdown, etc.).
///
-/// Recovery asymmetry between forward variants:
-/// - `ReplicaForwardFailed` is recoverable in-protocol. The dropped frame
-/// is a consensus message (`Prepare` / `PrepareOk` / view-change /
-/// `Commit`); VSR's WAL-driven retransmit re-sends it once the inbox
-/// drains.
-/// - `ClientForwardFailed` is terminal. Client Reply payloads live above
-/// the bus and have no retransmit path. A reply lost on full inbox is
-/// gone, the client times out, and request / response semantics break.
-/// Operators must size `inbox_capacity` for the worst-case cross-shard
-/// reply burst and alert on the drop-site `tracing` logs. The
-/// `frame_drops_total` `{variant="forward_client_send"}` counter is
-/// the structured complement, scrape-able once a per-shard exporter
-/// lands.
+/// `ReplicaForwardFailed` is recoverable in-protocol. The dropped frame is a
+/// consensus message (`Prepare` / `PrepareOk` / view-change / `Commit`), and
+/// VSR's WAL-driven retransmit re-sends it once the inbox drains. A client
+/// frame is never forwarded between shards.
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum SendError {
#[error("client {0} not found in local registry")]
ClientNotFound(u128),
- #[error("client {0}: no inter-shard forward fn installed")]
- ClientRouteMissing(u128),
-
- /// The owning shard's inter-shard inbox rejected the forward frame
- /// (full or disconnected). Unlike `ReplicaForwardFailed`, no VSR
- /// retransmit covers this drop: the client Reply payload was moved
- /// into the frame and is now gone. Callers must treat this as a
- /// final failure for the affected request; the client will time out.
- #[error("client {0}: inter-shard forward failed")]
- ClientForwardFailed(u128),
-
#[error("replica {0} not connected on this shard")]
ReplicaNotConnected(u8),
diff --git a/core/message_bus/src/installer/mod.rs b/core/message_bus/src/installer/mod.rs
index 8511eddaa..477067a82 100644
--- a/core/message_bus/src/installer/mod.rs
+++ b/core/message_bus/src/installer/mod.rs
@@ -175,8 +175,8 @@ pub trait ConnectionInstaller {
) -> impl Future<Output = Result<(), SendError>>;
/// [`Self::send_to_client_instance`] that waits for room in a full queue
- /// instead of closing the connection. A bus whose queues never fill
- /// sends at once.
+ /// instead of closing the connection, until the bus's write stall timeout
+ /// closes it. A bus whose queues never fill sends at once.
fn send_to_client_instance_waiting(
&self,
client_id: u128,
diff --git a/core/message_bus/src/lib.rs b/core/message_bus/src/lib.rs
index 0f2e12065..c574d34be 100644
--- a/core/message_bus/src/lib.rs
+++ b/core/message_bus/src/lib.rs
@@ -271,20 +271,6 @@ impl Default for ReplicaOwnerTable {
/// that outlive the caller and need independent ownership.
pub type ReplicaForwardFn = Box<dyn Fn(u8, u16, Frozen<MESSAGE_ALIGN>) -> Result<(), SendError>>;
-/// Callback for forwarding a client-bound message to the owning shard.
-///
-/// Provided by the shard layer at bus construction and installed via
-/// [`IggyMessageBus::set_client_forward_fn`]. Arguments:
-/// `(client_id, owning_shard_id, message)`. Returns `Ok(())` on successful
-/// enqueue into the inter-shard channel.
-///
-/// `owning_shard_id` is the value returned by [`client_id_owning_shard`];
-/// the slow path passes it pre-resolved so the closure does not redo the
-/// shift. `client_id` is carried so the receiver can rebuild the
-/// `ShardFrame::Lifecycle`'s `ForwardClientSend { client_id, msg }`
-/// payload.
-pub type ClientForwardFn = Box<dyn Fn(u128, u16, BusMessage) -> Result<(), SendError>>;
-
/// Callback invoked when a client connection metadata entry is removed.
pub type ClientConnectionLostFn = std::rc::Rc<dyn Fn(u128)>;
@@ -519,39 +505,34 @@ pub struct ReplicaReadMetrics {
/// retransmit. Real socket and HTTP targets require the installation
/// captured when dispatch admitted the request
/// ([`ConnectionInstaller::send_to_client_instance`]); a client id alone
-/// cannot prove reply ownership, so the owning shard refuses it. A
-/// `Backpressure` from the client's own queue closes that connection, so the
-/// client fails the pending request at once and recovers on a new
+/// cannot prove reply ownership, so [`IggyMessageBus`] refuses every frame
+/// sent by id with `ClientNotFound`, and never forwards it to another shard.
+/// A `Backpressure` from the client's own queue closes that connection, so
+/// the client fails the pending request at once and recovers on a new
/// connection. A reply sent with
/// [`ConnectionInstaller::send_to_client_instance_waiting`] waits for room
-/// instead. A `ClientForwardFailed` is terminal: the client never
-/// receives the reply and times out. A request the dispatch host relays to
-/// another shard gets its reply back over the request's own channel, and the
-/// shard holding the connection sends it as above.
-/// Operators must size `inbox_capacity` for the worst-case cross-shard
-/// reply burst. Each drop is logged at the drop site via `tracing`;
-/// alert on those logs today. The `frame_drops_total`
-/// `{variant="forward_client_send"}` counter is the structured
-/// complement and becomes scrape-able once a per-shard exporter lands.
-/// The symmetric `{variant="forward_replica_send"}` counter stays
-/// informational only.
+/// instead, up to `client_write_stall_timeout`, and then closes the
+/// connection the same way. A request the dispatch host relays to another
+/// shard gets its reply back over the request's own channel, and the shard
+/// holding the connection sends it as above. The
+/// `{variant="forward_replica_send"}` drop counter stays informational
+/// only.
///
/// # Setter install contract
///
-/// The three setters below do NOT share one install rule:
+/// The two setters below do NOT share one install rule:
/// - [`Self::set_connection_lost_fn`] is re-installable (`RefCell`
/// slot); it is a test observation hook some setups swap mid-test.
-/// - [`Self::set_replica_forward_fn`] and [`Self::set_client_forward_fn`]
-/// are one-shot (`OnceCell` slot) and panic on a second install;
-/// they are bootstrap-only wiring invariants.
+/// - [`Self::set_replica_forward_fn`] is one-shot (`OnceCell` slot) and
+/// panics on a second install, as a bootstrap-only wiring invariant.
///
/// A bus impl must preserve this divergence - see each method.
pub trait MessageBus {
/// Queue one frame for `client_id`. Takes anything that converts into a
/// [`BusMessage`]: a single `Frozen<MESSAGE_ALIGN>` buffer or an
- /// already fragmented frame. On [`IggyMessageBus`] socket and HTTP
- /// targets refuse a frame without its captured installation with
- /// `ClientNotFound` (see the recovery asymmetry above).
+ /// already fragmented frame. [`IggyMessageBus`] refuses every such frame
+ /// with `ClientNotFound`, because socket and HTTP targets need the
+ /// installation captured at admission (see the recovery asymmetry above).
fn send_to_client(
&self,
client_id: u128,
@@ -579,7 +560,7 @@ pub trait MessageBus {
/// Re-installing is allowed by design - the backing slot is a
/// `RefCell`, so a second call overwrites the first. This is the
/// deliberate exception to the install rules of
- /// [`Self::set_replica_forward_fn`] / [`Self::set_client_forward_fn`]:
+ /// [`Self::set_replica_forward_fn`]:
/// the notifier is a test observation hook some setups swap
/// mid-test, not a one-shot bootstrap invariant.
fn set_connection_lost_fn(&self, f: ConnectionLostFn);
@@ -599,17 +580,6 @@ pub trait MessageBus {
/// double-install is a wiring bug rather than a supported re-bind.
fn set_replica_forward_fn(&self, f: ReplicaForwardFn);
- /// Install the client-plane inter-shard forward closure.
- ///
- /// No default: every bus must explicitly opt in or stub. Same
- /// rationale as [`Self::set_replica_forward_fn`].
- ///
- /// # Panics
- ///
- /// Panics on a second install on the same bus, same one-shot
- /// `OnceCell` invariant as [`Self::set_replica_forward_fn`].
- fn set_client_forward_fn(&self, f: ClientForwardFn);
-
/// Register a detached `compio::runtime::spawn` handle so
/// [`shutdown`](IggyMessageBus::shutdown) can await it before the
/// runtime drops. Production [`IggyMessageBus`] pushes onto a
@@ -729,12 +699,11 @@ pub trait MessageBus {
/// on `Full` (returned as [`SendError::Backpressure`]) are recovered by
/// VSR retransmission for replica-bound consensus traffic. A client
/// reply refused on `Full` also closes that client's connection (see
-/// [`ConnectionRegistry::try_send_or_return`]). The one exception is
+/// [`ConnectionRegistry::try_send_to_instance`]). The one exception is
/// [`Self::send_to_client_instance_waiting`], which waits for room in the
-/// client's queue. Client Reply
-/// drops returned as [`SendError::ClientForwardFailed`] have no
-/// retransmit and are terminal; see the trait-level "Recovery
-/// asymmetry" section on [`MessageBus`].
+/// client's queue for up to `client_write_stall_timeout`. A client reply
+/// sent by id alone is refused, as the trait-level "Recovery asymmetry"
+/// section on [`MessageBus`] explains.
/// - The per-connection writer task batches up to `config.max_batch` messages into
/// a single `writev` syscall on the plaintext TCP plane (see
/// [`transports::tcp`]).
@@ -754,12 +723,6 @@ pub struct IggyMessageBus {
/// once at bootstrap via [`Self::set_replica_forward_fn`]; the slow
/// path reads through [`OnceCell::get`] without a runtime borrow.
replica_forward_fn: OnceCell<ReplicaForwardFn>,
- /// Forwards a client-bound message to the shard that owns the client's
- /// TCP connection (the owning shard is encoded in the top 16 bits of
- /// the client id). Empty on single-shard setups and tests. Installed
- /// once at bootstrap via [`Self::set_client_forward_fn`]; the slow
- /// path reads through [`OnceCell::get`] without a runtime borrow.
- client_forward_fn: OnceCell<ClientForwardFn>,
/// Optional disconnect notifier invoked by
/// `Self::notify_connection_lost` after it CAS-clears the owner
/// table. Production leaves this unset; tests install a callback to
@@ -904,7 +867,6 @@ impl IggyMessageBus {
background_tasks: RefCell::new(Vec::new()),
config,
replica_forward_fn: OnceCell::new(),
- client_forward_fn: OnceCell::new(),
connection_lost_fn: RefCell::new(None),
client_connection_lost_fn: RefCell::new(None),
client_meta: RefCell::new(ahash::AHashMap::new()),
@@ -1093,10 +1055,13 @@ impl IggyMessageBus {
/// [`Self::send_to_client_instance`] that waits for room in a full queue
/// instead of closing the connection. Unlike every other send, it yields.
+ /// A queue that stays full for `client_write_stall_timeout` closes the
+ /// connection, because its client stopped reading.
///
/// # Errors
///
- /// The errors of [`Self::send_to_client_instance`] except `Backpressure`.
+ /// The errors of [`Self::send_to_client_instance`]. `Backpressure` means
+ /// the wait stalled and the connection was closed.
#[allow(clippy::future_not_send)]
pub async fn send_to_client_instance_waiting(
&self,
@@ -1107,7 +1072,12 @@ impl IggyMessageBus {
let instance = self.reply_instance(client_id, instance)?;
let route = self
.clients
- .send_to_instance(client_id, instance, message)
+ .send_to_instance(
+ client_id,
+ instance,
+ message,
+ self.config.client_write_stall_timeout,
+ )
.await;
self.finish_local_reply(client_id, route)
}
@@ -1136,6 +1106,7 @@ impl IggyMessageBus {
.map_err(|_| SendError::ClientNotFound(client_id))
}
ReplyRoute::NoSlot(_message) => Err(SendError::ClientNotFound(client_id)),
+ ReplyRoute::Stalled => Err(SendError::Backpressure),
}
}
@@ -1255,25 +1226,6 @@ impl IggyMessageBus {
.expect("replica_forward_fn installed twice (bootstrap invariant)");
}
- /// Install the client-plane inter-shard forward closure.
- ///
- /// Shards invoke this from `send_to_client`'s slow path when the client
- /// connection lives on a different shard (top 16 bits of `client_id`).
- ///
- /// Takes `&self` so it can be called through an `Rc<IggyMessageBus>`
- /// wrapper. Call sites are bootstrap only.
- ///
- /// # Panics
- ///
- /// Panics if called more than once on the same bus. Same rationale as
- /// [`Self::set_replica_forward_fn`].
- pub fn set_client_forward_fn(&self, f: ClientForwardFn) {
- self.client_forward_fn
- .set(f)
- .ok()
- .expect("client_forward_fn installed twice (bootstrap invariant)");
- }
-
/// Get the owning shard for a replica, if any.
///
/// Reads the shard-shared [`ReplicaOwnerTable`], which is stamped
@@ -1470,10 +1422,6 @@ impl<T: MessageBus + ?Sized> MessageBus for std::rc::Rc<T> {
(**self).set_replica_forward_fn(f);
}
- fn set_client_forward_fn(&self, f: ClientForwardFn) {
- (**self).set_client_forward_fn(f);
- }
-
fn track_background(&self, handle: JoinHandle<()>) {
(**self).track_background(handle);
}
@@ -1509,24 +1457,12 @@ impl MessageBus for IggyMessageBus {
async fn send_to_client(
&self,
client_id: u128,
- message: impl Into<BusMessage>,
+ _message: impl Into<BusMessage>,
) -> Result<(), SendError> {
if self.is_shutting_down() {
return Err(SendError::BusShuttingDown);
}
- let message = message.into();
- // Owning shard is encoded in the top 16 bits of client_id.
- let owning_shard = client_id_owning_shard(client_id);
- if owning_shard == self.shard_id {
- let route = self.clients.try_send_or_return(client_id, message);
- return self.finish_local_reply(client_id, route);
- }
- let forward = self
- .client_forward_fn
- .get()
- .ok_or(SendError::ClientRouteMissing(client_id))?;
- forward(client_id, owning_shard, message)
- .map_err(|_| SendError::ClientForwardFailed(client_id))
+ Err(SendError::ClientNotFound(client_id))
}
async fn send_to_replica(
@@ -1579,10 +1515,6 @@ impl MessageBus for IggyMessageBus {
Self::set_replica_forward_fn(self, f);
}
- fn set_client_forward_fn(&self, f: ClientForwardFn) {
- Self::set_client_forward_fn(self, f);
- }
-
fn track_background(&self, handle: JoinHandle<()>) {
Self::track_background(self, handle);
}
@@ -1689,29 +1621,6 @@ mod tests {
})
}
- #[compio::test]
- #[allow(clippy::future_not_send)]
- async fn send_to_client_slow_path_forwards_to_owning_shard() {
- // Bus on shard 5; client id encodes owning shard = 7.
- let bus = IggyMessageBus::new(5);
- let captured: std::rc::Rc<RefCell<Option<u16>>> = std::rc::Rc::new(RefCell::new(None));
- let captured_clone = captured.clone();
- let client_id = (7u128 << 112) | 0x2a;
- let captured_id: std::rc::Rc<RefCell<Option<u128>>> = std::rc::Rc::new(RefCell::new(None));
- let captured_id_clone = captured_id.clone();
- bus.set_client_forward_fn(Box::new(move |id, target, _msg| {
- *captured_id_clone.borrow_mut() = Some(id);
- *captured_clone.borrow_mut() = Some(target);
- Ok(())
- }));
-
- bus.send_to_client(client_id, dummy_message().into_frozen())
- .await
- .expect("forward_fn should accept");
- assert_eq!(*captured.borrow(), Some(7));
- assert_eq!(*captured_id.borrow(), Some(client_id));
- }
-
#[compio::test]
#[allow(clippy::future_not_send)]
async fn send_to_client_fast_path_hits_local_when_owning_shard_matches() {
@@ -1836,17 +1745,19 @@ mod tests {
);
}
+ /// The owning shard refuses a frame without its installation too, so a
+ /// by-id frame for a client homed elsewhere is refused here, not forwarded.
#[compio::test]
#[allow(clippy::future_not_send)]
- async fn send_to_client_returns_route_missing_when_remote_and_no_forward_fn() {
- // shard_id != top-16-bits and no forward_fn installed.
+ async fn given_client_homed_on_another_shard_when_sending_by_id_should_refuse_without_forwarding()
+ {
let bus = IggyMessageBus::new(0);
let client_id = (9u128 << 112) | 1;
let err = bus
.send_to_client(client_id, dummy_message().into_frozen())
.await
.unwrap_err();
- assert!(matches!(err, SendError::ClientRouteMissing(_)));
+ assert!(matches!(err, SendError::ClientNotFound(id) if id == client_id));
}
#[compio::test]
diff --git a/core/message_bus/src/lifecycle/connection_registry.rs b/core/message_bus/src/lifecycle/connection_registry.rs
index 93da55093..d49f1414c 100644
--- a/core/message_bus/src/lifecycle/connection_registry.rs
+++ b/core/message_bus/src/lifecycle/connection_registry.rs
@@ -163,12 +163,15 @@ impl ReplyTarget {
/// so the caller can decode the request id and fire the matching oneshot
/// via [`ConnectionRegistry::fire_in_process`], keeping that decode off the
/// socket path. `NoSlot` means no entry matches the captured installation.
+/// `Stalled` means a waiting send found no room within its stall timeout: the
+/// reply is dropped and its connection closed.
#[derive(Debug)]
#[must_use]
pub enum ReplyRoute {
Delivered(Result<(), async_channel::TrySendError<BusMessage>>),
InProcess(BusMessage),
NoSlot(BusMessage),
+ Stalled,
}
/// Rejection reasons for [`ConnectionRegistry::install_reply_slot`].
@@ -248,7 +251,8 @@ struct Entry {
/// would close the broadcast channel and falsely wake readers waiting
/// on `conn_token.wait()`. Triggered by
/// [`ConnectionRegistry::try_send_to_instance`] when a client connection's
- /// queue refuses a reply; implicitly dropped on entry removal.
+ /// queue refuses a reply, and by [`ConnectionRegistry::send_to_instance`]
+ /// when it stalls. Dropped with the entry.
conn_shutdown: super::Shutdown,
}
@@ -326,16 +330,6 @@ where
.map(f)
}
- /// Route `msg` by `key` alone, handing it back when no target accepts it.
- ///
- /// Both socket and in-process targets require the installation captured
- /// at admission, so this returns [`ReplyRoute::NoSlot`]. A request's
- /// client id alone cannot identify the connection or HTTP waiter that
- /// owns its reply.
- pub fn try_send_or_return(&self, key: K, msg: BusMessage) -> ReplyRoute {
- self.route_reply(key, None, msg)
- }
-
/// Route `msg` to the installation `instance` names, handing it back when
/// no live target accepts it.
///
@@ -367,21 +361,39 @@ where
instance: InstanceToken,
msg: BusMessage,
) -> ReplyRoute {
- self.route_reply(key, Some(instance), msg)
+ let entries = self.entries.borrow();
+ let Some(entry) = entries.get(&key).filter(|entry| entry.token == instance) else {
+ return ReplyRoute::NoSlot(msg);
+ };
+ match &entry.reply {
+ ReplyTarget::Socket(sender) => {
+ let sent = sender.try_send(msg);
+ if matches!(sent, Err(async_channel::TrySendError::Full(_))) {
+ sender.close();
+ entry.conn_shutdown.trigger();
+ }
+ ReplyRoute::Delivered(sent)
+ }
+ ReplyTarget::InProcess { .. } => ReplyRoute::InProcess(msg),
+ }
}
- /// [`Self::try_send_to_instance`] that waits for room in a full socket
- /// queue instead of closing the connection.
+ /// [`Self::try_send_to_instance`] that waits up to `stall_timeout` for room
+ /// in a full socket queue before it closes the connection.
///
/// The wait ends when the writer takes a frame off the queue, or with
/// `Closed` when the connection goes away, a reinstall under `key`
- /// included, because only this installation's queue is waited on.
+ /// included, because only this installation's queue is waited on. A queue
+ /// that stays full for `stall_timeout` belongs to a client that stopped
+ /// reading. The reply is then dropped with [`ReplyRoute::Stalled`] and
+ /// this installation is shut down as a refused reply shuts it down.
#[allow(clippy::future_not_send)]
pub async fn send_to_instance(
&self,
key: K,
instance: InstanceToken,
msg: BusMessage,
+ stall_timeout: Duration,
) -> ReplyRoute {
// Only a full queue clones the `Sender` to wait on it, so a reply that
// finds room costs no atomic RMW on the channel state.
@@ -398,11 +410,20 @@ where
ReplyTarget::InProcess { .. } => return ReplyRoute::InProcess(msg),
}
};
+ let Ok(sent) = compio::time::timeout(stall_timeout, sender.send(msg)).await else {
+ sender.close();
+ if let Some(entry) = self
+ .entries
+ .borrow()
+ .get(&key)
+ .filter(|entry| entry.token == instance)
+ {
+ entry.conn_shutdown.trigger();
+ }
+ return ReplyRoute::Stalled;
+ };
ReplyRoute::Delivered(
- sender
- .send(msg)
- .await
- .map_err(|async_channel::SendError(msg)| async_channel::TrySendError::Closed(msg)),
+ sent.map_err(|async_channel::SendError(msg)| async_channel::TrySendError::Closed(msg)),
)
}
@@ -413,27 +434,6 @@ where
self.entries.borrow().get(&key).map(|entry| entry.token)
}
- fn route_reply(&self, key: K, instance: Option<InstanceToken>, msg: BusMessage) -> ReplyRoute {
- let entries = self.entries.borrow();
- let Some(entry) = entries
- .get(&key)
- .filter(|entry| instance == Some(entry.token))
- else {
- return ReplyRoute::NoSlot(msg);
- };
- match &entry.reply {
- ReplyTarget::Socket(sender) => {
- let sent = sender.try_send(msg);
- if matches!(sent, Err(async_channel::TrySendError::Full(_))) {
- sender.close();
- entry.conn_shutdown.trigger();
- }
- ReplyRoute::Delivered(sent)
- }
- ReplyTarget::InProcess { .. } => ReplyRoute::InProcess(msg),
- }
- }
-
/// Fire the in-process reply registered under `request` for `key`.
///
/// Companion to [`ReplyRoute::InProcess`]; the socket path never reaches
@@ -876,8 +876,8 @@ impl ReplicaRegistry {
/// `msg` back as `Err` when no slot is present.
///
/// Replicas only ever register socket reply targets, so unlike
- /// [`ConnectionRegistry::try_send_or_return`] (which returns a [`ReplyRoute`]
- /// also covering the in-process case) this stays a flat
+ /// [`ConnectionRegistry::try_send_to_instance`] (which returns a
+ /// [`ReplyRoute`] also covering the in-process case) this stays a flat
/// `Result<try-send outcome, BusMessage>`.
///
/// # Errors
@@ -1372,10 +1372,6 @@ mod tests {
.insert(1u8, tx, writer, reader, dummy_conn_shutdown())
.expect("insert ok");
- assert!(matches!(
- reg.try_send_or_return(1u8, make_bus_msg()),
- ReplyRoute::NoSlot(_)
- ));
let outer = reg.try_send_to_instance(1u8, instance, make_bus_msg());
assert!(matches!(outer, ReplyRoute::Delivered(Ok(()))));
// Receiver drains the message; closing the writer's rx end via
@@ -1384,15 +1380,14 @@ mod tests {
assert_eq!(received.total_len(), HEADER_SIZE);
}
- /// No slot for `key`: the message is handed back to the caller so the
- /// slow path (or no-op for clients) can decide what to do.
+ /// No slot for `key`: the message is handed back to the caller.
#[compio::test]
- async fn try_send_or_return_returns_msg_when_slot_missing() {
+ async fn try_send_to_instance_returns_msg_when_slot_missing() {
let reg: ConnectionRegistry<u8> = ConnectionRegistry::new();
let msg = make_bus_msg();
let original_len = msg.total_len();
- let outcome = reg.try_send_or_return(99u8, msg);
+ let outcome = reg.try_send_to_instance(99u8, InstanceToken(1), msg);
let ReplyRoute::NoSlot(returned) = outcome else {
panic!("missing slot should return msg");
};
@@ -1435,6 +1430,7 @@ mod tests {
/// full queue until the writer takes a frame, and never closes it.
#[compio::test]
async fn send_to_instance_waits_only_for_a_full_queue() {
+ const STALL_TIMEOUT: Duration = Duration::from_secs(30);
let reg: ConnectionRegistry<u8> = ConnectionRegistry::new();
let (_shutdown, token) = Shutdown::new();
let (tx, rx) = async_channel::bounded(1);
@@ -1445,10 +1441,11 @@ mod tests {
.expect("insert ok");
assert!(matches!(
- reg.send_to_instance(1u8, instance, make_bus_msg()).await,
+ reg.send_to_instance(1u8, instance, make_bus_msg(), STALL_TIMEOUT)
+ .await,
ReplyRoute::Delivered(Ok(()))
));
- let mut waiting = pin!(reg.send_to_instance(1u8, instance, make_bus_msg()));
+ let mut waiting = pin!(reg.send_to_instance(1u8, instance, make_bus_msg(), STALL_TIMEOUT));
assert!(poll!(waiting.as_mut()).is_pending());
rx.recv().await.expect("the first reply is queued");
assert!(matches!(waiting.await, ReplyRoute::Delivered(Ok(()))));
@@ -1456,6 +1453,60 @@ mod tests {
assert!(!rx.is_closed());
}
+ /// A client that leaves its socket unread keeps its queue full. A reply
+ /// that finds no room within the stall timeout closes that exact
+ /// connection, so the replies and tasks it pins are released, and every
+ /// reply still waiting on that queue ends with `Closed`.
+ #[compio::test]
+ async fn given_stalled_reply_queue_when_waiting_past_stall_timeout_should_close_that_connection()
+ {
+ const STALL: Duration = Duration::from_millis(50);
+ const TEST_BOUND: Duration = Duration::from_secs(5);
+ let reg: ConnectionRegistry<u8> = ConnectionRegistry::new();
+ let (stalled, stalled_rx, stalled_token) = insert_stalled(®, 1u8);
+ let (other, other_rx, other_token) = insert_stalled(®, 2u8);
+ assert!(matches!(
+ reg.send_to_instance(1u8, stalled, make_bus_msg(), STALL)
+ .await,
+ ReplyRoute::Delivered(Ok(()))
+ ));
+
+ let mut behind = pin!(reg.send_to_instance(1u8, stalled, make_bus_msg(), TEST_BOUND));
+ assert!(poll!(behind.as_mut()).is_pending());
+ let outcome = compio::time::timeout(
+ TEST_BOUND,
+ reg.send_to_instance(1u8, stalled, make_bus_msg(), STALL),
+ )
+ .await
+ .expect("a reply must stop waiting at the stall timeout");
+ assert!(matches!(outcome, ReplyRoute::Stalled));
+ assert!(
+ stalled_token.is_triggered(),
+ "the stalled connection must shut down"
+ );
+ assert!(stalled_rx.is_closed());
+ assert_eq!(
+ stalled_rx.len(),
+ 1,
+ "the reply queued before the stall stays for the writer"
+ );
+ assert!(matches!(
+ behind.await,
+ ReplyRoute::Delivered(Err(async_channel::TrySendError::Closed(_)))
+ ));
+
+ assert!(
+ !other_token.is_triggered(),
+ "an unrelated connection must stay open"
+ );
+ assert!(matches!(
+ reg.send_to_instance(2u8, other, make_bus_msg(), STALL)
+ .await,
+ ReplyRoute::Delivered(Ok(()))
+ ));
+ assert_eq!(other_rx.len(), 1);
+ }
+
/// A refused reply leaves its request unanswered, so the refusal must
/// close that exact connection: its shutdown fires and later replies
/// report `Closed`. An unrelated connection keeps delivering, and the
@@ -1573,7 +1624,7 @@ mod tests {
(token, rx, conn_token)
}
- /// Mirror of `try_send_or_return_returns_msg_when_slot_missing` for the
+ /// Mirror of `try_send_to_instance_returns_msg_when_slot_missing` for the
/// fixed-array `ReplicaRegistry`. Guards the slow-path forward in
/// `send_to_replica` against silent payload loss when no slot exists.
#[compio::test]
@@ -1594,10 +1645,6 @@ mod tests {
let instance = reg.insert_in_process(1u8).expect("fresh key");
let (guard, reply_rx) = reg.install_reply_slot(1u8, 42).expect("slot installs");
- assert!(matches!(
- reg.try_send_or_return(1u8, make_bus_msg()),
- ReplyRoute::NoSlot(_)
- ));
let msg = match reg.try_send_to_instance(1u8, instance, make_bus_msg()) {
ReplyRoute::InProcess(msg) => msg,
other => panic!("expected InProcess route, got {other:?}"),
diff --git a/core/message_bus/src/transports/mod.rs b/core/message_bus/src/transports/mod.rs
index a3b074e28..14ea38d3d 100644
--- a/core/message_bus/src/transports/mod.rs
+++ b/core/message_bus/src/transports/mod.rs
@@ -74,7 +74,7 @@
//! - fd-delegation (`F_DUPFD_CLOEXEC`) stays TCP-only and lives outside
//! this trait (see [`crate::fd_transfer`]). Other transports terminate
//! on shard 0 and forward `Frozen<MESSAGE_ALIGN>` over the inter-shard
-//! crossfire channel via [`crate::ReplicaForwardFn`] / [`crate::ClientForwardFn`].
+//! crossfire channel via [`crate::ReplicaForwardFn`].
//!
//! Per-transport impls live in
//! `transports/{tcp,tcp_tls,ws,wss,quic}.rs`.
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs
index 8ed99ac25..47cfbaffe 100644
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@ -3767,18 +3767,19 @@ where
let reply_buffers = freeze_client_reply(generic_reply);
emit_sim_event(SimEventKind::ClientReplyEmitted, &event);
+ // The production bus refuses it, and a resend gets the cached reply.
if let Err(e) = consensus
.message_bus()
.send_to_client(event.client_id, reply_buffers)
.await
{
- error!(
+ debug!(
client = event.client_id,
op = event.op,
request_id = event.request_id,
operation = ?event.operation,
%e,
- "client reply forward failed, no retransmit path; client will time out",
+ "client reply has no route by client id",
);
}
}
@@ -4288,7 +4289,7 @@ where
eligible: &[bool],
after: Option<IggyNamespace>,
) -> Option<PartitionPrimaryTransfer> {
- self.placement.borrow().primary_rebalance_candidate(
+ self.placement.borrow_mut().primary_rebalance_candidate(
self.mux_stm.streams(),
self.consensus.as_ref()?.replica_count(),
eligible,
@@ -5446,7 +5447,9 @@ mod tests {
use crate::stm::StateHandler;
use crate::stm::authz::GatedApply;
use crate::stm::consumer_group::JoinConsumerGroupRequest;
- use crate::stm::lifecycle::{CompleteLifecycleRequest, LifecycleCompletion};
+ use crate::stm::lifecycle::{
+ CompleteLifecycleRequest, LifecycleCompletion, apply_with_lifecycle_completion,
+ };
use crate::stm::stream::{Streams, StreamsInner};
use crate::stm::user::{PAT_TOKEN_HASH_BYTES, Users, UsersInner};
use consensus::{HistoryRecovery, LocalPipeline};
@@ -5463,9 +5466,7 @@ mod tests {
use iggy_common::{IggyTimestamp, UserStatus, variadic};
use journal::prepare_journal::{DEFAULT_SLOT_COUNT, PrepareJournal};
use journal::superblock::SuperblockContents;
- use message_bus::{
- BusMessage, ClientForwardFn, ConnectionLostFn, JoinHandle, ReplicaForwardFn, SendError,
- };
+ use message_bus::{BusMessage, ConnectionLostFn, JoinHandle, ReplicaForwardFn, SendError};
use server_common::MESSAGE_ALIGN;
use server_common::iobuf::Frozen;
use std::cell::RefCell;
@@ -6115,7 +6116,6 @@ mod tests {
}
fn set_connection_lost_fn(&self, _f: ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: ClientForwardFn) {}
}
/// Single-node metadata plane whose `prepare_request` is callable. `J` is
@@ -6524,17 +6524,17 @@ mod tests {
#[test]
fn given_partition_observations_when_creating_should_keep_highest_matching_incarnation() {
let mut metadata = placement_metadata();
- let mut streams = stream_with_one_partition();
- let partition = &mut streams
+ let streams = stream_with_one_partition_in_view(2);
+ let revision = streams
.items
- .get_mut(0)
+ .get(0)
.unwrap()
.topics
- .get_mut(0)
+ .get(0)
.unwrap()
- .partitions[0];
- partition.created_revision = 7;
- partition.created_view = 2;
+ .partitions[0]
+ .created_revision;
+ let mut recreated = streams.clone();
metadata.mux_stm = Rc::new(TestMux::new((
users_with_topic_manager(),
(streams.into(), ()),
@@ -6543,22 +6543,22 @@ mod tests {
metadata.observe_partition_views(&[
PartitionPlacementView {
namespace,
- created_revision: 7,
+ created_revision: revision,
view: 6,
},
PartitionPlacementView {
namespace,
- created_revision: 7,
+ created_revision: revision,
view: 4,
},
PartitionPlacementView {
namespace,
- created_revision: 6,
+ created_revision: revision - 1,
view: 100,
},
PartitionPlacementView {
namespace,
- created_revision: 8,
+ created_revision: revision + 1,
view: 101,
},
]);
@@ -6576,24 +6576,26 @@ mod tests {
.unwrap();
assert_eq!(assigned_views(&unchanged), [1]);
- let mut recreated = stream_with_one_partition();
- let partition = &mut recreated
- .items
- .get_mut(0)
- .unwrap()
- .topics
- .get_mut(0)
- .unwrap()
- .partitions[0];
- partition.created_revision = 8;
- partition.created_view = 2;
+ assert_eq!(
+ apply_with_lifecycle_completion(
+ &DeleteTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(0),
+ },
+ &mut recreated,
+ IggyTimestamp::now(),
+ )
+ .code,
+ 0
+ );
+ create_topic_in_view(&mut recreated, 2);
metadata.mux_stm = Rc::new(TestMux::new((
users_with_topic_manager(),
(recreated.into(), ()),
)));
metadata.observe_partition_views(&[PartitionPlacementView {
namespace,
- created_revision: 7,
+ created_revision: revision,
view: 102,
}]);
let next = metadata
@@ -6902,7 +6904,6 @@ mod tests {
}
fn set_connection_lost_fn(&self, _f: ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: ClientForwardFn) {}
}
/// Bus that records the client id of every `send_to_client`.
@@ -6933,7 +6934,6 @@ mod tests {
}
fn set_connection_lost_fn(&self, _f: ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: ClientForwardFn) {}
}
/// A view change or a boot re-pipeline rebuilds a pending expired-token
@@ -7885,16 +7885,25 @@ mod tests {
/// Stream 0 holding one topic with one partition.
fn stream_with_one_partition() -> StreamsInner {
+ stream_with_one_partition_in_view(0)
+ }
+
+ fn stream_with_one_partition_in_view(view: u32) -> StreamsInner {
let mut inner = StreamsInner::new();
- let timestamp = IggyTimestamp::now();
let _ = StateHandler::apply(
&CreateStreamRequest {
name: WireName::new("stream").unwrap(),
options: WireOptions::empty(),
},
&mut inner,
- timestamp,
+ IggyTimestamp::now(),
);
+ create_topic_in_view(&mut inner, view);
+ inner
+ }
+
+ /// Topic "existing" in stream 0, with one partition created in `view`.
+ fn create_topic_in_view(inner: &mut StreamsInner, view: u32) {
let _ = StateHandler::apply(
&PersistedCreateTopicRequest {
request: CreateTopicRequest {
@@ -7907,13 +7916,12 @@ mod tests {
partitions: vec![CreatedPartitionAssignment {
partition_id: 0,
consensus_group_id: 1,
- initial_view: 0,
+ initial_view: view,
}],
},
- &mut inner,
- timestamp,
+ inner,
+ IggyTimestamp::now(),
);
- inner
}
#[test]
@@ -8386,7 +8394,6 @@ mod tests {
}
fn set_connection_lost_fn(&self, _f: ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: ClientForwardFn) {}
}
/// `(target replica, op, prepare checksum)` of every `PrepareOk` sent on `bus`.
diff --git a/core/metadata/src/impls/placement.rs b/core/metadata/src/impls/placement.rs
index 85ecfcdfd..42b1d3196 100644
--- a/core/metadata/src/impls/placement.rs
+++ b/core/metadata/src/impls/placement.rs
@@ -16,7 +16,7 @@
// under the License.
use crate::impls::metadata::{PartitionPlacementView, PartitionPrimaryTransfer};
-use crate::stm::stream::{Partition, Streams, find_partition};
+use crate::stm::stream::{Partition, Streams, StreamsInner, find_partition};
use ahash::AHashMap;
use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment;
use iggy_binary_protocol::requests::partitions::CreatePartitionsWithAssignmentsRequest;
@@ -31,13 +31,32 @@ use server_common::sharding::IggyNamespace;
#[derive(Default)]
pub(super) struct PlacementState {
observations: AHashMap<IggyNamespace, ObservedView>,
+ /// Per replica, observed primaries minus created ones, for one replica count.
+ moved: Option<(u8, Vec<i64>)>,
+ /// The `namespace_revision` whose removals `observations` reflects.
+ synced_revision: Option<u64>,
+ /// Changes with every observation, so a scan can tell its input moved.
+ observed_generation: u64,
+ last_scan: Option<(ScanInputs, Option<PartitionPrimaryTransfer>)>,
reservations: AHashMap<(u64, u128), Vec<u64>>,
eligible: Option<(u32, Vec<bool>)>,
}
+#[derive(Clone, Copy, PartialEq, Eq)]
struct ObservedView {
report: PartitionPlacementView,
- present: bool,
+ created_view: u32,
+}
+
+/// Everything a rebalance scan reads. `revision` moves with the partition set
+/// and with every lifecycle intent, which hides partitions from the scan.
+#[derive(PartialEq, Eq)]
+struct ScanInputs {
+ revision: u64,
+ observed_generation: u64,
+ replicas: u8,
+ eligible: Vec<bool>,
+ after: Option<IggyNamespace>,
}
impl PlacementState {
@@ -49,12 +68,7 @@ impl PlacementState {
let () = streams.read(|inner| {
for report in reports {
let namespace = report.namespace;
- let Some(partition) = inner
- .items
- .get(namespace.stream_id())
- .and_then(|stream| stream.topics.get(namespace.topic_id()))
- .and_then(|topic| find_partition(&topic.partitions, namespace.partition_id()))
- else {
+ let Some(partition) = committed_partition(inner, namespace) else {
continue;
};
if partition.created_revision != report.created_revision {
@@ -63,120 +77,69 @@ impl PlacementState {
if report.view < partition.created_view {
continue;
}
- self.observations
- .entry(namespace)
- .and_modify(|previous| {
- if previous.report.created_revision == report.created_revision {
- previous.report.view = previous.report.view.max(report.view);
- } else {
- previous.report = *report;
- }
- })
- .or_insert(ObservedView {
- report: *report,
- present: true,
- });
+ let mut observed = ObservedView {
+ report: *report,
+ created_view: partition.created_view,
+ };
+ if let Some(previous) = self.observations.get(&namespace).copied() {
+ if previous.report.created_revision == report.created_revision {
+ observed.report.view = previous.report.view.max(report.view);
+ }
+ if observed == previous {
+ continue;
+ }
+ self.track(previous, -1);
+ }
+ self.track(observed, 1);
+ self.observations.insert(namespace, observed);
+ self.observed_generation += 1;
}
});
}
pub(super) fn committed_counts(&mut self, streams: &Streams, replicas: u8) -> Vec<u64> {
- let counts = streams.read(|inner| {
- let mut counts = vec![0; usize::from(replicas)];
- for observation in self.observations.values_mut() {
- observation.present = false;
+ streams.read(|inner| {
+ self.retire_removed(inner);
+ let mut counts = vec![0_u64; usize::from(replicas)];
+ for (&view, &partitions) in &inner.created_views {
+ counts[(view % u32::from(replicas)) as usize] += partitions;
}
- for (stream_id, stream) in &inner.items {
- for (topic_id, topic) in &stream.topics {
- for partition in &topic.partitions {
- let namespace = IggyNamespace::new(stream_id, topic_id, partition.id);
- let view = self
- .observations
- .get_mut(&namespace)
- .filter(|observation| {
- observation.report.created_revision == partition.created_revision
- })
- .map_or(partition.created_view, |observation| {
- observation.present = true;
- observation.report.view.max(partition.created_view)
- });
- counts[(view % u32::from(replicas)) as usize] += 1;
- }
- }
+ for (count, &moved) in counts.iter_mut().zip(self.moved(replicas)) {
+ *count = count.saturating_add_signed(moved);
}
counts
- });
- self.observations
- .retain(|_, observation| observation.present);
- counts
+ })
}
pub(super) fn primary_rebalance_candidate(
- &self,
+ &mut self,
streams: &Streams,
replicas: u8,
eligible: &[bool],
after: Option<IggyNamespace>,
) -> Option<PartitionPrimaryTransfer> {
- let replicas = usize::from(replicas);
+ let replica_count = usize::from(replicas);
let available = eligible.iter().filter(|&&available| available).count();
- if eligible.len() != replicas || available < 2 || available <= replicas / 2 {
+ if eligible.len() != replica_count || available < 2 || available <= replica_count / 2 {
return None;
}
streams.read(|inner| {
- let mut counts = vec![0_u64; replicas];
- let mut first: Vec<Option<PartitionPlacementView>> = vec![None; replicas];
- let mut next: Vec<Option<PartitionPlacementView>> = vec![None; replicas];
- for (stream_id, stream) in &inner.items {
- for (topic_id, topic) in &stream.topics {
- if inner.lifecycle_blocks(stream_id, Some(topic_id)) {
- continue;
- }
- for partition in &topic.partitions {
- let namespace = IggyNamespace::new(stream_id, topic_id, partition.id);
- let report = self.matching_view(namespace, partition);
- let primary = report.map_or(partition.created_view, |report| report.view)
- as usize
- % replicas;
- counts[primary] += 1;
- let Some(report) = report.filter(|_| eligible[primary]) else {
- continue;
- };
- if first[primary].is_none_or(|first| namespace < first.namespace) {
- first[primary] = Some(report);
- }
- if after.is_none_or(|cursor| namespace > cursor)
- && next[primary].is_none_or(|next| namespace < next.namespace)
- {
- next[primary] = Some(report);
- }
- }
- }
- }
- let mut available = eligible
- .iter()
- .enumerate()
- .filter_map(|(replica, &eligible)| eligible.then_some(replica));
- let mut source = available.next()?;
- let mut target = source;
- for replica in available {
- if counts[replica] > counts[source] {
- source = replica;
- }
- if counts[replica] < counts[target] {
- target = replica;
- }
- }
- if counts[source].saturating_sub(counts[target]) <= 1 {
- return None;
+ self.retire_removed(inner);
+ let inputs = ScanInputs {
+ revision: inner.revision,
+ observed_generation: self.observed_generation,
+ replicas,
+ eligible: eligible.to_vec(),
+ after,
+ };
+ if let Some((scanned, plan)) = &self.last_scan
+ && *scanned == inputs
+ {
+ return *plan;
}
- let report = next[source].or(first[source])?;
- Some(PartitionPrimaryTransfer {
- namespace: report.namespace,
- created_revision: report.created_revision,
- view: report.view,
- target: u8::try_from(target).ok()?,
- })
+ let plan = self.scan(inner, replica_count, eligible, after);
+ self.last_scan = Some((inputs, plan));
+ plan
})
}
@@ -284,6 +247,68 @@ impl PlacementState {
Ok(Some(highest_group_id))
}
+ fn scan(
+ &self,
+ inner: &StreamsInner,
+ replicas: usize,
+ eligible: &[bool],
+ after: Option<IggyNamespace>,
+ ) -> Option<PartitionPrimaryTransfer> {
+ let mut counts = vec![0_u64; replicas];
+ let mut first: Vec<Option<PartitionPlacementView>> = vec![None; replicas];
+ let mut next: Vec<Option<PartitionPlacementView>> = vec![None; replicas];
+ for (stream_id, stream) in &inner.items {
+ for (topic_id, topic) in &stream.topics {
+ if inner.lifecycle_blocks(stream_id, Some(topic_id)) {
+ continue;
+ }
+ for partition in &topic.partitions {
+ let namespace = IggyNamespace::new(stream_id, topic_id, partition.id);
+ let report = self.matching_view(namespace, partition);
+ let primary = report.map_or(partition.created_view, |report| report.view)
+ as usize
+ % replicas;
+ counts[primary] += 1;
+ let Some(report) = report.filter(|_| eligible[primary]) else {
+ continue;
+ };
+ if first[primary].is_none_or(|first| namespace < first.namespace) {
+ first[primary] = Some(report);
+ }
+ if after.is_none_or(|cursor| namespace > cursor)
+ && next[primary].is_none_or(|next| namespace < next.namespace)
+ {
+ next[primary] = Some(report);
+ }
+ }
+ }
+ }
+ let mut available = eligible
+ .iter()
+ .enumerate()
+ .filter_map(|(replica, &eligible)| eligible.then_some(replica));
+ let mut source = available.next()?;
+ let mut target = source;
+ for replica in available {
+ if counts[replica] > counts[source] {
+ source = replica;
+ }
+ if counts[replica] < counts[target] {
+ target = replica;
+ }
+ }
+ if counts[source].saturating_sub(counts[target]) <= 1 {
+ return None;
+ }
+ let report = next[source].or(first[source])?;
+ Some(PartitionPrimaryTransfer {
+ namespace: report.namespace,
+ created_revision: report.created_revision,
+ view: report.view,
+ target: u8::try_from(target).ok()?,
+ })
+ }
+
fn matching_view(
&self,
namespace: IggyNamespace,
@@ -295,6 +320,82 @@ impl PlacementState {
.filter(|report| report.created_revision == partition.created_revision)
.filter(|report| report.view >= partition.created_view)
}
+
+ /// Drops what was observed of partitions that commits removed since the
+ /// last call. When the tree no longer records all of those removals, every
+ /// observation is checked against the tree instead.
+ fn retire_removed(&mut self, inner: &StreamsInner) {
+ if self.synced_revision == Some(inner.namespace_revision) {
+ return;
+ }
+ let removals = self
+ .synced_revision
+ .and_then(|synced| inner.removed_partitions.since(synced));
+ if let Some(removals) = removals {
+ for removed in removals {
+ if self
+ .observations
+ .get(&removed.namespace)
+ .is_some_and(|observed| {
+ observed.report.created_revision == removed.created_revision
+ })
+ && let Some(observed) = self.observations.remove(&removed.namespace)
+ {
+ self.track(observed, -1);
+ self.observed_generation += 1;
+ }
+ }
+ } else {
+ self.observations.retain(|&namespace, observed| {
+ committed_partition(inner, namespace).is_some_and(|partition| {
+ partition.created_revision == observed.report.created_revision
+ })
+ });
+ self.moved = None;
+ self.observed_generation += 1;
+ }
+ self.synced_revision = Some(inner.namespace_revision);
+ }
+
+ fn moved(&mut self, replicas: u8) -> &[i64] {
+ if self
+ .moved
+ .as_ref()
+ .is_none_or(|(built_for, _)| *built_for != replicas)
+ {
+ let mut moved = vec![0; usize::from(replicas)];
+ for observed in self.observations.values() {
+ observed.shift(replicas, &mut moved, 1);
+ }
+ self.moved = Some((replicas, moved));
+ }
+ self.moved.as_ref().map_or(&[], |(_, moved)| moved)
+ }
+
+ fn track(&mut self, observed: ObservedView, by: i64) {
+ if let Some((replicas, moved)) = self.moved.as_mut() {
+ observed.shift(*replicas, moved, by);
+ }
+ }
+}
+
+impl ObservedView {
+ /// Counts this partition at its observed primary instead of its created
+ /// one, or back for a negative `by`.
+ fn shift(self, replicas: u8, moved: &mut [i64], by: i64) {
+ let replicas = u32::from(replicas);
+ let observed = self.report.view.max(self.created_view);
+ moved[(observed % replicas) as usize] += by;
+ moved[(self.created_view % replicas) as usize] -= by;
+ }
+}
+
+fn committed_partition(inner: &StreamsInner, namespace: IggyNamespace) -> Option<&Partition> {
+ inner
+ .items
+ .get(namespace.stream_id())
+ .and_then(|stream| stream.topics.get(namespace.topic_id()))
+ .and_then(|topic| find_partition(&topic.partitions, namespace.partition_id()))
}
#[cfg(test)]
@@ -302,7 +403,10 @@ mod tests {
use super::*;
use crate::stm::StateHandler;
use crate::stm::lifecycle::apply_with_lifecycle_completion;
+ use crate::stm::snapshot::Snapshotable;
+ use crate::stm::stream::{REMOVED_PARTITIONS_RETAINED, StreamsInner};
use iggy_binary_protocol::WireIdentifier;
+ use iggy_binary_protocol::requests::partitions::DeletePartitionsRequest;
use iggy_binary_protocol::requests::streams::DeleteStreamRequest;
use iggy_binary_protocol::requests::topics::DeleteTopicRequest;
use iggy_common::IggyTimestamp;
@@ -325,6 +429,186 @@ mod tests {
(streams, placement)
}
+ #[test]
+ fn given_creates_moves_deletes_and_recreates_when_counting_should_equal_the_full_walk() {
+ const REPLICAS: u8 = 3;
+ let (streams, mut placement) = observed_topics(&[0, 1, 2, 0, 1, 2]);
+ let wide = IggyNamespace::new(0, 6, 4);
+ streams.seed_namespace(wide, 100, 1);
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+
+ let moves = [
+ ((0, 0), 4),
+ ((1, 0), 5),
+ ((6, 1), 3),
+ ((6, 3), 2),
+ ((6, 4), 6),
+ ];
+ observe_views(&mut placement, &streams, &moves);
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+
+ let recreated = IggyNamespace::new(0, 1, 0);
+ let old_revision = streams.created_revision_for_namespace(recreated).unwrap();
+ let streams = apply_to(
+ &streams,
+ &DeleteTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(1),
+ },
+ );
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+
+ let streams = apply_to(
+ &streams,
+ &DeletePartitionsRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(6),
+ partitions_count: 2,
+ },
+ );
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+
+ streams.seed_namespace(recreated, 200, 2);
+ placement.observe(
+ &streams,
+ &[PartitionPlacementView {
+ namespace: recreated,
+ created_revision: old_revision,
+ view: 9,
+ }],
+ );
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+ observe_views(
+ &mut placement,
+ &streams,
+ &[((1, 0), 3), ((6, 2), 7), ((0, 0), 5)],
+ );
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+ assert_counts_match_walk(&mut placement, &streams, 5);
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+ }
+
+ #[test]
+ fn given_a_restored_tree_without_the_removals_when_counting_should_equal_the_full_walk() {
+ const REPLICAS: u8 = 3;
+ let (streams, mut placement) = observed_topics(&[0, 1, 2, 0]);
+ observe_views(&mut placement, &streams, &[((0, 0), 4), ((2, 0), 3)]);
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+
+ let deleted = apply_to(
+ &streams,
+ &DeleteTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(0),
+ },
+ );
+ let restored = Streams::from_snapshot(deleted.to_snapshot()).unwrap();
+
+ assert_counts_match_walk(&mut placement, &restored, REPLICAS);
+ }
+
+ #[test]
+ fn given_more_removals_than_the_tree_keeps_when_counting_should_equal_the_full_walk() {
+ const REPLICAS: u8 = 3;
+ let (streams, mut placement) = observed_topics(&[0, 1]);
+ let wide = IggyNamespace::new(0, 2, REMOVED_PARTITIONS_RETAINED);
+ streams.seed_namespace(wide, 100, 1);
+ let moves: Vec<_> = (0..=REMOVED_PARTITIONS_RETAINED)
+ .map(|partition| ((2, partition), 2))
+ .chain([((0, 0), 4)])
+ .collect();
+ observe_views(&mut placement, &streams, &moves);
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+
+ let streams = apply_to(
+ &streams,
+ &DeleteTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(2),
+ },
+ );
+ assert!(
+ streams.read(|inner| inner.removed_partitions.since(0).is_none()),
+ "the delete must outgrow what the tree keeps"
+ );
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+
+ let streams = apply_to(
+ &streams,
+ &DeleteTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(0),
+ },
+ );
+ assert_counts_match_walk(&mut placement, &streams, REPLICAS);
+ assert_eq!(placement.observations.len(), 1);
+ }
+
+ #[test]
+ fn given_unchanged_inputs_when_selecting_again_should_reuse_the_last_scan() {
+ let (streams, mut placement) = observed_topics(&[0, 0, 0]);
+ let first = placement.primary_rebalance_candidate(&streams, 3, &[true; 3], None);
+ assert!(first.is_some());
+ // A tree at the same revision cannot differ from the scanned one, so a
+ // changed tree that kept its revision exposes a skipped scan.
+ let mut inner = streams.read(Clone::clone);
+ for topic in [1, 2] {
+ inner
+ .items
+ .get_mut(0)
+ .unwrap()
+ .topics
+ .get_mut(topic)
+ .unwrap()
+ .partitions
+ .clear();
+ }
+ let same_revision = Streams::from(inner);
+
+ assert_eq!(
+ placement.primary_rebalance_candidate(&same_revision, 3, &[true; 3], None),
+ first
+ );
+ assert!(
+ PlacementState::default()
+ .primary_rebalance_candidate(&same_revision, 3, &[true; 3], None)
+ .is_none()
+ );
+ }
+
+ #[test]
+ fn given_a_new_lifecycle_intent_when_selecting_again_should_scan_again() {
+ let (streams, mut placement) = observed_topics(&[0, 0, 1, 2]);
+ assert!(
+ placement
+ .primary_rebalance_candidate(&streams, 3, &[true; 3], None)
+ .is_none()
+ );
+ let mut inner = streams.read(Clone::clone);
+ inner.apply_context.metadata_op = 100;
+ assert_eq!(
+ DeleteTopicRequest {
+ stream_id: WireIdentifier::numeric(0),
+ topic_id: WireIdentifier::numeric(2),
+ }
+ .apply(&mut inner, IggyTimestamp::from(1))
+ .code,
+ 0
+ );
+ assert_eq!(
+ inner.namespace_revision,
+ streams.read(|inner| inner.namespace_revision),
+ "a pending delete leaves the partition set alone"
+ );
+ let deleting = Streams::from(inner);
+
+ let transfer = placement
+ .primary_rebalance_candidate(&deleting, 3, &[true; 3], None)
+ .unwrap();
+
+ assert_eq!(transfer.target, 1);
+ }
+
#[test]
fn given_failed_and_returned_replica_when_rebalancing_should_restore_even_ownership() {
let views = (0..100).map(|topic| topic % 5).collect::<Vec<_>>();
@@ -379,7 +663,7 @@ mod tests {
#[test]
fn given_balanced_or_almost_balanced_primaries_when_selecting_should_not_transfer() {
for views in [&[][..], &[0, 1, 2], &[0, 1, 2, 0]] {
- let (streams, placement) = observed_topics(views);
+ let (streams, mut placement) = observed_topics(views);
assert!(
placement
.primary_rebalance_candidate(&streams, 3, &[true; 3], None)
@@ -390,7 +674,7 @@ mod tests {
#[test]
fn given_unsettled_failover_or_no_quorum_when_selecting_should_wait() {
- let (streams, placement) = observed_topics(&[0, 0, 0, 1, 2]);
+ let (streams, mut placement) = observed_topics(&[0, 0, 0, 1, 2]);
for eligible in [
&[false, true, true, true, true][..],
&[true, true, false, false, false],
@@ -417,7 +701,7 @@ mod tests {
#[test]
fn given_tied_load_and_recovering_target_when_selecting_should_rotate_candidates() {
- let (streams, placement) = observed_topics(&[0, 0, 0, 1, 1, 1]);
+ let (streams, mut placement) = observed_topics(&[0, 0, 0, 1, 1, 1]);
for (after, expected_topic) in [(None, 0), (Some(0), 1), (Some(1), 2), (Some(2), 0)] {
let transfer = placement
.primary_rebalance_candidate(
@@ -467,7 +751,7 @@ mod tests {
#[test]
fn given_pending_parent_deletion_when_selecting_should_exclude_its_partitions() {
- let (streams, placement) = observed_topics(&[0, 0, 0]);
+ let (streams, mut placement) = observed_topics(&[0, 0, 0]);
let namespace = IggyNamespace::new(0, 0, 0);
let revision = streams.created_revision_for_namespace(namespace).unwrap();
let mut inner = streams.read(Clone::clone);
@@ -492,6 +776,8 @@ mod tests {
.is_none()
);
+ // A sibling history with the same revision needs its own placement.
+ let (streams, mut placement) = observed_topics(&[0, 0, 0]);
let mut inner = streams.read(Clone::clone);
inner.apply_context.metadata_op = 101;
assert_eq!(
@@ -580,7 +866,7 @@ mod tests {
#[test]
fn given_an_unavailable_primary_when_other_partitions_are_imbalanced_should_move_a_ready_one() {
- let (streams, placement) = observed_topics(&[0, 1, 1, 1]);
+ let (streams, mut placement) = observed_topics(&[0, 1, 1, 1]);
let transfer = placement
.primary_rebalance_candidate(&streams, 3, &[false, true, true], None)
.unwrap();
@@ -602,4 +888,62 @@ mod tests {
placement.observe_replicas(2, &[]);
assert_eq!(placement.select(&mut [0, 5, 5], 2), Ok(0));
}
+
+ fn observe_views(
+ placement: &mut PlacementState,
+ streams: &Streams,
+ views: &[((usize, usize), u32)],
+ ) {
+ for &((topic, partition), view) in views {
+ let namespace = IggyNamespace::new(0, topic, partition);
+ placement.observe(
+ streams,
+ &[PartitionPlacementView {
+ namespace,
+ created_revision: streams.created_revision_for_namespace(namespace).unwrap(),
+ view,
+ }],
+ );
+ }
+ }
+
+ fn apply_to<R: StateHandler<State = StreamsInner>>(streams: &Streams, request: &R) -> Streams {
+ let mut inner = streams.read(Clone::clone);
+ assert_eq!(
+ apply_with_lifecycle_completion(request, &mut inner, IggyTimestamp::from(1)).code,
+ 0
+ );
+ Streams::from(inner)
+ }
+
+ fn assert_counts_match_walk(placement: &mut PlacementState, streams: &Streams, replicas: u8) {
+ let counted = placement.committed_counts(streams, replicas);
+ assert_eq!(counted, walked_counts(placement, streams, replicas));
+ }
+
+ /// Every committed partition at its observed view, or its created view
+ /// before an observation of its incarnation.
+ fn walked_counts(placement: &PlacementState, streams: &Streams, replicas: u8) -> Vec<u64> {
+ streams.read(|inner| {
+ let mut counts = vec![0; usize::from(replicas)];
+ for (stream_id, stream) in &inner.items {
+ for (topic_id, topic) in &stream.topics {
+ for partition in &topic.partitions {
+ let namespace = IggyNamespace::new(stream_id, topic_id, partition.id);
+ let view = placement
+ .observations
+ .get(&namespace)
+ .filter(|observed| {
+ observed.report.created_revision == partition.created_revision
+ })
+ .map_or(partition.created_view, |observed| {
+ observed.report.view.max(partition.created_view)
+ });
+ counts[(view % u32::from(replicas)) as usize] += 1;
+ }
+ }
+ }
+ counts
+ })
+ }
}
diff --git a/core/metadata/src/stm/stream.rs b/core/metadata/src/stm/stream.rs
index a4a5b5c81..6416bb5ad 100644
--- a/core/metadata/src/stm/stream.rs
+++ b/core/metadata/src/stm/stream.rs
@@ -35,7 +35,7 @@ use crate::{collect_handlers, define_state, impl_fill_restore};
use ahash::{AHashMap, AHashSet};
use bytes::{BufMut, Bytes, BytesMut};
use iggy_binary_protocol::codec::{WireDecode, WireEncode};
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::{BTreeMap, BTreeSet, VecDeque};
// Only `seed_namespace` (sim/test-gated) uses this at module scope, so keep the
// import under the same gate. The test module re-imports it independently.
#[cfg(any(test, feature = "simulator"))]
@@ -80,6 +80,10 @@ use server_common::sharding::{IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TO
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
+/// Removals [`RemovedPartitions`] keeps. A reader that falls further behind
+/// rebuilds from the tree, which a delete of this many partitions costs anyway.
+pub(crate) const REMOVED_PARTITIONS_RETAINED: usize = 4096;
+
/// Partition snapshot representation for serialization.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PartitionSnapshot {
@@ -784,6 +788,12 @@ define_state! {
pub(crate) pending_revocations: BTreeMap<(usize, usize, u64, usize), ConsumerGroupOwnershipTransition>,
// Locations are (stream, topic, group, member); never persisted.
pub(crate) consumer_group_members: AHashMap<u128, Vec<(usize, usize, u64, usize)>>,
+ // Committed partitions per created view, so placement counts them
+ // without a walk. Rebuilt on restore.
+ pub(crate) created_views: BTreeMap<u32, u64>,
+ // Partitions the newest commits removed, so placement retires what it
+ // observed of them without a walk. A restore starts it empty.
+ pub(crate) removed_partitions: RemovedPartitions,
// Shared aggregate stats, one `Arc` per stream/topic across both
// left-right buffers (see `StatsRegistry`). Not snapshotted -- rebuilt
// as streams/topics restore.
@@ -791,6 +801,48 @@ define_state! {
}
}
+/// Partitions that recent commits removed, oldest first, each stamped with
+/// the `namespace_revision` its commit set. Only the newest entries stay, so a
+/// reader that last looked before [`Self::since`] can answer must start over.
+#[derive(Debug, Clone, Default)]
+pub(crate) struct RemovedPartitions {
+ entries: VecDeque<RemovedPartition>,
+ /// Newest stamp whose entries are gone, evicted or never recorded.
+ forgotten_through: u64,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct RemovedPartition {
+ pub namespace_revision: u64,
+ pub namespace: IggyNamespace,
+ pub created_revision: u64,
+}
+
+impl RemovedPartitions {
+ /// Every partition removed after `namespace_revision`, or `None` when some
+ /// of those removals are no longer recorded.
+ pub(crate) fn since(
+ &self,
+ namespace_revision: u64,
+ ) -> Option<impl Iterator<Item = &RemovedPartition>> {
+ (self.forgotten_through <= namespace_revision).then(|| {
+ let first = self
+ .entries
+ .partition_point(|removed| removed.namespace_revision <= namespace_revision);
+ self.entries.range(first..)
+ })
+ }
+
+ fn push(&mut self, removed: RemovedPartition) {
+ if self.entries.len() == REMOVED_PARTITIONS_RETAINED
+ && let Some(evicted) = self.entries.pop_front()
+ {
+ self.forgotten_through = evicted.namespace_revision;
+ }
+ self.entries.push_back(removed);
+ }
+}
+
/// Server-originated request that advances a partition's delete watermark.
///
/// `up_to_offset` is resolved on the owning shard from a client
@@ -987,6 +1039,24 @@ impl StreamsInner {
.retain(|_, memberships| !memberships.is_empty());
}
+ /// Takes partitions a commit removed out of the placement indexes, after
+ /// that commit set `namespace_revision`.
+ fn retire_partitions(&mut self, stream_id: usize, topic_id: usize, partitions: &[Partition]) {
+ for partition in partitions {
+ if let Some(count) = self.created_views.get_mut(&partition.created_view) {
+ *count -= 1;
+ if *count == 0 {
+ self.created_views.remove(&partition.created_view);
+ }
+ }
+ self.removed_partitions.push(RemovedPartition {
+ namespace_revision: self.namespace_revision,
+ namespace: IggyNamespace::new(stream_id, topic_id, partition.id),
+ created_revision: partition.created_revision,
+ });
+ }
+ }
+
/// Resolve a wire stream identifier to its committed slab id, `None` when
/// the stream does not exist.
#[must_use]
@@ -2029,6 +2099,18 @@ impl Streams {
}
}
+fn count_created_views(items: &IdSlab<Stream>) -> BTreeMap<u32, u64> {
+ let mut created_views = BTreeMap::new();
+ for partition in items
+ .iter()
+ .flat_map(|(_, stream)| stream.topics.iter())
+ .flat_map(|(_, topic)| topic.partitions.iter())
+ {
+ *created_views.entry(partition.created_view).or_default() += 1;
+ }
+ created_views
+}
+
/// Whether the next insert's slab key still fits the packed namespace layout.
///
/// Takes `vacant_key()` rather than `len()` because the KEY is what gets packed
@@ -2192,10 +2274,13 @@ impl StateHandler for DeleteStreamRequest {
// Evict registry entries so a reused slab id starts with fresh stats.
// Before the removal: the committed tree is what names the entries.
state.stats_registry.remove_stream(stream_id, stream);
- state.items.remove(stream_id);
+ let removed = state.items.remove(stream_id);
state.index.remove(&name);
state.revision = state.revision.wrapping_add(1);
state.namespace_revision = state.revision;
+ for (topic_id, topic) in &removed.topics {
+ state.retire_partitions(stream_id, topic_id, &topic.partitions);
+ }
// The dropped stream may have held groups with pending revocations.
state.recompute_consumer_group_metadata();
ApplyReply {
@@ -2329,6 +2414,10 @@ impl StateHandler for CreateTopicWithAssignmentsRequest {
created_view: partition.initial_view,
deleted_up_to_offset: 0,
};
+ *state
+ .created_views
+ .entry(partition.created_view)
+ .or_default() += 1;
topic.partitions.push(partition);
}
}
@@ -2515,7 +2604,7 @@ impl StateHandler for DeleteTopicRequest {
.iter()
.map(|partition| partition.id)
.collect();
- stream.topics.remove(topic_id);
+ let removed = stream.topics.remove(topic_id);
stream.topic_index.remove(&name);
// Roll the topic and its partitions out of the stream total and evict
// their entries, so a reused slab id starts from zero.
@@ -2524,6 +2613,7 @@ impl StateHandler for DeleteTopicRequest {
.remove_topic(stream_id, topic_id, &partition_ids);
state.revision = state.revision.wrapping_add(1);
state.namespace_revision = state.revision;
+ state.retire_partitions(stream_id, topic_id, &removed.partitions);
// The dropped topic may have held groups with pending revocations.
state.recompute_consumer_group_metadata();
ApplyReply {
@@ -2616,6 +2706,10 @@ impl StateHandler for CreatePartitionsWithAssignmentsRequest {
return ApplyReply::err(CreatePartitionsResult::TopicNotFound);
};
for (resolved_id_usize, partition) in resolved.into_iter().zip(self.partitions.iter()) {
+ *state
+ .created_views
+ .entry(partition.initial_view)
+ .or_default() += 1;
topic.partitions.push(Partition {
id: resolved_id_usize,
consensus_group_id: partition.consensus_group_id,
@@ -2682,14 +2776,11 @@ impl StateHandler for DeletePartitionsRequest {
// applies as the historical ok no-op.
if count_to_delete > 0 {
let retained = topic.partitions.len() - count_to_delete;
- // Read the ids off the tail before it goes: `retained` is a count,
- // and only a primary that minted dense ids makes it double as an id
+ // Read the ids off the removed tail: `retained` is a count, and
+ // only a primary that minted dense ids makes it double as an id
// threshold.
- let removed_ids: Vec<usize> = topic.partitions[retained..]
- .iter()
- .map(|partition| partition.id)
- .collect();
- topic.partitions.truncate(retained);
+ let removed = topic.partitions.split_off(retained);
+ let removed_ids: Vec<usize> = removed.iter().map(|partition| partition.id).collect();
// Members assigned the removed partitions must give them up.
topic.rebalance_consumer_groups(state.apply_context.metadata_op, timestamp);
state.recompute_consumer_group_metadata();
@@ -2700,6 +2791,7 @@ impl StateHandler for DeletePartitionsRequest {
.remove_partitions(stream_id, topic_id, &removed_ids);
state.revision = state.revision.wrapping_add(1);
state.namespace_revision = state.revision;
+ state.retire_partitions(stream_id, topic_id, &removed);
}
ApplyReply::ok(Bytes::new())
}
@@ -2949,6 +3041,7 @@ impl StreamsInner {
.values()
.map(LifecycleIntent::scope)
.collect();
+ let created_views = count_created_views(&items);
let mut inner = Self {
index,
items,
@@ -2960,6 +3053,11 @@ impl StreamsInner {
// Recomputed from the restored groups just below.
pending_revocations: BTreeMap::new(),
consumer_group_members: AHashMap::new(),
+ created_views,
+ removed_partitions: RemovedPartitions {
+ entries: VecDeque::new(),
+ forgotten_through: snapshot.namespace_revision,
+ },
last_result: None,
apply_context: ApplyContext::default(),
stats_registry,
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs
index 94dbb6cc3..9530c4334 100644
--- a/core/partitions/src/iggy_partition.rs
+++ b/core/partitions/src/iggy_partition.rs
@@ -288,8 +288,6 @@ where
#[cfg(test)]
offset_dir_sync_count: Cell<usize>,
pub(crate) required_metadata_frontier: u64,
- #[cfg(feature = "simulator")]
- ideal_storage: bool,
/// Durable superblock for this partition's consensus group, recording
/// `(view, log_view)` across a crash so this replica can never
/// re-participate in a view older than one it advertised. `None` for
@@ -910,8 +908,6 @@ where
#[cfg(test)]
offset_dir_sync_count: Cell::new(0),
required_metadata_frontier: 0,
- #[cfg(feature = "simulator")]
- ideal_storage: false,
superblock: None,
superblock_lock: Rc::new(LocalGate::new()),
superblock_intent: Cell::new(None),
@@ -1388,13 +1384,6 @@ where
}
}
- /// The consensus simulator retains every journal write across crashes.
- /// This does not model filesystem barriers or physical storage failures.
- #[cfg(feature = "simulator")]
- pub const fn enable_ideal_storage(&mut self) {
- self.ideal_storage = true;
- }
-
#[must_use]
pub fn with_in_memory_storage(
stats: Arc<PartitionStats>,
@@ -1607,7 +1596,7 @@ where
&directory,
self.namespace().inner(),
self.created_revision(),
- journal::durable_storage::DiskStorage,
+ crate::WalStorage::default(),
capacity,
self.runtime_options
.preallocate_segments
@@ -1649,10 +1638,13 @@ where
.message_bus()
.spawn(Rc::clone(&persistence).run());
}
- persistence.drain_with_timeout().await.map_err(|error| {
- warn!(%error, "cannot enable partition segment persistence");
- IggyError::CannotSyncFile
- })?;
+ persistence
+ .drain_with_timeout(|timeout| self.consensus.message_bus().sleep(timeout))
+ .await
+ .map_err(|error| {
+ warn!(%error, "cannot enable partition segment persistence");
+ IggyError::CannotSyncFile
+ })?;
}
self.restore_certified_log_view(&persistence).await?;
if !self.materialization_missing && persistence.recovered_frontier() {
@@ -1727,7 +1719,7 @@ where
.spawn(Rc::clone(persistence).run());
}
persistence
- .drain_with_timeout()
+ .drain_with_timeout(|timeout| self.consensus.message_bus().sleep(timeout))
.await
.map_err(|_| IggyError::CannotSyncFile)?;
}
@@ -2321,10 +2313,6 @@ where
}
pub fn register_rebuilt_ack(&self, header: &PrepareHeader) -> bool {
- #[cfg(feature = "simulator")]
- if self.ideal_storage {
- return true;
- }
// State-transfer admission uses the accepted WAL head to protect every
// acknowledged op, including replicated prepares adopted in a new view.
let accepted = self
@@ -2983,6 +2971,39 @@ where
self.recovered_durable_offset = offset_space.committed_seeded.then_some(durable);
}
+ /// Attach the WAL the simulator reopened from its filesystem model, as
+ /// [`Self::open_persistence_with_recovered`] attaches one after segment
+ /// recovery. The adopted log stands in for the replayed prepares, so it must
+ /// hold exactly what the WAL holds above the applied prefix.
+ #[cfg(feature = "simulator")]
+ pub fn attach_simulated_persistence(&mut self, persistence: Rc<PartitionPersistence>) {
+ // New empty WALs carry view zero; positive certificates belong to recovered history.
+ if self.recovered_log_view.is_none()
+ && persistence.head() == 0
+ && matches!(persistence.certified_log_view(), None | Some(0))
+ {
+ persistence.certify_log_view(self.consensus.log_view(), 0, 0);
+ } else {
+ match persistence.certified_log_view() {
+ Some(view) if view >= self.consensus.log_view() => {
+ if view > self.consensus.view() {
+ self.consensus.set_view(view);
+ }
+ self.consensus.set_log_view(view);
+ }
+ _ => {
+ self.materialization_missing = true;
+ self.ensure_materialization_recovery();
+ }
+ }
+ }
+ if !self.materialization_missing && persistence.recovered_frontier() {
+ self.consensus.set_recovery_election_allowed(true);
+ }
+ self.persistence = Some(persistence);
+ self.start_persistence();
+ }
+
#[cfg(any(test, feature = "simulator"))]
#[must_use]
pub fn retained_consumer_offsets(&self, kind: ConsumerKind) -> Vec<(u32, u64)> {
@@ -8944,7 +8965,7 @@ where
persistence.truncate_from(from_op);
self.start_persistence();
persistence
- .drain_with_timeout()
+ .drain_with_timeout(|timeout| self.consensus.message_bus().sleep(timeout))
.await
.map_err(|_| IggyError::CannotSyncFile)?;
}
@@ -8979,7 +9000,7 @@ where
persistence.truncate_from(*ops.start());
self.start_persistence();
persistence
- .drain_with_timeout()
+ .drain_with_timeout(|timeout| self.consensus.message_bus().sleep(timeout))
.await
.map_err(|_| IggyError::CannotSyncFile)?;
}
@@ -9138,10 +9159,13 @@ where
self.start_persistence();
// Shutdown and transfer must finish the flush, but never drain while
// holding the lock used to append new messages.
- persistence.drain_with_timeout().await.map_err(|error| {
- warn!(%error, "cannot flush committed partition messages");
- IggyError::CannotSyncFile
- })?;
+ persistence
+ .drain_with_timeout(|timeout| self.consensus.message_bus().sleep(timeout))
+ .await
+ .map_err(|error| {
+ warn!(%error, "cannot flush committed partition messages");
+ IggyError::CannotSyncFile
+ })?;
}
if self
.commit_messages_inner(config, true, self.consensus.commit_max())
@@ -10654,7 +10678,7 @@ where
debug!(
namespace_raw = self.namespace().inner(),
client = header.client, op = header.op, %error,
- "client reply not routable by the bus; client will time out"
+ "client reply has no route by client id"
);
}
}
@@ -11542,7 +11566,7 @@ where
persistence.request_checkpoint();
self.checkpoint_persistence(config).await;
persistence
- .drain_with_timeout()
+ .drain_with_timeout(|timeout| self.consensus.message_bus().sleep(timeout))
.await
.map_err(|_| IggyError::CannotSyncFile)?;
if self.fatal.is_some() || persistence.checkpoint_op() != persistence.head() {
@@ -11689,10 +11713,13 @@ where
{
persistence.reanchor_segments(self.log.active_segment().start_offset);
self.start_persistence();
- persistence.drain_with_timeout().await.map_err(|error| {
- warn!(%error, "cannot reanchor the partition WAL at its reserved offset");
- IggyError::CannotSyncFile
- })?;
+ persistence
+ .drain_with_timeout(|timeout| self.consensus.message_bus().sleep(timeout))
+ .await
+ .map_err(|error| {
+ warn!(%error, "cannot reanchor the partition WAL at its reserved offset");
+ IggyError::CannotSyncFile
+ })?;
}
Ok(())
}
@@ -15218,7 +15245,10 @@ mod tests {
assert_eq!(batch.iter().next().unwrap().payload, b"replacement");
persistence.request_checkpoint();
partition.checkpoint_persistence(&config).await;
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
assert!(persistence.failure().is_none());
assert_eq!(persistence.checkpoint_op(), 2);
}
@@ -15406,7 +15436,10 @@ mod tests {
persistence.append(prepare.into_frozen(), true).unwrap();
}
partition.start_persistence();
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
let rotated = directory.path().join("00000000000000000001.log");
let metadata = std::fs::metadata(&rotated).unwrap();
@@ -16546,7 +16579,7 @@ mod tests {
.persistence
.as_ref()
.unwrap()
- .drain_with_timeout()
+ .drain_with_timeout(compio::runtime::time::sleep)
.await
.unwrap();
backup.drive_persistence().await;
@@ -16749,7 +16782,7 @@ mod tests {
.persistence
.as_ref()
.unwrap()
- .drain_with_timeout()
+ .drain_with_timeout(compio::runtime::time::sleep)
.await
.unwrap();
backup.drive_persistence().await;
@@ -16775,7 +16808,7 @@ mod tests {
.persistence
.as_ref()
.unwrap()
- .drain_with_timeout()
+ .drain_with_timeout(compio::runtime::time::sleep)
.await
.unwrap();
backup.drive_persistence().await;
@@ -16846,7 +16879,10 @@ mod tests {
assert!(!partition.register_rebuilt_ack(&header));
assert!(partition.pending_persisted_acks.borrow().contains_key(&1));
partition.start_persistence();
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
assert!(partition.register_rebuilt_ack(&header));
}
@@ -16890,7 +16926,7 @@ mod tests {
.persistence
.as_ref()
.unwrap()
- .drain_with_timeout()
+ .drain_with_timeout(compio::runtime::time::sleep)
.await
.unwrap();
partition.drive_persistence().await;
@@ -16995,7 +17031,10 @@ mod tests {
assert_eq!(partition.consensus().sequencer().current_sequence(), 3);
partition.on_replicate(third.clone()).await;
if let Some(persistence) = &partition.persistence {
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
}
partition.drive_persistence().await;
assert_eq!(
@@ -17071,7 +17110,10 @@ mod tests {
persistence.append(prepare.into_frozen(), true).unwrap();
assert!(!partition.register_rebuilt_ack(&header));
partition.start_persistence();
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
(directory, partition, header)
}
@@ -17107,7 +17149,10 @@ mod tests {
assert!(!partition.register_rebuilt_ack(&header));
persistence.certify_log_view(3, header.op, header.checksum);
partition.start_persistence();
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
let store = Rc::new(RecordingSuperblock::default());
partition.set_superblock(store.clone(), None);
assert!(partition.consensus().needs_superblock_persist());
@@ -21370,7 +21415,6 @@ mod tests {
fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {}
}
pub(super) type SentFrames = Rc<RefCell<Vec<(u128, Frozen<MESSAGE_ALIGN>)>>>;
@@ -21640,7 +21684,10 @@ mod tests {
let mut conclusion = RepairConclusion::InProgress;
for _ in 0..COMPLETION_STEPS_MAX {
conclusion = replica.complete_repair(&config).await;
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
replica.drive_persistence().await;
if conclusion == RepairConclusion::Done {
break;
@@ -22104,7 +22151,10 @@ mod tests {
);
}
if let Some(persistence) = &persistence {
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
assert!(partition.send_prepare_ok(&header).await);
}
partition.consensus().advance_commit_max(op);
@@ -22352,7 +22402,10 @@ mod tests {
assert_eq!(persistence.head(), 2);
assert_eq!(backup.consensus().sequencer().current_sequence(), 2);
assert!(persistence.failure().is_none());
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
assert!(persistence.is_durable_through(2));
backup.drive_persistence().await;
assert!(
@@ -22490,7 +22543,10 @@ mod tests {
}
settle_partition_io(partition, config).await;
let persistence = Rc::clone(partition.persistence.as_ref().unwrap());
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
for completion in completions.take() {
partition.on_persistence_completed(completion, config).await;
}
@@ -22788,7 +22844,10 @@ mod tests {
crate::PartitionIoStep::Progress => {}
crate::PartitionIoStep::Pending => {
if let Some(persistence) = &partition.persistence {
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
} else {
compio::time::sleep(std::time::Duration::from_millis(1)).await;
}
@@ -23017,7 +23076,7 @@ mod tests {
.persistence
.as_ref()
.unwrap()
- .drain_with_timeout()
+ .drain_with_timeout(compio::runtime::time::sleep)
.await
.unwrap();
}
@@ -24936,7 +24995,10 @@ mod tests {
let held = checksummed_segment_prepare(HELD, 0, 0, b"retained");
partition.on_replicate(held.clone()).await;
let persistence = partition.persistence.as_ref().unwrap().clone();
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
assert_eq!(persistence.head(), HELD);
assert_eq!(persistence.certified_log_view(), Some(0));
assert!(
@@ -25080,7 +25142,10 @@ mod tests {
partition.consensus().advance_commit_max(APPLIED);
for _ in 0..COMMIT_ROUNDS_MAX {
partition.commit_journal(&config).await;
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
partition.drive_persistence().await;
if partition.consensus().commit_min() == APPLIED {
break;
@@ -25108,7 +25173,10 @@ mod tests {
break;
}
partition.resume_io(&config).await;
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
}
assert!(!partition.view_truncation_pending());
} else {
@@ -25181,7 +25249,10 @@ mod tests {
let persistence = Rc::clone(donor.persistence.as_ref().unwrap());
persistence.request_checkpoint();
donor.checkpoint_persistence(&config).await;
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
assert!(donor.fatal().is_none(), "{:?}", donor.fatal());
assert_eq!(persistence.checkpoint_op(), commit_op);
drop(persistence);
@@ -25209,7 +25280,10 @@ mod tests {
// Each applied op waits on persistence, so drive it between commits.
for _ in 0..COMMIT_ROUNDS_MAX {
replica.commit_journal(config).await;
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
replica.drive_persistence().await;
if replica.consensus().commit_min() == commit_op {
break;
@@ -25338,7 +25412,10 @@ mod tests {
persistence.request_checkpoint();
partition.checkpoint_persistence(&config).await;
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
assert!(partition.fatal().is_none());
assert!(persistence.failure().is_none());
@@ -26516,7 +26593,10 @@ mod tests {
if partition.shutdown_io_complete() {
break;
}
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
}
assert!(
partition.shutdown_io_complete(),
@@ -31425,7 +31505,10 @@ mod tests {
let persistence = backup.persistence.as_ref().unwrap();
persistence.sync();
backup.start_persistence();
- persistence.drain_with_timeout().await.unwrap();
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await
+ .unwrap();
assert!(persistence.is_durable_through(1), "{durability:?}");
drop(backup);
diff --git a/core/partitions/src/io.rs b/core/partitions/src/io.rs
index f685f6ea0..ef09a2dec 100644
--- a/core/partitions/src/io.rs
+++ b/core/partitions/src/io.rs
@@ -276,7 +276,9 @@ impl PartitionTeardown {
self.io.drain().await?;
if let Some(persistence) = &self.persistence {
persistence.retire();
- persistence.drain_with_timeout().await?;
+ persistence
+ .drain_with_timeout(compio::runtime::time::sleep)
+ .await?;
}
Ok(())
})
diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs
index 293d9ca92..3e7545172 100644
--- a/core/partitions/src/lib.rs
+++ b/core/partitions/src/lib.rs
@@ -43,6 +43,14 @@ pub mod segment_anchor;
mod segment_recovery;
pub mod state_transfer;
mod types;
+#[cfg(feature = "simulator")]
+mod wal_storage;
+#[cfg(feature = "simulator")]
+pub use wal_storage::{InjectedFile, InjectedStorage, WalFile, WalStorage};
+
+/// The storage under every partition WAL.
+#[cfg(not(feature = "simulator"))]
+pub type WalStorage = ::journal::durable_storage::DiskStorage;
pub use consumer_offset_capacity::{AutoCommitReservation, ConsumerOffsetCapacityError};
use iggy_binary_protocol::PrepareHeader;
diff --git a/core/partitions/src/persistence.rs b/core/partitions/src/persistence.rs
index ce8079a65..32de4db7c 100644
--- a/core/partitions/src/persistence.rs
+++ b/core/partitions/src/persistence.rs
@@ -18,10 +18,11 @@
use crate::FILE_SYNC_CONCURRENCY;
use crate::offset_storage::{OffsetFilePermit, RetainedOffsetFile, RetainedOffsetFiles};
use futures::TryStreamExt;
+use futures::future::Either;
use iggy_binary_protocol::{Operation, PrepareHeader};
use iggy_common::ConsumerKind;
use journal::PartitionPrepareJournal;
-use journal::durable_storage::{DiskStorage, DurableFile, DurableStorage};
+use journal::durable_storage::{DurableFile, DurableStorage};
use journal::partition_journal::{PARTITION_WAL_BYTES_MAX, SegmentPosition, SegmentReference};
use server_common::Message;
use server_common::iobuf::Frozen;
@@ -126,7 +127,7 @@ impl FileSyncBarrier {
}
}
-pub struct PartitionPersistence<S: DurableStorage = DiskStorage> {
+pub struct PartitionPersistence<S: DurableStorage = crate::WalStorage> {
group: u64,
instance: u64,
recovered_frontier: bool,
@@ -490,7 +491,7 @@ impl PartitionPersistence {
group: u64,
incarnation: u64,
) -> io::Result<(Rc<Self>, Vec<Message<PrepareHeader>>)> {
- Self::open_with_storage(directory, group, incarnation, DiskStorage).await
+ Self::open_with_storage(directory, group, incarnation, crate::WalStorage::default()).await
}
}
@@ -839,20 +840,31 @@ impl<S: DurableStorage> PartitionPersistence<S> {
}
}
+ /// An injected file has no disk handle to lend, so it is checked out as absent.
pub(crate) fn checkout_offset_file(
&self,
path: &str,
- ) -> Option<(Option<S::File>, Rc<OffsetFilePermit>)> {
- self.offset_files.checkout(path)
+ ) -> Option<(Option<compio::fs::File>, Rc<OffsetFilePermit>)>
+ where
+ compio::fs::File: TryFrom<S::File>,
+ {
+ self.offset_files.checkout(path).map(|(file, permit)| {
+ (
+ file.and_then(|file| compio::fs::File::try_from(file).ok()),
+ permit,
+ )
+ })
}
pub(crate) fn return_offset_file(
&self,
path: &str,
- file: S::File,
+ file: compio::fs::File,
permit: Rc<OffsetFilePermit>,
- ) {
- self.offset_files.put(path, file, permit);
+ ) where
+ S::File: From<compio::fs::File>,
+ {
+ self.offset_files.put(path, S::File::from(file), permit);
}
pub fn retire_offset_file(&self, path: &str) {
@@ -1197,12 +1209,23 @@ impl<S: DurableStorage> PartitionPersistence<S> {
.await
}
+ /// Times out on the caller's clock, so a simulated owner drains in virtual time.
+ ///
/// # Errors
/// Returns a storage failure or a timeout without allowing a second writer.
- pub async fn drain_with_timeout(&self) -> io::Result<()> {
- compio::runtime::time::timeout(PERSISTENCE_DRAIN_TIMEOUT, self.drain())
- .await
- .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "partition WAL drain timed out"))?
+ pub async fn drain_with_timeout<F>(&self, sleep: impl FnOnce(Duration) -> F) -> io::Result<()>
+ where
+ F: Future<Output = ()>,
+ {
+ let drain = std::pin::pin!(self.drain());
+ let timeout = std::pin::pin!(sleep(PERSISTENCE_DRAIN_TIMEOUT));
+ match futures::future::select(drain, timeout).await {
+ Either::Left((result, _)) => result,
+ Either::Right(((), _)) => Err(io::Error::new(
+ io::ErrorKind::TimedOut,
+ "partition WAL drain timed out",
+ )),
+ }
}
/// Queue a barrier for preceding buffered writes; drain before claiming durability.
diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs
index 31c4e5a6b..5fc715914 100644
--- a/core/partitions/src/state_transfer.rs
+++ b/core/partitions/src/state_transfer.rs
@@ -4486,7 +4486,11 @@ where
},
crate::PartitionIoStep::Pending => {
if let Some(persistence) = &self.persistence {
- if let Err(source) = persistence.drain_with_timeout().await
+ if let Err(source) = persistence
+ .drain_with_timeout(|timeout| {
+ self.consensus().message_bus().sleep(timeout)
+ })
+ .await
&& let Some(crate::iggy_partition::PendingPartitionTransition::Install(
mut install,
)) = self.transition.take()
diff --git a/core/partitions/src/wal_storage.rs b/core/partitions/src/wal_storage.rs
new file mode 100644
index 000000000..d799aea41
--- /dev/null
+++ b/core/partitions/src/wal_storage.rs
@@ -0,0 +1,442 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use compio::fs::File;
+use futures::future::LocalBoxFuture;
+use journal::durable_storage::{
+ DiskStorage, DurableFile, DurableStorage, OpenMode, RegularFiles, StorageEntry,
+};
+use server_common::iobuf::{Frozen, Owned};
+use std::io;
+use std::path::{Path, PathBuf};
+use std::rc::Rc;
+
+/// The storage under a partition WAL. A partition names one storage type, so
+/// the simulator injects its filesystem model behind a trait object.
+#[derive(Clone, Default)]
+pub enum WalStorage {
+ #[default]
+ Disk,
+ Injected(Rc<dyn InjectedStorage>),
+}
+
+/// A file opened through a [`WalStorage`].
+pub enum WalFile {
+ Disk(File),
+ Injected(Box<dyn InjectedFile>),
+}
+
+/// [`DurableStorage`] as a trait object.
+pub trait InjectedStorage {
+ /// # Errors
+ /// As [`DurableStorage::writer_identity`].
+ fn writer_identity(&self, path: &Path) -> io::Result<Option<PathBuf>>;
+ fn open<'a>(
+ &'a self,
+ path: &'a Path,
+ mode: OpenMode,
+ ) -> LocalBoxFuture<'a, io::Result<Box<dyn InjectedFile>>>;
+ fn create_directories<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<()>>;
+ fn sync_directory<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<()>>;
+ fn rename<'a>(
+ &'a self,
+ source: &'a Path,
+ target: &'a Path,
+ ) -> LocalBoxFuture<'a, io::Result<()>>;
+ fn remove_file<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<()>>;
+ fn hard_link<'a>(
+ &'a self,
+ source: &'a Path,
+ target: &'a Path,
+ ) -> LocalBoxFuture<'a, io::Result<()>>;
+ fn exists<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<bool>>;
+ fn exists_following_links<'a>(&'a self, path: &'a Path)
+ -> LocalBoxFuture<'a, io::Result<bool>>;
+ fn entries<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<Vec<StorageEntry>>>;
+ fn regular_files<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<RegularFiles>>;
+ fn remove_tree<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<()>>;
+}
+
+/// [`DurableFile`] as a trait object.
+pub trait InjectedFile {
+ fn preallocate<'a>(&'a self, path: &'a Path, length: u64) -> LocalBoxFuture<'a, ()>;
+ fn read(&self, offset: u64, length: usize) -> LocalBoxFuture<'_, io::Result<Vec<u8>>>;
+ fn write(&mut self, offset: u64, bytes: Vec<u8>) -> LocalBoxFuture<'_, io::Result<()>>;
+ fn write_frozen(
+ &mut self,
+ offset: u64,
+ bytes: Frozen<4096>,
+ ) -> LocalBoxFuture<'_, io::Result<()>>;
+ fn write_frozen_vectored(
+ &mut self,
+ offset: u64,
+ buffers: Vec<Frozen<4096>>,
+ ) -> LocalBoxFuture<'_, io::Result<()>>;
+ fn write_aligned(
+ &mut self,
+ offset: u64,
+ bytes: Owned<4096>,
+ ) -> LocalBoxFuture<'_, io::Result<()>>;
+ fn read_aligned(
+ &self,
+ offset: u64,
+ length: usize,
+ ) -> LocalBoxFuture<'_, io::Result<Owned<4096>>>;
+ fn read_aligned_tail(
+ &self,
+ offset: u64,
+ bytes: Owned<4096>,
+ start: usize,
+ ) -> LocalBoxFuture<'_, io::Result<Owned<4096>>>;
+ fn length(&self) -> LocalBoxFuture<'_, io::Result<u64>>;
+ fn truncate(&self, length: u64) -> LocalBoxFuture<'_, io::Result<()>>;
+ fn sync(&self) -> LocalBoxFuture<'_, io::Result<()>>;
+}
+
+impl DurableStorage for WalStorage {
+ type File = WalFile;
+
+ fn writer_identity(&self, path: &Path) -> io::Result<Option<PathBuf>> {
+ match self {
+ Self::Disk => DurableStorage::writer_identity(&DiskStorage, path),
+ Self::Injected(storage) => storage.writer_identity(path),
+ }
+ }
+
+ async fn open(&self, path: &Path, mode: OpenMode) -> io::Result<WalFile> {
+ match self {
+ Self::Disk => DurableStorage::open(&DiskStorage, path, mode)
+ .await
+ .map(WalFile::Disk),
+ Self::Injected(storage) => storage.open(path, mode).await.map(WalFile::Injected),
+ }
+ }
+
+ async fn create_directories(&self, path: &Path) -> io::Result<()> {
+ match self {
+ Self::Disk => DurableStorage::create_directories(&DiskStorage, path).await,
+ Self::Injected(storage) => storage.create_directories(path).await,
+ }
+ }
+
+ async fn sync_directory(&self, path: &Path) -> io::Result<()> {
+ match self {
+ Self::Disk => DurableStorage::sync_directory(&DiskStorage, path).await,
+ Self::Injected(storage) => storage.sync_directory(path).await,
+ }
+ }
+
+ async fn rename(&self, source: &Path, target: &Path) -> io::Result<()> {
+ match self {
+ Self::Disk => DurableStorage::rename(&DiskStorage, source, target).await,
+ Self::Injected(storage) => storage.rename(source, target).await,
+ }
+ }
+
+ async fn remove_file(&self, path: &Path) -> io::Result<()> {
+ match self {
+ Self::Disk => DurableStorage::remove_file(&DiskStorage, path).await,
+ Self::Injected(storage) => storage.remove_file(path).await,
+ }
+ }
+
+ async fn hard_link(&self, source: &Path, target: &Path) -> io::Result<()> {
+ match self {
+ Self::Disk => DurableStorage::hard_link(&DiskStorage, source, target).await,
+ Self::Injected(storage) => storage.hard_link(source, target).await,
+ }
+ }
+
+ async fn exists(&self, path: &Path) -> io::Result<bool> {
+ match self {
+ Self::Disk => DurableStorage::exists(&DiskStorage, path).await,
+ Self::Injected(storage) => storage.exists(path).await,
+ }
+ }
+
+ async fn exists_following_links(&self, path: &Path) -> io::Result<bool> {
+ match self {
+ Self::Disk => DurableStorage::exists_following_links(&DiskStorage, path).await,
+ Self::Injected(storage) => storage.exists_following_links(path).await,
+ }
+ }
+
+ async fn entries(&self, path: &Path) -> io::Result<Vec<StorageEntry>> {
+ match self {
+ Self::Disk => DurableStorage::entries(&DiskStorage, path).await,
+ Self::Injected(storage) => storage.entries(path).await,
+ }
+ }
+
+ async fn regular_files(&self, path: &Path) -> io::Result<RegularFiles> {
+ match self {
+ Self::Disk => DurableStorage::regular_files(&DiskStorage, path).await,
+ Self::Injected(storage) => storage.regular_files(path).await,
+ }
+ }
+
+ async fn remove_tree(&self, path: &Path) -> io::Result<()> {
+ match self {
+ Self::Disk => DurableStorage::remove_tree(&DiskStorage, path).await,
+ Self::Injected(storage) => storage.remove_tree(path).await,
+ }
+ }
+}
+
+impl DurableFile for WalFile {
+ async fn preallocate(&self, path: &Path, length: u64) {
+ match self {
+ Self::Disk(file) => DurableFile::preallocate(file, path, length).await,
+ Self::Injected(file) => file.preallocate(path, length).await,
+ }
+ }
+
+ async fn read(&self, offset: u64, length: usize) -> io::Result<Vec<u8>> {
+ match self {
+ Self::Disk(file) => DurableFile::read(file, offset, length).await,
+ Self::Injected(file) => file.read(offset, length).await,
+ }
+ }
+
+ async fn write(&mut self, offset: u64, bytes: Vec<u8>) -> io::Result<()> {
+ match self {
+ Self::Disk(file) => DurableFile::write(file, offset, bytes).await,
+ Self::Injected(file) => file.write(offset, bytes).await,
+ }
+ }
+
+ async fn write_frozen(&mut self, offset: u64, bytes: Frozen<4096>) -> io::Result<()> {
+ match self {
+ Self::Disk(file) => DurableFile::write_frozen(file, offset, bytes).await,
+ Self::Injected(file) => file.write_frozen(offset, bytes).await,
+ }
+ }
+
+ async fn write_frozen_vectored(
+ &mut self,
+ offset: u64,
+ buffers: Vec<Frozen<4096>>,
+ ) -> io::Result<()> {
+ match self {
+ Self::Disk(file) => DurableFile::write_frozen_vectored(file, offset, buffers).await,
+ Self::Injected(file) => file.write_frozen_vectored(offset, buffers).await,
+ }
+ }
+
+ async fn write_aligned(&mut self, offset: u64, bytes: Owned<4096>) -> io::Result<()> {
+ match self {
+ Self::Disk(file) => DurableFile::write_aligned(file, offset, bytes).await,
+ Self::Injected(file) => file.write_aligned(offset, bytes).await,
+ }
+ }
+
+ async fn read_aligned(&self, offset: u64, length: usize) -> io::Result<Owned<4096>> {
+ match self {
+ Self::Disk(file) => DurableFile::read_aligned(file, offset, length).await,
+ Self::Injected(file) => file.read_aligned(offset, length).await,
+ }
+ }
+
+ async fn read_aligned_tail(
+ &self,
+ offset: u64,
+ bytes: Owned<4096>,
+ start: usize,
+ ) -> io::Result<Owned<4096>> {
+ match self {
+ Self::Disk(file) => DurableFile::read_aligned_tail(file, offset, bytes, start).await,
+ Self::Injected(file) => file.read_aligned_tail(offset, bytes, start).await,
+ }
+ }
+
+ async fn length(&self) -> io::Result<u64> {
+ match self {
+ Self::Disk(file) => DurableFile::length(file).await,
+ Self::Injected(file) => file.length().await,
+ }
+ }
+
+ async fn truncate(&self, length: u64) -> io::Result<()> {
+ match self {
+ Self::Disk(file) => DurableFile::truncate(file, length).await,
+ Self::Injected(file) => file.truncate(length).await,
+ }
+ }
+
+ async fn sync(&self) -> io::Result<()> {
+ match self {
+ Self::Disk(file) => DurableFile::sync(file).await,
+ Self::Injected(file) => file.sync().await,
+ }
+ }
+}
+
+impl From<File> for WalFile {
+ fn from(file: File) -> Self {
+ Self::Disk(file)
+ }
+}
+
+impl TryFrom<WalFile> for File {
+ type Error = WalFile;
+
+ fn try_from(file: WalFile) -> Result<Self, WalFile> {
+ match file {
+ WalFile::Disk(file) => Ok(file),
+ injected @ WalFile::Injected(_) => Err(injected),
+ }
+ }
+}
+
+impl<S> InjectedStorage for S
+where
+ S: DurableStorage,
+ S::File: 'static,
+{
+ fn writer_identity(&self, path: &Path) -> io::Result<Option<PathBuf>> {
+ DurableStorage::writer_identity(self, path)
+ }
+
+ fn open<'a>(
+ &'a self,
+ path: &'a Path,
+ mode: OpenMode,
+ ) -> LocalBoxFuture<'a, io::Result<Box<dyn InjectedFile>>> {
+ Box::pin(async move {
+ let file = DurableStorage::open(self, path, mode).await?;
+ Ok(Box::new(file) as Box<dyn InjectedFile>)
+ })
+ }
+
+ fn create_directories<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<()>> {
+ Box::pin(DurableStorage::create_directories(self, path))
+ }
+
+ fn sync_directory<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<()>> {
+ Box::pin(DurableStorage::sync_directory(self, path))
+ }
+
+ fn rename<'a>(
+ &'a self,
+ source: &'a Path,
+ target: &'a Path,
+ ) -> LocalBoxFuture<'a, io::Result<()>> {
+ Box::pin(DurableStorage::rename(self, source, target))
+ }
+
+ fn remove_file<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<()>> {
+ Box::pin(DurableStorage::remove_file(self, path))
+ }
+
+ fn hard_link<'a>(
+ &'a self,
+ source: &'a Path,
+ target: &'a Path,
+ ) -> LocalBoxFuture<'a, io::Result<()>> {
+ Box::pin(DurableStorage::hard_link(self, source, target))
+ }
+
+ fn exists<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<bool>> {
+ Box::pin(DurableStorage::exists(self, path))
+ }
+
+ fn exists_following_links<'a>(
+ &'a self,
+ path: &'a Path,
+ ) -> LocalBoxFuture<'a, io::Result<bool>> {
+ Box::pin(DurableStorage::exists_following_links(self, path))
+ }
+
+ fn entries<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<Vec<StorageEntry>>> {
+ Box::pin(DurableStorage::entries(self, path))
+ }
+
+ fn regular_files<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<RegularFiles>> {
+ Box::pin(DurableStorage::regular_files(self, path))
+ }
+
+ fn remove_tree<'a>(&'a self, path: &'a Path) -> LocalBoxFuture<'a, io::Result<()>> {
+ Box::pin(DurableStorage::remove_tree(self, path))
+ }
+}
+
+impl<F: DurableFile> InjectedFile for F {
+ fn preallocate<'a>(&'a self, path: &'a Path, length: u64) -> LocalBoxFuture<'a, ()> {
+ Box::pin(DurableFile::preallocate(self, path, length))
+ }
+
+ fn read(&self, offset: u64, length: usize) -> LocalBoxFuture<'_, io::Result<Vec<u8>>> {
+ Box::pin(DurableFile::read(self, offset, length))
+ }
+
+ fn write(&mut self, offset: u64, bytes: Vec<u8>) -> LocalBoxFuture<'_, io::Result<()>> {
+ Box::pin(DurableFile::write(self, offset, bytes))
+ }
+
+ fn write_frozen(
+ &mut self,
+ offset: u64,
+ bytes: Frozen<4096>,
+ ) -> LocalBoxFuture<'_, io::Result<()>> {
+ Box::pin(DurableFile::write_frozen(self, offset, bytes))
+ }
+
+ fn write_frozen_vectored(
+ &mut self,
+ offset: u64,
+ buffers: Vec<Frozen<4096>>,
+ ) -> LocalBoxFuture<'_, io::Result<()>> {
+ Box::pin(DurableFile::write_frozen_vectored(self, offset, buffers))
+ }
+
+ fn write_aligned(
+ &mut self,
+ offset: u64,
+ bytes: Owned<4096>,
+ ) -> LocalBoxFuture<'_, io::Result<()>> {
+ Box::pin(DurableFile::write_aligned(self, offset, bytes))
+ }
+
+ fn read_aligned(
+ &self,
+ offset: u64,
+ length: usize,
+ ) -> LocalBoxFuture<'_, io::Result<Owned<4096>>> {
+ Box::pin(DurableFile::read_aligned(self, offset, length))
+ }
+
+ fn read_aligned_tail(
+ &self,
+ offset: u64,
+ bytes: Owned<4096>,
+ start: usize,
+ ) -> LocalBoxFuture<'_, io::Result<Owned<4096>>> {
+ Box::pin(DurableFile::read_aligned_tail(self, offset, bytes, start))
+ }
+
+ fn length(&self) -> LocalBoxFuture<'_, io::Result<u64>> {
+ Box::pin(DurableFile::length(self))
+ }
+
+ fn truncate(&self, length: u64) -> LocalBoxFuture<'_, io::Result<()>> {
+ Box::pin(DurableFile::truncate(self, length))
+ }
+
+ fn sync(&self) -> LocalBoxFuture<'_, io::Result<()>> {
+ Box::pin(DurableFile::sync(self))
+ }
+}
diff --git a/core/sdk/src/links/mux.rs b/core/sdk/src/links/mux.rs
index 47dfd5bfc..f85f17ec3 100644
--- a/core/sdk/src/links/mux.rs
+++ b/core/sdk/src/links/mux.rs
@@ -23,7 +23,7 @@
use crate::links::{Frame, Reply};
use crate::vsr::{decode_eviction, eviction_reason, peek_command, reply_request_id, response_size};
-use bytes::{Buf, BufMut, BytesMut};
+use bytes::{Buf, BufMut, Bytes, BytesMut};
use futures_util::stream::{SplitSink, SplitStream};
use futures_util::{SinkExt, StreamExt};
use iggy_binary_protocol::consensus::{Command, EvictionReason, HEADER_SIZE};
@@ -273,16 +273,29 @@ impl MuxShared {
}
}
+ /// Dispatches each whole frame of `message` as a slice of it. Only a frame
+ /// that spans messages is copied, into `buffer`.
+ fn deliver_message(&self, buffer: &mut BytesMut, mut message: Bytes) -> Result<(), IggyError> {
+ if !buffer.is_empty() {
+ buffer.extend_from_slice(&message);
+ return self.deliver_buffered(buffer);
+ }
+ while let Some(&header) = message.first_chunk::<HEADER_SIZE>() {
+ let size = self.frame_size(&header)?;
+ if message.len() < size {
+ break;
+ }
+ let body = message.split_to(size).slice(HEADER_SIZE..);
+ self.dispatch(Reply { header, body })?;
+ }
+ buffer.extend_from_slice(&message);
+ Ok(())
+ }
+
/// Cuts every whole frame off the front of `buffer` and dispatches it.
fn deliver_buffered(&self, buffer: &mut BytesMut) -> Result<(), IggyError> {
while let Some(&header) = buffer.first_chunk::<HEADER_SIZE>() {
- let Ok(size) = response_size(&header) else {
- error!(
- "Received a frame shorter than its header from {}.",
- self.peer
- );
- return Err(IggyError::Disconnected);
- };
+ let size = self.frame_size(&header)?;
if buffer.len() < size {
break;
}
@@ -292,6 +305,16 @@ impl MuxShared {
Ok(())
}
+ fn frame_size(&self, header: &[u8; HEADER_SIZE]) -> Result<usize, IggyError> {
+ response_size(header).map_err(|_| {
+ error!(
+ "Received a frame shorter than its header from {}.",
+ self.peer
+ );
+ IggyError::Disconnected
+ })
+ }
+
/// Gives a reply to the request that waits for it. An error closes the link,
/// so an eviction interrupts every request sharing that connection.
fn dispatch(&self, reply: Reply) -> Result<(), IggyError> {
@@ -453,6 +476,8 @@ async fn write_websocket<S: AsyncRead + AsyncWrite + Unpin>(
shared: Arc<MuxShared>,
) {
while let Some(frame) = shared.next_frame(&mut frames).await {
+ // A message takes one contiguous payload. Two fragments would make the
+ // server join them with a copy of its own.
let mut message = BytesMut::with_capacity(HEADER_SIZE + frame.body.len());
message.put_slice(bytemuck::bytes_of(&frame.header));
message.put_slice(&frame.body);
@@ -484,8 +509,7 @@ async fn read_websocket<S: AsyncRead + AsyncWrite + Unpin>(
let reason = loop {
match stream.next().await {
Some(Ok(message @ (Message::Binary(_) | Message::Text(_)))) => {
- buffer.extend_from_slice(&message.into_data());
- if let Err(reason) = shared.deliver_buffered(&mut buffer) {
+ if let Err(reason) = shared.deliver_message(&mut buffer, message.into_data()) {
break reason;
}
}
@@ -515,7 +539,6 @@ mod tests {
use crate::links::test_support::{
CLIENT, DUPLEX_CAPACITY, LOCALHOST, read_request_header, reply, request, request_with_body,
};
- use bytes::Bytes;
use iggy_binary_protocol::consensus::{EvictionHeader, EvictionReason, RequestHeader};
use std::io::IoSlice;
use std::pin::Pin;
@@ -859,6 +882,33 @@ mod tests {
assert_eq!(second.unwrap().body[..], 2u64.to_le_bytes());
}
+ #[tokio::test]
+ async fn given_whole_replies_in_one_websocket_message_should_hand_over_slices_of_it() {
+ let (link, _server) = websocket_link().await;
+ let mut message = reply(1, &1u64.to_le_bytes());
+ message.extend_from_slice(&reply(2, &2u64.to_le_bytes()));
+ let message = Bytes::from(message);
+ let (first_waiter, first) = oneshot::channel();
+ let (second_waiter, second) = oneshot::channel();
+ let _first_entry = link.shared.wait_for(1, first_waiter).unwrap();
+ let _second_entry = link.shared.wait_for(2, second_waiter).unwrap();
+ let mut buffer = BytesMut::new();
+
+ link.shared
+ .deliver_message(&mut buffer, message.clone())
+ .unwrap();
+
+ let first = first.await.unwrap().unwrap();
+ let second = second.await.unwrap().unwrap();
+ let second_frame = message.len() / 2;
+ assert_eq!(first.body.as_ptr(), message[HEADER_SIZE..].as_ptr());
+ assert_eq!(
+ second.body.as_ptr(),
+ message[second_frame + HEADER_SIZE..].as_ptr()
+ );
+ assert!(buffer.is_empty());
+ }
+
#[tokio::test]
async fn given_a_websocket_link_when_shut_down_should_send_a_normal_close() {
let (link, mut server) = websocket_link().await;
diff --git a/core/server/README.md b/core/server/README.md
index 85692b4a0..ab38fb2b6 100644
--- a/core/server/README.md
+++ b/core/server/README.md
@@ -178,20 +178,20 @@ write can still commit. The body `id` 57 (`TransientNotCommitted`) or the body
again, because a resend is a new request that can apply twice. The Rust, C# and
Java SDKs never resend such a write, and the Rust SDK reports it as 57.
-Metadata forwards, partition relays and metadata-floor waits share a limit of
+Metadata forwards and partition relays share a limit of
128 concurrent forwards per node. New forwards at capacity return 503 `forward_busy` with
`Retry-After: 1`. Metadata forwarding has one 30-second retry budget beginning
before the request body is buffered, so a slow upload leaves less time for
forwarding. Partition requests share a 10-second deadline across their metadata
wait, local attempts, relays and reply waits.
-Socket replies use `message_bus.client_queue_capacity` per connection. A reply
-that cannot enter the non-waiting queue closes that connection, leaving the
-caller to recover any uncertain operation. Paths that wait for reply space hold
-their bounded permit until delivery or connection teardown. An administrative
-client that stops reading can retain that bounded queue until delivery, overflow
-or teardown; consumer-group heartbeat eviction does not apply to an admin-only
-connection.
+A node sends one metadata-floor read at a time and holds no forward slot for it. The read serves every linearizable read that arrived before it was sent, and the reads that arrive while it is in flight share the next one.
+
+Socket replies use `message_bus.client_queue_capacity` per connection. A reply that cannot enter the non-waiting queue closes that connection, leaving the caller to recover any uncertain operation. Paths that wait for reply space hold their bounded permit until delivery or connection teardown.
+
+A reply that waits longer than `message_bus.client_write_stall_timeout` (30 seconds by default) closes the connection, so a client that stops reading its socket, an administrative one included, holds its queue and request tasks for at most that long.
+
+Requests queued behind the ones a connection runs are capped at 1024 frames and at `message_bus.client_request_queue_bytes_max` (256 MiB by default). A request past either cap gets 58 (`TransientNotAccepted`), unless the queue is empty.
### Metadata freshness and counters
diff --git a/core/server/config.toml b/core/server/config.toml
index 59cba18c5..aec2d0a88 100644
--- a/core/server/config.toml
+++ b/core/server/config.toml
@@ -1148,6 +1148,20 @@ peer_queue_capacity = 4096
# connection whose outbound queue is full.
client_queue_capacity = 256
+# How long a reply waits for room in a connection's outbound queue before the
+# server closes that connection. The queue stays full only while the client
+# leaves its socket unread, so this bounds how long such a client pins its
+# queued replies and request tasks. The client then reconnects and recovers
+# its session. Must be > 0.
+client_write_stall_timeout = "30 s"
+
+# Bytes of requests that wait in the per-connection queue of 1024 described
+# above. A request that would take the queue past it gets the same retryable
+# error 58, unless the queue is empty, so one request of any legal size always
+# fits. The default holds the 256 requests one connection runs at once at
+# 1 MiB each. Must be > 0.
+client_request_queue_bytes_max = "256 MiB"
+
# Interval between outbound reconnect attempts to peers with peer_id > self_id.
# Also bounds each dial, so a peer that drops SYNs cannot stall the sweep.
reconnect_period = "1 s"
diff --git a/core/server/src/boot/mod.rs b/core/server/src/boot/mod.rs
index 8ffbe67b4..e6d6356c3 100644
--- a/core/server/src/boot/mod.rs
+++ b/core/server/src/boot/mod.rs
@@ -1097,7 +1097,7 @@ async fn shard_main(
// Await the watchdog even when the drain verdict is an error: the token
// has fired, so it either stands down within one poll interval or is
// mid-`bus.shutdown()`, and dropping it there truncates in-flight
- // `ClientForwardFailed` replies.
+ // client replies.
let pump_verdict = await_pump_drain(pump_handle.take(), config, shard_id).await;
if let Some(watchdog) = shutdown_watchdog.take() {
let _ = watchdog.await;
diff --git a/core/server/src/boot/threads.rs b/core/server/src/boot/threads.rs
index 0af669361..79ecef57f 100644
--- a/core/server/src/boot/threads.rs
+++ b/core/server/src/boot/threads.rs
@@ -819,7 +819,7 @@ pub(in crate::boot) async fn await_pump_drain(
/// The caller owns the returned handle and must await it on the exit paths
/// where shutdown is in progress (flag set or bus token triggered):
/// dropping it there cancels the watchdog mid-`bus.shutdown()`, truncating
-/// in-flight `ClientForwardFailed` replies (terminal per `SendError` docs).
+/// in-flight client replies.
/// It cannot go through `bus.track_background` instead: the watchdog itself
/// drives `bus.shutdown()`, and the bg-drain loop in `shutdown()` would
/// re-enter awaiting the watchdog's own pending shutdown call
diff --git a/core/server/src/consumer_group/liveness.rs b/core/server/src/consumer_group/liveness.rs
index 136eacc69..05c7b7aed 100644
--- a/core/server/src/consumer_group/liveness.rs
+++ b/core/server/src/consumer_group/liveness.rs
@@ -1663,16 +1663,19 @@ mod tests {
async fn given_failed_partition_when_retiring_should_allow_new_sessions() {
let (directory, shard) = shard_with_members(1).await;
let metadata = shard.plane.metadata();
+ let namespace = server_common::sharding::IggyNamespace::new(0, 0, 0);
+ let created_revision = metadata
+ .mux_stm
+ .streams()
+ .created_revision_for_namespace(namespace)
+ .unwrap();
assert!(
metadata
.client_table
.borrow_mut()
.end_session(CLIENT, 0, SESSION, SESSION)
);
- shard
- .plane
- .partitions()
- .fence(server_common::sharding::IggyNamespace::new(0, 0, 0), 1);
+ shard.plane.partitions().fence(namespace, created_revision);
let tracker = RefCell::new(ConsumerGroupLiveness::default());
let (_stop, receiver) = shard::channel(1);
let (_signal, shutdown) = Shutdown::new();
@@ -1696,11 +1699,11 @@ mod tests {
assert_eq!(
crate::partition_helpers::partition_retirement_fence(
directory.path().to_str().unwrap(),
- server_common::sharding::IggyNamespace::new(0, 0, 0),
+ namespace,
)
.await
.unwrap(),
- Some(1)
+ Some(created_revision)
);
metadata
.client_table
@@ -1726,7 +1729,12 @@ mod tests {
.end_session(CLIENT, 0, SESSION, SESSION)
);
let namespace = server_common::sharding::IggyNamespace::new(0, 0, 0);
- shard.plane.partitions().fence(namespace, 1);
+ let created_revision = metadata
+ .mux_stm
+ .streams()
+ .created_revision_for_namespace(namespace)
+ .unwrap();
+ shard.plane.partitions().fence(namespace, created_revision);
let obstruction = directory.path().join("partition-initialization");
std::fs::write(&obstruction, b"not a directory").unwrap();
let tracker = RefCell::new(ConsumerGroupLiveness::default());
diff --git a/core/server/src/dispatch/failure.rs b/core/server/src/dispatch/failure.rs
index 0bf04243e..e7cf10bfb 100644
--- a/core/server/src/dispatch/failure.rs
+++ b/core/server/src/dispatch/failure.rs
@@ -141,7 +141,8 @@ impl ClientOrigin {
/// frozen frame or the vectored poll reply.
///
/// A frame for a queued request waits for room in a full reply queue, so a
-/// client that reads its replies keeps its connection. Any other frame, the
+/// client that reads its replies keeps its connection. A queue that stays full
+/// for `message_bus.client_write_stall_timeout` closes it. Any other frame, the
/// overflow deny and the heartbeat eviction included, closes the connection
/// on a full queue: nothing bounds how many of them could wait.
#[allow(clippy::future_not_send)]
diff --git a/core/server/src/dispatch/mod.rs b/core/server/src/dispatch/mod.rs
index 00ab44ba3..037f7e74a 100644
--- a/core/server/src/dispatch/mod.rs
+++ b/core/server/src/dispatch/mod.rs
@@ -89,7 +89,37 @@ use std::sync::Arc;
use std::task::{Poll, Waker};
use tracing::{debug, warn};
-type ClientRequestQueues = Rc<RefCell<AHashMap<u128, VecDeque<QueuedClientRequest>>>>;
+type ClientRequestQueues = Rc<RefCell<AHashMap<u128, ClientRequestQueue>>>;
+
+/// The frames one connection has queued for its drain, and their bytes.
+#[derive(Default)]
+struct ClientRequestQueue {
+ requests: VecDeque<QueuedClientRequest>,
+ bytes: u64,
+}
+
+impl ClientRequestQueue {
+ fn push_back(&mut self, request: QueuedClientRequest) {
+ self.bytes += request.bytes();
+ self.requests.push_back(request);
+ }
+
+ fn remove(&mut self, index: usize) -> Option<QueuedClientRequest> {
+ let request = self.requests.remove(index)?;
+ self.bytes -= request.bytes();
+ Some(request)
+ }
+}
+
+impl FromIterator<QueuedClientRequest> for ClientRequestQueue {
+ fn from_iter<I: IntoIterator<Item = QueuedClientRequest>>(requests: I) -> Self {
+ let mut queue = Self::default();
+ for request in requests {
+ queue.push_back(request);
+ }
+ queue
+ }
+}
struct QueuedClientRequest {
origin: ClientOrigin,
@@ -107,6 +137,10 @@ impl QueuedClientRequest {
}
}
+ fn bytes(&self) -> u64 {
+ self.message.as_slice().len() as u64
+ }
+
fn progress_key(
&mut self,
metadata_frontier: u64,
@@ -140,7 +174,9 @@ impl QueuedClientRequest {
/// re-issue it anywhere, including here once the queue drains. Sized far above
/// any SDK's in-flight window, so it only ever fires under a genuine stall.
/// The deny does not wait for room in the reply queue, so with that queue
-/// full it closes the connection.
+/// full it closes the connection. `message_bus.client_request_queue_bytes_max`
+/// bounds the same queue by bytes and refuses a frame past it the same way,
+/// unless the queue is empty.
const MAX_QUEUED_CLIENT_REQUESTS: usize = 1024;
type ActiveClientRequests = Rc<RefCell<AHashMap<u128, Rc<ClientDrain>>>>;
type ConsumerProgress = (u64, u8, usize);
@@ -170,16 +206,23 @@ fn enqueue_client_request<B, MJ, S, SB>(
// connection that sent it.
let origin = ClientOrigin::capture(bus, client_id);
{
+ let request = QueuedClientRequest::new(origin.queued(), message);
+ let bytes_max = server_config
+ .message_bus
+ .client_request_queue_bytes_max
+ .as_bytes_u64();
let mut queues = queues.borrow_mut();
let queue = queues.entry(client_id).or_default();
- if queue.len() >= MAX_QUEUED_CLIENT_REQUESTS {
+ if queue.requests.len() >= MAX_QUEUED_CLIENT_REQUESTS
+ || (!queue.requests.is_empty() && queue.bytes + request.bytes() > bytes_max)
+ {
// Borrow released before the deny, which spawns onto this same
// task and would otherwise re-enter the table.
drop(queues);
- deny_overflowing_client_request(shard_handle, origin, message);
+ deny_overflowing_client_request(shard_handle, origin, request.message);
return;
}
- queue.push_back(QueuedClientRequest::new(origin.queued(), message));
+ queue.push_back(request);
}
let running = active.borrow().get(&client_id).cloned();
if let Some(drain) = running {
@@ -208,7 +251,8 @@ fn enqueue_client_request<B, MJ, S, SB>(
}
/// Answer a request that arrived with this client's queue already at
-/// [`MAX_QUEUED_CLIENT_REQUESTS`] with the retryable transient denial.
+/// [`MAX_QUEUED_CLIENT_REQUESTS`] or its byte budget with the retryable
+/// transient denial.
///
/// Spawned rather than awaited: the enqueue path is sync (it runs straight off
/// frame arrival) and the reply goes out on the bus. The header decodes as in
@@ -250,7 +294,6 @@ fn deny_overflowing_client_request<B, MJ, S, SB>(
debug!(
transport_client_id,
operation = ?request.header().operation,
- queued = MAX_QUEUED_CLIENT_REQUESTS,
"denying client request retryable: this connection's request queue is full"
);
send_deny_reply(
@@ -497,7 +540,7 @@ fn pop_next_client_request(
let queue = queues.get_mut(&client_id)?;
let mut waiting = false;
let mut barrier = false;
- let index = queue.iter_mut().position(|queued| {
+ let index = queue.requests.iter_mut().position(|queued| {
if is_ping(&queued.message) {
return true;
}
@@ -528,7 +571,7 @@ fn pop_next_client_request(
false
})?;
let message = queue.remove(index);
- if queue.is_empty() {
+ if queue.requests.is_empty() {
queues.remove(&client_id);
}
message
@@ -871,7 +914,7 @@ async fn handle_client_request<B, MJ, S, SB>(
if queues
.borrow()
.get(&transport_client_id)
- .is_some_and(|queue| !queue.is_empty())
+ .is_some_and(|queue| !queue.requests.is_empty())
{
return Poll::Ready(false);
}
@@ -1219,8 +1262,8 @@ mod tests {
WireConsumer,
};
use iggy_binary_protocol::{WireEncode, WireIdentifier, WireName, WireOptions};
- use iggy_common::Identifier;
use iggy_common::defaults::DEFAULT_ROOT_USER_ID;
+ use iggy_common::{Identifier, IggyByteSize};
use journal::prepare_journal::PrepareJournal;
use message_bus::installer::conn_info::ClientTransportKind;
use metadata::impls::metadata::IggySnapshot;
@@ -3141,7 +3184,7 @@ mod tests {
);
let queues = Rc::new(RefCell::new(AHashMap::from_iter([(
ROOT_TRANSPORT,
- VecDeque::from([queued]),
+ ClientRequestQueue::from_iter([queued]),
)])));
let drain = Rc::new(ClientDrain::new(&config));
let spawned_before = bus.spawned.get();
@@ -3173,7 +3216,7 @@ mod tests {
);
let queues = Rc::new(RefCell::new(AHashMap::from_iter([(
ROOT_TRANSPORT,
- VecDeque::from([queued]),
+ ClientRequestQueue::from_iter([queued]),
)])));
let drain = Rc::new(ClientDrain::new(&config));
let unrelated = drain.admit(Admission::Cap).await;
@@ -3434,7 +3477,7 @@ mod tests {
partition_id: Some(0),
}
.to_bytes();
- let mut reads = VecDeque::new();
+ let mut reads = ClientRequestQueue::default();
for (request, code, body) in [
(NEXT_REQUEST, POLL_MESSAGES_CODE, poll),
(OFFSET_REQUEST, GET_CONSUMER_OFFSET_CODE, offset),
@@ -3472,7 +3515,7 @@ mod tests {
queues
.borrow()
.get(&ROOT_TRANSPORT)
- .map_or(0, VecDeque::len),
+ .map_or(0, |queue| queue.requests.len()),
2,
"Next without auto-commit and GetConsumerOffset must both stay queued"
);
@@ -3562,7 +3605,7 @@ mod tests {
let metadata = drain.admit(Admission::MetadataLane).await;
let queues = Rc::new(RefCell::new(AHashMap::from_iter([(
ROOT_TRANSPORT,
- VecDeque::from(
+ ClientRequestQueue::from_iter(
[
held_get_me(READ_REQUEST, 0),
root_read(PING_CODE, PING_REQUEST).into_generic(),
@@ -3616,7 +3659,7 @@ mod tests {
};
let queues = Rc::new(RefCell::new(AHashMap::from_iter([(
ROOT_TRANSPORT,
- VecDeque::from(
+ ClientRequestQueue::from_iter(
[
wire_request(
Operation::StoreConsumerOffset,
@@ -3781,6 +3824,79 @@ mod tests {
);
}
+ /// Frames that would take a connection's queue past its byte budget get
+ /// the retryable deny at once, like frames past the frame cap. The frames
+ /// already queued are still served in order.
+ #[compio::test]
+ async fn given_a_queue_at_its_byte_budget_when_another_frame_arrives_should_deny_it_retryable()
+ {
+ const OVER_BUDGET: u64 = 5;
+ let ping_bytes = root_read(PING_CODE, OVER_BUDGET).as_slice().len() as u64;
+ let bus = SpyBus::default();
+ let shard = Rc::new(test_shard(&bus, 0, 1, FIRST_BOOT));
+ commit_root_session(&shard);
+ let shard_handle = unset_shard_handle();
+ *shard_handle.borrow_mut() = Some(Rc::downgrade(&shard));
+ let sessions = Rc::new(RefCell::new(SessionManager::new()));
+ bind_root_session(&shard, &sessions).await;
+ let mut server_config = ServerConfig::default();
+ server_config.message_bus.client_queue_capacity = 1;
+ server_config.message_bus.client_request_queue_bytes_max =
+ IggyByteSize::from(2 * ping_bytes);
+ let server_config = Arc::new(server_config);
+ let queues: ClientRequestQueues = Rc::default();
+ let active: ActiveClientRequests = Rc::default();
+ let enqueue = |message| {
+ enqueue_client_request(
+ &bus,
+ &shard_handle,
+ &sessions,
+ &server_config,
+ &queues,
+ &active,
+ ROOT_TRANSPORT,
+ message,
+ );
+ };
+
+ // The first read takes the only place under the cap and the second
+ // holds the drain, so the pings behind them stay queued.
+ enqueue(held_get_me(1, UNAPPLIED_OP));
+ enqueue(held_get_me(2, UNAPPLIED_OP));
+ settle().await;
+ for request in 3..=OVER_BUDGET {
+ enqueue(root_read(PING_CODE, request).into_generic());
+ }
+ await_replies(&bus, 1).await;
+ {
+ let replies = bus.client_replies.borrow();
+ assert_eq!(
+ replies.len(),
+ 1,
+ "only the frame past the budget is answered"
+ );
+ assert_eq!(reply_request(&replies[0].1), OVER_BUDGET);
+ assert_eq!(
+ reply_status(&replies[0].1),
+ IggyError::TransientNotAccepted.as_code()
+ );
+ }
+
+ shard
+ .plane
+ .metadata()
+ .applied_frontier()
+ .advance(UNAPPLIED_OP);
+ await_replies(&bus, 5).await;
+ let replies = bus.client_replies.borrow();
+ let served: Vec<u64> = replies[1..]
+ .iter()
+ .filter(|(_, frame)| reply_status(frame) == 0)
+ .map(|(_, frame)| reply_request(frame))
+ .collect();
+ assert_eq!(served, [1, 2, 3, 4]);
+ }
+
/// `BIND_SESSION` runs on the drain, so the replicated frame behind it is
/// served on the session it binds, not evicted for a missing session.
#[compio::test]
diff --git a/core/server/src/dispatch/partition.rs b/core/server/src/dispatch/partition.rs
index ca1b19120..d33c30314 100644
--- a/core/server/src/dispatch/partition.rs
+++ b/core/server/src/dispatch/partition.rs
@@ -1882,7 +1882,7 @@ mod tests {
&root.path().join("prepares-0"),
namespace.inner(),
0,
- journal::durable_storage::DiskStorage,
+ partitions::WalStorage::default(),
capacity,
false,
)
diff --git a/core/server/src/dispatch/test_support.rs b/core/server/src/dispatch/test_support.rs
index 8696b9fe6..7155743e8 100644
--- a/core/server/src/dispatch/test_support.rs
+++ b/core/server/src/dispatch/test_support.rs
@@ -41,9 +41,9 @@ use message_bus::installer::ConnectionInstaller;
use message_bus::installer::conn_info::ClientConnMeta;
use message_bus::replica::listener::MessageHandler;
use message_bus::{
- BusMessage, ClientConnectionLostFn, ClientForwardFn, ConnectionLostFn, ConnectionPermit,
- ConnectionRegistry, InstanceToken, JoinHandle, MessageBus, ReplicaForwardFn,
- ReplicaHandshakeDoneFn, SendError, SharedTlsServerConfig,
+ BusMessage, ClientConnectionLostFn, ConnectionLostFn, ConnectionPermit, ConnectionRegistry,
+ InstanceToken, JoinHandle, MessageBus, ReplicaForwardFn, ReplicaHandshakeDoneFn, SendError,
+ SharedTlsServerConfig,
};
use metadata::impls::metadata::{IggySnapshot, StreamsFrontend};
use metadata::stm::StateMachine as _;
@@ -180,7 +180,6 @@ impl MessageBus for SpyBus {
}
fn set_connection_lost_fn(&self, _f: ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: ClientForwardFn) {}
}
impl ConnectionInstaller for SpyBus {
diff --git a/core/server/src/http/error.rs b/core/server/src/http/error.rs
index 0dfd9fa77..1851322d2 100644
--- a/core/server/src/http/error.rs
+++ b/core/server/src/http/error.rs
@@ -509,8 +509,6 @@ pub(in crate::http) enum ReadError {
/// hazard (serving state the caller already saw replaced) reached from
/// different directions. Never a 2xx with stale state.
MetadataFrontierUnreached,
- /// A metadata floor read could not reserve forwarding capacity.
- ForwardBusy,
/// The metadata primary rejected this node's relay proof.
RelayRefused,
/// A partition read got no reply after submission to the owning shard.
@@ -529,7 +527,6 @@ impl IntoResponse for ReadError {
Self::NotPrimary => not_primary_response(),
Self::RedirectToPrimary(location) => primary_redirect_response(&location),
Self::RecoveryIncomplete | Self::MetadataFrontierUnreached => service_unavailable(),
- Self::ForwardBusy => forward_busy(),
Self::RelayRefused => relay_refused(),
Self::Timeout => (
StatusCode::GATEWAY_TIMEOUT,
diff --git a/core/server/src/http/extractor.rs b/core/server/src/http/extractor.rs
index 71440dc14..576a3b602 100644
--- a/core/server/src/http/extractor.rs
+++ b/core/server/src/http/extractor.rs
@@ -44,6 +44,17 @@ const BEARER: &str = "Bearer ";
const JWT_KEY_PREFIX: &str = "jwt:";
const PAT_KEY_PREFIX: &str = "pat:";
+/// What [`resolve_credential`] proved about a bearer: the JWT id when it is a
+/// JWT, the account behind it and its expiry in seconds. The partition route
+/// layer stores it in the request extensions for the extractor behind it.
+#[derive(Clone)]
+pub(in crate::http) struct VerifiedBearer {
+ pub jwt_id: Option<String>,
+ pub user_id: u32,
+ pub user_created_at: IggyTimestamp,
+ pub expiry: u64,
+}
+
/// Resolved caller identity for a protected write route: the shared VSR
/// session established once per presenting credential (it carries the
/// authenticated user id).
@@ -118,17 +129,17 @@ impl FromRequestParts<HttpState> for Credential {
state: &HttpState,
) -> Result<Self, Self::Rejection> {
let bearer = bearer_token(&parts.headers)?;
- // The JWT verify may await a `!Send` JWKS fetch through cyper, yet axum
- // requires this extractor future to be `Send`. `SendWrapper` bridges it,
- // sound for the reason `Authenticated` gives: the JWKS client only ever
- // runs on shard 0's thread.
- let (jwt_id, user_id, user_created_at, expiry) =
- SendWrapper::new(resolve_credential(state, bearer)).await?;
+ let verified = verified_bearer(parts, state, bearer).await?;
Ok(Self {
- key: credential_key(bearer, jwt_id, user_id, user_created_at),
- user_id,
- user_created_at,
- expiry,
+ key: credential_key(
+ bearer,
+ verified.jwt_id.as_deref(),
+ verified.user_id,
+ verified.user_created_at,
+ ),
+ user_id: verified.user_id,
+ user_created_at: verified.user_created_at,
+ expiry: verified.expiry,
})
}
}
@@ -172,14 +183,7 @@ impl FromRequestParts<HttpState> for Identity {
// returns feed the write path's session table; a read discards them -
// its read-your-writes floor is keyed by user id, not by credential
// (see `MetadataWatermarks`).
- // The verify is `!Send` (a trusted-issuer JWT may await a JWKS fetch),
- // so bridge it with `SendWrapper` - sound only because compio pins this
- // future to shard 0's single thread, the only thread the JWKS client
- // ever runs on (mirrors legacy `HttpSafeShard`). It holds no `RefCell`
- // borrow or `DashMap` guard across the `.await`, so a sibling task
- // scheduled on this thread meanwhile never observes a borrowed cell.
- let (_jwt_id, user_id, user_created_at, _expiry) =
- SendWrapper::new(resolve_credential(state, bearer)).await?;
+ let verified = verified_bearer(parts, state, bearer).await?;
let path_and_query = parts
.uri
.path_and_query()
@@ -196,8 +200,8 @@ impl FromRequestParts<HttpState> for Identity {
);
}
Ok(Self {
- user_id,
- user_created_at,
+ user_id: verified.user_id,
+ user_created_at: verified.user_created_at,
path_and_query,
client_ip,
})
@@ -217,8 +221,27 @@ pub(in crate::http) fn bearer_token(headers: &HeaderMap) -> Result<&str, IggyErr
.ok_or(IggyError::AccessTokenMissing)
}
-/// Map a bearer credential to its `(optional JWT id, user id, user created_at,
-/// expiry secs)`. Only `Authenticated` builds a session key from these facts.
+/// The facts of `bearer`: the ones a route layer stored when it verified the
+/// same bearer, so a request is verified once, or a fresh verify.
+async fn verified_bearer(
+ parts: &Parts,
+ state: &HttpState,
+ bearer: &str,
+) -> Result<VerifiedBearer, AuthError> {
+ if let Some(verified) = parts.extensions.get::<VerifiedBearer>() {
+ return Ok(verified.clone());
+ }
+ // The verify is `!Send` (a trusted-issuer JWT may await a JWKS fetch), so
+ // bridge it with `SendWrapper` - sound only because compio pins this
+ // future to shard 0's single thread, the only thread the JWKS client ever
+ // runs on (mirrors legacy `HttpSafeShard`). It holds no `RefCell` borrow
+ // or `DashMap` guard across the `.await`, so a sibling task scheduled on
+ // this thread meanwhile never observes a borrowed cell.
+ SendWrapper::new(resolve_credential(state, bearer)).await
+}
+
+/// Map a bearer credential to its [`VerifiedBearer`] facts. Only
+/// `Authenticated` builds a session key from them.
///
/// A JWT is the primary credential, keyed by its account and `jti`. A raw PAT
/// presented as the bearer is the documented fallback, keyed by the same
@@ -237,7 +260,7 @@ pub(in crate::http) fn bearer_token(headers: &HeaderMap) -> Result<&str, IggyErr
pub(in crate::http) async fn resolve_credential(
state: &HttpState,
bearer: &str,
-) -> Result<(Option<String>, u32, IggyTimestamp, u64), AuthError> {
+) -> Result<VerifiedBearer, AuthError> {
if let Ok(token) = state.jwt.decode(bearer).await {
let (user_id, user_created_at) = match token.subject {
TokenSubject::Account {
@@ -249,20 +272,30 @@ pub(in crate::http) async fn resolve_credential(
),
TokenSubject::TrustedIssuer(issuer) => state.trusted_issuer_user(&issuer)?,
};
- return Ok((Some(token.jti), user_id, user_created_at, token.exp));
+ return Ok(VerifiedBearer {
+ jwt_id: Some(token.jti),
+ user_id,
+ user_created_at,
+ expiry: token.exp,
+ });
}
let (user_id, user_created_at, expiry) =
verify_pat_credentials_with_expiry(&state.shard, bearer)
.map_err(|_| IggyError::Unauthenticated)?;
- Ok((None, user_id, user_created_at, expiry))
+ Ok(VerifiedBearer {
+ jwt_id: None,
+ user_id,
+ user_created_at,
+ expiry,
+ })
}
/// Table key of a credential [`resolve_credential`] resolved: the account and
/// `jti` of a JWT, or the PAT's hash, so the raw secret never becomes a key.
pub(in crate::http) fn credential_key(
bearer: &str,
- jwt_id: Option<String>,
+ jwt_id: Option<&str>,
user_id: u32,
user_created_at: IggyTimestamp,
) -> String {
@@ -283,3 +316,64 @@ pub(in crate::http) fn credential_key(
},
)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::http::reads::tests::read_state;
+ use axum::http::Request;
+
+ const LAYER_USER_ID: u32 = 7;
+
+ /// The partition route layer verifies the bearer before its handler runs,
+ /// so the extractor behind it reuses that proof. A bearer that would not
+ /// verify shows the extractor did not verify it again.
+ #[compio::test]
+ async fn given_a_bearer_the_route_layer_verified_when_extracting_should_not_verify_it_again() {
+ let state = read_state();
+ let verified = VerifiedBearer {
+ jwt_id: Some("layer-jti".to_owned()),
+ user_id: LAYER_USER_ID,
+ user_created_at: IggyTimestamp::from(1),
+ expiry: u64::MAX,
+ };
+
+ let mut parts = unverifiable_bearer_parts();
+ parts.extensions.insert(verified.clone());
+ let credential = Credential::from_request_parts(&mut parts, &state)
+ .await
+ .unwrap_or_else(|_| panic!("a verified bearer must not be verified again"));
+ assert_eq!(credential.user_id, LAYER_USER_ID);
+ assert_eq!(
+ credential.key,
+ credential_key(
+ "not-a-token",
+ verified.jwt_id.as_deref(),
+ LAYER_USER_ID,
+ verified.user_created_at
+ )
+ );
+ let identity = Identity::from_request_parts(&mut parts, &state)
+ .await
+ .unwrap_or_else(|_| panic!("a verified bearer must not be verified again"));
+ assert_eq!(identity.user_id, LAYER_USER_ID);
+
+ let mut unverified = unverifiable_bearer_parts();
+ assert!(
+ Credential::from_request_parts(&mut unverified, &state)
+ .await
+ .is_err(),
+ "without a layer the extractor still verifies the bearer"
+ );
+ }
+
+ fn unverifiable_bearer_parts() -> Parts {
+ Request::builder()
+ .uri("/streams/1/topics/1/messages")
+ .header(AUTHORIZATION, "Bearer not-a-token")
+ .body(())
+ .expect("request builds")
+ .into_parts()
+ .0
+ }
+}
diff --git a/core/server/src/http/forward.rs b/core/server/src/http/forward.rs
index 13ea9d7f6..8a88a3f05 100644
--- a/core/server/src/http/forward.rs
+++ b/core/server/src/http/forward.rs
@@ -78,14 +78,16 @@
//! from the local consensus view or from a refusing hop target's newer view,
//! never from a response `Location`, and the client follows no redirects.
-use std::cell::Cell;
+use std::cell::{Cell, RefCell};
use std::fmt;
+use std::future::{Future, poll_fn};
use std::net::SocketAddr;
use std::ops::ControlFlow;
use std::path::Path;
use std::rc::Rc;
use std::str::FromStr;
use std::sync::Arc;
+use std::task::{Poll, Waker};
use std::time::{Duration, Instant};
use axum::body::{Body, to_bytes};
@@ -189,8 +191,8 @@ const FORWARD_RETRY_INTERVAL: Duration = Duration::from_millis(50);
/// the caller gets a retryable 503.
const FORWARD_RETRY_DEADLINE: Duration = Duration::from_secs(30);
-/// Cap on concurrent forwards held by this node: metadata forwards, partition
-/// relays and metadata floor waits. Deliberately its own budget: a forward
+/// Cap on concurrent forwards held by this node: metadata forwards and
+/// partition relays. Deliberately its own budget: a forward
/// parks no reply slot and touches none of the shard bus machinery the
/// partition-write caps protect, and counting forwards against those caps
/// would let a slow primary starve this node's own direct clients.
@@ -485,6 +487,7 @@ pub(in crate::http) fn build_forward_state(
body_limit,
in_flight: Rc::new(Cell::new(0)),
relay_targets: RelayTargets::default(),
+ floor_reads: FloorReads::default(),
})
}
@@ -725,13 +728,14 @@ async fn relay_to_primary(
/// The metadata primary's commit floor, read before `deadline`: an op at or
/// above every metadata op acknowledged before this call. Refusals preserve
-/// whether forwarding is unavailable, its budget is full, or the primary
-/// rejected this node's proof.
+/// whether forwarding is unavailable or the primary rejected this node's
+/// proof.
///
-/// The reads leave this node like a forward, so the wait holds a forward slot
-/// ([`MAX_IN_FLIGHT_FORWARDS`]): a burst of linearizable reads cannot open
-/// unbounded connections to the metadata primary or resend to it every
-/// [`FORWARD_RETRY_INTERVAL`] without bound while it is not settled.
+/// The reads are single flight ([`FloorReads`]): a burst of linearizable reads
+/// sends one read per round trip to the metadata primary, and one per
+/// [`FORWARD_RETRY_INTERVAL`] while it is not settled. That is their own
+/// budget, so they hold no forward slot ([`MAX_IN_FLIGHT_FORWARDS`]) and
+/// relays cannot starve them.
pub(in crate::http) async fn metadata_floor(
state: &HttpInner,
deadline: Instant,
@@ -741,15 +745,17 @@ pub(in crate::http) async fn metadata_floor(
.relay_key
.as_ref()
.ok_or(ReadError::NotPrimary)?;
- let _guard = ForwardGuard::admit(&state.forward.in_flight).ok_or(ReadError::ForwardBusy)?;
loop {
- if let Some(socket) = primary_socket(state) {
+ let read = state.forward.floor_reads.read(|| async {
+ let Some(socket) = primary_socket(state) else {
+ return ControlFlow::Continue(());
+ };
state.metrics.metadata_floor_reads.inc();
- let read = read_metadata_floor(state, relay_key, socket);
- let remaining = deadline.saturating_duration_since(Instant::now());
- if let Ok(ControlFlow::Break(floor)) = compio::time::timeout(remaining, read).await {
- return floor.ok_or(ReadError::RelayRefused);
- }
+ read_metadata_floor(state, relay_key, socket).await
+ });
+ let remaining = deadline.saturating_duration_since(Instant::now());
+ if let Ok(ControlFlow::Break(floor)) = compio::time::timeout(remaining, read).await {
+ return floor.ok_or(ReadError::RelayRefused);
}
if deadline.saturating_duration_since(Instant::now()) <= FORWARD_RETRY_INTERVAL {
return Err(ReadError::NotPrimary);
@@ -890,7 +896,7 @@ async fn forward_or_pass(state: HttpState, request: Request, next: Next) -> Resp
// primary's applied op, and this node has to record it as this caller's
// read-your-writes floor (see `record_relayed_floor`).
let user_id = match resolve_credential(&state, bearer).await {
- Ok((_key, user_id, _user_created_at, _expiry)) => user_id,
+ Ok(verified) => verified.user_id,
Err(rejection) => return rejection.into_response(),
};
let deadline = Instant::now() + FORWARD_RETRY_DEADLINE;
@@ -966,7 +972,7 @@ async fn attach_partition_request(state: HttpState, request: Request, next: Next
Err(refusal) => return refusal.into_response(),
};
// Verify-only, like the metadata relay: an unauthenticated caller gets no
- // buffered body and no relay. The handler authenticates again.
+ // buffered body and no relay. The handler's extractor reuses the proof.
let bearer = match bearer_token(request.headers()) {
Ok(bearer) => bearer,
Err(error) => return CustomError::from(error).into_response(),
@@ -990,13 +996,17 @@ async fn attach_partition_request(state: HttpState, request: Request, next: Next
return not_admitted();
}
}
- let (jwt_id, user_id, user_created_at, _expiry) = match resolve_credential(&state, bearer).await
- {
- Ok(credential) => credential,
+ let verified = match resolve_credential(&state, bearer).await {
+ Ok(verified) => verified,
Err(rejection) => return rejection.into_response(),
};
let target = if matches!(hop, PartitionHop::Target(_)) {
- let credential = credential_key(bearer, jwt_id, user_id, user_created_at);
+ let credential = credential_key(
+ bearer,
+ verified.jwt_id.as_deref(),
+ verified.user_id,
+ verified.user_created_at,
+ );
match state.forward.relay_targets.admit(credential) {
Ok(guard) => Some(guard),
Err(refusal) => return relay_target_refused(refusal),
@@ -1037,6 +1047,7 @@ async fn attach_partition_request(state: HttpState, request: Request, next: Next
deadline,
};
parts.extensions.insert(Arc::new(partition_request));
+ parts.extensions.insert(verified);
next.run(Request::from_parts(parts, Body::from(body))).await
}
@@ -1605,6 +1616,80 @@ impl Drop for ForwardGuard {
}
}
+/// This node's metadata floor reads, single flight. A read already sent can
+/// miss an op acknowledged after it left, so it serves only the waiters that
+/// came before it was sent. The waiters that come while it is in flight share
+/// the next read, which the first of them sends once it ends.
+#[derive(Default)]
+pub(in crate::http) struct FloorReads {
+ /// Reads sent so far, which also numbers each read.
+ sent: Cell<u64>,
+ in_flight: Cell<bool>,
+ /// The number and outcome of the newest read that ended.
+ ended: Cell<Option<(u64, ControlFlow<Option<u64>>)>>,
+ /// Waiters parked until the read in flight ends.
+ parked: RefCell<Vec<Waker>>,
+}
+
+impl FloorReads {
+ /// The outcome of a read sent after this call. A waiter shares the read in
+ /// flight only when it was sent after the waiter came, and otherwise sends
+ /// the next one with `read` unless another waiter sent it first.
+ async fn read<F>(&self, read: impl FnOnce() -> F) -> ControlFlow<Option<u64>>
+ where
+ F: Future<Output = ControlFlow<Option<u64>>>,
+ {
+ let sent_before = self.sent.get();
+ loop {
+ if let Some((number, outcome)) = self.ended.get()
+ && number > sent_before
+ {
+ return outcome;
+ }
+ if !self.in_flight.get() {
+ break;
+ }
+ self.park().await;
+ }
+ let number = self.sent.get() + 1;
+ self.sent.set(number);
+ self.in_flight.set(true);
+ let sending = SentFloorRead { reads: self };
+ let outcome = read().await;
+ self.ended.set(Some((number, outcome)));
+ drop(sending);
+ outcome
+ }
+
+ async fn park(&self) {
+ let mut parked = false;
+ poll_fn(|context| {
+ if parked {
+ return Poll::Ready(());
+ }
+ parked = true;
+ self.parked.borrow_mut().push(context.waker().clone());
+ Poll::Pending
+ })
+ .await;
+ }
+}
+
+/// The read in flight. Its drop ends it, also when its waiter is dropped
+/// mid-read, and wakes the parked waiters so one of them sends the next read.
+struct SentFloorRead<'a> {
+ reads: &'a FloorReads,
+}
+
+impl Drop for SentFloorRead<'_> {
+ fn drop(&mut self) {
+ self.reads.in_flight.set(false);
+ for waker in self.reads.parked.take() {
+ waker.wake();
+ }
+ }
+}
+
/// Exact-DER pin against this node's own end-entity certificate. Presented-leaf
/// equality replaces chain building and name checks on purpose (config-listed
/// IPs rarely appear as SANs in operator certs); handshake signatures are still
@@ -1663,7 +1748,6 @@ mod tests {
use std::io::{Read, Write};
use std::pin::pin;
- use std::task::Poll;
use configs::cluster::{ClusterNodeConfig, ResolvedClusterNode, TransportPorts};
use futures::poll;
@@ -2170,23 +2254,7 @@ mod tests {
#[compio::test]
async fn given_a_rejected_proof_when_awaiting_the_metadata_floor_should_report_relay_refused() {
let (address, server) = answer_with(vec![refused_proof_reply(UnprovenRelay::Invalid)]);
- let mut state = read_state_on(None, Some(KEY_MATERIAL));
- let roster = Rc::get_mut(&mut Rc::get_mut(&mut state).unwrap().roster).unwrap();
- roster.enabled = true;
- roster.nodes.push(
- ResolvedClusterNode::try_from(ClusterNodeConfig {
- name: "primary".to_owned(),
- ip: address.ip().to_string(),
- advertised_address: None,
- advertised_addresses: Vec::new(),
- replica_id: 0,
- ports: TransportPorts {
- http: Some(address.port()),
- ..TransportPorts::default()
- },
- })
- .unwrap(),
- );
+ let state = state_with_metadata_primary_at(address);
let refusal = crate::http::reads::await_metadata_floor(
&state,
Instant::now() + PARTITION_REQUEST_DEADLINE,
@@ -2268,6 +2336,35 @@ mod tests {
}
}
+ /// A state whose metadata primary, replica 0, serves HTTP at `address`.
+ fn state_with_metadata_primary_at(address: SocketAddr) -> HttpState {
+ let mut state = read_state_on(None, Some(KEY_MATERIAL));
+ let roster = Rc::get_mut(&mut Rc::get_mut(&mut state).expect("one state owner").roster)
+ .expect("one roster owner");
+ roster.enabled = true;
+ roster.nodes.push(
+ ResolvedClusterNode::try_from(ClusterNodeConfig {
+ name: "primary".to_owned(),
+ ip: address.ip().to_string(),
+ advertised_address: None,
+ advertised_addresses: Vec::new(),
+ replica_id: 0,
+ ports: TransportPorts {
+ http: Some(address.port()),
+ ..TransportPorts::default()
+ },
+ })
+ .expect("a valid roster node"),
+ );
+ state
+ }
+
+ fn floor_reply(floor: u64) -> String {
+ format!(
+ "HTTP/1.1 204 No Content\r\n{APPLIED_OP_HEADER}: {floor}\r\nconnection: close\r\n\r\n"
+ )
+ }
+
fn refused_proof_reply(unproven: UnprovenRelay) -> String {
format!(
"HTTP/1.1 403 Forbidden\r\n{RELAY_REFUSAL_HEADER}: {}\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
@@ -2458,52 +2555,60 @@ mod tests {
assert!(ForwardGuard::admit(&in_flight).is_some());
}
+ /// A waiter whose deadline drops the read it sent must not strand the
+ /// waiters behind it: the next one sends its own read.
#[compio::test]
- async fn given_floor_read_burst_when_primary_is_unavailable_should_share_forward_capacity() {
- let state = read_state_on(None, Some(KEY_MATERIAL));
- let deadline = Instant::now() + PARTITION_REQUEST_DEADLINE;
- let mut waiting = Vec::new();
- for _ in 0..MAX_IN_FLIGHT_FORWARDS {
- let mut read = Box::pin(metadata_floor(&state, deadline));
- assert!(poll!(read.as_mut()).is_pending());
- waiting.push(read);
- }
- assert_eq!(state.forward.in_flight.get(), MAX_IN_FLIGHT_FORWARDS);
- assert!(ForwardGuard::admit(&state.forward.in_flight).is_none());
- let refusal = forward_busy();
- assert_eq!(refusal.status(), StatusCode::SERVICE_UNAVAILABLE);
- assert!(refusal.headers().contains_key(RETRY_AFTER));
- drop(waiting);
- assert_eq!(state.forward.in_flight.get(), 0);
- assert!(ForwardGuard::admit(&state.forward.in_flight).is_some());
+ async fn given_a_dropped_floor_read_when_waiters_remain_should_send_the_next_read() {
+ const FLOOR: u64 = 7;
+ let reads = FloorReads::default();
+ let mut dropped = Box::pin(reads.read(std::future::pending));
+ assert!(poll!(dropped.as_mut()).is_pending());
+ let mut waiting = Box::pin(reads.read(|| async { ControlFlow::Break(Some(FLOOR)) }));
+ assert!(poll!(waiting.as_mut()).is_pending());
+ drop(dropped);
+ assert_eq!(waiting.await, ControlFlow::Break(Some(FLOOR)));
}
+ /// Floor reads have their own budget, so relays that hold every forward
+ /// slot do not refuse a linearizable read its floor.
#[compio::test]
- async fn given_a_full_forward_budget_when_reading_the_metadata_floor_should_send_nothing() {
+ async fn given_a_full_forward_budget_when_reading_the_metadata_floor_should_still_wait_for_it()
+ {
let state = read_state_on(None, Some(KEY_MATERIAL));
- let in_flight = Rc::clone(&state.forward.in_flight);
- let deadline = Instant::now() + PARTITION_REQUEST_DEADLINE;
- in_flight.set(MAX_IN_FLIGHT_FORWARDS);
- let mut refused = pin!(metadata_floor(&state, deadline));
- let Poll::Ready(Err(refusal @ ReadError::ForwardBusy)) = poll!(refused.as_mut()) else {
- panic!("a full budget must retain its forwarding refusal");
- };
- let response = refusal.into_response();
- assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
- let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
- let body: serde_json::Value = serde_json::from_slice(&body).unwrap();
- assert_eq!(body["code"], "forward_busy");
+ state.forward.in_flight.set(MAX_IN_FLIGHT_FORWARDS);
+ let mut waiting = Box::pin(metadata_floor(
+ &state,
+ Instant::now() + PARTITION_REQUEST_DEADLINE,
+ ));
+ assert!(
+ poll!(waiting.as_mut()).is_pending(),
+ "a floor read must not take a forward slot"
+ );
+ assert_eq!(state.forward.in_flight.get(), MAX_IN_FLIGHT_FORWARDS);
+ }
- in_flight.set(MAX_IN_FLIGHT_FORWARDS - 1);
- let mut waiting = Box::pin(metadata_floor(&state, deadline));
- assert!(poll!(waiting.as_mut()).is_pending());
+ /// A floor read already sent may miss ops acknowledged after it left, so
+ /// it serves only the waiter that sent it. The two waiters that arrive
+ /// while it is in flight share the next read, and the metadata primary
+ /// answers two reads for three waiters.
+ #[compio::test]
+ async fn given_concurrent_floor_waiters_when_a_read_is_in_flight_should_share_the_next_read() {
+ const FIRST_FLOOR: u64 = 41;
+ const NEXT_FLOOR: u64 = 42;
+ let (address, server) =
+ answer_with(vec![floor_reply(FIRST_FLOOR), floor_reply(NEXT_FLOOR)]);
+ let state = state_with_metadata_primary_at(address);
+ let deadline = Instant::now() + Duration::from_secs(2);
+ let floors = futures::join!(
+ metadata_floor(&state, deadline),
+ metadata_floor(&state, deadline),
+ metadata_floor(&state, deadline),
+ );
+ server.join().expect("HTTP server finished");
assert_eq!(
- in_flight.get(),
- MAX_IN_FLIGHT_FORWARDS,
- "a floor wait holds a forward slot"
+ [floors.0.ok(), floors.1.ok(), floors.2.ok()],
+ [Some(FIRST_FLOOR), Some(NEXT_FLOOR), Some(NEXT_FLOOR)]
);
- drop(waiting);
- assert_eq!(in_flight.get(), MAX_IN_FLIGHT_FORWARDS - 1);
}
#[compio::test]
diff --git a/core/server/src/http/state.rs b/core/server/src/http/state.rs
index 3487d5ee6..0ba18879b 100644
--- a/core/server/src/http/state.rs
+++ b/core/server/src/http/state.rs
@@ -46,6 +46,7 @@ use crate::consumer_group::lease::{ConsumerGroupLiveness, SessionActivity};
use crate::dispatch::session_ops::{submit_logout_on_owner, submit_register_on_owner};
use crate::http::admission::RelayTargets;
use crate::http::error::{AuthError, ReadError, primary_redirect_location};
+use crate::http::forward::FloorReads;
use crate::http::relay_proof::RelayKey;
use crate::http::jwt::{JwtManager, TrustedIssuer};
@@ -195,6 +196,7 @@ pub(in crate::http) struct ForwardState {
pub(in crate::http) body_limit: usize,
pub(in crate::http) in_flight: Rc<Cell<u32>>,
pub(in crate::http) relay_targets: RelayTargets,
+ pub(in crate::http) floor_reads: FloorReads,
}
impl ForwardState {
diff --git a/core/server/src/partition_helpers.rs b/core/server/src/partition_helpers.rs
index 7638807fe..46d2088ea 100644
--- a/core/server/src/partition_helpers.rs
+++ b/core/server/src/partition_helpers.rs
@@ -684,7 +684,7 @@ async fn load_partition(
&directory,
namespace.inner(),
partition_metadata.created_revision,
- DiskStorage,
+ partitions::WalStorage::default(),
config.partition.wal_bytes_max.as_bytes_u64(),
runtime_options
.preallocate_segments
diff --git a/core/shard/src/builder.rs b/core/shard/src/builder.rs
index 5a7ccdcc3..ea92fe66a 100644
--- a/core/shard/src/builder.rs
+++ b/core/shard/src/builder.rs
@@ -156,18 +156,15 @@ where
count: self.senders.len(),
})?;
- // Install the cross-shard forward closures. The bus' slow path
- // (`send_to_replica` / `send_to_client`) calls these when the
- // destination connection lives on a different shard; the closure
- // wraps the message into a `LifecycleFrame::Forward{Replica,
- // Client}Send` and `try_send`s it to the owning shard's inbox.
- // The receiving shard's router unwraps the frame and re-enters
- // the bus' fast path locally. The `senders` Vec is canonically
- // ordered (`senders[i].shard_id() == i`).
+ // Install the cross-shard replica forward closure. The bus' slow path
+ // (`send_to_replica`) calls it when the replica connection lives on a
+ // different shard. The closure wraps the message into a
+ // `LifecycleFrame::ForwardReplicaSend` and `try_send`s it to the
+ // owning shard's inbox. The receiving shard's router unwraps the frame
+ // and re-enters the bus' fast path locally. The `senders` Vec is
+ // canonically ordered (`senders[i].shard_id() == i`).
let senders_for_replica: Rc<Vec<TaggedSender>> = Rc::new(self.senders.clone());
- let senders_for_client = Rc::clone(&senders_for_replica);
let metrics_for_replica = self.metrics.clone();
- let metrics_for_client = self.metrics.clone();
self.bus
.set_replica_forward_fn(Box::new(move |replica_id, owning_shard, msg| {
senders_for_replica.get(usize::from(owning_shard)).map_or(
@@ -187,28 +184,6 @@ where
},
)
}));
- self.bus
- .set_client_forward_fn(Box::new(move |client_id, owning_shard, msg| {
- senders_for_client.get(usize::from(owning_shard)).map_or(
- Err(SendError::ClientForwardFailed(client_id)),
- |sender| {
- let frame = ShardFrame::lifecycle(LifecycleFrame::ForwardClientSend {
- client_id,
- msg,
- });
- // Reply lane: a forwarded client reply is terminal on
- // drop, so it must not compete with consensus frames
- // for main-lane capacity.
- sender.reply_sender().try_send(frame).map_err(|err| {
- metrics_for_client.record_frame_drop(
- frame_drop_variant::FORWARD_CLIENT_SEND,
- classify_try_send_err(&err),
- );
- SendError::ClientForwardFailed(client_id)
- })
- },
- )
- }));
// On shard 0: build the coordinator BEFORE the shard, so the
// coordinator can be passed to `IggyShard::new` as a ctor arg.
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index a4a830e10..68c47fb72 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -886,11 +886,11 @@ pub enum LifecycleFrame {
replica_id: u8,
msg: Frozen<MESSAGE_ALIGN>,
},
- /// A reply by client id (`MessageBus::send_to_client`) for the receiving
- /// shard's pump to send: one for a client homed there (top 16 bits of
- /// `client_id`), or a deny a synchronous caller staged. By id it reaches
- /// only an in-process (HTTP) reply slot: a client socket takes a reply
- /// only from the shard that holds it, addressed to its installation.
+ /// A deny a synchronous caller staged for its own shard's pump to send by
+ /// client id (`MessageBus::send_to_client`). The production bus refuses
+ /// every frame sent by id, because a client socket or HTTP reply slot
+ /// takes a reply only addressed to its installation. Only a bus that
+ /// delivers by id, such as the raw simulator, answers with it.
ForwardClientSend {
client_id: u128,
msg: BusMessage,
@@ -2912,6 +2912,9 @@ where
// the floor with the partition value.
self.metrics
.record_partition_prepare_gap_drops(partition.take_prepare_gap_drops());
+ self.metrics.record_partition_applied_log_contradictions(
+ partition.consensus().take_applied_log_contradictions(),
+ );
// Roll whatever this partition still counts out of its
// parent topic and stream. Here and not in the
// reconciler's teardown: a handler suspended mid-append
@@ -4242,8 +4245,8 @@ where
/// Deny a client partition request with `TransientNotAccepted`: the frame
/// never reached journal admission, so the SDK can safely replay it
/// anywhere, and partition rebuild completes well inside the replay
- /// budget. Sent directly over the bus; delivery failure is terminal for
- /// this reply (the client recovers via its own read-timeout).
+ /// budget. Sent by client id, which only a raw request without a reply
+ /// channel reaches, and which the production bus refuses.
#[allow(clippy::future_not_send)]
async fn deny_partition_request_transient(&self, request_header: &RoutedRequestHeader) {
let reply = build_deny_reply_from_request_header(
@@ -4263,12 +4266,12 @@ where
crate::metrics::frame_drop_variant::PARTITION,
crate::metrics::frame_drop_reason::DELIVERY_FAILED,
);
- tracing::warn!(
+ tracing::debug!(
shard = self.id,
client = request_header.client,
operation = ?request_header.operation,
error = %error,
- "failed to send transient deny for partition request"
+ "transient deny for partition request has no route by client id"
);
return;
}
@@ -4305,12 +4308,12 @@ where
crate::metrics::frame_drop_variant::CONSENSUS,
crate::metrics::frame_drop_reason::DELIVERY_FAILED,
);
- tracing::warn!(
+ tracing::debug!(
shard = self.id,
client = request_header.client,
operation = ?request_header.operation,
error = %error,
- "failed to send deny for unroutable request"
+ "deny for unroutable request has no route by client id"
);
}
}
@@ -4643,6 +4646,10 @@ where
/// reports `commit_offset` 0, which reads as a regression rather than a
/// harness that discarded the log.
///
+ /// `persistence` is this group's WAL, reopened by the caller from its
+ /// filesystem model. Its certificate restores `log_view` as boot's does, and
+ /// every acknowledgement and view-scoped send waits for it.
+ ///
/// `materialisation` carries the committed `created_revision` and this
/// partition's assigned initial view; see `fresh_group_start`.
///
@@ -4655,8 +4662,8 @@ where
// while `partitions` builds as a plain dependency, so `RetainedPartitionLog`
// and `adopt_retained_log` are configured out and the crate does not compile.
// The feature forwards to `partitions/simulator` instead.
- /// Persisted policies use ideal storage here: journal and receipts survive
- /// together. Filesystem barriers and torn writes belong to storage fault tests.
+ /// The WAL runs on the simulator's filesystem model. The retained log and
+ /// receipts stand in for segment and receipt files, which it does not model.
#[cfg(feature = "simulator")]
pub fn init_partition(
&self,
@@ -4664,6 +4671,7 @@ where
superblock: Option<Rc<SB>>,
recovered_state: Option<&consensus::VsrState>,
retained: Option<partitions::RetainedPartitionState>,
+ persistence: Rc<partitions::PartitionPersistence>,
materialisation: PartitionMaterialisation,
) where
B: MessageBus + Clone + 'static,
@@ -4725,7 +4733,6 @@ where
consensus,
partitions.config().segment_size,
);
- partition.enable_ideal_storage();
partition.set_created_revision(epoch);
let runtime_options = self.plane.metadata().mux_stm.streams().read(|inner| {
inner
@@ -4764,6 +4771,7 @@ where
// the restore at all, a simulator replica rebuilt against a retained
// store resumes minting at 0 while its group is at N.
partition.restore_offset_frontier(recovered_state);
+ partition.attach_simulated_persistence(persistence);
partition.consensus().resume_adopted_view();
// Match production's solo reservation handling. A replicated WAL may
// retain a suffix the next view discards; anchoring at its dirty offset
@@ -8243,6 +8251,11 @@ where
if gap_drops > 0 {
self.metrics.record_partition_prepare_gap_drops(gap_drops);
}
+ let contradictions = partition.consensus().take_applied_log_contradictions();
+ if contradictions > 0 {
+ self.metrics
+ .record_partition_applied_log_contradictions(contradictions);
+ }
// A fenced partition must not tick: its consensus would emit
// view-scoped sends for a log the cluster has already passed.
if let Some(fault) = partition.fatal() {
@@ -8762,9 +8775,9 @@ where
}
/// Re-read a bounded range on retry so a full inbox retains no stale payload.
- /// Each interval visits every namespace once, in batches over consecutive ticks,
- /// up to `PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX` namespaces per tick of the
- /// interval. A shard with more namespaces reports each one every few intervals.
+ /// Each interval visits every namespace once, in batches over consecutive ticks.
+ /// A batch holds `PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN` namespaces, or more
+ /// when the shard owns too many for that pace to finish within the interval.
fn publish_partition_views(&self, namespaces: &[IggyNamespace]) {
if namespaces.is_empty() {
self.partition_placement_cursor.set(None);
@@ -8785,9 +8798,13 @@ where
let start = self.partition_placement_cursor.get().map_or(0, |cursor| {
namespaces.partition_point(|namespace| *namespace < cursor) % namespaces.len()
});
- let visited = owed
- .min(namespaces.len())
- .min(PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX);
+ let batch = namespaces
+ .len()
+ .div_ceil(
+ usize::try_from(PARTITION_PLACEMENT_REPORT_INTERVAL_TICKS).unwrap_or(usize::MAX),
+ )
+ .max(PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN);
+ let visited = owed.min(namespaces.len()).min(batch);
let partitions = self.plane.partitions();
let mut reports = Vec::with_capacity(visited);
for namespace in namespaces[start..]
@@ -10921,6 +10938,11 @@ where
// Placed before the transfer-stall block below: that block's exhausted
// branch returns early, so a detector after it would be skipped on the
// tick that abandons a transfer.
+ let contradictions = consensus.take_applied_log_contradictions();
+ if contradictions > 0 {
+ self.metrics
+ .record_metadata_applied_log_contradictions(contradictions);
+ }
if let Some(journal) = metadata.journal.as_ref() {
let gap_drops = metadata.take_prepare_gap_drops();
if gap_drops > 0 {
@@ -11538,9 +11560,9 @@ const PARTITION_WALKS_PER_TICK_MAX: usize = 16;
const PARTITION_PLACEMENT_REPORT_INTERVAL_TICKS: u128 =
primary_rebalance::PROBE_INTERVAL.as_millis() / CONSENSUS_TICK_INTERVAL.as_millis();
-/// One lifecycle batch per tick until the interval's pass ends; the cursor also
-/// bounds visits to ineligible groups.
-const PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX: usize = 64;
+/// The smallest lifecycle batch sent per tick until the interval's pass ends.
+/// The cursor also bounds visits to ineligible groups.
+const PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN: usize = 64;
/// Floor under the gap detector's debounce, in ticks.
///
@@ -15474,12 +15496,12 @@ mod partition_placement_tests {
async fn given_more_groups_than_one_batch_when_an_interval_starts_should_report_each_group_within_it()
{
let (shard, inbox) = owner(1);
- let count = PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX * 2 + 3;
+ let count = PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN * 2 + 3;
for id in 0..count {
insert(&shard, id, 7, 0);
}
let batches =
- u64::try_from(count.div_ceil(PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX)).unwrap();
+ u64::try_from(count.div_ceil(PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN)).unwrap();
let mut observed = BTreeSet::new();
for tick in 1..=batches {
shard.ticks.set(tick);
@@ -15499,6 +15521,44 @@ mod partition_placement_tests {
);
}
+ #[compio::test]
+ async fn given_more_groups_than_an_interval_of_batches_when_an_interval_runs_should_report_each_group_within_it()
+ {
+ let (shard, inbox) = owner(1);
+ let interval = u64::try_from(PARTITION_PLACEMENT_REPORT_INTERVAL_TICKS).unwrap();
+ let count = PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN * usize::try_from(interval).unwrap()
+ + PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN
+ + 3;
+ for id in 0..count {
+ insert(&shard, id, 7, 0);
+ }
+ let mut observed = BTreeSet::new();
+ for tick in 1..=interval {
+ shard.ticks.set(tick);
+ publish(&shard);
+ if !inbox.is_empty() {
+ observed.extend(
+ receive(&inbox)
+ .into_iter()
+ .map(|report| report.namespace.partition_id()),
+ );
+ }
+ }
+ assert_eq!(
+ observed.len(),
+ count,
+ "every group reports within one interval"
+ );
+
+ shard.ticks.set(interval + 1);
+ publish(&shard);
+ assert_eq!(
+ receive(&inbox)[0].namespace,
+ IggyNamespace::new(0, 0, 0),
+ "the next interval starts a new pass at the front"
+ );
+ }
+
#[compio::test]
async fn given_owned_groups_when_ticking_should_report_only_normal_serving_views_to_shard_zero()
{
@@ -15545,15 +15605,15 @@ mod partition_placement_tests {
async fn given_more_groups_than_one_batch_when_publishing_should_bound_batches_and_cover_the_tail()
{
let (shard, inbox) = owner(1);
- let count = PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX * 2 + 3;
+ let count = PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN * 2 + 3;
for id in 0..count {
insert(&shard, id, 7, 0);
}
let mut observed = BTreeSet::new();
- for _ in 0..count.div_ceil(PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX) {
+ for _ in 0..count.div_ceil(PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN) {
publish(&shard);
let reports = receive(&inbox);
- assert_eq!(reports.len(), PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX);
+ assert_eq!(reports.len(), PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN);
observed.extend(
reports
.into_iter()
@@ -15615,9 +15675,9 @@ mod partition_placement_tests {
async fn given_an_ineligible_prefix_when_publishing_should_bound_visits_and_reach_live_groups()
{
let (shard, inbox) = owner(1);
- for id in 0..=PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX {
+ for id in 0..=PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN {
let namespace = insert(&shard, id, 7, 0);
- if id < PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX {
+ if id < PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN {
shard
.plane
.partitions()
@@ -15638,7 +15698,7 @@ mod partition_placement_tests {
assert_eq!(reports.len(), 1);
assert_eq!(
reports[0].namespace.partition_id(),
- PARTITION_PLACEMENT_REPORTS_PER_TICK_MAX
+ PARTITION_PLACEMENT_REPORTS_PER_TICK_MIN
);
}
diff --git a/core/shard/src/metrics.rs b/core/shard/src/metrics.rs
index 06e4d215f..edd31b022 100644
--- a/core/shard/src/metrics.rs
+++ b/core/shard/src/metrics.rs
@@ -110,15 +110,11 @@ impl From<Command> for FrameCommandLabel {
/// Variant labels used in `frame_drops_total`. Exposed as constants to
/// catch typos at compile time and to keep the cardinality bounded.
///
-/// `FORWARD_CLIENT_SEND` ticks when the cross-shard client-reply forward
-/// closure fails. Unlike `CONSENSUS` drops (which VSR retransmit
-/// recovers), a `FORWARD_CLIENT_SEND` drop is terminal: the client never
-/// receives the reply and request / response semantics break above the
-/// bus. Operators should alert on this pair in the scrape (backed by the
-/// drop-site `tracing` logs) and size `inbox_capacity` for the worst-case
-/// cross-shard reply burst.
-/// `FORWARD_REPLICA_SEND` is the symmetric variant for replica forwards;
-/// VSR retransmit covers its loss so it stays informational.
+/// `FORWARD_CLIENT_SEND` ticks when the pump cannot send a deny a synchronous
+/// caller staged by client id. The production bus refuses every frame sent by
+/// id, so only a raw request without a reply channel ends up here.
+/// `FORWARD_REPLICA_SEND` ticks when a replica forward fails. VSR retransmit
+/// covers its loss, so it stays informational.
///
/// `PARTITION` covers the partition plane: a frame shed because the namespace
/// had not materialised and its park buffer was at capacity
@@ -293,6 +289,8 @@ pub struct ShardMetrics {
partition_requests_denied_transient_total: Counter,
partition_prepare_gap_drops_total: Counter,
metadata_prepare_gap_drops_total: Counter,
+ partition_applied_log_contradictions_total: Counter,
+ metadata_applied_log_contradictions_total: Counter,
metadata_read_frontier_refusals_total: Counter,
client_requests_denied_queue_full_total: Counter,
replica_socket_reads_total: Counter,
@@ -370,6 +368,8 @@ impl ShardMetrics {
partition_requests_denied_transient_total: Counter::default(),
partition_prepare_gap_drops_total: Counter::default(),
metadata_prepare_gap_drops_total: Counter::default(),
+ partition_applied_log_contradictions_total: Counter::default(),
+ metadata_applied_log_contradictions_total: Counter::default(),
metadata_read_frontier_refusals_total: Counter::default(),
client_requests_denied_queue_full_total: Counter::default(),
replica_socket_reads_total: Counter::default(),
@@ -838,6 +838,40 @@ impl ShardMetrics {
self.metadata_prepare_gap_drops_total.get()
}
+ /// Add the views whose logs a partition refused for contradicting an op
+ /// it applied, drained from `VsrConsensus::take_applied_log_contradictions`.
+ ///
+ /// A refusing replica keeps refusing every later log of that quorum, so it
+ /// drops out of its group. Any non-zero value is a safety violation that
+ /// needs an operator. Shard-scoped for the reason given at
+ /// [`Self::record_partition_prepare_gap_drops`].
+ pub fn record_partition_applied_log_contradictions(&self, views: u64) {
+ self.partition_applied_log_contradictions_total
+ .inc_by(views);
+ }
+
+ /// Snapshot of `partition_applied_log_contradictions_total`. Test/simulator
+ /// accessor.
+ #[cfg(any(test, feature = "simulator"))]
+ #[must_use]
+ pub fn partition_applied_log_contradictions_value(&self) -> u64 {
+ self.partition_applied_log_contradictions_total.get()
+ }
+
+ /// The metadata sibling of
+ /// [`Self::record_partition_applied_log_contradictions`].
+ pub fn record_metadata_applied_log_contradictions(&self, views: u64) {
+ self.metadata_applied_log_contradictions_total.inc_by(views);
+ }
+
+ /// Snapshot of `metadata_applied_log_contradictions_total`. Test/simulator
+ /// accessor.
+ #[cfg(any(test, feature = "simulator"))]
+ #[must_use]
+ pub fn metadata_applied_log_contradictions_value(&self) -> u64 {
+ self.metadata_applied_log_contradictions_total.get()
+ }
+
/// Snapshot of `partition_frames_rejected_stale_total`. Test/simulator
/// accessor, readable from any crate under those cfgs so the crates that
/// drive the reconciler can assert a reject did not happen.
@@ -998,6 +1032,18 @@ impl ShardMetrics {
"replicated metadata prepares dropped out of order by a backup's gap check",
self.metadata_prepare_gap_drops_total.clone(),
);
+ registry.register(
+ "partition_applied_log_contradictions",
+ "views whose partition log this replica refused for naming another prepare at an op \
+ it applied; non-zero is a safety violation",
+ self.partition_applied_log_contradictions_total.clone(),
+ );
+ registry.register(
+ "metadata_applied_log_contradictions",
+ "views whose metadata log this replica refused for naming another prepare at an op \
+ it applied; non-zero is a safety violation",
+ self.metadata_applied_log_contradictions_total.clone(),
+ );
registry.register(
"metadata_read_frontier_refusals",
"metadata reads refused because this node never applied the caller's committed op",
diff --git a/core/shard/src/partition_io.rs b/core/shard/src/partition_io.rs
index e8a184df4..30fc68d7e 100644
--- a/core/shard/src/partition_io.rs
+++ b/core/shard/src/partition_io.rs
@@ -5570,15 +5570,20 @@ mod tests {
let namespace = IggyNamespace::new(0, 0, 0);
let prepares = primary_prepares(&Rc::new(IggyMessageBus::new(0)), namespace, HEAD).await;
let local = usize::try_from(LOCAL_HEAD).unwrap();
- let backup = HeadlessReplica::journaled(
+ let backup = Box::pin(HeadlessReplica::journaled(
HEADLESS_BACKUP,
namespace,
&prepares[..local],
Some((directory.path(), &store)),
- )
+ ))
+ .await;
+ let candidate = Box::pin(HeadlessReplica::journaled(
+ HEADLESS_CANDIDATE,
+ namespace,
+ &prepares,
+ None,
+ ))
.await;
- let candidate =
- HeadlessReplica::journaled(HEADLESS_CANDIDATE, namespace, &prepares, None).await;
let start_view = start_view_suffix(
namespace,
HEADLESS_VIEW,
@@ -5638,20 +5643,24 @@ mod tests {
let prepares =
primary_prepares(&Rc::new(IggyMessageBus::new(0)), namespace, HELD_HEAD).await;
let adopted = usize::try_from(ADOPTED_HEAD).unwrap();
- let backup = HeadlessReplica::journaled(
+ let backup = Box::pin(HeadlessReplica::journaled(
HEADLESS_BACKUP,
namespace,
&prepares,
Some((directory.path(), &store)),
- )
+ ))
.await;
assert_eq!(
backup.partition(namespace).log.journal().inner.last_op(),
Some(HELD_HEAD)
);
- let candidate =
- HeadlessReplica::journaled(HEADLESS_CANDIDATE, namespace, &prepares[..adopted], None)
- .await;
+ let candidate = Box::pin(HeadlessReplica::journaled(
+ HEADLESS_CANDIDATE,
+ namespace,
+ &prepares[..adopted],
+ None,
+ ))
+ .await;
if head_committed {
for replica in [&backup, &candidate] {
replica
@@ -5725,14 +5734,19 @@ mod tests {
header.request += 1;
header.checksum = header.identity_checksum();
});
- let backup =
- HeadlessReplica::journaled(HEADLESS_BACKUP, namespace, &prepares[..prefix], None).await;
- let elect = HeadlessReplica::journaled(
+ let backup = Box::pin(HeadlessReplica::journaled(
+ HEADLESS_BACKUP,
+ namespace,
+ &prepares[..prefix],
+ None,
+ ))
+ .await;
+ let elect = Box::pin(HeadlessReplica::journaled(
HEADLESS_CANDIDATE,
namespace,
&prepares,
Some((directory, store)),
- )
+ ))
.await;
backup
.owner
@@ -6018,14 +6032,19 @@ mod tests {
header.view = HEADLESS_VIEW;
header.checksum = header.identity_checksum();
});
- let backup =
- HeadlessReplica::journaled(HEADLESS_BACKUP, namespace, &prepares[..1], None).await;
- let elect = HeadlessReplica::journaled(
+ let backup = Box::pin(HeadlessReplica::journaled(
+ HEADLESS_BACKUP,
+ namespace,
+ &prepares[..1],
+ None,
+ ))
+ .await;
+ let elect = Box::pin(HeadlessReplica::journaled(
HEADLESS_CANDIDATE,
namespace,
&prepares[..adopted],
Some((directory.path(), &store)),
- )
+ ))
.await;
let backup_consensus = || backup.partition(namespace).consensus();
let elect_partition = || elect.partition(namespace);
@@ -6326,6 +6345,102 @@ mod tests {
assert_eq!(partition.consensus().applied_log_contradictions(), 1);
}
+ #[compio::test]
+ async fn given_a_refused_contradicting_start_view_when_ticking_should_export_it_once() {
+ const APPLIED: u64 = 2;
+ const NEW_VIEW: u32 = 3;
+ let bus = Rc::new(IggyMessageBus::new(0));
+ let (owner, _sender) = test_owner(&bus, None);
+ let namespace = IggyNamespace::new(0, 0, 0);
+ let prepares = primary_prepares(&bus, namespace, APPLIED).await;
+ host_applied_backup(&owner, &bus, namespace, &prepares).await;
+ let mut forged = *prepares.last().unwrap().header();
+ forged.timestamp += 1;
+ forged.checksum = forged.identity_checksum();
+ owner
+ .on_start_view(start_view_suffix(
+ namespace,
+ NEW_VIEW,
+ APPLIED,
+ APPLIED,
+ &[forged],
+ ))
+ .await;
+
+ for _ in 0..2 {
+ assert!(owner.tick_partitions(&mut Vec::new()).await.is_none());
+ }
+
+ assert_eq!(
+ owner.metrics.partition_applied_log_contradictions_value(),
+ 1
+ );
+ let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap();
+ assert_eq!(
+ partition.consensus().applied_log_contradictions(),
+ 1,
+ "the export must leave the total for the simulator invariants"
+ );
+ }
+
+ #[compio::test]
+ async fn given_a_refused_contradicting_metadata_log_when_ticking_should_export_it_once() {
+ let bus = Rc::new(IggyMessageBus::new(0));
+ let (mut owner, _sender) = test_owner(&bus, None);
+ let consensus = VsrConsensus::new(
+ 1,
+ 0,
+ 1,
+ METADATA_GROUP,
+ Rc::clone(&bus),
+ LocalPipeline::new(),
+ );
+ consensus.init();
+ owner.plane.inner_mut().0 = IggyMetadata::new(
+ Some(consensus),
+ None,
+ None,
+ None,
+ IoTestMetadata::new((
+ UsersInner::new().into(),
+ (StreamsInner::default().into(), ()),
+ )),
+ None,
+ );
+ let announced = PrepareHeader {
+ checksum: 1,
+ ..PrepareHeader::default()
+ };
+ let consensus = owner.plane.inner().0.consensus.as_ref().unwrap();
+ assert!(consensus.contradicts_applied_log(
+ consensus::PlaneKind::Metadata,
+ consensus::ContradictionSource::StartView,
+ 1,
+ [&announced],
+ |_| Some(PrepareHeader {
+ checksum: 2,
+ ..announced
+ }),
+ ));
+
+ for _ in 0..2 {
+ owner.tick_metadata().await;
+ }
+
+ assert_eq!(owner.metrics.metadata_applied_log_contradictions_value(), 1);
+ assert_eq!(
+ owner
+ .plane
+ .inner()
+ .0
+ .consensus
+ .as_ref()
+ .unwrap()
+ .applied_log_contradictions(),
+ 1
+ );
+ }
+
#[compio::test]
async fn given_a_start_view_that_matches_the_applied_ops_when_adopting_should_adopt_it() {
const APPLIED: u64 = 2;
diff --git a/core/shard/src/poll/timeout_tests.rs b/core/shard/src/poll/timeout_tests.rs
index 59edbf011..addfbf6ed 100644
--- a/core/shard/src/poll/timeout_tests.rs
+++ b/core/shard/src/poll/timeout_tests.rs
@@ -24,8 +24,7 @@ use consensus::PartitionsHandle;
use futures::channel::oneshot;
use iggy_common::{IggyError, PollingStrategy};
use message_bus::{
- BusMessage, ClientForwardFn, ConnectionLostFn, JoinHandle, MessageBus, ReplicaForwardFn,
- SendError,
+ BusMessage, ConnectionLostFn, JoinHandle, MessageBus, ReplicaForwardFn, SendError,
};
use metadata::IggyMetadata;
use partitions::{IggyPartitions, PollFragments, PollingArgs, PollingConsumer};
@@ -547,6 +546,5 @@ impl MessageBus for PollTestBus {
fn set_connection_lost_fn(&self, _f: ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: ClientForwardFn) {}
fn track_background(&self, _handle: JoinHandle<()>) {}
}
diff --git a/core/shard/src/primary_rebalance/tests.rs b/core/shard/src/primary_rebalance/tests.rs
index 8bbd782f1..f30157277 100644
--- a/core/shard/src/primary_rebalance/tests.rs
+++ b/core/shard/src/primary_rebalance/tests.rs
@@ -95,7 +95,6 @@ impl MessageBus for RecordingBus {
fn set_connection_lost_fn(&self, _callback: message_bus::ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _callback: message_bus::ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _callback: message_bus::ClientForwardFn) {}
}
type TestShard<SB = PingPongSuperblock> = IggyShard<
diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs
index 8a012fcec..9be4d7e95 100644
--- a/core/shard/src/router.rs
+++ b/core/shard/src/router.rs
@@ -943,20 +943,17 @@ where
}
LifecycleFrame::ForwardClientSend { client_id, msg } => {
if let Err(e) = self.bus.send_to_client(client_id, msg).await {
- // Terminal per `SendError` docs: the client never
- // receives the reply and request / response semantics
- // break above the bus. Bump the receiver-side counter
- // (sender side already bumps in `builder.rs`) and raise
- // to `warn!` so it surfaces in default operator logs.
+ // The production bus refuses every frame sent by id, so
+ // this is the expected outcome there, not a fault.
self.metrics.record_frame_drop(
frame_drop_variant::FORWARD_CLIENT_SEND,
frame_drop_reason::DELIVERY_FAILED,
);
- tracing::warn!(
+ tracing::debug!(
shard = self.id,
client_id,
error = ?e,
- "forward-client-send delivery failed"
+ "staged client deny has no route by client id"
);
}
}
diff --git a/core/simulator/src/bus.rs b/core/simulator/src/bus.rs
index c31ab7383..15ffd1d82 100644
--- a/core/simulator/src/bus.rs
+++ b/core/simulator/src/bus.rs
@@ -254,7 +254,6 @@ impl MessageBus for SimOutbox {
fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {}
fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {}
- fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {}
fn sleep(&self, duration: Duration) -> impl Future<Output = ()> {
self.timer.sleep(duration)
@@ -301,6 +300,7 @@ impl MessageBus for SharedSimOutbox {
if self.0.shell {
return Err(SendError::ClientNotFound(client_id));
}
+ // Raw mode dispatches client packets with no installation to reply to.
self.0.send_to_client(client_id, data).await
}
@@ -320,10 +320,6 @@ impl MessageBus for SharedSimOutbox {
self.0.set_replica_forward_fn(f);
}
- fn set_client_forward_fn(&self, f: message_bus::ClientForwardFn) {
- self.0.set_client_forward_fn(f);
- }
-
fn sleep(&self, duration: Duration) -> impl Future<Output = ()> {
self.0.sleep(duration)
}
diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs
index 0832192d8..80c6351e0 100644
--- a/core/simulator/src/lib.rs
+++ b/core/simulator/src/lib.rs
@@ -27,6 +27,8 @@ pub mod seeds;
pub mod storage;
pub mod workload;
+mod partition_wal;
+
use bus::SimOutbox;
use client::SimClient;
use clock::Clock;
@@ -37,7 +39,8 @@ use deps::{MemStorage, SimJournal};
use executor::{DetExecutor, RunOutcome, TaskId};
use iggy_binary_protocol::primitives::partition_history::{ConsumerGroupOwner, PartitionContext};
use iggy_binary_protocol::{
- Command, GenericHeader, PrepareHeader, ReplyHeader, RequestHeader, RoutedRequestHeader,
+ Command, CommitHeader, DoViewChangeHeader, GenericHeader, PrepareHeader, ReplyHeader,
+ RequestHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader,
};
use iggy_common::IggyError;
use message_bus::MessageBus;
@@ -55,7 +58,7 @@ use rand_xoshiro::rand_core::SeedableRng;
use replica::{Replica, SIM_INBOX_CAPACITY, new_shard};
use seeds::SimSeeds;
use server_common::Message;
-use server_common::sharding::{IggyNamespace, PartitionLocation, ShardId};
+use server_common::sharding::{IggyNamespace, METADATA_GROUP, PartitionLocation, ShardId};
use shard::shards_table::{ShardsTable, calculate_shard_assignment};
use shard::{CONSENSUS_TICK_INTERVAL, PartitionMaterialisation};
use std::cell::RefCell;
@@ -65,6 +68,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
+use storage::{Crash, SimStorage};
/// Poll budget per [`DetExecutor::run_until_stalled`]. Pumps are event-driven, so
/// hitting it means a task is spin-waking: a bug, panicked with the seed.
@@ -103,6 +107,9 @@ pub struct SimReplica {
/// persist gate marks every view durable without writing, leaving the gate, its
/// write-failure fence and view recovery unexercised.
pub partition_superblocks: RefCell<HashMap<IggyNamespace, (u64, Rc<SimSuperblock>)>>,
+ /// The disk under every partition WAL of this replica, retained across a
+ /// restart like `superblock`. A crash keeps only what the WAL synced.
+ pub partition_storage: SimStorage,
/// One retained message log per partition group, plus the offsets recovered from
/// it. A real server's messages are in segment files; the simulator has none, so
/// without this a rebuilt partition comes back empty and the monotonicity
@@ -236,6 +243,10 @@ pub struct Simulator {
/// their committed per-assignment views. Later seeds reuse the same choice
/// so replicas cannot initialize one group with different views.
partition_created_views: HashMap<IggyNamespace, u32>,
+ /// Highest view in which each replica sent a view-scoped message of a
+ /// partition group, keyed with the incarnation it hosted when the message
+ /// left. A restart may forget a later view only if it sent nothing in it.
+ partition_sent_views: HashMap<(u8, IggyNamespace, u64), u32>,
/// Replies a setup handshake pulled off the wire that were not its own.
///
/// `await_setup_reply` steps the whole simulator, so it sees every client's
@@ -564,6 +575,7 @@ impl Simulator {
metadata_journal,
metadata_incarnation,
partition_superblocks: RefCell::new(HashMap::new()),
+ partition_storage: SimStorage::default(),
partition_logs: RefCell::new(HashMap::new()),
deferred_partitions: RefCell::new(BTreeSet::new()),
data_dir: replica_data_dir,
@@ -587,6 +599,7 @@ impl Simulator {
entry_rng: Xoshiro256PlusPlus::seed_from_u64(SimSeeds::derive(seed).entry_shard),
evicted: Vec::new(),
partition_created_views: HashMap::new(),
+ partition_sent_views: HashMap::new(),
deferred_client_replies: Vec::new(),
seed,
shell,
@@ -993,6 +1006,14 @@ impl Simulator {
let message = match envelope.payload {
bus::EnvelopePayload::Replica(m) | bus::EnvelopePayload::Client(m) => m,
};
+ if let Some((namespace, incarnation, view)) = view_scoped_partition_frame(&message)
+ {
+ let sent = self
+ .partition_sent_views
+ .entry((i as u8, namespace, incarnation))
+ .or_default();
+ *sent = (*sent).max(view);
+ }
self.network.submit(from, to, message);
}
}
@@ -1388,6 +1409,12 @@ impl Simulator {
// Discard any unsent messages (never reached the wire).
self.outboxes[replica_index as usize].drain();
+ // Power loss: the WAL keeps only what it synced. Handles the dead
+ // process held go stale, as they do across a real restart.
+ self.replicas[replica_index as usize]
+ .partition_storage
+ .crash(Crash::PowerLoss);
+
// Block all network links to/from this process.
self.network
.process_disable(ProcessId::Replica(replica_index));
@@ -1432,6 +1459,7 @@ impl Simulator {
// partition reads the record in its directory.
let partition_superblocks =
std::mem::take(&mut *self.replicas[idx].partition_superblocks.borrow_mut());
+ let partition_storage = self.replicas[idx].partition_storage.clone();
// Take each live partition's log while its shard still stands: a real
// server's messages are in segment files its boot recovers the offset counter
// from, so rebuilding with nothing would model total data loss rather than a
@@ -1548,6 +1576,7 @@ impl Simulator {
metadata_journal,
metadata_incarnation,
partition_superblocks: RefCell::new(partition_superblocks),
+ partition_storage,
partition_logs: RefCell::new(partition_logs),
deferred_partitions: RefCell::new(BTreeSet::new()),
data_dir: replica_data_dir,
@@ -1762,6 +1791,19 @@ impl Simulator {
/// Consensus view for a replica's partition-plane group, or `None` if that
/// replica does not host the namespace.
#[must_use]
+ /// Highest view in which `replica_idx` sent a view-scoped message of
+ /// `incarnation` of `namespace`, or `None` when it sent none.
+ pub(crate) fn partition_sent_view(
+ &self,
+ replica_idx: u8,
+ namespace: IggyNamespace,
+ incarnation: u64,
+ ) -> Option<u32> {
+ self.partition_sent_views
+ .get(&(replica_idx, namespace, incarnation))
+ .copied()
+ }
+
pub(crate) fn consensus_view(
&self,
replica_idx: usize,
@@ -1986,6 +2028,7 @@ fn materialise_partition(
Ordering::Less => {
stores.remove(&namespace);
replica.partition_logs.borrow_mut().remove(&namespace);
+ partition_wal::remove(&replica.partition_storage, namespace, stored);
}
Ordering::Equal => {}
}
@@ -2008,15 +2051,26 @@ fn materialise_partition(
// Removed rather than cloned: the rebuilt partition becomes its sole owner, and
// a second materialisation with no restart between would otherwise resurrect a
// log the live partition has moved past.
- let retained = replica.partition_logs.borrow_mut().remove(&namespace);
- replica.shards[usize::from(owner)].init_partition(
- namespace,
- Some(superblock),
- recovered_state.as_ref(),
- retained,
- PartitionMaterialisation::new(epoch, created_view, first_initialization)
- .with_consumer_offsets_max(consumer_offsets_max),
- );
+ let mut retained = replica.partition_logs.borrow_mut().remove(&namespace);
+ let owner_shard = &replica.shards[usize::from(owner)];
+ // The live group holds this incarnation's WAL lease; a second open is refused.
+ if !owner_shard.plane.partitions().contains(&namespace) {
+ let persistence = partition_wal::reopen(
+ &replica.partition_storage,
+ namespace,
+ epoch,
+ retained.as_mut(),
+ );
+ owner_shard.init_partition(
+ namespace,
+ Some(superblock),
+ recovered_state.as_ref(),
+ retained,
+ persistence,
+ PartitionMaterialisation::new(epoch, created_view, first_initialization)
+ .with_consumer_offsets_max(consumer_offsets_max),
+ );
+ }
for shard in &replica.shards {
shard.shards_table().insert(
namespace,
@@ -2026,6 +2080,47 @@ fn materialise_partition(
true
}
+/// The partition group, incarnation and view of a message the persist gate
+/// holds until the replica's view is durable, or `None` for any other message.
+/// `PrepareOk` names no incarnation, so it is left out.
+fn view_scoped_partition_frame(
+ message: &Message<GenericHeader>,
+) -> Option<(IggyNamespace, u64, u32)> {
+ let (group, incarnation) = match message.header().command {
+ Command::Prepare => {
+ let header = message.try_as_typed::<PrepareHeader>().ok()?.header();
+ (header.group, header.partition_incarnation)
+ }
+ Command::Commit => {
+ let header = message.try_as_typed::<CommitHeader>().ok()?.header();
+ (header.group, header.partition_incarnation)
+ }
+ Command::StartViewChange => {
+ let header = message
+ .try_as_typed::<StartViewChangeHeader>()
+ .ok()?
+ .header();
+ (header.group, header.partition_incarnation)
+ }
+ Command::DoViewChange => {
+ let header = message.try_as_typed::<DoViewChangeHeader>().ok()?.header();
+ (header.group, header.partition_incarnation)
+ }
+ Command::StartView => {
+ let header = message.try_as_typed::<StartViewHeader>().ok()?.header();
+ (header.group, header.partition_incarnation)
+ }
+ _ => return None,
+ };
+ (group != METADATA_GROUP).then(|| {
+ (
+ IggyNamespace::from_raw(group),
+ incarnation,
+ message.header().view,
+ )
+ })
+}
+
/// How long the reconciler lets a revocation wait for its drain, in microseconds.
pub(crate) fn rebalancing_timeout_micros() -> u64 {
configs::server::ServerConfig::default()
@@ -3861,6 +3956,7 @@ mod tests {
None,
None,
None,
+ crate::partition_wal::reopen(&replica.partition_storage, fixture, named + 1, None),
PartitionMaterialisation::new(named + 1, created_view, true),
);
assert_eq!(sim.partition_incarnation(FORGED, fixture), Some(named + 1));
@@ -8249,12 +8345,15 @@ mod tests {
});
executor.run_until_stalled(POLL_BUDGET); // borrow acquired; task parks
let grow = Rc::clone(&sim.replicas[0].shards[0]);
+ let persistence =
+ crate::partition_wal::reopen(&SimStorage::default(), ns_grow, 0, None);
executor.spawn(async move {
grow.init_partition(
ns_grow,
None,
None,
None,
+ persistence,
PartitionMaterialisation::new(0, 0, true),
);
});
@@ -8290,12 +8389,14 @@ mod tests {
});
executor.run_until_stalled(POLL_BUDGET);
let grow = Rc::clone(&sim.replicas[0].shards[0]);
+ let persistence = crate::partition_wal::reopen(&SimStorage::default(), ns_grow, 0, None);
executor.spawn(async move {
grow.init_partition(
ns_grow,
None,
None,
None,
+ persistence,
PartitionMaterialisation::new(0, 0, true),
);
});
diff --git a/core/simulator/src/partition_wal.rs b/core/simulator/src/partition_wal.rs
new file mode 100644
index 000000000..7459624bf
--- /dev/null
+++ b/core/simulator/src/partition_wal.rs
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::storage::SimStorage;
+use futures::executor::block_on;
+use iggy_binary_protocol::Operation;
+use journal::Journal;
+use journal::durable_storage::DurableStorage;
+use partitions::{PartitionPersistence, RetainedPartitionState, WalStorage};
+use server_common::send_messages::decode_prepare_slice_trusted;
+use server_common::sharding::IggyNamespace;
+use std::path::{Path, PathBuf};
+use std::rc::Rc;
+
+/// Open the WAL of one partition incarnation on a replica's disk, creating its
+/// directory on first use, and fit the log a restart carries over to it.
+/// Without that log the incarnation starts on an empty WAL.
+///
+/// Each incarnation has its own directory, so the writer lease of a deleted
+/// incarnation never fences the open of its successor.
+pub fn reopen(
+ storage: &SimStorage,
+ namespace: IggyNamespace,
+ incarnation: u64,
+ retained: Option<&mut RetainedPartitionState>,
+) -> Rc<PartitionPersistence> {
+ let directory = PathBuf::from(format!("/partition-{}-{incarnation}", namespace.inner()));
+ // The harness replays no WAL into an empty partition, so a group that comes
+ // back without its log comes back without its WAL too, as a lost disk would.
+ let lost = retained.is_none();
+ block_on(async {
+ if lost {
+ storage.remove_tree(&directory).await?;
+ }
+ storage.create_directories(&directory).await?;
+ storage.sync_directory(Path::new("/")).await?;
+ storage.sync_directory(&directory).await
+ })
+ .expect("the simulated disk creates a partition directory");
+ let (persistence, _) = block_on(PartitionPersistence::open_with_storage(
+ &directory.join(format!("prepares-{incarnation}")),
+ namespace.inner(),
+ incarnation,
+ WalStorage::Injected(Rc::new(storage.clone())),
+ ))
+ .expect("the simulated disk reopens a partition WAL");
+ if let Some(retained) = retained {
+ let kept = fit_to_wal(retained, &persistence);
+ if kept < persistence.head() {
+ persistence.truncate_from(kept + 1);
+ if persistence.start() {
+ block_on(Rc::clone(&persistence).run());
+ }
+ }
+ }
+ persistence
+}
+
+/// Remove the directory of a deleted incarnation.
+pub fn remove(storage: &SimStorage, namespace: IggyNamespace, incarnation: u64) {
+ let directory = PathBuf::from(format!("/partition-{}-{incarnation}", namespace.inner()));
+ block_on(storage.remove_tree(&directory)).expect("the simulated disk removes a partition");
+}
+
+/// Keep the applied prefix of `retained` and the journal suffix the WAL still
+/// holds, and return the head that survives. A crash takes what the WAL had not
+/// synced, so an op above the applied prefix that the WAL lost is lost here too.
+/// Production recovers that suffix from the WAL itself, so an entry the journal
+/// does not hold at the same op is truncated from the WAL by the caller.
+fn fit_to_wal(retained: &mut RetainedPartitionState, persistence: &PartitionPersistence) -> u64 {
+ let journal = &retained.log.journal().inner;
+ let held = |op: u64| journal.repair_header(op).map(|header| header.checksum);
+ let first_divergent = (1..=retained.applied_op.min(persistence.head()))
+ .find(|&op| held(op).is_some_and(|checksum| persistence.checksum(op) != Some(checksum)));
+ let mut kept = retained.applied_op;
+ while kept < retained.head_op
+ && first_divergent.is_none()
+ && held(kept + 1).is_some_and(|checksum| persistence.checksum(kept + 1) == Some(checksum))
+ {
+ kept += 1;
+ }
+ if kept < retained.head_op {
+ retained.prepare_checksum = held(kept)
+ .or_else(|| persistence.checksum(kept))
+ .unwrap_or_default();
+ let mut write_offset = retained.durable_offset;
+ for op in retained.applied_op + 1..=kept {
+ if journal
+ .repair_header(op)
+ .is_some_and(|header| header.operation == Operation::SendMessages)
+ && let Some(entry) = journal.repair_entry(op)
+ && let Ok(batch) = decode_prepare_slice_trusted(entry.as_slice())
+ && batch.header.message_count > 0
+ {
+ write_offset = write_offset
+ .max(batch.header.base_offset + u64::from(batch.header.message_count) - 1);
+ }
+ }
+ block_on(journal.remove_range(kept + 1..=u64::MAX))
+ .expect("the retained journal drops a suffix");
+ retained.head_op = kept;
+ retained.write_offset = write_offset;
+ }
+ first_divergent.map_or(kept, |op| op - 1)
+}
diff --git a/core/simulator/src/workload/invariants.rs b/core/simulator/src/workload/invariants.rs
index 97c5d3dc1..3b7f30489 100644
--- a/core/simulator/src/workload/invariants.rs
+++ b/core/simulator/src/workload/invariants.rs
@@ -66,6 +66,9 @@ const COMMIT_PREFIX_APPLIED_HEAD_TICKS: u32 = 2_000;
pub struct Invariants {
commit_offset: HashMap<(u8, IggyNamespace), u64>,
view: HashMap<(u8, IggyNamespace), u64>,
+ /// Metadata incarnation each replica ran under at the last check, so a
+ /// restart is told apart from a live regression of `view`.
+ replica_incarnation: HashMap<u8, u128>,
/// Partition incarnation the marks of each `(replica, namespace)` belong to. A
/// recreated topic keeps its namespaces and starts their offsets and views
/// afresh, so a new incarnation resets them.
@@ -92,7 +95,9 @@ impl Invariants {
///
/// Per live `(replica, namespace)`:
/// - partition `commit_offset` never regresses,
- /// - consensus `view` never regresses (a view change only advances it).
+ /// - consensus `view` never regresses while the replica runs (a view change
+ /// only advances it), and never falls below a view the replica sent a
+ /// view-scoped message in.
///
/// Per live replica, no metadata or partition log contradicted an op it applied.
///
@@ -104,12 +109,14 @@ impl Invariants {
/// - partition commits agree within each fixture incarnation, including
/// history witnessed before a peer replaced that incarnation.
///
- /// Crashed replicas are skipped and their last-seen marks retained, which
- /// holds across a restart for both quantities: the superblock carries `view`,
- /// and `Simulator::retain_partition_logs` carries each partition's log so a
+ /// Crashed replicas are skipped and their last-seen marks retained.
+ /// `Simulator::retain_partition_logs` carries each partition's log so a
/// rebuilt partition recovers its offsets instead of reporting zero. Without
/// that the check trips on a discarded log and calls it a regression. A
- /// replica hosting a new incarnation of a namespace starts its marks again.
+ /// restart recovers the view its superblock recorded, which the WAL gate can
+ /// hold below the view the replica entered, so across a restart only the
+ /// views it sent in bound the recovered one. A replica hosting a new
+ /// incarnation of a namespace starts its marks again.
///
/// # Panics
/// On any regression or in-flight overflow. The workload seed is in the
@@ -126,6 +133,14 @@ impl Invariants {
}
self.check_recovery_barrier(sim, seed, replica_idx);
check_applied_log_contradictions(sim, seed, replica_idx);
+ let incarnation = sim.replicas[usize::from(replica_idx)].metadata_incarnation;
+ if self
+ .replica_incarnation
+ .insert(replica_idx, incarnation)
+ .is_some_and(|previous| previous != incarnation)
+ {
+ self.view.retain(|(replica, _), _| *replica != replica_idx);
+ }
self.check_commit_prefix_contiguity(
seed,
replica_idx,
@@ -153,6 +168,21 @@ impl Invariants {
if let Some(&prev) = self.view.get(&(replica_idx, ns)) {
assert_no_regression(seed, "consensus view", replica_idx, ns, prev, view);
}
+ if let Some(sent) = sim
+ .partition_incarnation(usize::from(replica_idx), ns)
+ .and_then(|incarnation| {
+ sim.partition_sent_view(replica_idx, ns, incarnation)
+ })
+ {
+ assert_no_regression(
+ seed,
+ "consensus view below one it sent in",
+ replica_idx,
+ ns,
+ u64::from(sent),
+ view,
+ );
+ }
self.view.insert((replica_idx, ns), view);
}
}
diff --git a/core/system_stats/src/lib.rs b/core/system_stats/src/lib.rs
index 432e92ab1..dfa80c869 100644
--- a/core/system_stats/src/lib.rs
+++ b/core/system_stats/src/lib.rs
@@ -227,9 +227,14 @@ fn allowed_cores_cpu_usage(sys: &System, allowed: &[usize]) -> Option<f32> {
#[cfg(test)]
mod tests {
- use super::*;
+ use std::sync::Mutex;
+
use sysinfo::{Pid, ProcessesToUpdate};
+ use super::*;
+
+ static OPEN_FILES_TEST_LOCK: Mutex<()> = Mutex::new(());
+
#[test]
fn given_fresh_system_when_capturing_probe_should_sample_own_process() {
let mut sys = System::new();
@@ -257,6 +262,7 @@ mod tests {
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[test]
fn given_extra_open_files_when_counting_should_include_them() {
+ let _guard = OPEN_FILES_TEST_LOCK.lock().unwrap();
const EXTRA: u64 = 64;
let manifest = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml");
let before = count_open_files().expect("open files countable");
@@ -274,6 +280,7 @@ mod tests {
#[cfg(target_os = "linux")]
#[test]
fn given_fd_dir_size_when_kernel_reports_it_should_match_the_scan() {
+ let _guard = OPEN_FILES_TEST_LOCK.lock().unwrap();
let Some(from_size) = count_open_files_without_scan() else {
return;
};
diff --git a/foreign/cpp/BUILD.bazel b/foreign/cpp/BUILD.bazel
index c5d655ba1..417d7d711 100644
--- a/foreign/cpp/BUILD.bazel
+++ b/foreign/cpp/BUILD.bazel
@@ -196,6 +196,7 @@ cc_binary(
"tests/bdd/background_steps.cpp",
"tests/bdd/messaging_steps.cpp",
"tests/bdd/raw_command_steps.cpp",
+ "tests/bdd/session_contract_steps.cpp",
"tests/bdd/world.hpp",
],
copts = [
diff --git a/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestContext.cs b/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestContext.cs
index 27a08f555..91de7b66a 100644
--- a/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestContext.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestContext.cs
@@ -27,6 +27,7 @@ public class TestContext
public string TcpUrl => TestEnvironment.TcpAddress;
public string LeaderTcpUrl => TestEnvironment.LeaderTcpAddress;
public string FollowerTcpUrl => TestEnvironment.FollowerTcpAddress;
+ public string NoQuorumTcpUrl => TestEnvironment.NoQuorumTcpAddress;
public Dictionary<string, IIggyClient> Clients { get; } = new();
public StreamResponse? CreatedStream { get; set; }
public TopicResponse? CreatedTopic { get; set; }
@@ -36,4 +37,5 @@ public class TestContext
public uint? LastStreamId { get; set; }
public byte[]? LastRawResponse { get; set; }
public Exception? LastRawError { get; set; }
+ public Exception? LastCallError { get; set; }
}
diff --git a/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestEnvironment.cs b/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestEnvironment.cs
index cfb89a964..a3af10a2c 100644
--- a/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestEnvironment.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestEnvironment.cs
@@ -27,6 +27,7 @@ public static class TestEnvironment
public static string TcpAddress => Require("IGGY_TCP_ADDRESS");
public static string LeaderTcpAddress => Require("IGGY_TCP_ADDRESS_LEADER");
public static string FollowerTcpAddress => Require("IGGY_TCP_ADDRESS_FOLLOWER");
+ public static string NoQuorumTcpAddress => Require("IGGY_TCP_ADDRESS_NO_QUORUM");
public static string RootUsername => Require("IGGY_ROOT_USERNAME");
public static string RootPassword => Require("IGGY_ROOT_PASSWORD");
diff --git a/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestHooks.cs b/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestHooks.cs
index d63f2cd6b..23b8e442e 100644
--- a/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestHooks.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.BDD/Context/TestHooks.cs
@@ -38,6 +38,7 @@ public class TestHooks
_context.LastStreamId = null;
_context.LastRawResponse = null;
_context.LastRawError = null;
+ _context.LastCallError = null;
}
[AfterScenario]
diff --git a/foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj b/foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj
index 7d7785453..e52a301a0 100644
--- a/foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj
+++ b/foreign/csharp/Iggy_SDK.Tests.BDD/Iggy_SDK.Tests.BDD.csproj
@@ -63,6 +63,9 @@ under the License.
<ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\raw_command.feature">
<Link>Features\raw_command.feature</Link>
</ReqnrollFeatureFile>
+ <ReqnrollFeatureFile Include="..\..\..\bdd\scenarios\session_contract.feature">
+ <Link>Features\session_contract.feature</Link>
+ </ReqnrollFeatureFile>
</ItemGroup>
<ItemGroup>
diff --git a/foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/SessionContractSteps.cs b/foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/SessionContractSteps.cs
new file mode 100644
index 000000000..0dc622214
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK.Tests.BDD/StepDefinitions/SessionContractSteps.cs
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Configuration;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Factory;
+using Reqnroll;
+using Shouldly;
+using TestContext = Apache.Iggy.Tests.BDD.Context.TestContext;
+using TestEnvironment = Apache.Iggy.Tests.BDD.Context.TestEnvironment;
+
+namespace Apache.Iggy.Tests.BDD.StepDefinitions;
+
+[Binding]
+public class SessionContractSteps
+{
+ private readonly TestContext _context;
+
+ public SessionContractSteps(TestContext context)
+ {
+ _context = context;
+ }
+
+ [Given(@"I have a cluster node that cannot reach quorum")]
+ public async Task GivenIHaveAClusterNodeThatCannotReachQuorum()
+ {
+ _context.IggyClient = IggyClientFactory.CreateClient(new IggyClientConfigurator
+ {
+ BaseAddress = _context.NoQuorumTcpUrl,
+ Protocol = Protocol.Tcp
+ });
+
+ await _context.IggyClient.ConnectAsync();
+ }
+
+ [When(@"I sign out")]
+ public async Task WhenISignOut()
+ {
+ await _context.IggyClient.LogoutUserAsync();
+ }
+
+ [When(@"I request the stream with ID (\d+)")]
+ public async Task WhenIRequestTheStreamWithId(uint streamId)
+ {
+ await RecordCallError(() => _context.IggyClient.GetStreamByIdAsync(Identifier.Numeric(streamId)));
+ }
+
+ [When(@"I sign in to that node as the root user")]
+ public async Task WhenISignInToThatNodeAsTheRootUser()
+ {
+ await RecordCallError(() =>
+ _context.IggyClient.LoginUserAsync(TestEnvironment.RootUsername, TestEnvironment.RootPassword));
+ }
+
+ [Then(@"the call should fail with error code (\d+)")]
+ public void ThenTheCallShouldFailWithErrorCode(int code)
+ {
+ var error = _context.LastCallError.ShouldBeOfType<IggyInvalidStatusCodeException>();
+ error.StatusCode.ShouldBe(code);
+ }
+
+ private async Task RecordCallError(Func<Task> call)
+ {
+ try
+ {
+ await call();
+ _context.LastCallError = null;
+ }
+ catch (Exception error)
+ {
+ _context.LastCallError = error;
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs
index a5b2d3660..75a76c509 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs
@@ -440,7 +440,8 @@ public class IggyConsumerTests
offset.ShouldNotBeNull();
receivedCount.ShouldBe(5);
lastOffset.ShouldBe(4ul);
- offset.StoredOffset.ShouldBe(lastOffset);
+ // The loop broke inside the body of offset 4, so that message is delivered again.
+ offset.StoredOffset.ShouldBe(3ul);
}
[Test]
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs
index a1cf6421b..f8d7c2f4c 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs
@@ -129,7 +129,8 @@ public class IggyTypedConsumerTests
offset.ShouldNotBeNull();
count.ShouldBe(5);
- offset.StoredOffset.ShouldBe(4ul);
+ // The loop broke inside the body of offset 4, so that message is delivered again.
+ offset.StoredOffset.ShouldBe(3ul);
}
[Test]
diff --git a/foreign/csharp/Iggy_SDK/Consumers/AutoCommitMode.cs b/foreign/csharp/Iggy_SDK/Consumers/AutoCommitMode.cs
index 06c324ebe..c9ea5e34a 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/AutoCommitMode.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/AutoCommitMode.cs
@@ -28,7 +28,9 @@ public enum AutoCommitMode
Auto,
/// <summary>
- /// Set offset after receive message
+ /// Stores the offset of a message once the receive loop asks for the next message. Neither this store nor the
+ /// final checkpoint on dispose covers a message whose loop body threw, broke or was still running, so that
+ /// message is delivered again.
/// </summary>
AfterReceive,
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
index f463f71cc..171e91fad 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
@@ -55,7 +55,7 @@ public partial class IggyConsumer
continue;
}
- if (!RecordDelivery(message.CurrentOffset, message.PartitionId, message.Context))
+ if (!BeginDelivery(message.CurrentOffset, message.PartitionId, message.Context))
{
message.Dispose();
yield break;
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
index 7d0a465c7..3a1dcda87 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
@@ -234,7 +234,11 @@ public partial class IggyConsumer : IAsyncDisposable
}
}
- private protected bool RecordDelivery(ulong offset, uint partition, PartitionContext context)
+ // Under AfterReceive a message counts as delivered once its loop body completes, in CommitDeliveredAsync.
+ private protected bool BeginDelivery(ulong offset, uint partition, PartitionContext context) =>
+ _config.AutoCommitMode == AutoCommitMode.AfterReceive ? !IsDisposed : RecordDelivery(offset, partition, context);
+
+ private bool RecordDelivery(ulong offset, uint partition, PartitionContext context)
{
lock (_deliveryGate)
{
@@ -255,7 +259,9 @@ public partial class IggyConsumer : IAsyncDisposable
first.Incarnation == second.Incarnation && first.OwnerGeneration == second.OwnerGeneration;
private protected Task CommitDeliveredAsync(ulong offset, uint partition, PartitionContext context, CancellationToken ct) =>
- StoreOffsetCoreAsync(offset, partition, context, ct, afterReceive: true);
+ RecordDelivery(offset, partition, context)
+ ? StoreOffsetCoreAsync(offset, partition, context, ct, afterReceive: true)
+ : Task.CompletedTask;
/// <summary>
/// Initializes the consumer by logging in (if needed) and setting up consumer groups
@@ -336,7 +342,7 @@ public partial class IggyConsumer : IAsyncDisposable
continue;
}
- if (!RecordDelivery(message.CurrentOffset, message.PartitionId, message.Context))
+ if (!BeginDelivery(message.CurrentOffset, message.PartitionId, message.Context))
{
yield break;
}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs
index c79e6768c..416675f7d 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs
@@ -78,7 +78,7 @@ public class IggyConsumer<T> : IggyConsumer
continue;
}
- if (!RecordDelivery(message.Header.Offset, message.PartitionId, message.Context))
+ if (!BeginDelivery(message.Header.Offset, message.PartitionId, message.Context))
{
yield break;
}
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs
index 5a1480c03..1329f0afe 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.PollRouting.cs
@@ -274,7 +274,7 @@ public sealed partial class TcpMessageStream
{
captured ??= new PartitionContext(0, 0, _metadataWatermark);
}
- EnsureOperationSession(generation);
+ EnsureOperationSession(code, generation);
if (operation is null && (retainable || progress))
{
var lane = target is { } selected
@@ -316,7 +316,7 @@ public sealed partial class TcpMessageStream
{
connection = ReadyReplicaLinks().FirstOrDefault() ?? throw new NotConnectedException();
}
- EnsureOperationSession(generation);
+ EnsureOperationSession(code, generation);
if (operation is not null && !_mutations.Owns(operation))
{
throw PollNotAccepted();
@@ -515,7 +515,7 @@ public sealed partial class TcpMessageStream
{
if (!ResumesUnderNewSession(generation))
{
- EnsureOperationSession(generation);
+ EnsureOperationSession(code, generation);
}
lastWasNotPrimary = false;
terminal = false;
@@ -631,11 +631,11 @@ public sealed partial class TcpMessageStream
return Task.CompletedTask;
}
- private void EnsureOperationSession(ulong generation)
+ private void EnsureOperationSession(int code, ulong generation)
{
lock (_pollRoutingGate)
{
- if (_signInWithdrawn && _consensusSession.IsBound)
+ if (_signInWithdrawn && code != CommandCodes.PING_CODE)
{
throw _consensusSession.Ended(generation)
?? VsrError.Exception(VsrError.UNAUTHENTICATED, "An explicit sign-in is required.");
diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/PartitionIncarnationTests.Disposal.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/PartitionIncarnationTests.Disposal.cs
index 54bff3e4a..f401f0990 100644
--- a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/PartitionIncarnationTests.Disposal.cs
+++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/PartitionIncarnationTests.Disposal.cs
@@ -119,14 +119,47 @@ public sealed partial class PartitionIncarnationTests
Assert.Equal(1, owner.DisposeCount);
}
await consumer.DisposeAsync();
- Assert.Equal(mode == AutoCommitMode.Disabled ? [] : [(0ul, Original)], stores);
- Assert.Equal(mode == AutoCommitMode.Disabled ? ["leave", "logout", "dispose"] : ["store", "leave", "logout", "dispose"], events);
+ Assert.Equal(mode == AutoCommitMode.Auto ? [(0ul, Original)] : [], stores);
+ Assert.Equal(mode == AutoCommitMode.Auto ? ["store", "leave", "logout", "dispose"] : ["leave", "logout", "dispose"], events);
if (receive == "plain")
{
rental.Dispose();
}
}
+ [Fact]
+ public async Task given_after_receive_when_a_loop_body_throws_should_not_commit_its_message_on_dispose()
+ {
+ var stores = new ConcurrentQueue<(ulong Offset, PartitionContext Context)>();
+ var client = ClientMock(stores);
+ client.Setup(mock => mock.PollMessagesAsync(It.IsAny<Identifier>(), It.IsAny<Identifier>(),
+ It.IsAny<uint?>(), It.IsAny<Consumer>(), It.IsAny<Func<uint, PollingStrategy>>(), It.IsAny<uint>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
+ .ReturnsAsync(new PolledMessages
+ {
+ Context = Original,
+ PartitionId = PartitionId,
+ CurrentOffset = 1,
+ Messages = [.. Batch(0, Original).Messages, .. Batch(1, Original).Messages]
+ });
+ await using var consumer = new IggyConsumer(client.Object, Config(), NullLoggerFactory.Instance);
+ using var timeout = Timeout();
+ await consumer.InitAsync(timeout.Token);
+
+ await Assert.ThrowsAsync<InvalidOperationException>(async () =>
+ {
+ await foreach (var message in consumer.ReceiveAsync(timeout.Token))
+ {
+ if (message.CurrentOffset == 1)
+ {
+ throw new InvalidOperationException("Processing failed.");
+ }
+ }
+ });
+ await consumer.DisposeAsync();
+
+ Assert.Equal([(0ul, Original)], stores);
+ }
+
[Theory]
[InlineData(false)]
[InlineData(true)]
@@ -147,7 +180,7 @@ public sealed partial class PartitionIncarnationTests
events.Enqueue("store");
return alreadyStored ? Task.CompletedTask : Task.FromException(new IOException("Checkpoint failed."));
});
- var config = Config();
+ var config = Config(AutoCommitMode.Auto);
config.Consumer = Consumer.Group("group");
await using var consumer = new IggyConsumer(client.Object, config, NullLoggerFactory.Instance);
using var timeout = Timeout();
diff --git a/foreign/csharp/README.md b/foreign/csharp/README.md
index bd86b2836..9de45a3c8 100644
--- a/foreign/csharp/README.md
+++ b/foreign/csharp/README.md
@@ -889,6 +889,8 @@ keeps the delivered message's partition context. `AutoCommitMode.Disabled` skips
checkpoint. The final stores share one 30-second budget; failures are logged and cleanup continues.
Disposing from inside a receive loop is supported, including rented and deserialized messages.
+Under `AutoCommitMode.AfterReceive` a message counts as delivered once its loop body completes. A message whose body threw, broke or was still running when the consumer was disposed is not checkpointed and is delivered again.
+
For automatic deserialization:
```c#
diff --git a/foreign/go/README.md b/foreign/go/README.md
index 43498da63..c0e0f1bd9 100644
--- a/foreign/go/README.md
+++ b/foreign/go/README.md
@@ -69,14 +69,11 @@ new registration. While the client registers that new session by itself, a call
that it did not send waits for the session. So does a call that starts while
`Connect` signs in. Then the call goes out under the new session. If no session
comes before the call deadline, the call fails with 58, and the caller can retry
-it. If registration ends with a refusal, every call that waits for it fails with
-40 and the refusal as its cause. A `Connect` whose own sign-in is refused returns
-the refusal itself. The client also logs the refusal. A refusal of the current
-credentials with code 4, 40, 42, 43, 44, 47, 53 or 54 blocks automatic sign-in
-until another explicit sign-in. `Connect`, including after `Disconnect`, cannot
-unblock it. If a committed password change already updated the same owner's
-credentials, refusal of the older password retries the updated credentials.
-Other refusals keep the last sign-in, so a later `Connect` can retry it.
+it.
+
+If the server refuses the current credentials of that registration, with code 4, 40, 42, 43, 44, 47, 53 or 54, every call that waits for it fails with 40 and the refusal as its cause. Such a refusal blocks automatic sign-in until another explicit sign-in. `Connect`, including after `Disconnect`, cannot unblock it. If a committed password change already updated the same owner's credentials, refusal of the older password retries the updated credentials.
+
+Other refusals keep the last sign-in and leave the session owed. After a session loss the client keeps registering, and a call that waits for the session fails with 58 at its deadline. A call that waits for a `Connect` fails with 58 and the refusal as its cause, and a later `Connect` can retry the sign-in. A `Connect` whose own sign-in is refused returns the refusal itself. The client also logs every refusal.
A sign-in, a logout or a disconnect by the application ends every call that
started before it. A call that did not change server state fails with 40. A
diff --git a/foreign/go/client/tcp/tcp_connect_test.go b/foreign/go/client/tcp/tcp_connect_test.go
index 2e593910a..739fbea1d 100644
--- a/foreign/go/client/tcp/tcp_connect_test.go
+++ b/foreign/go/client/tcp/tcp_connect_test.go
@@ -103,6 +103,29 @@ func TestConnectBoundsSilentSignInWithoutCallerDeadline(t *testing.T) {
})
}
+func TestExplicitSilentSignInFailsAsNotAcceptedAtItsDeadline(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ serverConnection, connection := net.Pipe()
+ defer func() { _ = serverConnection.Close() }()
+ client := newTestClient(t, connection)
+ server := serve(serverConnection, func(_ int, read request) []byte { return nil })
+ done := make(chan error, 1)
+ go func() {
+ _, err := client.LoginUser(context.Background(), "iggy", "secret")
+ done <- err
+ }()
+ time.Sleep(responseReadTimeout)
+ synctest.Wait()
+ select {
+ case err := <-done:
+ require.ErrorIs(t, err, ierror.ErrTransientNotAccepted)
+ default:
+ t.Fatal("LoginUser outlived its sign-in deadline")
+ }
+ require.Len(t, server.recorded(), 1)
+ })
+}
+
func TestConnectDoesNotSpendSignInBudgetWaitingForDial(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
client := newDialingClient(t, "127.0.0.1:1")
@@ -799,6 +822,56 @@ func TestCallsThatStartDuringConnectWaitForItsSession(t *testing.T) {
}
}
+// The server refuses the sign-in of Connect for a reason other than its
+// credentials. Connect returns the refusal, and the calls that wait for it end
+// with 58 and the refusal as their cause. The client keeps its sign-in.
+func TestCallsThatWaitForAConnectRefusedForAnotherReasonEndWith58(t *testing.T) {
+ registered, release := make(chan struct{}), make(chan struct{})
+ unblock := sync.OnceFunc(func() { close(release) })
+ defer unblock()
+ newerProtocol := vsr.ProtocolVersion + 1
+ var refused atomic.Bool
+ var server *testListener
+ server = listenVSR(t, nil, func(_, _ int, read request) []byte {
+ if read.operation() == vsr.OperationRegister && refused.CompareAndSwap(false, true) {
+ close(registered)
+ <-release
+ return evictionFrame(vsr.EvictionIncompatibleProtocol, newerProtocol, newerProtocol)
+ }
+ return singleNodeHandler(t, func() string { return server.address() })(0, 0, read)
+ })
+ client := newDialingClient(t, server.address(), WithAutoLogin(NewUsernamePasswordCredentials("iggy", "iggy")))
+ epoch := client.epoch.Load()
+ connected := make(chan error, 1)
+ go func() { connected <- client.Connect(context.Background()) }()
+ <-registered
+ ctx, cancel := context.WithTimeout(context.Background(), managedLinkTestBudget)
+ defer cancel()
+ created, pinged := make(chan error, 1), make(chan error, 1)
+ go func() {
+ _, err := client.do(ctx, &command.CreateStream{Name: "waiting"})
+ created <- err
+ }()
+ go func() { pinged <- client.Ping(ctx) }()
+ require.Never(t, func() bool { return len(created) != 0 || len(pinged) != 0 }, recoveryWaitDeadline, managedLinkPoll,
+ "the calls wait for the session of the Connect")
+
+ unblock()
+ err := <-connected
+ var refusal ierror.IggyError
+ require.ErrorAs(t, err, &refusal)
+ require.Equal(t, ierror.IncompatibleProtocolVersionCode, refusal.Code(), "Connect returns its own refusal")
+ for _, call := range []chan error{created, pinged} {
+ err := <-call
+ require.ErrorIs(t, err, ierror.ErrTransientNotAccepted)
+ require.ErrorIs(t, err, ierror.ErrIncompatibleProtocolVersion, "the call keeps the refusal as its cause")
+ }
+ require.Same(t, epoch, client.epoch.Load(), "the refusal keeps the epoch")
+ require.Empty(t, createStreams(server.recorded()), "the create never went out")
+ require.NoError(t, client.Connect(ctx), "the client keeps its sign-in")
+ require.True(t, sessionBound(client))
+}
+
func TestLoginUserRacingDisconnectCannotRememberItsCredentials(t *testing.T) {
serverConnection, connection := net.Pipe()
defer func() { _ = serverConnection.Close() }()
diff --git a/foreign/go/client/tcp/tcp_core.go b/foreign/go/client/tcp/tcp_core.go
index f54aa49c4..8db23df83 100644
--- a/foreign/go/client/tcp/tcp_core.go
+++ b/foreign/go/client/tcp/tcp_core.go
@@ -422,6 +422,17 @@ func appendCommandFrame(buf []byte, cmd command.Command) ([]byte, error) {
}
func (c *IggyTcpClient) Connect(ctx context.Context) error {
+ err := c.connect(ctx)
+ var refusal *signInRefusal
+ if errors.As(err, &refusal) {
+ return refusal.err
+ }
+ return err
+}
+
+// connect is Connect, except that a refused sign-in stays a signInRefusal, so
+// the recovery and the calls that wait for the attempt can tell it apart.
+func (c *IggyTcpClient) connect(ctx context.Context) error {
if ctx == nil {
return ierror.ErrNilContext
}
@@ -463,9 +474,7 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error {
}
}
var refusal *signInRefusal
- if errors.As(err, &refusal) {
- err = refusal.err
- } else if ended := epoch.err(); ended != nil {
+ if ended := epoch.err(); ended != nil && !errors.As(err, &refusal) {
err = ended
}
c.connectMu.Lock()
diff --git a/foreign/go/client/tcp/tcp_driver.go b/foreign/go/client/tcp/tcp_driver.go
index f7bbf42ad..7c0d4a07f 100644
--- a/foreign/go/client/tcp/tcp_driver.go
+++ b/foreign/go/client/tcp/tcp_driver.go
@@ -20,6 +20,7 @@ package tcp
import (
"context"
"errors"
+ "fmt"
"time"
iggcon "github.com/apache/iggy/foreign/go/contracts"
@@ -635,8 +636,10 @@ func (c *IggyTcpClient) sessionRecovers(ctx context.Context, err error) bool {
// waitForRecovery waits while an automatic recovery replaces a lost session,
// or a Connect signs in, so a call that starts or restarts meanwhile runs
// under the new session. A sign-in that binds no session by the deadline ends
-// the call with 58. A sign-in, logout or disconnect of the caller, and a
-// refused automatic sign-in, end the calls that started before them with 40.
+// the call with 58, and so does a Connect that the server refuses for a reason
+// other than its credentials. A sign-in, logout or disconnect of the caller,
+// and a refusal of the credentials of an automatic sign-in, end the calls that
+// started before them with 40.
func (c *IggyTcpClient) waitForRecovery(ctx context.Context) error {
epoch := c.callEpoch(ctx)
for {
@@ -649,11 +652,12 @@ func (c *IggyTcpClient) waitForRecovery(ctx context.Context) error {
return err
}
var signInEnded <-chan struct{}
+ var attempt *connectAttempt
if unbound && recovery != nil && recovery.Err() == nil {
signInEnded = recovery.Done()
} else if unbound && signsIn {
c.connectMu.Lock()
- if attempt := c.connectionAttempt; attempt != nil {
+ if attempt = c.connectionAttempt; attempt != nil {
signInEnded = attempt.done
}
c.connectMu.Unlock()
@@ -663,6 +667,10 @@ func (c *IggyTcpClient) waitForRecovery(ctx context.Context) error {
}
select {
case <-signInEnded:
+ var refusal *signInRefusal
+ if attempt != nil && errors.As(attempt.err, &refusal) && !epoch.over() {
+ return fmt.Errorf("%w: %w", ierror.ErrTransientNotAccepted, refusal.err)
+ }
case <-epoch.ended:
case <-ctx.Done():
return ierror.ErrTransientNotAccepted
diff --git a/foreign/go/client/tcp/tcp_replicas_test.go b/foreign/go/client/tcp/tcp_replicas_test.go
index 5557aac85..ad1cd1c90 100644
--- a/foreign/go/client/tcp/tcp_replicas_test.go
+++ b/foreign/go/client/tcp/tcp_replicas_test.go
@@ -1047,44 +1047,41 @@ func (l *errorLog) holds(target error) bool {
return slices.ContainsFunc(l.errors, func(err error) bool { return errors.Is(err, target) })
}
-// The server refuses the sign-in of a recovery with an eviction or with a
-// status. A create and a read wait for the recovery, and both end with 40,
-// also when no link is left, with the refusal as their cause. A credential
-// refusal blocks automatic sign-in until the caller explicitly signs in again.
+// The server refuses the credentials of the sign-in of a recovery with an
+// eviction or with a status. A create and a read wait for the recovery, and
+// both end with 40, also when no link is left, with the refusal as their
+// cause. The refusal blocks automatic sign-in until the caller explicitly
+// signs in again.
func TestSessionRecovery_ARefusedSignInEndsTheCallsThatWaitForIt(t *testing.T) {
// The connect signs in as iggy and the caller as other, so the recovery
// sends the third sign-in.
const recoverySignIn = 3
- newerProtocol := vsr.ProtocolVersion + 1
for _, refusal := range []struct {
- name string
- answer []byte
- cause error
- forgets bool
- token bool
+ name string
+ answer []byte
+ cause error
+ token bool
}{
- {"evicted_invalid_credentials", evictionFrame(vsr.EvictionInvalidCredentials, 0, 0), ierror.ErrInvalidCredentials, true, false},
- {"evicted_user_inactive", evictionFrame(vsr.EvictionUserInactive, 0, 0), ierror.ErrUnauthenticated, true, false},
- {"evicted_session_error", evictionFrame(vsr.EvictionSessionError, 0, 0), ierror.ErrUnauthenticated, true, false},
- {"evicted_invalid_token", evictionFrame(vsr.EvictionInvalidToken, 0, 0), ierror.ErrInvalidPersonalAccessToken, true, true},
- {"evicted_incompatible_protocol", evictionFrame(vsr.EvictionIncompatibleProtocol, newerProtocol, newerProtocol),
- ierror.ErrIncompatibleProtocolVersion, false, false},
+ {"evicted_invalid_credentials", evictionFrame(vsr.EvictionInvalidCredentials, 0, 0), ierror.ErrInvalidCredentials, false},
+ {"evicted_user_inactive", evictionFrame(vsr.EvictionUserInactive, 0, 0), ierror.ErrUnauthenticated, false},
+ {"evicted_session_error", evictionFrame(vsr.EvictionSessionError, 0, 0), ierror.ErrUnauthenticated, false},
+ {"evicted_invalid_token", evictionFrame(vsr.EvictionInvalidToken, 0, 0), ierror.ErrInvalidPersonalAccessToken, true},
{"status_invalid_credentials", statusReplyFrame(vsr.OperationRegister, uint32(ierror.InvalidCredentialsCode), nil),
- ierror.ErrInvalidCredentials, true, false},
+ ierror.ErrInvalidCredentials, false},
{"status_invalid_token", statusReplyFrame(vsr.OperationRegister, uint32(ierror.InvalidPersonalAccessTokenCode), nil),
- ierror.ErrInvalidPersonalAccessToken, true, true},
+ ierror.ErrInvalidPersonalAccessToken, true},
{"status_expired_token", statusReplyFrame(vsr.OperationRegister, uint32(ierror.PersonalAccessTokenExpiredCode), nil),
- ierror.ErrPersonalAccessTokenExpired, true, true},
+ ierror.ErrPersonalAccessTokenExpired, true},
{"status_invalid_format", statusReplyFrame(vsr.OperationRegister, uint32(ierror.InvalidFormatCode), nil),
- ierror.ErrInvalidFormat, true, false},
+ ierror.ErrInvalidFormat, false},
{"status_invalid_username", statusReplyFrame(vsr.OperationRegister, uint32(ierror.InvalidUsernameCode), nil),
- ierror.ErrInvalidUsername, true, false},
+ ierror.ErrInvalidUsername, false},
{"status_invalid_password", statusReplyFrame(vsr.OperationRegister, uint32(ierror.InvalidPasswordCode), nil),
- ierror.ErrInvalidPassword, true, false},
+ ierror.ErrInvalidPassword, false},
{"status_user_inactive", statusReplyFrame(vsr.OperationRegister, uint32(ierror.UserInactiveCode), nil),
- ierror.ErrUserInactive, true, false},
+ ierror.ErrUserInactive, false},
{"status_unauthenticated", statusReplyFrame(vsr.OperationRegister, uint32(ierror.UnauthenticatedCode), nil),
- ierror.ErrUnauthenticated, true, false},
+ ierror.ErrUnauthenticated, false},
} {
t.Run(refusal.name, func(t *testing.T) {
server, _ := lostSessionServer(t, func(number int, _ request) []byte {
@@ -1139,23 +1136,103 @@ func TestSessionRecovery_ARefusedSignInEndsTheCallsThatWaitForIt(t *testing.T) {
require.True(t, logged.holds(refusal.cause), "the log keeps the refusal")
require.Empty(t, requestsFor(server.recorded(), vsr.OperationCreateStream), "the create never went out")
require.NoError(t, client.Connect(ctx))
- if refusal.forgets {
- require.NoError(t, client.Disconnect())
- require.NoError(t, client.Connect(ctx))
- require.Len(t, registerRequests(server), recoverySignIn, "a refused credential must not fall back to AutoLogin")
- require.False(t, sessionBound(client))
- _, err := client.LoginUser(ctx, "retry", "password")
- require.NoError(t, err)
+ require.NoError(t, client.Disconnect())
+ require.NoError(t, client.Connect(ctx))
+ require.Len(t, registerRequests(server), recoverySignIn, "a refused credential must not fall back to AutoLogin")
+ require.False(t, sessionBound(client))
+ _, err := client.LoginUser(ctx, "retry", "password")
+ require.NoError(t, err)
+ var signIns []string
+ for _, read := range registerRequests(server) {
+ signIns = append(signIns, registerUsername(read.payload))
}
- connectedAs := caller
- if refusal.forgets {
- connectedAs = "retry"
+ require.Equal(t, []string{"iggy", caller, caller, "retry"}, signIns)
+ })
+ }
+}
+
+// The server refuses the sign-in of a recovery for a reason other than its
+// credentials. The session stays owed, so the calls that wait for the recovery
+// keep waiting and end with 58 with their context, and the recovery signs in
+// again as the caller once the server accepts it.
+func TestSessionRecovery_ASignInRefusedForAnotherReasonKeepsTheSessionOwed(t *testing.T) {
+ // The connect signs in as iggy and the caller as other, so the recovery
+ // sends the third sign-in.
+ const recoverySignIn = 3
+ newerProtocol := vsr.ProtocolVersion + 1
+ for _, refusal := range []struct {
+ name string
+ answer []byte
+ cause error
+ }{
+ {"evicted_incompatible_protocol", evictionFrame(vsr.EvictionIncompatibleProtocol, newerProtocol, newerProtocol),
+ ierror.ErrIncompatibleProtocolVersion},
+ {"evicted_release_too_low", evictionFrame(vsr.EvictionClientReleaseTooLow, 0, 0), ierror.ErrInvalidCommand},
+ {"status_stale_client", statusReplyFrame(vsr.OperationRegister, uint32(ierror.StaleClientCode), nil), ierror.ErrStaleClient},
+ } {
+ t.Run(refusal.name, func(t *testing.T) {
+ var refuse atomic.Bool
+ refuse.Store(true)
+ server, _ := lostSessionServer(t, func(number int, _ request) []byte {
+ if number >= recoverySignIn && refuse.Load() {
+ // Each answer goes out on its own link and gets its own request id.
+ return slices.Clone(refusal.answer)
+ }
+ return registerReplyFrame(7, managedLinkSession+uint64(number))
+ })
+ client := newDialingClient(t, server.address(), WithAutoLogin(NewUsernamePasswordCredentials("iggy", "iggy")))
+ logged := &errorLog{}
+ client.logger = slog.New(logged)
+ ctx, cancel := context.WithTimeout(context.Background(), managedLinkTestBudget)
+ defer cancel()
+ require.NoError(t, client.Connect(ctx))
+ _, err := client.LoginUser(ctx, "other", "password")
+ require.NoError(t, err)
+ epoch := client.epoch.Load()
+ release := holdRecovery(t, client)
+ recoverNow(client)
+ // Ending the wait stands in for the deadline of the calls.
+ wait, endWait := context.WithCancel(ctx)
+ defer endWait()
+ callCtx, endCalls := client.operationContext(wait)
+ defer endCalls()
+ created, pinged := make(chan error, 1), make(chan error, 1)
+ go func() {
+ _, err := client.do(callCtx, &command.CreateStream{Name: "waiting"})
+ created <- err
+ }()
+ go func() { pinged <- client.Ping(callCtx) }()
+
+ release()
+ require.Eventually(t, func() bool { return logged.holds(refusal.cause) }, managedLinkTestBudget, managedLinkPoll,
+ "the log keeps the refusal")
+ require.Never(t, func() bool { return len(created) != 0 || len(pinged) != 0 }, recoveryWaitDeadline, managedLinkPoll,
+ "the calls still wait for the owed session")
+ endWait()
+ for _, call := range []chan error{created, pinged} {
+ select {
+ case err := <-call:
+ require.ErrorIs(t, err, ierror.ErrTransientNotAccepted)
+ case <-ctx.Done():
+ t.Fatal("a call still waits")
+ }
}
+ require.Same(t, epoch, client.epoch.Load(), "the refusal keeps the epoch of the caller")
+ require.Empty(t, requestsFor(server.recorded(), vsr.OperationCreateStream), "the create never went out")
+
+ refuse.Store(false)
+ require.Eventually(t, func() bool { return sessionBound(client) }, managedLinkTestBudget, managedLinkPoll,
+ "the recovery signs in again")
+ require.NoError(t, client.Ping(context.Background()))
var signIns []string
for _, read := range registerRequests(server) {
signIns = append(signIns, registerUsername(read.payload))
}
- require.Equal(t, []string{"iggy", caller, caller, connectedAs}, signIns)
+ require.Greater(t, len(signIns), recoverySignIn, "the recovery signs in again after the refusal")
+ require.Equal(t, "iggy", signIns[0])
+ for _, signIn := range signIns[1:] {
+ require.Equal(t, "other", signIn, "the recovery signs in as the caller")
+ }
})
}
}
diff --git a/foreign/go/client/tcp/tcp_session_management.go b/foreign/go/client/tcp/tcp_session_management.go
index 8fc44c951..d8bea8751 100644
--- a/foreign/go/client/tcp/tcp_session_management.go
+++ b/foreign/go/client/tcp/tcp_session_management.go
@@ -106,6 +106,9 @@ func (c *IggyTcpClient) register(ctx context.Context, code command.Code, credent
err = ended
} else if callerCtx != nil && callerCtx.Err() != nil && unanswered(err) {
err = callerCtx.Err()
+ } else if callerCtx != nil && callerCtx.Err() == nil && (errors.Is(err, context.DeadlineExceeded) ||
+ errors.Is(err, ierror.ErrTransientNotCommitted) && errors.Is(ctx.Err(), context.DeadlineExceeded)) {
+ err = ierror.ErrTransientNotAccepted
}
c.mtx.Lock()
var stopped func(bool)
@@ -346,7 +349,7 @@ func (c *IggyTcpClient) establishSession(ctx context.Context, link *tcpLink) err
var refusal *signInRefusal
if errors.As(err, &refusal) {
err = refusal.err
- retry, ended := c.endRefusedSignIn(epoch, credentials, err)
+ retry, refused := c.endRefusedSignIn(epoch, credentials, err)
if retry {
link.close(err)
link, err = c.links.connect(ctx)
@@ -355,8 +358,8 @@ func (c *IggyTcpClient) establishSession(ctx context.Context, link *tcpLink) err
}
continue
}
- if ended {
- // Connect returns its own refusal, not the 40 of the epoch it ended.
+ if refused {
+ // Connect returns its own refusal, also when it ended the epoch.
link.close(err)
return refusal
}
@@ -463,7 +466,7 @@ func (c *IggyTcpClient) invalidateSessionLocked() {
}
// signInEpoch lasts from one sign-in, logout or disconnect of the caller, or
-// one terminal automatic sign-in refusal, to the next. A call keeps its
+// one refusal of the credentials of an automatic sign-in, to the next. A call keeps its
// starting epoch, so only the SDK's own recovery continues it, and no call goes out
// under a later sign-in. The client replaces its epoch only under mtx, and a
// call reads it without the lock as it starts.
@@ -529,13 +532,16 @@ func (c *IggyTcpClient) recoverSession(expected uint64) {
}
// The session loss stopped every link and the heartbeat, so only
// this task brings the client back. A sign-in that the server never
- // answered goes again, until an answer, a sign-in by the caller or a
- // disconnect ends the recovery.
+ // answered, or refused for a reason other than its credentials, goes
+ // again, until a session, a refusal of the credentials, a sign-in by
+ // the caller or a disconnect ends the recovery.
for retry := uint32(0); ; retry++ {
signInCtx, endSignIn := context.WithTimeout(ctx, responseReadTimeout)
- err := c.Connect(signInCtx)
+ err := c.connect(signInCtx)
endSignIn()
- if err == nil || !c.config.reconnection.enabled || !unanswered(err) ||
+ var refusal *signInRefusal
+ owed := unanswered(err) || errors.As(err, &refusal) && !credentialRefused(refusal.err)
+ if err == nil || !c.config.reconnection.enabled || !owed ||
c.pause(ctx, reconnectDelay(c.config.reconnection.interval, retry)) != nil {
return
}
@@ -552,25 +558,27 @@ func (r *signInRefusal) Error() string { return r.err.Error() }
func (r *signInRefusal) Unwrap() error { return r.err }
// endRefusedSignIn retries only credentials updated by this same owner.
-// A refusal of its current credentials blocks fallback until an explicit login.
-// ended reports whether the refusal ended the epoch. It is false when a newer
-// owner ended the epoch first.
-func (c *IggyTcpClient) endRefusedSignIn(epoch *signInEpoch, attempted Credentials, refusal error) (retry, ended bool) {
+// A refusal of its current credentials ends the epoch and blocks fallback until
+// an explicit login. Any other refusal keeps the epoch, so the session stays
+// owed. refused reports whether the refusal answers the current epoch. It is
+// false when a newer owner ended the epoch first.
+func (c *IggyTcpClient) endRefusedSignIn(epoch *signInEpoch, attempted Credentials, refusal error) (retry, refused bool) {
c.logger.Error("The server refused the automatic sign-in", slog.Any("error", refusal))
c.mtx.Lock()
defer c.mtx.Unlock()
if c.epoch.Load() != epoch {
return false, false
}
- if credentialRefused(refusal) {
- current, enabled := c.signInCredentialsLocked()
- if enabled && current != attempted {
- return true, false
- }
- c.rememberedLogin = AutoLogin{}
- c.autoLoginBlocked = true
- c.loggedOut = true
+ if !credentialRefused(refusal) {
+ return false, true
}
+ current, enabled := c.signInCredentialsLocked()
+ if enabled && current != attempted {
+ return true, false
+ }
+ c.rememberedLogin = AutoLogin{}
+ c.autoLoginBlocked = true
+ c.loggedOut = true
c.endSignInEpochLocked(refusal)
return false, true
}
diff --git a/foreign/go/errors/errors.yaml b/foreign/go/errors/errors.yaml
index efecaa624..677336c97 100644
--- a/foreign/go/errors/errors.yaml
+++ b/foreign/go/errors/errors.yaml
@@ -237,7 +237,7 @@
fields: []
- name: TransientNotCommitted
code: 57
- format: "request transiently not committed; retry"
+ format: "request outcome unknown; it may have applied"
fields: []
- name: TransientNotAccepted
code: 58
@@ -1161,10 +1161,8 @@
type: uint32
- name: ConsumerGroupPartitionNotOwned
code: 5009
- format: "consumer group member with VSR client ID: %d does not own partition: %d at the current generation (rebalance in progress)."
+ format: "consumer group member does not own partition: %d at the current generation (rebalance in progress)."
fields:
- - name: ClientId
- type: uint32
- name: PartitionId
type: uint32
- name: TooManyConsumerGroups
diff --git a/foreign/go/errors/errors_gen.go b/foreign/go/errors/errors_gen.go
index 23bee9486..d400e7c62 100644
--- a/foreign/go/errors/errors_gen.go
+++ b/foreign/go/errors/errors_gen.go
@@ -496,7 +496,7 @@ func (e InvalidPersonalAccessTokenExpiry) Is(target error) bool {
type TransientNotCommitted struct{}
-func (e TransientNotCommitted) Error() string { return "request transiently not committed; retry" }
+func (e TransientNotCommitted) Error() string { return "request outcome unknown; it may have applied" }
func (e TransientNotCommitted) Code() Code { return 57 }
func (e TransientNotCommitted) Is(target error) bool {
_, ok := target.(TransientNotCommitted)
@@ -2378,12 +2378,11 @@ func (e CannotDeleteConsumerGroupInfo) Is(target error) bool {
}
type ConsumerGroupPartitionNotOwned struct {
- ClientId uint32
PartitionId uint32
}
func (e ConsumerGroupPartitionNotOwned) Error() string {
- return fmt.Sprintf("consumer group member with VSR client ID: %d does not own partition: %d at the current generation (rebalance in progress).", e.ClientId, e.PartitionId)
+ return fmt.Sprintf("consumer group member does not own partition: %d at the current generation (rebalance in progress).", e.PartitionId)
}
func (e ConsumerGroupPartitionNotOwned) Code() Code { return 5009 }
func (e ConsumerGroupPartitionNotOwned) Is(target error) bool {
diff --git a/foreign/go/errors/errors_test.go b/foreign/go/errors/errors_test.go
index 21604a0b4..b63594239 100644
--- a/foreign/go/errors/errors_test.go
+++ b/foreign/go/errors/errors_test.go
@@ -56,7 +56,7 @@ func TestIggyError_ConsensusErrors(t *testing.T) {
err: TransientNotCommitted{},
sentinel: ErrTransientNotCommitted,
code: TransientNotCommittedCode,
- message: "request transiently not committed; retry",
+ message: "request outcome unknown; it may have applied",
},
{
err: TransientNotAccepted{},
@@ -77,10 +77,10 @@ func TestIggyError_ConsensusErrors(t *testing.T) {
message: "partition history changed after the request was resolved",
},
{
- err: ConsumerGroupPartitionNotOwned{ClientId: 4, PartitionId: 9},
+ err: ConsumerGroupPartitionNotOwned{PartitionId: 9},
sentinel: ErrConsumerGroupPartitionNotOwned,
code: ConsumerGroupPartitionNotOwnedCode,
- message: "consumer group member with VSR client ID: 4 does not own partition: 9 at the current generation (rebalance in progress).",
+ message: "consumer group member does not own partition: 9 at the current generation (rebalance in progress).",
},
{
err: TooManyConsumerGroups{},
diff --git a/foreign/java/README.md b/foreign/java/README.md
index d0dd3334c..dfb137d0c 100644
--- a/foreign/java/README.md
+++ b/foreign/java/README.md
@@ -115,6 +115,8 @@ automatic sign-in. If it fails, the client does not sign in again as the
previous user. While the explicit sign-in runs, a call that finds no signed-in
connection ends with 40 (`UNAUTHENTICATED`) at once.
+After a logout, a refused automatic sign-in or a failed explicit sign-in, a call that needs a session fails with 40 (`UNAUTHENTICATED`) until the caller signs in again. Before the first sign-in, such a call fails with `IggyNotConnectedException`.
+
A committed password change or rename of the signed-in user updates the
credentials of the automatic sign-in. It also updates the builder credentials
that `login()` uses when they name that user. This holds for typed calls,
@@ -341,6 +343,8 @@ stale-client eviction or a malformed reply, the call replays the same request on
connection within its deadline. If the server refuses to bind the session again, the client
registers a new session and drops the operation.
+A call that loses its connection before it sends its request, or that cannot open one, waits for a connection within its deadline. If no connection carries it by then, the call ends with 58 (`TRANSIENT_NOT_ACCEPTED`).
+
An eviction and a reply can carry the same status, for example 30 (`STALE_CLIENT`) or 40
(`UNAUTHENTICATED`). Only the eviction is a lost connection, so the call does not replay a
request that a reply refused. A refused first attempt ends with that error. After an earlier
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java
index 245d99c4d..2fb058533 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java
@@ -66,6 +66,7 @@ import java.util.concurrent.CompletionException;
import java.util.concurrent.Executor;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
@@ -1316,7 +1317,11 @@ public class AsyncIggyTcpClient {
}
previous.whenComplete((prior, priorError) -> gate.complete(null));
if (error != null) {
- callerFuture.completeExceptionally(error);
+ Throwable failure = unwrap(error);
+ callerFuture.completeExceptionally(
+ explicit && (failure instanceof IggyTimeoutException || failure instanceof TimeoutException)
+ ? AsyncTcpConnection.notAccepted()
+ : error);
} else {
callerFuture.complete(identity);
}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java
index ce7a4213c..bfdfb2452 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java
@@ -42,7 +42,6 @@ import io.netty.util.concurrent.DefaultThreadFactory;
import io.netty.util.concurrent.Future;
import io.netty.util.concurrent.FutureListener;
import io.netty.util.concurrent.GlobalEventExecutor;
-import io.netty.util.concurrent.ScheduledFuture;
import org.apache.iggy.client.ConnectionInfo;
import org.apache.iggy.client.async.tcp.vsr.ConsensusSession;
import org.apache.iggy.client.async.tcp.vsr.VsrFrameDecoder;
@@ -144,13 +143,9 @@ public class AsyncTcpConnection {
private final IntConsumer sessionResetListener;
private final BiConsumer<AsyncTcpConnection, Throwable> connectionFailureListener;
private final long requestTimeoutNanos;
- private final long heartbeatIntervalNanos;
- private final Object heartbeatLock = new Object();
+ private final ConnectionHeartbeat heartbeat;
// Shared with the connections that bind this session, see bindSharedSession.
private volatile SessionLogin login = new SessionLogin();
- private ScheduledFuture<?> heartbeatTask;
- private boolean heartbeatRunning;
- private volatile Runnable heartbeatListener = () -> {};
private BooleanSupplier signInOwed = () -> false;
private volatile CredentialsListener credentialsListener = (source, previousUsername, username, password) -> {};
private volatile LongConsumer signInRefusedListener = lineage -> {};
@@ -202,7 +197,6 @@ public class AsyncTcpConnection {
this.sessionResetListener = sessionResetListener;
this.connectionFailureListener = connectionFailureListener;
this.requestTimeoutNanos = toTimeoutNanos(requestTimeout.orElse(DEFAULT_REQUEST_TIMEOUT));
- this.heartbeatIntervalNanos = toTimeoutNanos(heartbeatInterval);
SslContext sslContext = null;
if (enableTls) {
try {
@@ -219,6 +213,8 @@ public class AsyncTcpConnection {
this.ownsEventLoopGroup = sharedEventLoopGroup.isEmpty();
this.eventLoopGroup = sharedEventLoopGroup.orElseGet(() -> newEventLoopGroup(ioThreads));
this.eventLoop = eventLoopGroup.next();
+ this.heartbeat = new ConnectionHeartbeat(
+ eventLoop, toTimeoutNanos(heartbeatInterval), this::sendHeartbeat, isClosed::get);
long dialTimeoutMillis =
connectionTimeout.orElse(DEFAULT_CONNECTION_TIMEOUT).toMillis();
@@ -262,7 +258,7 @@ public class AsyncTcpConnection {
if (f.isSuccess()) {
channelPool.release(f.getNow()).addListener(release -> {
if (release.isSuccess()) {
- startHeartbeat();
+ heartbeat.start();
future.complete(null);
} else {
future.completeExceptionally(release.cause());
@@ -280,65 +276,16 @@ public class AsyncTcpConnection {
return future;
}
- private void startHeartbeat() {
- synchronized (heartbeatLock) {
- if (heartbeatRunning || isClosed.get()) {
- return;
- }
- heartbeatRunning = true;
- scheduleNextHeartbeat();
- }
- }
-
- private void scheduleNextHeartbeat() {
- synchronized (heartbeatLock) {
- if (!heartbeatRunning || isClosed.get()) {
- return;
- }
- try {
- heartbeatTask = eventLoop.schedule(this::sendHeartbeat, heartbeatIntervalNanos, TimeUnit.NANOSECONDS);
- } catch (RejectedExecutionException loopGone) {
- // Only a caller-owned group shuts down under a live connection.
- heartbeatRunning = false;
- log.warn("Event loop rejected the heartbeat, stopping it: {}", loopGone.getMessage());
- }
- }
- }
-
- private void sendHeartbeat() {
- synchronized (heartbeatLock) {
- heartbeatTask = null;
- if (!heartbeatRunning || isClosed.get()) {
- return;
- }
- }
- CompletableFuture<ByteBuf> heartbeat;
- try {
- TransientFailoverState state = new TransientFailoverState();
- state.heartbeat = true;
- heartbeat = send(CommandCode.System.PING.getValue(), Unpooled.EMPTY_BUFFER, 0, state);
- } catch (RuntimeException error) {
- log.warn("Failed to send heartbeat: {}", error.getMessage());
- scheduleNextHeartbeat();
- return;
- }
- heartbeat.whenComplete((response, error) -> {
- if (response != null) {
- response.release();
- }
- if (error != null && !isClosed.get()) {
- log.warn("Heartbeat failed: {}", error.getMessage());
- }
- scheduleNextHeartbeat();
- if (error == null) {
- heartbeatListener.run();
- }
- });
+ /** Sends a ping whose timed-out reply counts as a lost link, see completeRequest. */
+ private CompletableFuture<ByteBuf> sendHeartbeat() {
+ TransientFailoverState state = new TransientFailoverState();
+ state.heartbeat = true;
+ return send(CommandCode.System.PING.getValue(), Unpooled.EMPTY_BUFFER, 0, state);
}
/** Runs after each heartbeat that the server answered. */
void onHeartbeat(Runnable listener, BooleanSupplier owed) {
- heartbeatListener = listener;
+ heartbeat.onAnswered(listener);
signInOwed = owed;
}
@@ -402,20 +349,8 @@ public class AsyncTcpConnection {
signInRefusedListener = listener;
}
- private void stopHeartbeat() {
- synchronized (heartbeatLock) {
- heartbeatRunning = false;
- if (heartbeatTask != null) {
- heartbeatTask.cancel(false);
- heartbeatTask = null;
- }
- }
- }
-
boolean heartbeatScheduled() {
- synchronized (heartbeatLock) {
- return heartbeatTask != null;
- }
+ return heartbeat.scheduled();
}
EventLoop eventLoop() {
@@ -634,7 +569,7 @@ public class AsyncTcpConnection {
payload.release();
notifyConnectionFailure(f.cause());
if (settles) {
- call.fail(mapAcquireException(f.cause()));
+ completeFailedRequest(call, mapAcquireException(f.cause()));
}
} else if (!settles || call.callerFuture().isDone()) {
payload.release();
@@ -810,42 +745,41 @@ public class AsyncTcpConnection {
channel.close();
notifyConnectionFailure(new IggyConnectionException("The heartbeat reply timed out", cause));
}
- completeFailedRequest(commandCode, failoverPayload, requestDeadlineNanos, callerFuture, failoverState, error);
+ if (cause instanceof IggyConnectionException || cause instanceof java.io.IOException) {
+ notifyConnectionFailure(cause);
+ }
+ completeFailedRequest(
+ new Call(commandCode, failoverPayload, requestDeadlineNanos, callerFuture, failoverState), error);
}
/**
- * Retries unsent calls after session recovery and transient refusals after a leader recheck.
- * Routed calls and terminal failures return to their caller.
+ * Retries unsent calls after session recovery, and transient refusals and links lost before the frame went out
+ * after a leader recheck, as in Rust. Routed calls and terminal failures return to their caller.
*/
@SuppressWarnings({"checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity"})
- private void completeFailedRequest(
- int commandCode,
- ByteBuf failoverPayload,
- long requestDeadlineNanos,
- CompletableFuture<ByteBuf> callerFuture,
- TransientFailoverState failoverState,
- Throwable error) {
+ private void completeFailedRequest(Call call, Throwable error) {
Throwable cause = unwrap(error);
- if (cause instanceof IggyConnectionException || cause instanceof java.io.IOException) {
- notifyConnectionFailure(cause);
- }
+ TransientFailoverState failoverState = call.state();
if (failoverState.routed) {
- releaseIfPresent(failoverPayload);
- callerFuture.completeExceptionally(failoverState.linkFailed ? new LinkFailure(error) : error);
+ releaseIfPresent(call.failoverPayload());
+ call.callerFuture().completeExceptionally(failoverState.linkFailed ? new LinkFailure(error) : error);
return;
}
- Call call = new Call(commandCode, failoverPayload, requestDeadlineNanos, callerFuture, failoverState);
IggyServerException serverError = findServerError(error);
boolean notPrimaryInARow = failoverState.notPrimary;
failoverState.notPrimary = serverError != null && serverError.getRawErrorCode() == NOT_PRIMARY;
- if (failoverPayload == null || call.over()) {
+ // Only a link that failed a registered frame can have carried it.
+ boolean unsentLoss = lostLink(failoverState, cause) && !failoverState.linkFailed;
+ if (call.failoverPayload() == null || call.over()) {
call.end(error);
} else if (failoverState.ticket != null && failoverState.ticket.takeRestart()) {
restartUnderSession(call);
- } else if (!isRefusal(serverError) && !(failoverState.uncertain && lostLink(failoverState, cause))) {
+ } else if (!isRefusal(serverError)
+ && !unsentLoss
+ && !(failoverState.uncertain && lostLink(failoverState, cause))) {
call.end(error);
} else {
- failoverState.refused |= isRefusal(serverError);
+ failoverState.refused |= isRefusal(serverError) || unsentLoss;
if (!failoverState.notPrimary || notPrimaryInARow) {
pauseBeforeRecheck(call, cause);
} else {
@@ -1069,7 +1003,7 @@ public class AsyncTcpConnection {
Channel channel, long requestDeadlineNanos, TransientFailoverState callerState) {
return authenticationSnapshot()
.map(current -> sendAuthenticationFrame(channel, current, requestDeadlineNanos, callerState))
- .orElseGet(() -> CompletableFuture.failedFuture(new IggyNotConnectedException(NOT_AUTHENTICATED)));
+ .orElseGet(() -> CompletableFuture.failedFuture(IggyAuthenticator.notSignedIn(this)));
}
/** Lets the client redial after a pool acquire failure or expired reply. */
@@ -1365,7 +1299,10 @@ public class AsyncTcpConnection {
&& exchange.failoverState().ticket != null));
exchange.failoverState().uncertain = nextUncertain;
exchange.failoverState().everUncertain |= wasUncertain || nextUncertain;
- if (exchange.failoverState().ticket != null && (isRefusal(serverError) || PollRouter.isNamespaceMiss(error))) {
+ if (exchange.failoverState().ticket != null
+ && (isRefusal(serverError)
+ || (serverError != null && serverError.getRawErrorCode() == LIFECYCLE_BUSY)
+ || PollRouter.isNamespaceMiss(error))) {
exchange.failoverState().ticket.settleSent(nextUncertain);
}
if (shouldRetryTransient(
@@ -1704,7 +1641,7 @@ public class AsyncTcpConnection {
if (!isClosed.compareAndSet(false, true)) {
return CompletableFuture.completedFuture(null);
}
- stopHeartbeat();
+ heartbeat.stop();
if (consensusSession.owner() == consensusSession) {
consensusSession.reset();
}
@@ -1766,8 +1703,9 @@ public class AsyncTcpConnection {
/**
* Ends the call with {@code error}, or with its uncertainty when an attempt can have taken effect. A call
- * that a 58 or 89 refusal sent to another attempt and that its deadline ends without another answer reports
- * 58, as the deadline of the Rust driver reports the refusal of a call that no node admitted.
+ * that a 58 or 89 refusal or a link lost before its frame went out sent to another attempt, and that its
+ * deadline ends without another answer, reports 58, as the deadline of the Rust driver reports the refusal
+ * of a call that no node admitted.
*/
void end(Throwable error) {
releaseIfPresent(failoverPayload);
@@ -1814,7 +1752,7 @@ public class AsyncTcpConnection {
private boolean everUncertain;
// The last attempt got NOT_PRIMARY, so another one in a row pauses first, see pauseBeforeRecheck.
private boolean notPrimary;
- // A 58 or 89 refusal sent the call to another attempt, see Call#end.
+ // A 58 or 89 refusal or a link lost before the frame went out sent the call to another attempt, see Call#end.
private boolean refused;
private long pauseCeilingNanos = RETRY_BACKOFF_BASE.toNanos();
private ConsensusSession.LaneKey lane;
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConnectionHeartbeat.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConnectionHeartbeat.java
new file mode 100644
index 000000000..ee1b42e0d
--- /dev/null
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConnectionHeartbeat.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iggy.client.async.tcp;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.EventLoop;
+import io.netty.util.concurrent.ScheduledFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+import java.util.function.Supplier;
+
+/** Pings a connection at a fixed interval on its event loop until the connection closes. */
+final class ConnectionHeartbeat {
+ private static final Logger log = LoggerFactory.getLogger(ConnectionHeartbeat.class);
+
+ private final EventLoop eventLoop;
+ private final long intervalNanos;
+ private final Supplier<CompletableFuture<ByteBuf>> ping;
+ private final BooleanSupplier closed;
+ private final Object lock = new Object();
+ private ScheduledFuture<?> task;
+ private boolean running;
+ private volatile Runnable listener = () -> {};
+
+ ConnectionHeartbeat(
+ EventLoop eventLoop,
+ long intervalNanos,
+ Supplier<CompletableFuture<ByteBuf>> ping,
+ BooleanSupplier closed) {
+ this.eventLoop = eventLoop;
+ this.intervalNanos = intervalNanos;
+ this.ping = ping;
+ this.closed = closed;
+ }
+
+ void start() {
+ synchronized (lock) {
+ if (running || closed.getAsBoolean()) {
+ return;
+ }
+ running = true;
+ scheduleNext();
+ }
+ }
+
+ private void scheduleNext() {
+ synchronized (lock) {
+ if (!running || closed.getAsBoolean()) {
+ return;
+ }
+ try {
+ task = eventLoop.schedule(this::send, intervalNanos, TimeUnit.NANOSECONDS);
+ } catch (RejectedExecutionException loopGone) {
+ // Only a caller-owned group shuts down under a live connection.
+ running = false;
+ log.warn("Event loop rejected the heartbeat, stopping it: {}", loopGone.getMessage());
+ }
+ }
+ }
+
+ private void send() {
+ synchronized (lock) {
+ task = null;
+ if (!running || closed.getAsBoolean()) {
+ return;
+ }
+ }
+ CompletableFuture<ByteBuf> heartbeat;
+ try {
+ heartbeat = ping.get();
+ } catch (RuntimeException error) {
+ log.warn("Failed to send heartbeat: {}", error.getMessage());
+ scheduleNext();
+ return;
+ }
+ heartbeat.whenComplete((response, error) -> {
+ if (response != null) {
+ response.release();
+ }
+ if (error != null && !closed.getAsBoolean()) {
+ log.warn("Heartbeat failed: {}", error.getMessage());
+ }
+ scheduleNext();
+ if (error == null) {
+ listener.run();
+ }
+ });
+ }
+
+ /** Runs after each heartbeat that the server answered. */
+ void onAnswered(Runnable answered) {
+ listener = answered;
+ }
+
+ void stop() {
+ synchronized (lock) {
+ running = false;
+ if (task != null) {
+ task.cancel(false);
+ task = null;
+ }
+ }
+ }
+
+ boolean scheduled() {
+ synchronized (lock) {
+ return task != null;
+ }
+ }
+}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java
index b2091cbd0..d976b7f18 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java
@@ -118,7 +118,7 @@ final class IggyAuthenticator {
return CompletableFuture.failedFuture(new IggyServerException(IggyErrorCode.UNAUTHENTICATED.getCode()));
}
if (!signInOwed.getAsBoolean()) {
- return CompletableFuture.failedFuture(new IggyNotConnectedException(AsyncTcpConnection.NOT_AUTHENTICATED));
+ return CompletableFuture.failedFuture(notSignedIn(connection));
}
// Register binds the session before publishing the connection's authentication state.
return connection
@@ -134,7 +134,7 @@ final class IggyAuthenticator {
AsyncTcpConnection.TransientFailoverState state) {
var authentication = connection.authenticationSnapshot();
if (authentication.isEmpty()) {
- return CompletableFuture.failedFuture(new IggyNotConnectedException(AsyncTcpConnection.NOT_AUTHENTICATED));
+ return CompletableFuture.failedFuture(notSignedIn(connection));
}
var snapshot = authentication.get();
return ensureAuthenticated(
@@ -144,6 +144,17 @@ final class IggyAuthenticator {
ignored -> connection.sendAuthenticationFrame(channel, snapshot, deadline, state));
}
+ /**
+ * The error of a call that needs a session while no sign-in can bring one: 40 once the caller began a sign-in,
+ * as in Rust, so after a logout, a refused automatic sign-in or a failed sign-in. Before that, the caller has to
+ * sign in first.
+ */
+ static RuntimeException notSignedIn(AsyncTcpConnection connection) {
+ return connection.sessionOwner().beganSignIn()
+ ? new IggyServerException(IggyErrorCode.UNAUTHENTICATED.getCode())
+ : new IggyNotConnectedException(AsyncTcpConnection.NOT_AUTHENTICATED);
+ }
+
static void setAuthGeneration(Channel channel, long generation) {
channel.attr(AUTH_GENERATION_KEY).set(generation);
}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java
index 73d02a887..ce6ab8261 100644
--- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java
+++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java
@@ -69,6 +69,8 @@ public final class ConsensusSession {
private long lineage;
// The lineage in which a sign-in of the caller runs, see signingIn().
private long signInLineage = NO_SIGN_IN;
+ // See beganSignIn().
+ private boolean signInBegun;
// The automatic sign-in that the server refused, and the lineage that the refusal ended.
private Throwable signInRefusal;
private long refusedLineage;
@@ -353,10 +355,22 @@ public final class ConsensusSession {
ConsensusSession owner = owner();
synchronized (owner) {
owner.signInLineage = ++owner.lineage;
+ owner.signInBegun = true;
}
owner.changed();
}
+ /**
+ * Whether the caller ever began a sign-in. From then on, a call that finds no session and none on its way ends
+ * with 40, as in Rust, instead of asking for a first sign-in.
+ */
+ public boolean beganSignIn() {
+ ConsensusSession owner = owner();
+ synchronized (owner) {
+ return owner.signInBegun;
+ }
+ }
+
public void finishSignIn() {
ConsensusSession owner = owner();
synchronized (owner) {
@@ -904,9 +918,9 @@ public final class ConsensusSession {
}
/**
- * After a 58 or 89 refusal or a namespace miss, counts the frame as sent only while an earlier attempt can
- * have taken effect, as in Rust. So a call that no node admitted starts again under a new session instead of
- * ending uncertain.
+ * After a 58 or 89 refusal, an 88 lifecycle refusal or a namespace miss, counts the frame as sent only while
+ * an earlier attempt can have taken effect, as in Rust. So a call that no node admitted starts again under a
+ * new session instead of ending uncertain.
*/
public void settleSent(boolean uncertain) {
synchronized (owner) {
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java
index ffc5dbe1e..938c7e2a5 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java
@@ -21,7 +21,9 @@ package org.apache.iggy.client.async;
import org.apache.iggy.client.BaseIntegrationTest;
import org.apache.iggy.client.async.tcp.AsyncIggyTcpClient;
+import org.apache.iggy.exception.IggyErrorCode;
import org.apache.iggy.exception.IggyNotConnectedException;
+import org.apache.iggy.exception.IggyServerException;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.message.Message;
import org.apache.iggy.message.Partitioning;
@@ -129,10 +131,8 @@ class AsyncConnectionPoolAuthTest extends BaseIntegrationTest {
client.users().logout().get(5, TimeUnit.SECONDS);
log.info("Logout successful");
- // then - commands should fail after logout
- assertThatThrownBy(() -> client.streams().getStreams().get(5, TimeUnit.SECONDS))
- .isInstanceOf(ExecutionException.class)
- .hasCauseInstanceOf(IggyNotConnectedException.class);
+ // then - commands should fail with 40 after logout
+ assertUnauthenticated(client.streams().getStreams());
log.info("Commands correctly rejected after logout");
// when - re-login
@@ -261,10 +261,18 @@ class AsyncConnectionPoolAuthTest extends BaseIntegrationTest {
client.users().logout().get(5, TimeUnit.SECONDS);
// then - verify commands are rejected
- assertThatThrownBy(() -> client.streams().getStreams().get(5, TimeUnit.SECONDS))
- .isInstanceOf(ExecutionException.class)
- .hasCauseInstanceOf(IggyNotConnectedException.class);
+ assertUnauthenticated(client.streams().getStreams());
log.info("Cycle {}: logout and rejection verified", i + 1);
}
}
+
+ private static void assertUnauthenticated(CompletableFuture<?> call) {
+ assertThatThrownBy(() -> call.get(5, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class)
+ .cause()
+ .isInstanceOfSatisfying(
+ IggyServerException.class,
+ error ->
+ assertThat(error.getRawErrorCode()).isEqualTo(IggyErrorCode.UNAUTHENTICATED.getCode()));
+ }
}
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java
index a6755ed6c..1361b1ba0 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java
@@ -925,8 +925,8 @@ class AsyncIggyTcpClientEndpointFailoverTest {
node.rejectBindings = true;
lost.set(true);
node.dropConnections();
- assertThatThrownBy(() -> client.sendBinaryRequest(GET_STATS_CODE, new byte[0])
- .get(5, TimeUnit.SECONDS))
+ // A roster read never waits for a session, so it ends when the sign-in on the next link hangs up.
+ assertThatThrownBy(() -> client.system().getClusterMetadata().get(5, TimeUnit.SECONDS))
.isInstanceOf(ExecutionException.class);
held.get(RECOVERY.toMillis(), TimeUnit.MILLISECONDS);
node.rejectBindings = false;
@@ -1024,8 +1024,8 @@ class AsyncIggyTcpClientEndpointFailoverTest {
node.rejectBindings = true;
lost.set(true);
node.dropConnections();
- assertThatThrownBy(() -> client.sendBinaryRequest(GET_STATS_CODE, new byte[0])
- .get(5, TimeUnit.SECONDS))
+ // A roster read never waits for a session, so it ends when the sign-in on the next link hangs up.
+ assertThatThrownBy(() -> client.system().getClusterMetadata().get(5, TimeUnit.SECONDS))
.isInstanceOf(ExecutionException.class);
held.get(RECOVERY.toMillis(), TimeUnit.MILLISECONDS);
node.rejectBindings = false;
@@ -1598,7 +1598,8 @@ class AsyncIggyTcpClientEndpointFailoverTest {
/**
* Ends the session and returns the sign-ins that restore it: the one on the next link of the connection,
- * which hangs up, and the one that the redial replays.
+ * which hangs up, and the one that the redial replays. A roster read never waits for a session, so it ends
+ * when that sign-in hangs up.
*/
private static List<String> signInsAfterTheSessionIsLost(
AsyncIggyTcpClient client, MockNode node, AtomicBoolean hangUpOnRegister, List<String> signIns)
@@ -1607,8 +1608,7 @@ class AsyncIggyTcpClientEndpointFailoverTest {
node.rejectBindings = true;
hangUpOnRegister.set(true);
node.dropConnections();
- assertThatThrownBy(() ->
- client.sendBinaryRequest(GET_STATS_CODE, new byte[0]).get(5, TimeUnit.SECONDS))
+ assertThatThrownBy(() -> client.system().getClusterMetadata().get(5, TimeUnit.SECONDS))
.isInstanceOf(ExecutionException.class);
AsyncIggyTcpClientEventLoopGroupTest.await(() -> signIns.size() >= before + 2, "the redial sign-in", RECOVERY);
List<String> all = List.copyOf(signIns);
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientFailoverSignInTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientFailoverSignInTest.java
index b90315d5b..f1077ded5 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientFailoverSignInTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientFailoverSignInTest.java
@@ -256,10 +256,7 @@ class AsyncIggyTcpClientFailoverSignInTest {
AsyncIggyTcpClientSessionRulesTest.assertEndedByTheRefusal(first, refusal);
AsyncIggyTcpClientSessionRulesTest.assertEndedByTheRefusal(second, refusal);
- assertThatThrownBy(() -> client.sendBinaryRequest(GET_STATS_CODE, new byte[0])
- .get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS))
- .cause()
- .isInstanceOf(IggyNotConnectedException.class);
+ AsyncIggyTcpClientSessionRulesTest.assertSignedOut(client);
assertThat(survivorSignIns)
.as("the sign-ins with the refused token")
.hasValue(1);
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLostLinkReplayTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLostLinkReplayTest.java
index 9d7f91a36..5892ecf2a 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLostLinkReplayTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLostLinkReplayTest.java
@@ -25,6 +25,7 @@ import org.apache.iggy.client.ConnectionInfo;
import org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.Request;
import org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.Response;
import org.apache.iggy.exception.IggyErrorCode;
+import org.apache.iggy.serde.CommandCode;
import org.junit.jupiter.api.Test;
import java.net.InetAddress;
@@ -33,8 +34,10 @@ import java.time.Duration;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.BIND_SESSION_CODE;
@@ -49,6 +52,7 @@ import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailov
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.serve;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.threeNodeMetadata;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/**
* A request that replays after a lost link is not routed, so it belongs on the metadata primary. The replay tries
@@ -58,6 +62,7 @@ class AsyncIggyTcpClientLostLinkReplayTest {
private static final int TEST_TIMEOUT_SECONDS = 5;
private static final int BACKLOG = 4;
private static final int MAX_REPLAY_ROUNDS = 100;
+ private static final int PING_CODE = CommandCode.System.PING.getValue();
@Test
void shouldReplayALostMutationOnTheMetadataPrimaryBeforeTheNextRosterNode() throws Exception {
@@ -178,6 +183,106 @@ class AsyncIggyTcpClientLostLinkReplayTest {
}
}
+ /**
+ * The link of the client drops, and the next one drops during the bind that a mutation waits for. The mutation
+ * sent nothing, so it waits for another link within its deadline and goes out there once, as in Rust.
+ */
+ @Test
+ void shouldSendAnUnsentMutationOnTheNextLinkWhenItsLinkDropsDuringTheBind() throws Exception {
+ // The link of the sign-in, the link whose bind drops and the link that carries the mutation.
+ int links = 3;
+ try (ServerSocket socket = new ServerSocket(0, links, InetAddress.getLoopbackAddress())) {
+ AtomicInteger binds = new AtomicInteger();
+ List<Request> writes = new CopyOnWriteArrayList<>();
+ var serving = serve(socket, links, request -> {
+ if (request.is(BIND_SESSION_CODE, OPERATION_NON_REPLICATED)) {
+ return binds.incrementAndGet() == 1
+ ? Response.disconnect()
+ : Response.success(OPERATION_NON_REPLICATED, bindBody(request));
+ }
+ if (request.operation() == OPERATION_CREATE_STREAM) {
+ writes.add(request);
+ return Response.success(
+ OPERATION_CREATE_STREAM, Unpooled.buffer().writeIntLE(0));
+ }
+ return answerSignIn(request, socket);
+ });
+ AsyncIggyTcpClient client = client(socket);
+ try {
+ client.connect().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ client.login().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ dropTheLink(client);
+
+ client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]).get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ assertThat(binds).hasValue(2);
+ assertThat(writes).hasSize(1);
+ } finally {
+ client.close().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+ serving.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+ }
+
+ /** No link opens after the link of the client drops, so a mutation that sent nothing ends with 58 at its deadline. */
+ @Test
+ void shouldEndAnUnsentMutationWith58WhenNoLinkOpensBeforeItsDeadline() throws Exception {
+ try (ServerSocket socket = new ServerSocket(0, BACKLOG, InetAddress.getLoopbackAddress())) {
+ List<Request> writes = new CopyOnWriteArrayList<>();
+ var serving = serve(socket, request -> {
+ if (request.operation() == OPERATION_CREATE_STREAM) {
+ writes.add(request);
+ return Response.success(
+ OPERATION_CREATE_STREAM, Unpooled.buffer().writeIntLE(0));
+ }
+ return answerSignIn(request, socket);
+ });
+ AsyncIggyTcpClient client = AsyncIggyTcpClient.builder()
+ .host(socket.getInetAddress().getHostAddress())
+ .port(socket.getLocalPort())
+ .credentials("iggy", "iggy")
+ .requestTimeout(Duration.ofSeconds(1))
+ .heartbeatInterval(Duration.ofHours(1))
+ .build();
+ try {
+ client.connect().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ client.login().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ dropTheLink(client);
+ serving.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ socket.close();
+
+ AsyncIggyTcpClientTransientFailoverTest.assertServerError(
+ client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]),
+ IggyErrorCode.TRANSIENT_NOT_ACCEPTED.getCode());
+ assertThat(writes).isEmpty();
+ } finally {
+ client.close().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ /** Sends a ping that the node answers by dropping the link. The ping went out, so it fails at once. */
+ private static void dropTheLink(AsyncIggyTcpClient client) {
+ assertThatThrownBy(() ->
+ client.sendBinaryRequest(PING_CODE, new byte[0]).get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class);
+ }
+
+ private static Response answerSignIn(Request request, ServerSocket socket) {
+ if (request.operation() == OPERATION_REGISTER) {
+ return Response.success(OPERATION_REGISTER, registerBody(1));
+ }
+ if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) {
+ return Response.success(
+ OPERATION_NON_REPLICATED,
+ AsyncIggyTcpClientTransientFailoverTest.singleNodeMetadata(socket.getLocalPort()));
+ }
+ if (request.is(PING_CODE, OPERATION_NON_REPLICATED)) {
+ return Response.disconnect();
+ }
+ throw new IllegalStateException("Unexpected request " + request);
+ }
+
private static Response answerDataSlot(Request request, Supplier<ByteBuf> roster) {
if (request.is(BIND_SESSION_CODE, OPERATION_NON_REPLICATED)) {
return Response.success(OPERATION_NON_REPLICATED, bindBody(request));
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientSessionRulesTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientSessionRulesTest.java
index c8044fa8c..1b3ec6c8c 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientSessionRulesTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientSessionRulesTest.java
@@ -40,7 +40,6 @@ import org.apache.iggy.exception.IggyMissingCredentialsException;
import org.apache.iggy.exception.IggyNotConnectedException;
import org.apache.iggy.exception.IggyOperationException;
import org.apache.iggy.exception.IggyServerException;
-import org.apache.iggy.exception.IggyTimeoutException;
import org.apache.iggy.identifier.ConsumerId;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.identifier.TopicId;
@@ -68,7 +67,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -877,10 +875,7 @@ class AsyncIggyTcpClientSessionRulesTest {
assertEndedByTheRefusal(signingIn, refusal);
assertEndedByTheRefusal(waiting, refusal);
- assertThatThrownBy(() -> client.sendBinaryRequest(GET_STATS_CODE, new byte[0])
- .get(TIMEOUT_SECONDS, TimeUnit.SECONDS))
- .cause()
- .isInstanceOf(IggyNotConnectedException.class);
+ assertSignedOut(client);
assertThat(registers).as("the sign-ins with the token").hasValue(2);
} finally {
refuse.complete(null);
@@ -950,6 +945,52 @@ class AsyncIggyTcpClientSessionRulesTest {
}
}
+ /**
+ * Before its first sign-in, a call asks the caller to sign in. After a logout it fails with 40 until the caller
+ * signs in again, as in Rust.
+ */
+ @Test
+ void shouldEndCallsWith40AfterALogoutUntilTheCallerSignsInAgain() throws Exception {
+ try (ServerSocket socket = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) {
+ AtomicInteger registers = new AtomicInteger();
+ AtomicInteger reads = new AtomicInteger();
+ var server = serve(socket, request -> {
+ if (request.operation() == VsrOperation.REGISTER) {
+ return Response.success(
+ VsrOperation.REGISTER, registerBody(FIRST_SESSION + registers.getAndIncrement()));
+ }
+ if (request.operation() == VsrOperation.LOGOUT) {
+ return Response.success(VsrOperation.LOGOUT, Unpooled.EMPTY_BUFFER);
+ }
+ if (request.commandCode() == GET_STATS_CODE) {
+ reads.incrementAndGet();
+ return Response.success(VsrOperation.NON_REPLICATED, Unpooled.EMPTY_BUFFER);
+ }
+ return metadata(socket, request);
+ });
+ AsyncIggyTcpClient client = client(socket);
+ try {
+ client.connect().get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ assertThatThrownBy(() -> client.sendBinaryRequest(GET_STATS_CODE, new byte[0])
+ .get(TIMEOUT_SECONDS, TimeUnit.SECONDS))
+ .cause()
+ .isInstanceOf(IggyNotConnectedException.class);
+ client.users().login("alice", "alice-password").get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ client.users().logout().get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ assertSignedOut(client);
+ assertThat(reads).hasValue(0);
+
+ client.users().login("alice", "alice-password").get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ client.sendBinaryRequest(GET_STATS_CODE, new byte[0]).get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ assertThat(reads).hasValue(1);
+ } finally {
+ client.close().get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+ assertThat(server).isNotCompletedExceptionally();
+ }
+ }
+
@ParameterizedTest(name = "failure={0}")
@EnumSource(ExplicitFailure.class)
@SuppressWarnings({"checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity"})
@@ -1006,9 +1047,11 @@ class AsyncIggyTcpClientSessionRulesTest {
if (cause instanceof IggyServerException refusal) {
assertThat(refusal.getRawErrorCode())
.isEqualTo(
- failure == ExplicitFailure.LOGOUT_REFUSED
- ? IggyErrorCode.UNAUTHENTICATED.getCode()
- : IggyErrorCode.INVALID_CREDENTIALS.getCode());
+ switch (failure) {
+ case LOGOUT_REFUSED -> IggyErrorCode.UNAUTHENTICATED.getCode();
+ case REGISTER_REFUSED -> IggyErrorCode.INVALID_CREDENTIALS.getCode();
+ default -> IggyErrorCode.TRANSIENT_NOT_ACCEPTED.getCode();
+ });
}
assertSignedOut(client);
assertThat(reads).hasValue(0);
@@ -1057,8 +1100,7 @@ class AsyncIggyTcpClientSessionRulesTest {
switch (ending) {
case QUEUED_TIMEOUT -> {
var later = client.users().login("bob", "bob-password");
- assertThatThrownBy(() -> later.get(TIMEOUT_SECONDS, TimeUnit.SECONDS))
- .hasCauseInstanceOf(TimeoutException.class);
+ assertServerError(later, IggyErrorCode.TRANSIENT_NOT_ACCEPTED.getCode());
}
case CANCELLED -> assertThat(first.cancel(false)).isTrue();
default -> {
@@ -1079,11 +1121,10 @@ class AsyncIggyTcpClientSessionRulesTest {
}
}
- private static void assertSignedOut(AsyncIggyTcpClient client) {
- assertThatThrownBy(() ->
- client.sendBinaryRequest(GET_STATS_CODE, new byte[0]).get(TIMEOUT_SECONDS, TimeUnit.SECONDS))
- .cause()
- .isInstanceOf(IggyNotConnectedException.class);
+ /** A call fails with 40 until the caller signs in again, as in Rust. */
+ static void assertSignedOut(AsyncIggyTcpClient client) {
+ assertServerError(
+ client.sendBinaryRequest(GET_STATS_CODE, new byte[0]), IggyErrorCode.UNAUTHENTICATED.getCode());
}
private enum SettlementEnd {
@@ -1097,9 +1138,9 @@ class AsyncIggyTcpClientSessionRulesTest {
INVALID_TOKEN(IggyInvalidArgumentException.class),
MISSING_CREDENTIALS(IggyMissingCredentialsException.class),
LOGOUT_REFUSED(IggyServerException.class),
- LOGOUT_TIMEOUT(IggyTimeoutException.class),
+ LOGOUT_TIMEOUT(IggyServerException.class),
REGISTER_REFUSED(IggyServerException.class),
- REGISTER_TIMEOUT(IggyTimeoutException.class);
+ REGISTER_TIMEOUT(IggyServerException.class);
private final Class<? extends Throwable> errorType;
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionLifecycleBusyTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionLifecycleBusyTest.java
index e8a1ca3e2..dbeddc535 100644
--- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionLifecycleBusyTest.java
+++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionLifecycleBusyTest.java
@@ -29,6 +29,8 @@ import org.apache.iggy.exception.IggyErrorCode;
import org.apache.iggy.identifier.ConsumerId;
import org.apache.iggy.identifier.StreamId;
import org.apache.iggy.identifier.TopicId;
+import org.apache.iggy.message.Message;
+import org.apache.iggy.message.Partitioning;
import org.apache.iggy.partition.PartitionContext;
import org.apache.iggy.serde.CommandCode;
import org.junit.jupiter.api.Test;
@@ -46,12 +48,16 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.BIND_SESSION_CODE;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.GET_CLUSTER_METADATA_CODE;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.GET_PARTITION_ROUTE_CODE;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.OPERATION_NON_REPLICATED;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.OPERATION_REGISTER;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.assertServerError;
+import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.bindBody;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.client;
+import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.clusterMetadata;
+import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.pollRoute;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.registerBody;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.serve;
import static org.apache.iggy.client.async.tcp.AsyncIggyTcpClientTransientFailoverTest.singleNodeMetadata;
@@ -66,8 +72,10 @@ class AsyncTcpConnectionLifecycleBusyTest {
private static final int PING_CODE = 1;
private static final int LOGIN_CODE = 38;
private static final int POLL_CODE = CommandCode.Messages.POLL.getValue();
+ private static final int GET_STATS_CODE = CommandCode.System.GET_STATS.getValue();
private static final int JOIN_GROUP_CODE = 604;
private static final int OPERATION_JOIN_GROUP = 148;
+ private static final int OPERATION_SEND_MESSAGES = 160;
private static final int TEST_TIMEOUT_SECONDS = 5;
private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(1);
// Pauses doubling from 50 ms start attempts near 0, 50, 150, 350 and 750 ms
@@ -486,6 +494,100 @@ class AsyncTcpConnectionLifecycleBusyTest {
}
}
+ /**
+ * The partition primary refuses a send with 88, and the client loses its session before the retry: the metadata
+ * primary dropped the link after it routed the send and refuses the bind on the next link, so the client
+ * registers a new session. The refusal proved that nothing applied the send, so it goes out again under that
+ * session with a new request id and does not end uncertain.
+ */
+ @Test
+ @SuppressWarnings({"checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity"})
+ void shouldSendALifecycleRefusalAgainUnderTheNextSessionWhenTheSessionIsLostDuringThePause() throws Exception {
+ InetAddress loopback = InetAddress.getLoopbackAddress();
+ // The link that routes the send and the link that finds the session lost.
+ int leaderLinks = 2;
+ try (ServerSocket leader = new ServerSocket(0, leaderLinks, loopback);
+ ServerSocket primary = new ServerSocket(0, 1, loopback)) {
+ List<Request> registers = new CopyOnWriteArrayList<>();
+ AtomicInteger lookups = new AtomicInteger();
+ var leaderServed = serve(leader, leaderLinks, request -> {
+ if (request.operation() == OPERATION_REGISTER) {
+ registers.add(request);
+ return Response.success(OPERATION_REGISTER, registerBody(registers.size()));
+ }
+ if (request.is(BIND_SESSION_CODE, OPERATION_NON_REPLICATED)) {
+ return Response.error(OPERATION_NON_REPLICATED, IggyErrorCode.UNAUTHENTICATED.getCode());
+ }
+ if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) {
+ return Response.success(
+ OPERATION_NON_REPLICATED,
+ clusterMetadata(leader.getLocalPort(), primary.getLocalPort(), leader.getLocalPort()));
+ }
+ if (request.is(GET_STATS_CODE, OPERATION_NON_REPLICATED)) {
+ return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER);
+ }
+ if (!request.is(GET_PARTITION_ROUTE_CODE, OPERATION_NON_REPLICATED)) {
+ throw new IllegalStateException("Unexpected leader request: " + request);
+ }
+ ByteBuf route = pollRoute(request, 1, 1, primary.getLocalPort());
+ return lookups.incrementAndGet() == 1
+ ? Response.successAndDisconnect(OPERATION_NON_REPLICATED, route)
+ : Response.success(OPERATION_NON_REPLICATED, route);
+ });
+ List<Request> sends = new CopyOnWriteArrayList<>();
+ CompletableFuture<Void> firstSend = new CompletableFuture<>();
+ CompletableFuture<Void> refuse = new CompletableFuture<>();
+ var primaryServed = serve(primary, request -> {
+ if (request.is(BIND_SESSION_CODE, OPERATION_NON_REPLICATED)) {
+ return Response.success(OPERATION_NON_REPLICATED, bindBody(request));
+ }
+ if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) {
+ return Response.success(
+ OPERATION_NON_REPLICATED,
+ clusterMetadata(leader.getLocalPort(), primary.getLocalPort(), leader.getLocalPort()));
+ }
+ if (request.operation() != OPERATION_SEND_MESSAGES) {
+ throw new IllegalStateException("Unexpected primary request: " + request);
+ }
+ sends.add(request);
+ if (sends.size() > 1) {
+ return Response.success(OPERATION_SEND_MESSAGES, Unpooled.EMPTY_BUFFER);
+ }
+ firstSend.complete(null);
+ refuse.join();
+ return Response.error(OPERATION_SEND_MESSAGES, LIFECYCLE_BUSY);
+ });
+ AsyncIggyTcpClient client = client(leader);
+ try {
+ client.connect().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ client.login().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ var sent = client.messages()
+ .sendMessages(
+ StreamId.of(1L),
+ TopicId.of(1L),
+ Partitioning.partitionId(0L),
+ List.of(Message.of("m")));
+ firstSend.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ client.sendBinaryRequest(GET_STATS_CODE, new byte[0]).get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ refuse.complete(null);
+ sent.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ assertThat(registers).hasSize(2);
+ assertThat(sends).hasSize(2);
+ Request retried = sends.get(1);
+ assertThat(retried.requestId()).isNotEqualTo(sends.get(0).requestId());
+ assertThat(retried.clientLow()).isEqualTo(registers.get(1).clientLow());
+ assertThat(retried.clientHigh()).isEqualTo(registers.get(1).clientHigh());
+ } finally {
+ refuse.complete(null);
+ client.close().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+ leaderServed.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ primaryServed.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+ }
+
/** The server commits a lifecycle refusal in the result section, as the metadata plane does. */
private static Response lifecycleBusy() {
return Response.success(OPERATION_JOIN_GROUP, transientResult(LIFECYCLE_BUSY));
diff --git a/foreign/node/README.md b/foreign/node/README.md
index 771f0a63b..f3dda9387 100644
--- a/foreign/node/README.md
+++ b/foreign/node/README.md
@@ -107,12 +107,12 @@ after the server refuses those credentials, it does not sign in by itself, and
a call fails with status 40 until the caller signs in again. A sign-in or
logout of the caller ends each earlier call that has to wait for a new session:
with status 40 if it sent nothing, or with `OperationUncertainError` if its
-outcome is unknown. A typed send and a group poll can send more than one
+outcome is unknown. An auto-commit or Next poll, which has no operation ID, ends with status 57 instead. A typed send and a group poll can send more than one
command. If the caller signs in or logs out before the next command of such a
call, the call fails with status 40. Only a call that waits for the client's own sign-in goes on
under the new session. If the server refuses that sign-in, the call fails with
status 40, and its `cause` holds the refusal. A call whose deadline ends while
-it waits for a session fails with status 58.
+it waits for a session fails with status 58. So does a call that replicas refuse with 89 until its deadline.
A reply with status 30 (`StaleClient`) says that the server no longer attaches
the link to the session. This happens, for example, after the user was deleted
@@ -170,7 +170,7 @@ discards retained operations. It does not wait for outstanding calls to finish.
Await those calls before destroying the client when their results are needed.
Auto-commit and Next polls hold their partition lane only during the active call.
-Error 57 or a lost reply ends the call immediately with unknown progress and no
+Error 57, a lost reply or another unknown outcome ends the call immediately with status 57 and no
retained operation ID. These polls never replay after uncertainty. Known
non-admission errors 58/89 remain retryable; pure reads may retry lost replies.
Before uncertainty, lifecycle error 88 retries with a fresh ID within the same
diff --git a/foreign/node/src/bdd/README.md b/foreign/node/src/bdd/README.md
index 4bdd5c81b..c584e9734 100644
--- a/foreign/node/src/bdd/README.md
+++ b/foreign/node/src/bdd/README.md
@@ -10,6 +10,7 @@ the bdd test suite has no defaults and fails when any of these is missing:
- `IGGY_TCP_ADDRESS="host:port"` - server address
- `IGGY_ROOT_USERNAME` / `IGGY_ROOT_PASSWORD` - root credentials
+- `IGGY_TCP_ADDRESS_NO_QUORUM="host:port"` - cluster node that cannot reach quorum, read only by the session contract scenarios
## Run (recommended)
diff --git a/foreign/node/src/bdd/env.ts b/foreign/node/src/bdd/env.ts
index 2c8bfc288..4bcacf3c7 100644
--- a/foreign/node/src/bdd/env.ts
+++ b/foreign/node/src/bdd/env.ts
@@ -25,14 +25,18 @@ const requiredEnv = (name: string): string => {
return value;
};
-export const getServerAddress = (): [string, number] => {
- const address = requiredEnv('IGGY_TCP_ADDRESS');
+const getAddress = (name: string): [string, number] => {
+ const address = requiredEnv(name);
const [host, port] = address.split(':');
if (!host || !port)
- throw new Error(`IGGY_TCP_ADDRESS must be "host:port", got "${address}"`);
+ throw new Error(`${name} must be "host:port", got "${address}"`);
return [host, parseInt(port, 10)];
};
+export const getServerAddress = (): [string, number] => getAddress('IGGY_TCP_ADDRESS');
+
+export const getNoQuorumAddress = (): [string, number] => getAddress('IGGY_TCP_ADDRESS_NO_QUORUM');
+
export const getRootCredentials = () => ({
username: requiredEnv('IGGY_ROOT_USERNAME'),
password: requiredEnv('IGGY_ROOT_PASSWORD')
diff --git a/foreign/node/src/bdd/session.ts b/foreign/node/src/bdd/session.ts
new file mode 100644
index 000000000..4bc65bfc3
--- /dev/null
+++ b/foreign/node/src/bdd/session.ts
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import assert from 'node:assert/strict';
+import { Given, Then, When } from '@cucumber/cucumber';
+import { Client } from '../client/index.js';
+import { OPERATION_TIMEOUT_MS } from '../client/client.operation.js';
+import type { TestWorld } from './world.js';
+import { getNoQuorumAddress, getRootCredentials } from './env.js';
+
+const recordCallError = async (world: TestWorld, call: () => Promise<unknown>) => {
+ try {
+ await call();
+ world.callError = undefined;
+ } catch (error) {
+ world.callError = error instanceof Error ? error : new Error(String(error));
+ }
+};
+
+Given('I have a cluster node that cannot reach quorum', function (this: TestWorld) {
+ const [host, port] = getNoQuorumAddress();
+ this.client = new Client({
+ transport: 'TCP',
+ options: { host, port },
+ credentials: getRootCredentials()
+ });
+});
+
+When('I sign out', async function (this: TestWorld) {
+ assert.ok(await this.client.session.logout());
+});
+
+When('I request the stream with ID {int}', async function (this: TestWorld, streamId: number) {
+ await recordCallError(this, () => this.client.stream.get({ streamId }));
+});
+
+// The sign-in runs until the SDK deadline, past cucumber's default step timeout.
+When(
+ 'I sign in to that node as the root user',
+ { timeout: 2 * OPERATION_TIMEOUT_MS },
+ async function (this: TestWorld) {
+ await recordCallError(this, () => this.client.session.login(getRootCredentials()));
+ }
+);
+
+Then('the call should fail with error code {int}', function (this: TestWorld, code: number) {
+ assert.ok(this.callError, 'the call should have failed');
+ assert.equal((this.callError as { errorCode?: number }).errorCode, code, this.callError.message);
+});
diff --git a/foreign/node/src/bdd/world.ts b/foreign/node/src/bdd/world.ts
index 0a1a83ad0..cd689b491 100644
--- a/foreign/node/src/bdd/world.ts
+++ b/foreign/node/src/bdd/world.ts
@@ -27,5 +27,6 @@ export interface TestWorld {
sendMessages: CreateMessage[],
polledMessages: Message[],
rawResponse?: Buffer,
- rawError?: Error
+ rawError?: Error,
+ callError?: Error
};
diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts
index 9bdba9154..6503e07fc 100644
--- a/foreign/node/src/client/client.socket.test.ts
+++ b/foreign/node/src/client/client.socket.test.ts
@@ -55,8 +55,7 @@ import { VsrEvictionError } from '../wire/vsr/reply.js';
import { IggyConnection, type LinkAttempt } from './client.connection.js';
import { OPERATION_TIMEOUT_MS } from './client.operation.js';
import {
- CommandResponseStream, LaneBlockedError, NotConnectedError, OperationNotRetainedError, OperationUncertainError,
- VsrResponseTimeoutError
+ CommandResponseStream, LaneBlockedError, NotConnectedError, OperationNotRetainedError, OperationUncertainError
} from './client.socket.js';
import type { ClientConfig } from './client.type.js';
import { SimpleClient } from './client.js';
@@ -1006,7 +1005,7 @@ describe('primary auto-commit polling', () => {
await assert.rejects(first, (error: unknown) =>
failure === 'stale session'
? error instanceof ResponseError && error.errorCode === STALE_CLIENT
- : error instanceof OperationUncertainError && error.operationId === undefined);
+ : error instanceof ResponseError && error.errorCode === TRANSIENT_NOT_COMMITTED);
assert.equal(polls, 1);
assert.equal(resets, 0);
if (failure === 'partial reply') {
@@ -1219,7 +1218,7 @@ describe('primary auto-commit polling', () => {
try {
const polling = cluster.client.sendCommand(POLL_MESSAGES.code, groupPollPayload());
const failed = assert.rejects(polling, (error: unknown) =>
- error instanceof OperationUncertainError && error.operationId === undefined);
+ error instanceof ResponseError && error.errorCode === TRANSIENT_NOT_COMMITTED);
await pendingAdmission;
await cluster.client.sendCommand(LOGOUT.code, LOGOUT.serialize());
await failed;
@@ -3560,7 +3559,7 @@ describe('native mutation recovery', () => {
for (const strategy of ['auto-commit', 'Next'] as const) {
for (const failure of ['not committed', 'cancel'] as const) {
- it(`ends uncertain ${strategy} polls after ${failure} and releases the transient lane`, async () => {
+ it(`ends uncertain ${strategy} polls with 57 after ${failure} and releases the transient lane`, async () => {
const cancel = new AbortController();
let polls = 0;
const cluster = await startPollCluster((frame, socket) => {
@@ -3576,7 +3575,8 @@ describe('native mutation recovery', () => {
autocommit: strategy === 'auto-commit' });
try {
await assert.rejects(cluster.client.sendCommand(COMMAND_CODE.PollMessages, body,
- { signal: cancel.signal }), (error: unknown) => error instanceof OperationUncertainError && error.operationId === undefined);
+ { signal: cancel.signal }), (error: unknown) => error instanceof ResponseError &&
+ error.errorCode === TRANSIENT_NOT_COMMITTED);
assert.equal(polls, 1);
await cluster.client.sendCommand(COMMAND_CODE.SendMessages, explicitSend);
} finally { await cluster.close(); }
@@ -3884,7 +3884,8 @@ describe('native replica and selection lifecycle', () => {
const client = new CommandResponseStream(vsrConfig(server.port));
try {
await client.authenticate(vsrConfig(server.port).credentials);
- await assert.rejects(client.sendCommand(LOGIN.code, LOGIN.serialize({ username: 'other', password: 'secret' })), OperationUncertainError);
+ await assert.rejects(client.sendCommand(LOGIN.code, LOGIN.serialize({ username: 'other', password: 'secret' })),
+ (error: unknown) => error instanceof ResponseError && error.errorCode === TRANSIENT_NOT_COMMITTED);
assert.equal(framesOf(server, Operation.Register).length, 1);
assert.equal(framesOf(server, Operation.Logout).length, 1);
} finally { client.destroy(); await server.close(); }
@@ -4039,6 +4040,30 @@ describe('native retry pacing', () => {
} finally { context.mock.timers.reset(); client.destroy(); await server.close(); }
});
+ it('ends a call that gets 89 until its deadline with 58', async (context) => {
+ const server = await startVsrServer((frame, socket) => singleNodeHandler(server.port)(frame, socket));
+ const client = new CommandResponseStream({ ...vsrConfig(server.port), heartbeatInterval: 0 });
+ let lookups = 0;
+ let refusals = 0;
+ try {
+ await client.authenticate(vsrConfig(server.port).credentials);
+ context.mock.method(IggyConnection.prototype, 'exchange', async (encode: () => Buffer): Promise<LinkAttempt> => {
+ const frame = encode();
+ if (frame.readUInt32LE(REQUEST_OFFSET.reserved) === COMMAND_CODE.GetPartitionRoute) {
+ const route = routingBody(1n, 1n, 0);
+ route.writeUInt32LE(lookups++, 16);
+ return { started: true, response: replyFrame(frame, Operation.NonReplicated, route) };
+ }
+ refusals += 1;
+ return { started: true, response: replyFrame(frame, Operation.StoreConsumerOffset, Buffer.alloc(0), 89) };
+ });
+ await assert.rejects(client.sendCommand(COMMAND_CODE.StoreOffset, storeOffsetPayload,
+ { deadline: Date.now() + 300 }), (error: unknown) => error instanceof ResponseError &&
+ error.errorCode === TRANSIENT_NOT_ACCEPTED && error.fromServer);
+ assert.ok(refusals > 1);
+ } finally { client.destroy(); await server.close(); }
+ });
+
for (const [sample, delays] of [
[0, [0]],
[0.5, [500, 1_000, 2_000, 4_000, 5_000, 5_000]],
@@ -5158,7 +5183,8 @@ describe('explicit sign-in ownership', () => {
{ deadline: Date.now() + 150, signal: abort.signal }).catch((error: unknown) => error);
if (typeof failure === 'number') assert.ok(error instanceof ResponseError && error.errorCode === failure);
else if (failure === 'eviction') assert.ok(error instanceof VsrEvictionError);
- else if (failure === 'timeout') assert.ok(error instanceof VsrResponseTimeoutError);
+ else if (failure === 'timeout') assert.ok(error instanceof ResponseError &&
+ error.errorCode === TRANSIENT_NOT_ACCEPTED);
else assert.equal(error, failure === 'cancel' ? cancellation : configuration);
dial?.mock.restore();
refuse = false;
@@ -5214,7 +5240,7 @@ describe('explicit sign-in ownership', () => {
const queued = client.sendCommand(LOGIN.code, LOGIN.serialize(caller), { deadline: Date.now() + 50 })
.catch((error: unknown) => { failure = error; });
await waitUntil(() => failure !== undefined);
- assert.ok(failure instanceof VsrResponseTimeoutError);
+ assert.ok(failure instanceof ResponseError && failure.errorCode === TRANSIENT_NOT_ACCEPTED);
held!();
await queued;
assert.ok(isSignedOut(await older));
@@ -5259,7 +5285,8 @@ describe('explicit sign-in ownership', () => {
{ deadline: Date.now() + 100, signal: abort.signal }).catch((error: unknown) => error);
if (failure === 'refusal') assert.ok(error instanceof ResponseError && error.errorCode === LIFECYCLE_BUSY);
else if (failure === 'closed link') assert.ok(error instanceof Error && error.message === CLOSED_LINK);
- else if (failure === 'timeout') assert.ok(error instanceof VsrResponseTimeoutError);
+ else if (failure === 'timeout') assert.ok(error instanceof ResponseError &&
+ error.errorCode === TRANSIENT_NOT_ACCEPTED);
else assert.equal(error, cancellation);
assert.equal(client.isAuthenticated, false);
await waitUntil(() => server.sockets.size === 0);
@@ -5605,7 +5632,7 @@ describe('consumer stream', () => {
await cluster.primary.reopen();
assert.equal((await ended()).length, 1);
assert.equal(polls, 2);
- assert.ok(refusals[0] instanceof OperationUncertainError &&
+ assert.ok(refusals[0] instanceof ResponseError && refusals[0].errorCode === TRANSIENT_NOT_COMMITTED &&
refusals[0].cause instanceof VsrEvictionError && refusals[0].cause.errorCode === UNAUTHENTICATED,
String(refusals[0]));
for (const refusal of refusals.slice(1)) assert.ok(refusal instanceof NotConnectedError, String(refusal));
diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts
index 8421fa9a1..ee40fc772 100644
--- a/foreign/node/src/client/client.socket.ts
+++ b/foreign/node/src/client/client.socket.ts
@@ -388,8 +388,7 @@ export class CommandResponseStream extends EventEmitter {
}
sent ||= attempt.started;
if (generation !== this.routingGeneration && attempt.started && (replicated || progress))
- throw new OperationUncertainError(operation?.retainable ? operation.id : undefined,
- attempt.error ?? responseError(command, TRANSIENT_NOT_ACCEPTED));
+ throw uncertainOutcome(command, operation, attempt.error ?? responseError(command, TRANSIENT_NOT_ACCEPTED));
// A committed rejection proves the outcome even when it also ends the session.
if (operation?.uncertain && attempt.committed && attempt.error) {
operation.uncertain = false;
@@ -404,7 +403,7 @@ export class CommandResponseStream extends EventEmitter {
(attempt.started && (replicated || progress) && attempt.error instanceof VsrEvictionError);
this.recoverSession(generation);
if (uncertain)
- throw new OperationUncertainError(operation?.retainable ? operation.id : undefined, attempt.error);
+ throw uncertainOutcome(command, operation, attempt.error);
// An eviction fails every exchange of its link, also one never written.
if (restartable && exchanged && !attempt.started && generation !== this.routingGeneration) continue;
throw attempt.error;
@@ -439,13 +438,18 @@ export class CommandResponseStream extends EventEmitter {
notPrimary = code === NOT_PRIMARY;
if (code === TRANSIENT_NOT_COMMITTED) {
if (operation?.retainable) operation.uncertain = true;
- else if (!safeRead) throw new OperationUncertainError(undefined, error);
+ else if (!safeRead) throw error;
} else if (code === OPERATION_NOT_RETAINED) {
if (operation?.retainable) operation.uncertain = true;
- throw new OperationUncertainError(operation?.retainable ? operation.id : undefined, error);
... 678 lines suppressed ...