You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by yu...@apache.org on 2026/08/26 03:14:40 UTC
(fluss) branch main updated: [FIP-49][gateway] Add the Gateway backend runtime, cluster discovery, and metadata listing (#4078)
This is an automated email from the ASF dual-hosted git repository.
luoyuxia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new 1e74ea267 [FIP-49][gateway] Add the Gateway backend runtime, cluster discovery, and metadata listing (#4078)
1e74ea267 is described below
commit 1e74ea267f5d01da8de112e292914b6536537acc
Author: Junbo Wang <be...@gmail.com>
AuthorDate: Wed Aug 26 11:14:36 2026 +0800
[FIP-49][gateway] Add the Gateway backend runtime, cluster discovery, and metadata listing (#4078)
---
.github/workflows/client-integration.yml | 52 +-
fluss-gateway/Cargo.lock | 13 +
fluss-gateway/Cargo.toml | 16 +-
fluss-gateway/DEPENDENCIES.rust.tsv | 1 +
fluss-gateway/README.md | 43 +-
fluss-gateway/justfile | 6 +
fluss-gateway/openapi.yaml | 253 +++++++++
fluss-gateway/src/backend/client.rs | 272 +++++++++
fluss-gateway/src/backend/connection.rs | 764 ++++++++++++++++++++++++++
fluss-gateway/src/backend/context.rs | 218 ++++++++
fluss-gateway/src/backend/errors.rs | 215 ++++++++
fluss-gateway/src/backend/fake.rs | 227 ++++++++
fluss-gateway/src/backend/mod.rs | 75 +++
fluss-gateway/src/backend/types.rs | 77 +++
fluss-gateway/src/config.rs | 439 ++++++++++-----
fluss-gateway/src/error.rs | 3 +
fluss-gateway/src/lib.rs | 1 +
fluss-gateway/src/lifecycle.rs | 190 +++++--
fluss-gateway/src/observability.rs | 54 ++
fluss-gateway/src/protocol/mod.rs | 3 +
fluss-gateway/src/protocol/rest/clusters.rs | 149 +++++
fluss-gateway/src/protocol/rest/health.rs | 26 +-
fluss-gateway/src/protocol/rest/metadata.rs | 370 +++++++++++++
fluss-gateway/src/protocol/rest/mod.rs | 393 ++++++++++---
fluss-gateway/src/protocol/rest/openapi.rs | 23 +
fluss-gateway/src/protocol/rest/pagination.rs | 349 ++++++++++++
fluss-gateway/tests/e2e_cluster.rs | 283 ++++++++++
fluss-gateway/tests/http_api.rs | 45 ++
fluss-gateway/tests/process.rs | 4 +-
29 files changed, 4288 insertions(+), 276 deletions(-)
diff --git a/.github/workflows/client-integration.yml b/.github/workflows/client-integration.yml
index 069579480..2fb99c56f 100644
--- a/.github/workflows/client-integration.yml
+++ b/.github/workflows/client-integration.yml
@@ -17,7 +17,7 @@
# Client integration tests against a SAME-REVISION server (FIP-40 §3.2).
# Builds the Fluss server image from this source tree ONCE, caches + saves it,
-# then fans out the Rust / Python / C++ / Elixir integration suites against that
+# then fans out the Rust / Python / C++ / Elixir / Gateway integration suites against that
# fluss:dev image (build-once-fan-out, à la Temporal/PyFlink). The image build is
# cached on server/proto hashes, so client-only PRs reuse it instead of rebuilding.
@@ -37,6 +37,7 @@ on:
- 'fluss-rust/bindings/**'
- 'fluss-rust/Cargo.toml'
- 'fluss-rust/Cargo.lock'
+ - 'fluss-gateway/**'
- '.github/workflows/client-integration.yml'
pull_request:
branches:
@@ -51,6 +52,7 @@ on:
- 'fluss-rust/bindings/**'
- 'fluss-rust/Cargo.toml'
- 'fluss-rust/Cargo.lock'
+ - 'fluss-gateway/**'
- '.github/workflows/client-integration.yml'
workflow_dispatch:
@@ -60,8 +62,8 @@ concurrency:
jobs:
# Decide which client suites to run, mirroring the per-binding scoping the
- # standalone fluss-rust repo had: a binding suite runs only when its own
- # binding, the core fluss-rs crate, or the server/proto changed. On non-PR
+ # standalone fluss-rust repo had: a client suite runs only when its own
+ # sources, the core fluss-rs crate, or the server/proto changed. On non-PR
# events (push to main, manual) everything runs.
detect-changes:
runs-on: ubuntu-latest
@@ -70,6 +72,7 @@ jobs:
python: ${{ steps.filter.outputs.python }}
cpp: ${{ steps.filter.outputs.cpp }}
elixir: ${{ steps.filter.outputs.elixir }}
+ gateway: ${{ steps.filter.outputs.gateway }}
steps:
- uses: actions/checkout@v6
with:
@@ -86,19 +89,21 @@ jobs:
has() { echo "$changed" | grep -qE "$1"; }
# a change to this workflow itself should exercise every suite
has '^\.github/workflows/client-integration\.yml' && all=true || true
- protocol=false; core=false; py=false; cpp=false; ex=false
+ protocol=false; core=false; py=false; cpp=false; ex=false; gateway=false
has '^(fluss-rpc/src/main/proto/|fluss-server/|fluss-common/|fluss-dist/|docker/fluss/)' && protocol=true || true
has '^(fluss-rust/crates/|fluss-rust/Cargo\.)' && core=true || true
has '^fluss-rust/bindings/python/' && py=true || true
has '^fluss-rust/bindings/cpp/' && cpp=true || true
has '^fluss-rust/bindings/elixir/' && ex=true || true
- # a suite runs if: non-PR (all) OR core crate OR server/proto OR its own binding changed
+ has '^fluss-gateway/' && gateway=true || true
+ # a suite runs if: non-PR (all) OR core crate OR server/proto OR its own sources changed
gate() { if [ "$all" = true ] || [ "$core" = true ] || [ "$protocol" = true ] || [ "$1" = true ]; then echo true; else echo false; fi; }
{
echo "rust=$(gate false)"
echo "python=$(gate $py)"
echo "cpp=$(gate $cpp)"
echo "elixir=$(gate $ex)"
+ echo "gateway=$(gate $gateway)"
} >> "$GITHUB_OUTPUT"
# Build the server image from THIS source tree once; cache it on server/proto
@@ -106,7 +111,7 @@ jobs:
# is uploaded as an artifact and loaded by every client integration job.
build-server-image:
needs: detect-changes
- if: needs.detect-changes.outputs.rust == 'true' || needs.detect-changes.outputs.python == 'true' || needs.detect-changes.outputs.cpp == 'true' || needs.detect-changes.outputs.elixir == 'true'
+ if: needs.detect-changes.outputs.rust == 'true' || needs.detect-changes.outputs.python == 'true' || needs.detect-changes.outputs.cpp == 'true' || needs.detect-changes.outputs.elixir == 'true' || needs.detect-changes.outputs.gateway == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
@@ -172,6 +177,41 @@ jobs:
RUST_LOG: DEBUG
RUST_BACKTRACE: full
+ gateway-integration:
+ needs: [detect-changes, build-server-image]
+ if: needs.detect-changes.outputs.gateway == 'true'
+ timeout-minutes: 60
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: fluss-gateway
+ steps:
+ - uses: actions/checkout@v6
+ - uses: actions/download-artifact@v4
+ with:
+ name: fluss-dev-image
+ path: /tmp
+ - name: Load server image
+ run: docker load -i /tmp/fluss-dev.tar
+ - name: Install protobuf compiler
+ run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+ - name: Rust Cache
+ uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+ with:
+ workspaces: |
+ fluss-gateway
+ fluss-rust
+ - name: Build Fluss test cluster helper
+ run: cargo build --locked --manifest-path ../fluss-rust/Cargo.toml -p fluss-test-cluster
+ - name: Integration Test
+ run: cargo test --locked --features integration_tests --test e2e_cluster
+ env:
+ FLUSS_TEST_CLUSTER_BIN: ${{ github.workspace }}/fluss-rust/target/debug/fluss-test-cluster
+ FLUSS_IMAGE: fluss
+ FLUSS_VERSION: dev
+ RUST_LOG: DEBUG
+ RUST_BACKTRACE: full
+
python-integration:
needs: [detect-changes, build-server-image]
if: needs.detect-changes.outputs.python == 'true'
diff --git a/fluss-gateway/Cargo.lock b/fluss-gateway/Cargo.lock
index b94e7604f..ec76ecd9f 100644
--- a/fluss-gateway/Cargo.lock
+++ b/fluss-gateway/Cargo.lock
@@ -312,6 +312,17 @@ dependencies = [
"regex-syntax",
]
+[[package]]
+name = "async-trait"
+version = "0.1.92"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.3",
+]
+
[[package]]
name = "atoi"
version = "2.0.0"
@@ -791,7 +802,9 @@ dependencies = [
name = "fluss-gateway"
version = "1.0.0"
dependencies = [
+ "async-trait",
"axum",
+ "base64 0.22.1",
"clap",
"fluss-rs",
"futures-util",
diff --git a/fluss-gateway/Cargo.toml b/fluss-gateway/Cargo.toml
index e64a702a5..eaaae24ec 100644
--- a/fluss-gateway/Cargo.toml
+++ b/fluss-gateway/Cargo.toml
@@ -26,8 +26,7 @@ name = "fluss-gateway"
edition = "2024"
version = "1.0.0"
license = "Apache-2.0"
-# The shipped dependency set only needs 1.85; the declared floor stays at 1.88 to match fluss-rust,
-# which the gateway depends on once the FlussBackend capability lands.
+# The floor matches fluss-rust, which the native backend links against.
rust-version = "1.88"
authors = ["Apache Fluss <de...@fluss.apache.org>"]
repository = "https://github.com/apache/fluss"
@@ -38,6 +37,9 @@ repository = "https://github.com/apache/fluss"
publish = false
description = "Stateless REST gateway for Apache Fluss"
+[features]
+integration_tests = []
+
[[bin]]
name = "fluss-gateway"
path = "src/main.rs"
@@ -47,11 +49,15 @@ path = "src/main.rs"
name = "fluss_gateway"
[dependencies]
+# `FlussBackend` is used behind `Arc<dyn ...>`, which native `async fn` in traits cannot be.
+async-trait = "0.1"
axum = { version = "0.8", default-features = false, features = ["http1", "matched-path", "tokio"] }
+# Encodes the self-describing keyset page token; already in the graph transitively.
+base64 = "0.22"
clap = { version = "4.5.37", features = ["derive"] }
# Only FutureExt/join_all are used; avoid the larger futures facade crate.
futures-util = "0.3"
-# Build the native connection settings owned by the Gateway from the typed per-cluster configuration.
+# The native backend and the connection settings owned by the Gateway.
fluss = { package = "fluss-rs", path = "../fluss-rust/crates/fluss" }
# The listeners follow axum's official serve-with-hyper example: axum::serve does not expose the
# hyper builder that the header read timeout lives on. The plain http1 builder (not auto) has no
@@ -93,5 +99,9 @@ tower = { version = "0.5", features = ["util"] }
[[test]]
name = "http_api"
+[[test]]
+name = "e2e_cluster"
+required-features = ["integration_tests"]
+
[[test]]
name = "process"
diff --git a/fluss-gateway/DEPENDENCIES.rust.tsv b/fluss-gateway/DEPENDENCIES.rust.tsv
index 1ba94a438..cefc1de24 100644
--- a/fluss-gateway/DEPENDENCIES.rust.tsv
+++ b/fluss-gateway/DEPENDENCIES.rust.tsv
@@ -22,6 +22,7 @@ arrow-row@59.2.0 X
arrow-schema@59.2.0 X
arrow-select@59.2.0 X
arrow-string@59.2.0 X
+async-trait@0.1.92 X X
atoi@2.0.0 X
atomic-waker@1.1.2 X X
autocfg@1.5.1 X X
diff --git a/fluss-gateway/README.md b/fluss-gateway/README.md
index 3c5c09723..8c57973fd 100644
--- a/fluss-gateway/README.md
+++ b/fluss-gateway/README.md
@@ -30,11 +30,26 @@ lock file or its generated dependency inventories.
## Status
-This module currently contains only the scaffolding: the manifest with
-placeholder bin/lib targets, the pinned toolchain, lint and license
-configuration, and the developer recipes. The runtime — configuration,
-lifecycle management, the REST layer, and the test suites — lands in follow-up
-pull requests.
+The gateway currently serves the process runtime and REST metadata discovery:
+configuration loading and validation (`gateway.yaml` with flat dotted keys), the
+HTTP listener with its request-id, body-size and deadline middleware,
+`GET /health`, `GET /ready`, `GET /v1/openapi.json`, `GET /v1/clusters`,
+`GET /v1/clusters/{cluster}/databases`,
+`GET /v1/clusters/{cluster}/databases/{database}/tables`, the Prometheus
+listener, and the backend runtime that owns one shared service connection per
+configured cluster. A connection is opened lazily on the first request that
+needs it, shared by every request to that cluster, released after the configured
+`connection.idle-timeout`, and drained during shutdown. Concurrent cold requests
+serialize behind one connection attempt. Cancelling that request cancels its
+attempt and lets the next waiter retry; bootstrap timeout and retry behavior remain
+owned by `fluss-rust`.
+Connections use Fluss's default plaintext protocol unless
+`connection.security.protocol: sasl` selects the configured service account. A
+broken transport is left to the native client, which reconnects the affected
+server on its own.
+`connection.identity-mode: user` is refused at startup until Fluss supports
+act-as. The describe-table, partition, DDL, write, and lookup APIs, and client
+authentication, land in follow-up pull requests.
## Prerequisites
@@ -51,12 +66,30 @@ Run everything from this directory, or use the `just` recipes:
```bash
just build # cargo build --all-targets
just test # cargo test --all-targets
+just test-e2e # real Gateway + Dockerized Fluss cluster
just fmt-check # cargo fmt --all -- --check
just clippy # cargo clippy --all-targets -- -D warnings
just doc # RUSTDOCFLAGS="-D warnings" cargo doc --no-deps
just licenses # cargo deny check licenses
```
+The E2E test requires Docker. It builds and invokes the `fluss-test-cluster`
+helper from the `fluss-rust` workspace, creates catalog objects in a real Fluss
+cluster, then verifies the cluster, database, and table REST APIs through both
+its plaintext and SASL endpoints. Set `FLUSS_IMAGE` and `FLUSS_VERSION` to
+select the Fluss image; CI builds and uses `fluss:dev` from the same source
+revision.
+
+Plaintext is the default and must not carry service credentials. To use
+SASL/PLAIN, configure all three options explicitly:
+
+```yaml
+gateway.cluster.default.connection.security.protocol: sasl
+gateway.cluster.default.connection.service.account: gateway_svc
+gateway.cluster.default.connection.service.secret: change-me
+gateway.cluster.default.connection.idle-timeout: 10m
+```
+
The MSRV can be verified locally with `cargo +1.88.0 check --all-targets`.
## CI
diff --git a/fluss-gateway/justfile b/fluss-gateway/justfile
index 9172a6c1c..f81798893 100644
--- a/fluss-gateway/justfile
+++ b/fluss-gateway/justfile
@@ -30,6 +30,12 @@ build-release:
test:
cargo test --all-targets
+# Run the full HTTP-to-Fluss integration test. Requires Docker and a locally available Fluss image;
+# FLUSS_IMAGE and FLUSS_VERSION select it (the client integration CI uses fluss:dev).
+test-e2e:
+ cargo build --manifest-path ../fluss-rust/Cargo.toml -p fluss-test-cluster
+ FLUSS_TEST_CLUSTER_BIN=../fluss-rust/target/debug/fluss-test-cluster cargo test --features integration_tests --test e2e_cluster
+
# Format the gateway sources in place.
fmt:
cargo fmt --all
diff --git a/fluss-gateway/openapi.yaml b/fluss-gateway/openapi.yaml
index c7736a209..19a43af22 100644
--- a/fluss-gateway/openapi.yaml
+++ b/fluss-gateway/openapi.yaml
@@ -1,5 +1,30 @@
components:
schemas:
+ ClustersResponse:
+ description: 'Response of `GET /v1/clusters`: the IDs used in the cluster-scoped paths of the other APIs.'
+ properties:
+ clusters:
+ items:
+ type: string
+ type: array
+ required:
+ - clusters
+ type: object
+ DatabasesResponse:
+ description: Response of `GET /v1/clusters/{cluster}/databases`.
+ properties:
+ databases:
+ items:
+ type: string
+ type: array
+ next_page_token:
+ description: Present only while more entries follow.
+ type:
+ - string
+ - 'null'
+ required:
+ - databases
+ type: object
ErrorBody:
description: Body of the REST error envelope.
properties:
@@ -81,6 +106,21 @@ components:
required:
- status
type: object
+ TablesResponse:
+ description: Response of `GET /v1/clusters/{cluster}/databases/{database}/tables`.
+ properties:
+ next_page_token:
+ description: Present only while more entries follow.
+ type:
+ - string
+ - 'null'
+ tables:
+ items:
+ type: string
+ type: array
+ required:
+ - tables
+ type: object
info:
description: Stateless REST gateway for Apache Fluss
license:
@@ -100,6 +140,12 @@ paths:
schema:
$ref: '#/components/schemas/HealthResponse'
description: Gateway liveness and uptime
+ '400':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: This operation accepts no query parameters
'405':
content:
application/json:
@@ -124,6 +170,12 @@ paths:
schema:
$ref: '#/components/schemas/ReadyResponse'
description: The gateway accepts application traffic
+ '400':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: This operation accepts no query parameters
'405':
content:
application/json:
@@ -139,12 +191,213 @@ paths:
summary: Readiness for load balancers and readiness probes.
tags:
- health
+ /v1/clusters:
+ get:
+ description: |-
+ This is a configuration echo: it touches no connection, so it answers at the same speed whether
+ Fluss is up or down, and a caller polling it cannot turn into a burst of connection attempts. It
+ deliberately carries **no** reachability field — the gateway has no background probe, so any such
+ field would report the outcome of some earlier request rather than the current state. Whether a
+ cluster can be served shows up in the errors of its own requests and in the `connections_*` metrics.
+ operationId: listClusters
+ responses:
+ '200':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ClustersResponse'
+ description: Configured cluster IDs in lexical order
+ '400':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: This operation accepts no query parameters
+ '405':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Wrong method for this route
+ '413':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Request body above the configured limit
+ '503':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Gateway starting or shutting down
+ '504':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Request deadline exceeded
+ summary: Lists the configured clusters.
+ tags:
+ - metadata
+ /v1/clusters/{cluster}/databases:
+ get:
+ operationId: listDatabases
+ parameters:
+ - description: Configured cluster ID
+ in: path
+ name: cluster
+ required: true
+ schema:
+ type: string
+ - description: Maximum entries to return. Defaults to 100, capped at 1000.
+ in: query
+ name: max_results
+ required: false
+ schema:
+ maximum: 1000
+ minimum: 1
+ type: integer
+ - description: Opaque token from the `next_page_token` of the previous response.
+ in: query
+ name: page_token
+ required: false
+ schema:
+ type: string
+ responses:
+ '200':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/DatabasesResponse'
+ description: Databases in lexical order
+ '400':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Invalid page parameter or page token
+ '404':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Unknown cluster
+ '429':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Metadata concurrency limit exceeded
+ '500':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Fluss backend failure
+ '503':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Fluss is unavailable, or the gateway is starting or shutting down
+ '504':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Request deadline exceeded
+ summary: Lists the databases of one configured cluster.
+ tags:
+ - metadata
+ /v1/clusters/{cluster}/databases/{database}/tables:
+ get:
+ operationId: listTables
+ parameters:
+ - description: Configured cluster ID
+ in: path
+ name: cluster
+ required: true
+ schema:
+ type: string
+ - description: Database name
+ in: path
+ name: database
+ required: true
+ schema:
+ type: string
+ - description: Maximum entries to return. Defaults to 100, capped at 1000.
+ in: query
+ name: max_results
+ required: false
+ schema:
+ maximum: 1000
+ minimum: 1
+ type: integer
+ - description: Opaque token from the `next_page_token` of the previous response.
+ in: query
+ name: page_token
+ required: false
+ schema:
+ type: string
+ responses:
+ '200':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/TablesResponse'
+ description: Tables in lexical order
+ '400':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Invalid page parameter or page token
+ '404':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Unknown cluster or database
+ '429':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Metadata concurrency limit exceeded
+ '500':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Fluss backend failure
+ '503':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Fluss is unavailable, or the gateway is starting or shutting down
+ '504':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: Request deadline exceeded
+ summary: Lists the tables of one database.
+ tags:
+ - metadata
/v1/openapi.json:
get:
operationId: getOpenApi
responses:
'200':
description: OpenAPI 3.1 document
+ '400':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ErrorEnvelope'
+ description: This operation accepts no query parameters
'405':
content:
application/json:
diff --git a/fluss-gateway/src/backend/client.rs b/fluss-gateway/src/backend/client.rs
new file mode 100644
index 000000000..03c88bfb3
--- /dev/null
+++ b/fluss-gateway/src/backend/client.rs
@@ -0,0 +1,272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! The production [`FlussBackend`] over `fluss-rs`.
+//!
+//! It owns one `ConnectionCache` per configured cluster, which is the whole of the gateway's connection
+//! management: routing a request to its cluster and sharing its service-mode connection. None of that
+//! is visible above this module.
+//!
+//! This module and [`crate::backend::connection`] own native client handles, connection lifecycle, and
+//! transport error translation. Protocol adapters may still reuse stable `fluss-rs` domain types, such
+//! as metadata descriptors, when they fit the API instead of defining duplicate models.
+
+use crate::backend::FlussBackend;
+use crate::backend::connection::{ConnectionCache, NativeConnector};
+use crate::backend::context::RequestContext;
+use crate::backend::errors::map_fluss_error;
+use crate::backend::types::ClusterId;
+use crate::backend::unknown_cluster;
+use crate::config::GatewayConfig;
+use crate::error::GatewayResult;
+use async_trait::async_trait;
+use fluss::client::FlussAdmin;
+use fluss::error::Error as FlussClientError;
+use futures_util::future::join_all;
+use std::collections::BTreeMap;
+use std::sync::Arc;
+use std::time::Duration;
+
+pub struct NativeFlussBackend {
+ /// The cluster routing table. Built from configuration and never mutated, so it needs no lock and
+ /// no request can add, remove, or reorder a cluster.
+ caches: BTreeMap<ClusterId, ConnectionCache<NativeConnector>>,
+}
+
+impl NativeFlussBackend {
+ /// Performs no I/O: the process starts while Fluss is down, and the first request of a cluster
+ /// dials it.
+ pub fn from_config(config: &GatewayConfig) -> Self {
+ let caches = config
+ .clusters
+ .iter()
+ .map(|(id, cluster)| {
+ let id = ClusterId::try_from(id.as_str())
+ .expect("configuration validation accepted every cluster ID");
+ let cache =
+ ConnectionCache::new(id.clone(), cluster, NativeConnector::new(cluster));
+ (id, cache)
+ })
+ .collect();
+ Self { caches }
+ }
+
+ /// Closes every connection of every cluster within `timeout`. Idempotent.
+ ///
+ /// Concurrent because the budget is the whole shutdown's, not each cluster's: closing sequentially
+ /// would let one slow cluster consume the time the others need.
+ pub(crate) async fn close(&self, timeout: Duration) -> GatewayResult<()> {
+ let closes = join_all(self.caches.values().map(|cache| cache.close(timeout))).await;
+ let mut first_failure = None;
+ for (id, result) in self.caches.keys().zip(closes) {
+ if let Err(error) = result {
+ log::warn!("failed to close the connections of cluster `{id}`: {error}");
+ first_failure = first_failure.or(Some(error));
+ }
+ }
+ first_failure.map_or(Ok(()), Err)
+ }
+
+ /// Runs one idle scan for every configured cluster. Each cache logs its own best-effort close
+ /// failures, so a slow or unavailable cluster cannot suppress cleanup of the others.
+ pub(crate) async fn clean_expired_connections(&self) {
+ join_all(self.caches.values().map(ConnectionCache::clean_expired)).await;
+ }
+
+ /// The only place `cluster_not_found` originates.
+ fn cache_for(&self, ctx: &RequestContext) -> GatewayResult<&ConnectionCache<NativeConnector>> {
+ self.caches
+ .get(ctx.cluster_id())
+ .ok_or_else(|| unknown_cluster(ctx.cluster_id().as_str()))
+ }
+
+ /// The single entry point of every admin call: route, run under the request budget, take the
+ /// cluster's service connection, then classify the failure.
+ ///
+ /// A failure never evicts the connection. `fluss-rs` reports a broken transport per server and
+ /// reconnects that server on the next use, so the logical client recovers on its own; discarding it
+ /// would only throw away its cluster metadata and cached sub-clients.
+ ///
+ async fn admin_call<T, F, Fut>(
+ &self,
+ ctx: &RequestContext,
+ what: &'static str,
+ operation: F,
+ ) -> GatewayResult<T>
+ where
+ F: FnOnce(Arc<FlussAdmin>) -> Fut,
+ Fut: Future<Output = Result<T, FlussClientError>>,
+ {
+ let cache = self.cache_for(ctx)?;
+ ctx.run(async {
+ let connection = cache.connection(ctx).await?;
+ let result = match connection.get_admin() {
+ Ok(admin) => operation(admin).await,
+ Err(error) => Err(error),
+ };
+ result.map_err(|native| map_fluss_error(what, native))
+ })
+ .await
+ }
+}
+
+#[async_trait]
+impl FlussBackend for NativeFlussBackend {
+ fn clusters(&self) -> Vec<ClusterId> {
+ self.caches.keys().cloned().collect()
+ }
+
+ fn has_cluster(&self, id: &str) -> bool {
+ ClusterId::try_from(id).is_ok_and(|id| self.caches.contains_key(&id))
+ }
+
+ async fn list_databases(&self, ctx: &RequestContext) -> GatewayResult<Vec<String>> {
+ self.admin_call(ctx, "list the databases", |admin| async move {
+ admin.list_databases().await
+ })
+ .await
+ }
+
+ async fn list_tables(
+ &self,
+ ctx: &RequestContext,
+ database: &str,
+ ) -> GatewayResult<Vec<String>> {
+ let database = database.to_string();
+ self.admin_call(ctx, "list the tables", |admin| async move {
+ admin.list_tables(&database).await
+ })
+ .await
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::config::{ClusterConfig, ConfigDuration};
+ use crate::error::ErrorKind;
+
+ fn backend(clusters: &[(&str, ClusterConfig)]) -> NativeFlussBackend {
+ NativeFlussBackend::from_config(&GatewayConfig {
+ clusters: clusters
+ .iter()
+ .map(|(id, config)| ((*id).to_string(), config.clone()))
+ .collect(),
+ ..GatewayConfig::default()
+ })
+ }
+
+ fn service_cluster() -> ClusterConfig {
+ ClusterConfig::default()
+ }
+
+ /// Every configured cluster is routable in lexical order, and a malformed or unconfigured ID is
+ /// the same 404 — resolved from configuration alone, without any connection attempt.
+ #[tokio::test]
+ async fn routing_answers_only_from_configuration() {
+ let backend = backend(&[("zeta", service_cluster()), ("alpha", service_cluster())]);
+
+ assert_eq!(
+ backend
+ .clusters()
+ .iter()
+ .map(ClusterId::as_str)
+ .collect::<Vec<_>>(),
+ ["alpha", "zeta"]
+ );
+ assert!(backend.has_cluster("alpha"));
+ for unknown in ["beta", "Not A Cluster", ""] {
+ assert!(!backend.has_cluster(unknown), "{unknown:?}");
+ }
+
+ let ctx = RequestContext::for_test("beta", Duration::from_secs(5));
+ assert_eq!(
+ failure(backend.list_databases(&ctx).await),
+ (ErrorKind::NotFound, "cluster_not_found")
+ );
+ }
+
+ /// The backend constructs without touching the network, so both ways a dial can fail surface on
+ /// the first request rather than at startup. Shutdown then has nothing to close.
+ #[tokio::test]
+ async fn a_dial_failure_reaches_the_caller_classified() {
+ let unreachable = ClusterConfig {
+ // Port 1 has no listener in any test environment; the connect timeout bounds the attempt.
+ bootstrap_servers: "127.0.0.1:1".to_string(),
+ connect_timeout: ConfigDuration::from_millis(200),
+ ..service_cluster()
+ };
+ let illegal = ClusterConfig {
+ bootstrap_servers: "not-a-host-port".to_string(),
+ ..service_cluster()
+ };
+ let backend = backend(&[("down", unreachable), ("bad_address", illegal)]);
+
+ for (cluster, expected) in [
+ ("down", ErrorKind::Unavailable),
+ ("bad_address", ErrorKind::InvalidArgument),
+ ] {
+ let ctx = RequestContext::for_test(cluster, Duration::from_secs(5));
+ assert_eq!(
+ failure(backend.list_databases(&ctx).await).0,
+ expected,
+ "{cluster}"
+ );
+ }
+ backend.close(Duration::from_secs(1)).await.unwrap();
+ }
+
+ fn failure<T>(result: GatewayResult<T>) -> (ErrorKind, &'static str) {
+ let error = result.err().expect("the call fails");
+ (error.kind(), error.code())
+ }
+
+ /// A server that accepts the connection and never answers must still end at the deadline.
+ ///
+ /// Nothing below the gateway ends this wait: `fluss-rs` has no per-RPC timeout, and the connect
+ /// timeout is already satisfied by the accepted TCP connection. Only [`RequestContext::run`] does.
+ /// The abandoned RPC stays registered on the native connection — see the note on
+ /// `NativeConnector::dial`.
+ #[tokio::test]
+ async fn a_server_that_never_answers_ends_at_the_request_deadline() {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("a bound listener");
+ let address = listener.local_addr().expect("a bound address");
+ tokio::spawn(async move {
+ let mut accepted = Vec::new();
+ while let Ok((stream, _)) = listener.accept().await {
+ // Held open and never answered.
+ accepted.push(stream);
+ }
+ });
+
+ let backend = backend(&[(
+ "default",
+ ClusterConfig {
+ bootstrap_servers: address.to_string(),
+ ..service_cluster()
+ },
+ )]);
+ let ctx = RequestContext::for_test("default", Duration::from_millis(300));
+
+ assert_eq!(
+ failure(backend.list_databases(&ctx).await).0,
+ ErrorKind::DeadlineExceeded
+ );
+ }
+}
diff --git a/fluss-gateway/src/backend/connection.rs b/fluss-gateway/src/backend/connection.rs
new file mode 100644
index 000000000..25c0edcf2
--- /dev/null
+++ b/fluss-gateway/src/backend/connection.rs
@@ -0,0 +1,764 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! The connection cache for one configured Fluss cluster.
+//!
+//! Connections are indexed by effective Fluss user and multiplexed by `fluss-rs`. Service identity
+//! mode is the one-entry case: `ConnectionCache::key` always returns the configured service user.
+//! Concurrent cold requests for the same user serialize behind one dial. A cancelled request cancels
+//! its dial, releases the dial lock, and lets the next waiter retry; dial failures are not cached.
+//!
+//! Idle cleanup is based on the last acquisition time rather than a lease count. Configuration keeps
+//! the idle timeout above the longest backend operation, so no request can still be using a connection
+//! once it becomes eligible. Shutdown similarly relies on the process lifecycle draining request and
+//! cleaner tasks before calling `ConnectionCache::close`.
+
+use crate::backend::context::RequestContext;
+use crate::backend::errors::map_fluss_error;
+use crate::backend::types::ClusterId;
+use crate::config::{ClusterConfig, ConnectionSecurityProtocol, IdentityMode};
+use crate::error::{GatewayError, GatewayResult};
+use crate::observability;
+use fluss::client::FlussConnection;
+use fluss::config::Config as NativeClientConfig;
+use fluss::error::Error as FlussClientError;
+use futures_util::future::join_all;
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, OnceLock, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
+use std::time::Duration;
+use tokio::time::Instant;
+
+/// How often the process-owned cleaner scans every cluster cache.
+pub(crate) const CLEANUP_INTERVAL: Duration = Duration::from_secs(30);
+
+/// Per-connection budget for an idle close. Idle cleanup is best effort and must not stall later scans.
+const CLEANER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5);
+
+/// Private cache key for a service connection without Fluss authentication.
+///
+/// A configured SASL service account and every future request-derived principal are non-blank, so this
+/// sentinel cannot collide with an authenticated effective user.
+const PLAINTEXT_SERVICE_USER: &str = "";
+
+/// The two Fluss-side users of one authenticated native connection.
+///
+/// `gateway_user` authenticates the connection. `fluss_user` is the effective user Fluss authorizes.
+/// They are the same configured service account today. Future user identity mode keeps the Gateway
+/// account in the first field and maps the request [`crate::backend::context::Principal`] into the
+/// second immediately before selecting a connection.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) struct FlussConnectionIdentity {
+ gateway_user: Arc<str>,
+ fluss_user: Arc<str>,
+}
+
+impl FlussConnectionIdentity {
+ fn service(config: &ClusterConfig) -> Option<Self> {
+ if config.security_protocol == ConnectionSecurityProtocol::Plaintext {
+ return None;
+ }
+ let service_user: Arc<str> = Arc::from(
+ config
+ .service_account()
+ .expect("validated SASL configuration has a service account"),
+ );
+ Some(Self {
+ gateway_user: service_user.clone(),
+ fluss_user: service_user,
+ })
+ }
+
+ fn gateway_user(&self) -> &str {
+ &self.gateway_user
+ }
+
+ fn fluss_user(&self) -> &str {
+ &self.fluss_user
+ }
+}
+
+/// Opens and releases connections for one cluster.
+///
+/// The cache owns sharing and cleanup; this trait owns what a connection is, which lets those
+/// algorithms be tested without a cluster.
+pub(crate) trait Connector: Send + Sync + 'static {
+ type Conn: Send + Sync + 'static;
+
+ fn dial(
+ &self,
+ identity: Option<&FlussConnectionIdentity>,
+ ) -> impl Future<Output = Result<Self::Conn, FlussClientError>> + Send;
+
+ /// Releases a connection, draining pending writes within `timeout`.
+ fn close(
+ &self,
+ connection: Arc<Self::Conn>,
+ timeout: Duration,
+ ) -> impl Future<Output = Result<(), FlussClientError>> + Send;
+}
+
+/// Dials the native Fluss client with the cluster's configured security protocol.
+pub(crate) struct NativeConnector {
+ config: NativeClientConfig,
+}
+
+impl NativeConnector {
+ pub(crate) fn new(config: &ClusterConfig) -> Self {
+ Self {
+ config: config.native_client_config(),
+ }
+ }
+}
+
+impl Connector for NativeConnector {
+ type Conn = FlussConnection;
+
+ async fn dial(
+ &self,
+ identity: Option<&FlussConnectionIdentity>,
+ ) -> Result<FlussConnection, FlussClientError> {
+ if let Some(identity) = identity {
+ if identity.gateway_user() != identity.fluss_user() {
+ return Err(FlussClientError::UnsupportedOperation {
+ message: "user identity mode requires SASL authorization ID support"
+ .to_string(),
+ });
+ }
+ debug_assert_eq!(self.config.security_sasl_username, identity.gateway_user());
+ } else {
+ debug_assert!(!self.config.is_sasl_enabled());
+ }
+ FlussConnection::new(self.config.clone()).await
+ }
+
+ async fn close(
+ &self,
+ connection: Arc<FlussConnection>,
+ timeout: Duration,
+ ) -> Result<(), FlussClientError> {
+ connection.close(timeout).await
+ }
+}
+
+/// One effective user's cache entry.
+#[derive(Debug)]
+struct CachedConnection<C> {
+ /// Installed once for this entry and never replaced. Eviction removes the entire entry.
+ connection: OnceLock<Arc<C>>,
+ /// Serializes cold dials for this user without blocking dials for other users.
+ dialing: tokio::sync::Mutex<()>,
+ /// Milliseconds relative to [`ConnectionCache::origin`], updated whenever a request acquires the
+ /// entry. `fetch_max` prevents an older concurrent reader from moving the timestamp backwards.
+ last_access_ms: AtomicU64,
+}
+
+impl<C> CachedConnection<C> {
+ fn new(now_ms: u64) -> Self {
+ Self {
+ connection: OnceLock::new(),
+ dialing: tokio::sync::Mutex::new(()),
+ last_access_ms: AtomicU64::new(now_ms),
+ }
+ }
+
+ fn touch(&self, now_ms: u64) {
+ self.last_access_ms.fetch_max(now_ms, Ordering::Relaxed);
+ }
+}
+
+struct CacheState<C> {
+ closed: bool,
+ entries: HashMap<Arc<str>, Arc<CachedConnection<C>>>,
+}
+
+/// The connections of one configured Fluss cluster, indexed by effective Fluss user.
+pub(crate) struct ConnectionCache<K: Connector> {
+ cluster: ClusterId,
+ /// Service-mode cache key: the service account under SASL, or a private sentinel for plaintext.
+ gateway_user: Arc<str>,
+ identity: Option<FlussConnectionIdentity>,
+ connector: K,
+ idle_timeout: Duration,
+ origin: Instant,
+ state: RwLock<CacheState<K::Conn>>,
+}
+
+impl<K: Connector> ConnectionCache<K> {
+ pub(crate) fn new(cluster: ClusterId, config: &ClusterConfig, connector: K) -> Self {
+ assert_eq!(
+ config.identity_mode,
+ IdentityMode::Service,
+ "validated configuration must reject user identity mode"
+ );
+ let identity = FlussConnectionIdentity::service(config);
+ let gateway_user = identity
+ .as_ref()
+ .map(|identity| identity.fluss_user.clone())
+ .unwrap_or_else(|| Arc::from(PLAINTEXT_SERVICE_USER));
+ Self {
+ cluster,
+ gateway_user,
+ identity,
+ connector,
+ idle_timeout: config.connection_idle_timeout.get(),
+ origin: Instant::now(),
+ state: RwLock::new(CacheState {
+ closed: false,
+ entries: HashMap::new(),
+ }),
+ }
+ }
+
+ /// Returns the shared connection for the request's effective Fluss user.
+ ///
+ /// The dial belongs to the calling request. Cancelling the request drops the dial future and its
+ /// mutex guard; the next waiter then retries. A native dial failure is likewise not cached.
+ pub(crate) async fn connection(&self, ctx: &RequestContext) -> GatewayResult<Arc<K::Conn>> {
+ let key = self.key(ctx);
+ if let Some(connection) = self.hit(&key)? {
+ return Ok(connection);
+ }
+ self.dial_for(&key).await
+ }
+
+ /// Removes and closes connections whose last acquisition is beyond the configured idle timeout.
+ pub(crate) async fn clean_expired(&self) {
+ let connections = {
+ let mut state = self.write_state();
+ if state.closed {
+ return;
+ }
+ self.take_expired(&mut state)
+ };
+ if connections.is_empty() {
+ return;
+ }
+
+ let results = join_all(
+ connections
+ .iter()
+ .cloned()
+ .map(|connection| self.connector.close(connection, CLEANER_CLOSE_TIMEOUT)),
+ )
+ .await;
+ for result in results {
+ if let Err(error) = result {
+ log::warn!(
+ "failed to drain an idle Fluss connection of cluster `{}`: {error}",
+ self.cluster
+ );
+ }
+ observability::connection_closed(self.cluster.as_str(), "idle");
+ }
+ observability::connections_active(self.cluster.as_str(), self.live_count());
+ }
+
+ /// Closes and removes every cached connection. Idempotent.
+ ///
+ /// The process lifecycle must drain every request and stop the cleaner before calling this method.
+ /// Consequently no dial or operation can race with the map drain; `closed` only prevents accidental
+ /// requests after that lifecycle boundary from opening a connection the process would not own.
+ pub(crate) async fn close(&self, timeout: Duration) -> GatewayResult<()> {
+ let connections = {
+ let mut state = self.write_state();
+ if state.closed {
+ return Ok(());
+ }
+ state.closed = true;
+ state
+ .entries
+ .drain()
+ .filter_map(|(_, entry)| entry.connection.get().cloned())
+ .collect::<Vec<_>>()
+ };
+
+ let closes = connections.len();
+ let results = join_all(
+ connections
+ .into_iter()
+ .map(|connection| self.connector.close(connection, timeout)),
+ )
+ .await;
+ let mut first_failure = None;
+ for result in results {
+ if let Err(error) = result {
+ first_failure = first_failure
+ .or_else(|| Some(map_fluss_error("close the Fluss connection", error)));
+ }
+ observability::connection_closed(self.cluster.as_str(), "shutdown");
+ }
+ if closes > 0 {
+ observability::connections_active(self.cluster.as_str(), 0);
+ }
+ first_failure.map_or(Ok(()), Err)
+ }
+
+ async fn dial_for(&self, key: &Arc<str>) -> GatewayResult<Arc<K::Conn>> {
+ let entry = self.admit(key)?;
+ // TODO: Share one failed dial with its current waiters without caching it for later requests;
+ // see #4101.
+ let _dialing = entry.dialing.lock().await;
+
+ // The caller that won the previous dial may have installed the connection while this request
+ // waited for the user-local mutex.
+ if let Some(connection) = entry.connection.get() {
+ entry.touch(self.elapsed_ms());
+ return Ok(connection.clone());
+ }
+
+ let connection = self
+ .connector
+ .dial(self.identity.as_ref())
+ .await
+ .map_err(|native| map_fluss_error("connect to Fluss", native))?;
+ Ok(self.install(&entry, connection))
+ }
+
+ fn install(&self, entry: &Arc<CachedConnection<K::Conn>>, connection: K::Conn) -> Arc<K::Conn> {
+ let connection = Arc::new(connection);
+ let installed = entry.connection.set(connection.clone());
+ assert!(
+ installed.is_ok(),
+ "a user-local dial mutex must allow only one connection installation"
+ );
+ entry.touch(self.elapsed_ms());
+ observability::connection_created(self.cluster.as_str());
+ observability::connections_active(self.cluster.as_str(), self.live_count());
+ if let Some(identity) = &self.identity {
+ log::info!(
+ "connected to Fluss cluster `{}` as Fluss user `{}` through Gateway user `{}`",
+ self.cluster,
+ identity.fluss_user(),
+ identity.gateway_user()
+ );
+ } else {
+ log::info!(
+ "connected to Fluss cluster `{}` without Fluss authentication",
+ self.cluster
+ );
+ }
+ connection
+ }
+
+ fn hit(&self, key: &str) -> GatewayResult<Option<Arc<K::Conn>>> {
+ let state = self.read_state();
+ if state.closed {
+ return Err(GatewayError::unavailable(
+ "the Fluss connection cache is closed",
+ ));
+ }
+ let Some(entry) = state.entries.get(key) else {
+ return Ok(None);
+ };
+ let Some(connection) = entry.connection.get() else {
+ return Ok(None);
+ };
+ entry.touch(self.elapsed_ms());
+ Ok(Some(connection.clone()))
+ }
+
+ fn admit(&self, key: &Arc<str>) -> GatewayResult<Arc<CachedConnection<K::Conn>>> {
+ {
+ let state = self.read_state();
+ if state.closed {
+ return Err(GatewayError::unavailable(
+ "the Fluss connection cache is closed",
+ ));
+ }
+ if let Some(entry) = state.entries.get(key) {
+ // Refresh under the state lock so the cleaner cannot remove an entry being redialed.
+ entry.touch(self.elapsed_ms());
+ return Ok(entry.clone());
+ }
+ }
+
+ let mut state = self.write_state();
+ if state.closed {
+ return Err(GatewayError::unavailable(
+ "the Fluss connection cache is closed",
+ ));
+ }
+ if let Some(entry) = state.entries.get(key) {
+ entry.touch(self.elapsed_ms());
+ return Ok(entry.clone());
+ }
+ let entry = Arc::new(CachedConnection::new(self.elapsed_ms()));
+ state.entries.insert(key.clone(), entry.clone());
+ Ok(entry)
+ }
+
+ /// The only identity-mode branch point. Service mode always has exactly one key; future user mode
+ /// maps the request principal into an effective Fluss user here.
+ fn key(&self, _ctx: &RequestContext) -> Arc<str> {
+ self.gateway_user.clone()
+ }
+
+ fn take_expired(&self, state: &mut CacheState<K::Conn>) -> Vec<Arc<K::Conn>> {
+ let now_ms = self.elapsed_ms();
+ let threshold_ms = u64::try_from(self.idle_timeout.as_millis()).unwrap_or(u64::MAX);
+ let mut expired = Vec::new();
+ state.entries.retain(|_, entry| {
+ if now_ms.saturating_sub(entry.last_access_ms.load(Ordering::Relaxed)) < threshold_ms {
+ return true;
+ }
+ if let Some(connection) = entry.connection.get() {
+ expired.push(connection.clone());
+ }
+ false
+ });
+ expired
+ }
+
+ fn live_count(&self) -> usize {
+ self.read_state()
+ .entries
+ .values()
+ .filter(|entry| entry.connection.get().is_some())
+ .count()
+ }
+
+ fn elapsed_ms(&self) -> u64 {
+ u64::try_from(self.origin.elapsed().as_millis()).unwrap_or(u64::MAX)
+ }
+
+ fn read_state(&self) -> RwLockReadGuard<'_, CacheState<K::Conn>> {
+ self.state.read().unwrap_or_else(PoisonError::into_inner)
+ }
+
+ fn write_state(&self) -> RwLockWriteGuard<'_, CacheState<K::Conn>> {
+ self.state.write().unwrap_or_else(PoisonError::into_inner)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::backend::errors::tests::api_failure;
+ use crate::config::ConfigDuration;
+ use crate::error::ErrorKind;
+ use fluss::error::FlussError;
+ use futures_util::future::join_all;
+ use std::sync::Mutex;
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ #[derive(Debug)]
+ struct FakeConnection;
+
+ #[derive(Clone)]
+ enum Outcome {
+ Connect,
+ Fail(FlussError),
+ Hang,
+ }
+
+ struct Dialer {
+ dials: Mutex<Vec<Option<FlussConnectionIdentity>>>,
+ closes: AtomicUsize,
+ outcome: Mutex<Outcome>,
+ latency: Duration,
+ active_dials: AtomicUsize,
+ max_active_dials: AtomicUsize,
+ }
+
+ impl Dialer {
+ fn new(outcome: Outcome, latency: Duration) -> Arc<Self> {
+ Arc::new(Self {
+ dials: Mutex::new(Vec::new()),
+ closes: AtomicUsize::new(0),
+ outcome: Mutex::new(outcome),
+ latency,
+ active_dials: AtomicUsize::new(0),
+ max_active_dials: AtomicUsize::new(0),
+ })
+ }
+
+ fn connecting() -> Arc<Self> {
+ Self::new(Outcome::Connect, Duration::ZERO)
+ }
+
+ fn set(&self, outcome: Outcome) {
+ *self.outcome.lock().unwrap() = outcome;
+ }
+
+ fn dialed(&self) -> Vec<Option<FlussConnectionIdentity>> {
+ self.dials.lock().unwrap().clone()
+ }
+
+ fn closes(&self) -> usize {
+ self.closes.load(Ordering::SeqCst)
+ }
+
+ fn max_active_dials(&self) -> usize {
+ self.max_active_dials.load(Ordering::SeqCst)
+ }
+ }
+
+ struct ActiveDial(Arc<Dialer>);
+
+ impl Drop for ActiveDial {
+ fn drop(&mut self) {
+ self.0.active_dials.fetch_sub(1, Ordering::SeqCst);
+ }
+ }
+
+ impl Connector for Arc<Dialer> {
+ type Conn = FakeConnection;
+
+ fn dial(
+ &self,
+ identity: Option<&FlussConnectionIdentity>,
+ ) -> impl Future<Output = Result<FakeConnection, FlussClientError>> + Send {
+ let dialer = self.clone();
+ let identity = identity.cloned();
+ async move {
+ dialer.dials.lock().unwrap().push(identity);
+ let active = dialer.active_dials.fetch_add(1, Ordering::SeqCst) + 1;
+ dialer.max_active_dials.fetch_max(active, Ordering::SeqCst);
+ let _active = ActiveDial(dialer.clone());
+ let outcome = dialer.outcome.lock().unwrap().clone();
+ if !dialer.latency.is_zero() {
+ tokio::time::sleep(dialer.latency).await;
+ }
+ match outcome {
+ Outcome::Connect => Ok(FakeConnection),
+ Outcome::Fail(error) => Err(api_failure(error)),
+ Outcome::Hang => std::future::pending().await,
+ }
+ }
+ }
+
+ fn close(
+ &self,
+ _connection: Arc<FakeConnection>,
+ _timeout: Duration,
+ ) -> impl Future<Output = Result<(), FlussClientError>> + Send {
+ self.closes.fetch_add(1, Ordering::SeqCst);
+ std::future::ready(Ok(()))
+ }
+ }
+
+ fn cache(dialer: &Arc<Dialer>, config: ClusterConfig) -> ConnectionCache<Arc<Dialer>> {
+ ConnectionCache::new(
+ ClusterId::try_from("default").unwrap(),
+ &config,
+ dialer.clone(),
+ )
+ }
+
+ fn service_cache(dialer: &Arc<Dialer>) -> ConnectionCache<Arc<Dialer>> {
+ cache(dialer, service_config())
+ }
+
+ fn service_config() -> ClusterConfig {
+ ClusterConfig {
+ security_protocol: ConnectionSecurityProtocol::Sasl,
+ service_account: Some("gateway_svc".to_string()),
+ service_secret: Some(crate::config::Secret::new("secret")),
+ ..ClusterConfig::default()
+ }
+ }
+
+ fn context() -> RequestContext {
+ RequestContext::for_test("default", Duration::from_secs(5))
+ }
+
+ #[tokio::test]
+ async fn concurrent_first_requests_share_a_single_dial() {
+ let dialer = Dialer::new(Outcome::Connect, Duration::from_millis(50));
+ let cache = Arc::new(service_cache(&dialer));
+
+ let connections = join_all((0..32).map(|_| {
+ let cache = cache.clone();
+ async move { cache.connection(&context()).await.unwrap() }
+ }))
+ .await;
+
+ assert_eq!(dialer.dialed().len(), 1);
+ assert_eq!(dialer.max_active_dials(), 1);
+ assert!(
+ connections
+ .windows(2)
+ .all(|pair| Arc::ptr_eq(&pair[0], &pair[1])),
+ "every caller must share the connection"
+ );
+ assert_eq!(cache.live_count(), 1);
+ }
+
+ #[tokio::test]
+ async fn concurrent_failures_are_serialized_but_not_cached() {
+ let dialer = Dialer::new(
+ Outcome::Fail(FlussError::NetworkException),
+ Duration::from_millis(10),
+ );
+ let cache = Arc::new(service_cache(&dialer));
+
+ let failures = join_all((0..8).map(|_| {
+ let cache = cache.clone();
+ async move { cache.connection(&context()).await.unwrap_err() }
+ }))
+ .await;
+
+ assert!(
+ failures
+ .iter()
+ .all(|failure| failure.kind() == ErrorKind::Unavailable)
+ );
+ assert_eq!(dialer.dialed().len(), 8);
+ assert_eq!(dialer.max_active_dials(), 1);
+ assert_eq!(cache.live_count(), 0);
+ }
+
+ #[tokio::test]
+ async fn a_cancelled_caller_releases_the_dial_for_the_next_request() {
+ let dialer = Dialer::new(Outcome::Hang, Duration::ZERO);
+ let cache = service_cache(&dialer);
+
+ let cancelled =
+ tokio::time::timeout(Duration::from_millis(20), cache.connection(&context())).await;
+ assert!(cancelled.is_err());
+
+ dialer.set(Outcome::Connect);
+ assert!(cache.connection(&context()).await.is_ok());
+ assert_eq!(dialer.dialed().len(), 2);
+ assert_eq!(dialer.max_active_dials(), 1);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn an_expired_failed_entry_is_not_removed_while_redialing() {
+ let dialer = Dialer::new(Outcome::Fail(FlussError::NetworkException), Duration::ZERO);
+ let cache = Arc::new(cache(
+ &dialer,
+ ClusterConfig {
+ connection_idle_timeout: ConfigDuration::from_secs(2),
+ ..service_config()
+ },
+ ));
+
+ assert!(cache.connection(&context()).await.is_err());
+ tokio::time::advance(Duration::from_secs(2)).await;
+
+ dialer.set(Outcome::Hang);
+ let retry_cache = cache.clone();
+ let retry = tokio::spawn(async move { retry_cache.connection(&context()).await });
+ while dialer.dialed().len() < 2 {
+ tokio::task::yield_now().await;
+ }
+
+ cache.clean_expired().await;
+ assert_eq!(cache.read_state().entries.len(), 1);
+
+ let follower_cache = cache.clone();
+ let follower = tokio::spawn(async move { follower_cache.connection(&context()).await });
+ tokio::task::yield_now().await;
+ assert_eq!(dialer.dialed().len(), 2);
+ assert_eq!(dialer.max_active_dials(), 1);
+
+ retry.abort();
+ follower.abort();
+ assert!(retry.await.unwrap_err().is_cancelled());
+ assert!(follower.await.unwrap_err().is_cancelled());
+ dialer.set(Outcome::Connect);
+ assert!(cache.connection(&context()).await.is_ok());
+ assert_eq!(dialer.max_active_dials(), 1);
+ assert_eq!(cache.live_count(), 1);
+ }
+
+ #[tokio::test]
+ async fn service_mode_uses_the_gateway_user_as_its_single_key() {
+ let dialer = Dialer::connecting();
+ let cache = service_cache(&dialer);
+ let first = context();
+ let second = RequestContext::for_test("default", Duration::from_secs(1));
+
+ assert_eq!(&*cache.key(&first), "gateway_svc");
+ assert_eq!(cache.key(&first), cache.key(&second));
+ drop(cache.connection(&first).await.unwrap());
+ assert_eq!(
+ dialer.dialed(),
+ [Some(FlussConnectionIdentity {
+ gateway_user: Arc::from("gateway_svc"),
+ fluss_user: Arc::from("gateway_svc"),
+ })]
+ );
+ }
+
+ #[tokio::test]
+ async fn plaintext_service_mode_uses_the_unauthenticated_key() {
+ let dialer = Dialer::connecting();
+ let cache = cache(&dialer, ClusterConfig::default());
+
+ assert_eq!(&*cache.key(&context()), PLAINTEXT_SERVICE_USER);
+ drop(cache.connection(&context()).await.unwrap());
+ assert_eq!(dialer.dialed(), [None]);
+ }
+
+ #[test]
+ fn access_time_never_moves_backwards() {
+ let entry = CachedConnection::<FakeConnection>::new(10);
+
+ entry.touch(20);
+ entry.touch(15);
+
+ assert_eq!(entry.last_access_ms.load(Ordering::Relaxed), 20);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn idle_cleanup_releases_and_recreates_the_service_connection() {
+ let dialer = Dialer::connecting();
+ let cache = cache(
+ &dialer,
+ ClusterConfig {
+ connection_idle_timeout: ConfigDuration::from_secs(2),
+ ..service_config()
+ },
+ );
+ drop(cache.connection(&context()).await.unwrap());
+
+ tokio::time::advance(Duration::from_millis(1999)).await;
+ cache.clean_expired().await;
+ assert_eq!(dialer.closes(), 0);
+ assert_eq!(cache.live_count(), 1);
+
+ tokio::time::advance(Duration::from_millis(1)).await;
+ cache.clean_expired().await;
+ assert_eq!(dialer.closes(), 1);
+ assert_eq!(cache.live_count(), 0);
+
+ drop(cache.connection(&context()).await.unwrap());
+ assert_eq!(dialer.dialed().len(), 2);
+ }
+
+ #[tokio::test]
+ async fn close_drains_connections_and_permanently_closes_the_cache() {
+ let dialer = Dialer::connecting();
+ let cache = service_cache(&dialer);
+ drop(cache.connection(&context()).await.unwrap());
+
+ cache.close(Duration::from_secs(1)).await.unwrap();
+ assert_eq!(dialer.closes(), 1);
+ assert_eq!(cache.live_count(), 0);
+
+ cache.close(Duration::from_secs(1)).await.unwrap();
+ assert_eq!(dialer.closes(), 1);
+
+ let error = cache.connection(&context()).await.unwrap_err();
+ assert_eq!(error.kind(), ErrorKind::Unavailable);
+ assert_eq!(error.message(), "the Fluss connection cache is closed");
+ assert_eq!(dialer.dialed().len(), 1);
+ }
+}
diff --git a/fluss-gateway/src/backend/context.rs b/fluss-gateway/src/backend/context.rs
new file mode 100644
index 000000000..7eb9a8bae
--- /dev/null
+++ b/fluss-gateway/src/backend/context.rs
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! The per-request context every [`crate::backend::FlussBackend`] call carries.
+//!
+//! It is the whole protocol-neutral description of one request: who asked, which cluster, how long
+//! there is left, and whether the caller is still waiting. Nothing in it outlives the call.
+
+use crate::backend::types::ClusterId;
+use crate::error::{GatewayError, GatewayResult};
+use std::collections::BTreeMap;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio_util::sync::CancellationToken;
+
+/// The caller identity produced by the Gateway authenticator.
+///
+/// A principal describes only the client-to-Gateway request. It is deliberately not hashable and
+/// carries no Fluss connection identity: the native backend maps it immediately before choosing a
+/// connection when user identity mode is implemented.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Principal {
+ name: Arc<str>,
+ attributes: Arc<BTreeMap<String, Vec<String>>>,
+}
+
+impl Principal {
+ pub fn new(name: impl Into<Arc<str>>, attributes: BTreeMap<String, Vec<String>>) -> Self {
+ Self {
+ name: name.into(),
+ attributes: Arc::new(attributes),
+ }
+ }
+
+ pub fn name(&self) -> &str {
+ &self.name
+ }
+
+ pub fn attributes(&self) -> &BTreeMap<String, Vec<String>> {
+ &self.attributes
+ }
+}
+
+/// One request's deadline, cancellation signal, principal, and cluster.
+#[derive(Clone, Debug)]
+pub struct RequestContext {
+ request_id: Arc<str>,
+ cluster_id: ClusterId,
+ /// Absolute end of this request's budget, assigned by the REST middleware.
+ deadline: Instant,
+ /// Cancelled when the caller goes away or the process drains.
+ cancellation: CancellationToken,
+ /// `None` means the request has not been authenticated. It is not represented by a reserved
+ /// principal name, so a real caller named `anonymous` remains unambiguous.
+ principal: Option<Principal>,
+}
+
+impl RequestContext {
+ pub fn new(
+ request_id: impl Into<Arc<str>>,
+ cluster_id: ClusterId,
+ deadline: Instant,
+ cancellation: CancellationToken,
+ principal: Option<Principal>,
+ ) -> Self {
+ Self {
+ request_id: request_id.into(),
+ cluster_id,
+ deadline,
+ cancellation,
+ principal,
+ }
+ }
+
+ /// Runs one backend operation under this request's deadline and cancellation signal.
+ ///
+ /// Every wait a backend call can produce — queueing behind a connection attempt, the RPC itself —
+ /// happens inside this future, so the caller is always answered within its budget. Abandoning the
+ /// future is not the same as stopping the work: until `fluss-rs` accepts a per-RPC timeout, an RPC
+ /// that outran the deadline stays in flight on its connection.
+ pub async fn run<T, F>(&self, operation: F) -> GatewayResult<T>
+ where
+ F: Future<Output = GatewayResult<T>>,
+ {
+ self.ensure_active()?;
+ tokio::select! {
+ biased;
+ () = self.cancellation.cancelled() => Err(cancelled()),
+ () = tokio::time::sleep_until(self.deadline.into()) => Err(expired()),
+ result = operation => result,
+ }
+ }
+
+ /// Fails fast when the caller is already gone or the budget is already spent.
+ pub fn ensure_active(&self) -> GatewayResult<()> {
+ if self.cancellation.is_cancelled() {
+ return Err(cancelled());
+ }
+ if Instant::now() >= self.deadline {
+ return Err(expired());
+ }
+ Ok(())
+ }
+
+ /// What is left of the request's budget, which is what bounds connection retries.
+ pub fn remaining(&self) -> Duration {
+ self.deadline.saturating_duration_since(Instant::now())
+ }
+
+ pub fn cancellation(&self) -> &CancellationToken {
+ &self.cancellation
+ }
+
+ /// The authenticated caller, once authentication is implemented.
+ pub fn principal(&self) -> Option<&Principal> {
+ self.principal.as_ref()
+ }
+
+ pub fn cluster_id(&self) -> &ClusterId {
+ &self.cluster_id
+ }
+
+ pub fn request_id(&self) -> &str {
+ &self.request_id
+ }
+
+ /// An anonymous context with `budget` left, for the tests of the layers below the adapter.
+ #[cfg(test)]
+ pub(crate) fn for_test(cluster: &str, budget: Duration) -> Self {
+ Self::new(
+ "test-request",
+ ClusterId::try_from(cluster).expect("valid test cluster ID"),
+ Instant::now() + budget,
+ CancellationToken::new(),
+ None,
+ )
+ }
+}
+
+fn cancelled() -> GatewayError {
+ GatewayError::cancelled("the request was cancelled")
+}
+
+fn expired() -> GatewayError {
+ GatewayError::deadline_exceeded("request deadline exceeded")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::error::ErrorKind;
+
+ #[test]
+ fn an_authenticated_principal_is_distinct_from_an_unauthenticated_request() {
+ let principal = Principal::new(
+ "anonymous",
+ BTreeMap::from([("tenant".to_string(), vec!["sales".to_string()])]),
+ );
+ let ctx = RequestContext::new(
+ "request",
+ ClusterId::try_from("default").unwrap(),
+ Instant::now() + Duration::from_secs(1),
+ CancellationToken::new(),
+ Some(principal),
+ );
+
+ assert_eq!(ctx.principal().unwrap().name(), "anonymous");
+ assert_eq!(ctx.principal().unwrap().attributes()["tenant"], ["sales"]);
+ assert!(
+ RequestContext::for_test("default", Duration::from_secs(1))
+ .principal()
+ .is_none()
+ );
+ }
+
+ /// The three outcomes `run` has to keep apart, and the fact that the operation is abandoned in two
+ /// of them rather than left running.
+ #[tokio::test]
+ async fn run_is_bounded_by_the_deadline_and_the_cancellation_signal() {
+ let ctx = RequestContext::for_test("default", Duration::from_secs(5));
+ assert_eq!(ctx.run(async { Ok(7) }).await.unwrap(), 7);
+
+ let expired = RequestContext::for_test("default", Duration::from_millis(20));
+ let error: GatewayError = expired
+ .run(async {
+ std::future::pending::<()>().await;
+ Ok(())
+ })
+ .await
+ .unwrap_err();
+ assert_eq!(error.kind(), ErrorKind::DeadlineExceeded);
+ // An already-spent budget is refused without entering the operation at all.
+ assert_eq!(
+ expired.ensure_active().unwrap_err().kind(),
+ ErrorKind::DeadlineExceeded
+ );
+ assert_eq!(expired.remaining(), Duration::ZERO);
+
+ let ctx = RequestContext::for_test("default", Duration::from_secs(5));
+ ctx.cancellation().cancel();
+ let error: GatewayError = ctx.run(async { Ok(()) }).await.unwrap_err();
+ assert_eq!(error.kind(), ErrorKind::Cancelled);
+ }
+}
diff --git a/fluss-gateway/src/backend/errors.rs b/fluss-gateway/src/backend/errors.rs
new file mode 100644
index 000000000..ee7c45e1f
--- /dev/null
+++ b/fluss-gateway/src/backend/errors.rs
@@ -0,0 +1,215 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Classification of native `fluss-rs` failures.
+//!
+//! Its own module because both layers above it need it: the pool classifies a failed dial, and the
+//! backend classifies a failed operation. Nothing outside this module inspects a `fluss` error type.
+
+use crate::error::{GatewayError, Resource};
+use fluss::error::{Error as FlussClientError, FlussError};
+
+/// Classifies one native failure, with `what` naming the attempted operation in the client-facing
+/// message.
+///
+/// The rules the gateway commits to: an error code Fluss defines for a resource keeps that meaning,
+/// service authentication or authorization failures are backend deployment faults, a retriable
+/// transport failure is a transient outage, and anything the client cannot classify is reported as a
+/// backend failure rather than a gateway-internal one. Messages carry the operation, never the native
+/// error text, which can contain addresses and payload detail; that goes to the log.
+pub(crate) fn map_fluss_error(what: &str, error: FlussClientError) -> GatewayError {
+ if let Some(api_error) = error.api_error()
+ && let Some(mapped) = map_api_error(what, api_error)
+ {
+ return mapped;
+ }
+ match &error {
+ FlussClientError::UnsupportedOperation { .. }
+ | FlussClientError::UnsupportedVersion { .. } => {
+ log::warn!("Fluss does not support the request while trying to {what}: {error}");
+ GatewayError::unsupported(format!(
+ "Fluss does not support the request while trying to {what}"
+ ))
+ }
+ FlussClientError::IllegalArgument { .. } => GatewayError::invalid_argument(format!(
+ "Fluss rejected the request while trying to {what}"
+ )),
+ _ if error.is_retriable() => {
+ log::warn!("Fluss is temporarily unavailable while trying to {what}: {error}");
+ GatewayError::unavailable(format!("Fluss is unavailable while trying to {what}"))
+ }
+ _ => {
+ log::error!("the Fluss request failed while trying to {what}: {error}");
+ // An unclassifiable native failure is the backend's, not the gateway's (FIP-49 `backend`).
+ GatewayError::backend(format!("Fluss failed while trying to {what}"))
+ }
+ }
+}
+
+/// Maps the protocol error codes that carry a meaning of their own. `None` falls through to the
+/// transport-level classification.
+fn map_api_error(what: &str, api_error: FlussError) -> Option<GatewayError> {
+ Some(match api_error {
+ FlussError::DatabaseNotExist => GatewayError::not_found(format!(
+ "the database does not exist while trying to {what}"
+ ))
+ .with_resource(Resource::Database),
+ FlussError::TableNotExist => {
+ GatewayError::not_found(format!("the table does not exist while trying to {what}"))
+ .with_resource(Resource::Table)
+ }
+ FlussError::DatabaseAlreadyExist => GatewayError::already_exists(format!(
+ "the database already exists while trying to {what}"
+ ))
+ .with_resource(Resource::Database),
+ FlussError::TableAlreadyExist => {
+ GatewayError::already_exists(format!("the table already exists while trying to {what}"))
+ .with_resource(Resource::Table)
+ }
+ FlussError::DatabaseNotEmpty => {
+ GatewayError::failed_precondition(format!("the database is not empty, {what} refused"))
+ .with_resource(Resource::Database)
+ }
+ FlussError::InvalidDatabaseException | FlussError::InvalidTableException => {
+ GatewayError::invalid_argument(format!(
+ "Fluss rejected the name while trying to {what}"
+ ))
+ }
+ // In service mode, both failures are deployment faults, including protocol mismatches.
+ // User mode may later map a caller-specific authorization failure to 403.
+ FlussError::AuthenticateException => {
+ log::error!("Fluss rejected the gateway connection while trying to {what}");
+ GatewayError::backend(format!("Fluss rejected the gateway while trying to {what}"))
+ }
+ FlussError::AuthorizationException => {
+ log::error!("Fluss denied the gateway while trying to {what}");
+ GatewayError::backend(format!("Fluss denied the gateway while trying to {what}"))
+ }
+ _ => return None,
+ })
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+ use super::*;
+ use crate::error::ErrorKind;
+
+ /// A native failure carrying one Fluss protocol error code.
+ pub(crate) fn api_failure(error: FlussError) -> FlussClientError {
+ FlussClientError::FlussAPIError {
+ api_error: fluss::error::ApiError {
+ code: error.code(),
+ message: "server detail".to_string(),
+ },
+ }
+ }
+
+ /// Every mapped condition keeps the class and the wire code a caller can act on, and no message
+ /// quotes the native error text.
+ #[test]
+ fn native_failures_map_to_their_gateway_class_and_code() {
+ let cases = [
+ (
+ api_failure(FlussError::DatabaseNotExist),
+ ErrorKind::NotFound,
+ "database_not_found",
+ ),
+ (
+ api_failure(FlussError::TableNotExist),
+ ErrorKind::NotFound,
+ "table_not_found",
+ ),
+ (
+ api_failure(FlussError::DatabaseAlreadyExist),
+ ErrorKind::AlreadyExists,
+ "database_already_exists",
+ ),
+ (
+ api_failure(FlussError::TableAlreadyExist),
+ ErrorKind::AlreadyExists,
+ "table_already_exists",
+ ),
+ (
+ api_failure(FlussError::DatabaseNotEmpty),
+ ErrorKind::FailedPrecondition,
+ "database_not_empty",
+ ),
+ (
+ api_failure(FlussError::InvalidDatabaseException),
+ ErrorKind::InvalidArgument,
+ "invalid_argument",
+ ),
+ (
+ api_failure(FlussError::AuthenticateException),
+ ErrorKind::Backend,
+ "backend",
+ ),
+ (
+ api_failure(FlussError::AuthorizationException),
+ ErrorKind::Backend,
+ "backend",
+ ),
+ (
+ FlussClientError::UnsupportedVersion {
+ message: "server detail".to_string(),
+ },
+ ErrorKind::Unsupported,
+ "unsupported",
+ ),
+ (
+ FlussClientError::IllegalArgument {
+ message: "server detail".to_string(),
+ },
+ ErrorKind::InvalidArgument,
+ "invalid_argument",
+ ),
+ (
+ api_failure(FlussError::NetworkException),
+ ErrorKind::Unavailable,
+ "unavailable",
+ ),
+ (
+ api_failure(FlussError::NotLeaderOrFollower),
+ ErrorKind::Unavailable,
+ "unavailable",
+ ),
+ (
+ FlussClientError::RowConvertError {
+ message: "server detail".to_string(),
+ },
+ ErrorKind::Backend,
+ "backend",
+ ),
+ ];
+ for (native, expected_kind, expected_code) in cases {
+ let rendered = native.to_string();
+ let mapped = map_fluss_error("list the databases", native);
+ assert_eq!(mapped.kind(), expected_kind, "{rendered}");
+ assert_eq!(mapped.code(), expected_code, "{rendered}");
+ assert!(
+ mapped.message().contains("list the databases"),
+ "{}",
+ mapped.message()
+ );
+ assert!(
+ !mapped.message().contains("server detail"),
+ "the native detail must stay in the log: {}",
+ mapped.message()
+ );
+ }
+ }
+}
diff --git a/fluss-gateway/src/backend/fake.rs b/fluss-gateway/src/backend/fake.rs
new file mode 100644
index 000000000..2eafc86fc
--- /dev/null
+++ b/fluss-gateway/src/backend/fake.rs
@@ -0,0 +1,227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! In-memory [`FlussBackend`] for the protocol tests (FIP-49 test plan).
+//!
+//! It needs no Fluss cluster and is compiled only under `cfg(test)`, so it can never be part of a
+//! shippable path. Replacing the whole backend also replaces the connection pool, which is why the
+//! pool has its own tests against an injected connector.
+
+use crate::backend::context::RequestContext;
+use crate::backend::types::ClusterId;
+use crate::backend::{FlussBackend, unknown_cluster};
+use crate::error::{GatewayError, GatewayResult, Resource};
+use async_trait::async_trait;
+use std::collections::BTreeMap;
+use std::sync::{Mutex, MutexGuard, PoisonError};
+use std::time::Duration;
+
+/// The injected behaviour of one fixture backend.
+struct FakeState {
+ databases: BTreeMap<String, Vec<String>>,
+ /// Returned by every catalog call instead of the fixture content.
+ failure: Option<GatewayError>,
+ /// Delay before answering, for the deadline tests.
+ latency: Duration,
+}
+
+/// A backend whose clusters and catalog are fixed and whose failures are injected by the test.
+///
+/// The catalog is shared by every configured cluster: the tests that need cluster isolation are the
+/// pool's, not the protocol's.
+pub struct FakeFlussBackend {
+ clusters: Vec<ClusterId>,
+ state: Mutex<FakeState>,
+}
+
+impl Default for FakeFlussBackend {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl FakeFlussBackend {
+ /// One cluster named `default` with an empty catalog.
+ pub fn new() -> Self {
+ Self::with_catalog(&[])
+ }
+
+ /// One cluster named `default` over a catalog of databases and their tables, in any order.
+ pub fn with_catalog(databases: &[(&str, &[&str])]) -> Self {
+ Self {
+ clusters: vec![cluster_id("default")],
+ state: Mutex::new(FakeState {
+ databases: databases
+ .iter()
+ .map(|(database, tables)| {
+ (
+ (*database).to_string(),
+ tables.iter().map(|table| (*table).to_string()).collect(),
+ )
+ })
+ .collect(),
+ failure: None,
+ latency: Duration::ZERO,
+ }),
+ }
+ }
+
+ /// Several clusters with an empty catalog, for the discovery tests.
+ pub fn with_clusters(ids: &[&str]) -> Self {
+ let mut clusters: Vec<ClusterId> = ids.iter().map(|id| cluster_id(id)).collect();
+ clusters.sort();
+ Self {
+ clusters,
+ ..Self::with_catalog(&[])
+ }
+ }
+
+ /// Makes every catalog call fail, the way an unreachable or refusing cluster would.
+ pub fn fail_with(&self, error: GatewayError) {
+ self.state().failure = Some(error);
+ }
+
+ /// Delays every catalog call, so a test can drive a request into its deadline.
+ pub fn set_latency(&self, latency: Duration) {
+ self.state().latency = latency;
+ }
+
+ /// Runs one fixture answer under the request's budget, like the production backend does.
+ async fn answer<T>(
+ &self,
+ ctx: &RequestContext,
+ produce: impl FnOnce(&FakeState) -> GatewayResult<T>,
+ ) -> GatewayResult<T> {
+ if !self.has_cluster(ctx.cluster_id().as_str()) {
+ return Err(unknown_cluster(ctx.cluster_id().as_str()));
+ }
+ let latency = self.state().latency;
+ ctx.run(async move {
+ if !latency.is_zero() {
+ tokio::time::sleep(latency).await;
+ }
+ let state = self.state();
+ match &state.failure {
+ Some(failure) => Err(failure.clone()),
+ None => produce(&state),
+ }
+ })
+ .await
+ }
+
+ fn state(&self) -> MutexGuard<'_, FakeState> {
+ self.state.lock().unwrap_or_else(PoisonError::into_inner)
+ }
+}
+
+fn cluster_id(id: &str) -> ClusterId {
+ ClusterId::try_from(id).expect("valid fixture cluster ID")
+}
+
+#[async_trait]
+impl FlussBackend for FakeFlussBackend {
+ fn clusters(&self) -> Vec<ClusterId> {
+ self.clusters.clone()
+ }
+
+ fn has_cluster(&self, id: &str) -> bool {
+ self.clusters.iter().any(|cluster| cluster.as_str() == id)
+ }
+
+ async fn list_databases(&self, ctx: &RequestContext) -> GatewayResult<Vec<String>> {
+ self.answer(ctx, |state| Ok(state.databases.keys().cloned().collect()))
+ .await
+ }
+
+ async fn list_tables(
+ &self,
+ ctx: &RequestContext,
+ database: &str,
+ ) -> GatewayResult<Vec<String>> {
+ self.answer(ctx, |state| {
+ state.databases.get(database).cloned().ok_or_else(|| {
+ GatewayError::not_found(format!("database `{database}` does not exist"))
+ .with_resource(Resource::Database)
+ })
+ })
+ .await
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::error::ErrorKind;
+
+ #[tokio::test]
+ async fn the_fixture_answers_its_catalog_until_a_failure_is_injected() {
+ let backend = FakeFlussBackend::with_catalog(&[("sales", &["orders"]), ("ops", &[])]);
+ let ctx = RequestContext::for_test("default", Duration::from_secs(5));
+
+ assert_eq!(
+ backend.list_databases(&ctx).await.unwrap(),
+ ["ops", "sales"]
+ );
+ assert_eq!(
+ backend.list_tables(&ctx, "sales").await.unwrap(),
+ ["orders"]
+ );
+ assert_eq!(
+ backend.list_tables(&ctx, "nope").await.unwrap_err().code(),
+ "database_not_found"
+ );
+
+ // A cluster the fixture does not serve is the same 404 the production backend answers.
+ let elsewhere = RequestContext::for_test("other", Duration::from_secs(5));
+ assert_eq!(
+ backend.list_databases(&elsewhere).await.unwrap_err().code(),
+ "cluster_not_found"
+ );
+
+ backend.fail_with(GatewayError::unavailable("cluster is down"));
+ assert_eq!(
+ backend.list_databases(&ctx).await.unwrap_err().kind(),
+ ErrorKind::Unavailable
+ );
+ }
+
+ /// An injected delay is bounded by the request, not by the fixture.
+ #[tokio::test]
+ async fn a_slow_answer_runs_into_the_request_deadline() {
+ let backend = FakeFlussBackend::new();
+ backend.set_latency(Duration::from_secs(30));
+ let ctx = RequestContext::for_test("default", Duration::from_millis(20));
+
+ assert_eq!(
+ backend.list_databases(&ctx).await.unwrap_err().kind(),
+ ErrorKind::DeadlineExceeded
+ );
+ }
+
+ #[test]
+ fn several_fixture_clusters_are_discovered_in_order() {
+ let backend = FakeFlussBackend::with_clusters(&["zeta", "alpha"]);
+ assert_eq!(
+ backend
+ .clusters()
+ .iter()
+ .map(ClusterId::as_str)
+ .collect::<Vec<_>>(),
+ ["alpha", "zeta"]
+ );
+ }
+}
diff --git a/fluss-gateway/src/backend/mod.rs b/fluss-gateway/src/backend/mod.rs
new file mode 100644
index 000000000..12f2bbd26
--- /dev/null
+++ b/fluss-gateway/src/backend/mod.rs
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! The gateway's HTTP-independent backend component.
+//!
+//! Protocol adapters reach Fluss only through [`FlussBackend`], which is their single dependency: the
+//! cluster a request names, the identity it acts as, and the connection that serves it are all resolved
+//! below this trait. That boundary is what lets the REST layer be tested without a cluster and lets a
+//! second protocol be added without reimplementing any of it.
+//!
+//! A backend **owns its connections**. Shutdown is a method on the concrete type rather than on this
+//! trait, so a protocol adapter cannot reach it.
+//!
+//! Every capability method is a complete request-response operation: the contract deliberately exposes
+//! **no** way to open a stream, scanner, cursor, or any other handle that would outlive the call,
+//! because the gateway keeps no request-spanning state. There is also no cluster-health method —
+//! reachability is not a queryable state, it is the outcome of the next request.
+
+pub mod client;
+pub mod connection;
+pub mod context;
+pub mod errors;
+pub mod types;
+
+#[cfg(test)]
+pub mod fake;
+
+use crate::backend::context::RequestContext;
+use crate::backend::types::ClusterId;
+use crate::error::{GatewayError, GatewayResult, Resource};
+use async_trait::async_trait;
+
+/// The backend capabilities the protocol adapters depend on.
+///
+/// Implementations never return HTTP or JSON types: the adapter owns status mapping, and the backend
+/// owns the translation from native failures into [`GatewayError`].
+///
+/// The remaining FIP-49 capabilities — describe table, partitions, DDL, write, lookup — are appended
+/// here in the same shape, so carrying them changes neither the connection layer nor the protocol layer.
+#[async_trait]
+pub trait FlussBackend: Send + Sync + 'static {
+ /// The clusters this gateway serves, in lexical ID order.
+ ///
+ /// A configuration echo, which is why it is synchronous, cannot fail, and takes no request.
+ fn clusters(&self) -> Vec<ClusterId>;
+
+ /// Whether `id` names a cluster this gateway serves. Allocation-free, for per-request validation.
+ fn has_cluster(&self, id: &str) -> bool;
+
+ async fn list_databases(&self, ctx: &RequestContext) -> GatewayResult<Vec<String>>;
+
+ async fn list_tables(&self, ctx: &RequestContext, database: &str)
+ -> GatewayResult<Vec<String>>;
+}
+
+/// The answer to a cluster this gateway does not serve.
+///
+/// A malformed ID and an unconfigured one are the same answer to a caller, so they share one error.
+pub fn unknown_cluster(id: &str) -> GatewayError {
+ GatewayError::not_found(format!("unknown cluster `{id}`")).with_resource(Resource::Cluster)
+}
diff --git a/fluss-gateway/src/backend/types.rs b/fluss-gateway/src/backend/types.rs
new file mode 100644
index 000000000..88f3c1e01
--- /dev/null
+++ b/fluss-gateway/src/backend/types.rs
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Protocol-neutral identifiers of the backend component.
+
+use crate::error::GatewayError;
+use std::fmt;
+use std::sync::Arc;
+
+/// Validated cluster identifier, shared by configuration, the registry, and the protocol adapters.
+///
+/// Parsing a caller-supplied ID here means an unconfigured or malformed cluster reaches the registry
+/// as a value it can look up, never as a raw string that could be interpolated somewhere else. The
+/// accepted shape is the one configuration validation enforces, so a request can only name a cluster
+/// that an operator could have configured.
+///
+/// Backed by an `Arc<str>` because every request clones the ID into its context and its pool key.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct ClusterId(Arc<str>);
+
+impl ClusterId {
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+impl TryFrom<&str> for ClusterId {
+ type Error = GatewayError;
+
+ /// Rejects an ID no operator could have configured.
+ fn try_from(value: &str) -> Result<Self, Self::Error> {
+ if !crate::config::valid_cluster_id(value) {
+ return Err(GatewayError::invalid_argument(
+ "cluster ID must be at most 63 characters, start with a lowercase letter, and \
+ contain only lowercase letters, digits, or underscores",
+ ));
+ }
+ Ok(Self(Arc::from(value)))
+ }
+}
+
+impl fmt::Display for ClusterId {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str(self.as_str())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::error::ErrorKind;
+
+ #[test]
+ fn accepts_only_ids_configuration_would_accept() {
+ for id in ["default", "a", "east_1", &"c".repeat(63)] {
+ assert_eq!(ClusterId::try_from(id).unwrap().as_str(), id);
+ }
+ for id in ["", "Default", "1east", "east-1", "east.1", &"c".repeat(64)] {
+ let error = ClusterId::try_from(id).unwrap_err();
+ assert_eq!(error.kind(), ErrorKind::InvalidArgument, "{id:?}");
+ }
+ }
+}
diff --git a/fluss-gateway/src/config.rs b/fluss-gateway/src/config.rs
index bc8792136..75b123b52 100644
--- a/fluss-gateway/src/config.rs
+++ b/fluss-gateway/src/config.rs
@@ -215,8 +215,10 @@ impl<'de> Deserialize<'de> for ByteSize {
const INSTANCE_ID_KEY: &str = "gateway.instance-id";
const REST_LISTEN_KEY: &str = "gateway.rest.listen";
const REST_HEADER_READ_TIMEOUT_KEY: &str = "gateway.rest.header-read-timeout";
-const REST_REQUEST_TIMEOUT_KEY: &str = "gateway.rest.write.request-timeout";
+const REST_REQUEST_TIMEOUT_KEY: &str = "gateway.rest.request-timeout";
const REST_MAX_REQUEST_BYTES_KEY: &str = "gateway.rest.write.max-request-bytes";
+const REST_METADATA_MAX_CONCURRENT_REQUESTS_KEY: &str =
+ "gateway.rest.metadata.max-concurrent-requests";
const METRICS_ENABLED_KEY: &str = "gateway.metrics.enabled";
const METRICS_EXPORTERS_KEY: &str = "gateway.metrics.exporters";
const METRICS_LISTEN_KEY: &str = "gateway.metrics.exporter.prometheus.listen";
@@ -247,6 +249,20 @@ const REST_PREFIX_LOOKUP_MAX_CONCURRENT_REQUESTS_KEY: &str =
const DEFAULT_REST_LISTEN: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080);
const DEFAULT_REST_HEADER_READ_TIMEOUT: ConfigDuration = ConfigDuration::from_secs(10);
const DEFAULT_REST_REQUEST_TIMEOUT: ConfigDuration = ConfigDuration::from_secs(30);
+const DEFAULT_REST_METADATA_MAX_CONCURRENT_REQUESTS: u32 = 16;
+
+/// Time reserved for encoding and sending an HTTP response after the handler deadline.
+pub(crate) const REST_RESPONSE_GRACE: Duration = Duration::from_secs(1);
+
+/// Applies the response grace exactly as the REST deadline middleware does.
+pub(crate) fn rest_handler_timeout(request_timeout: Duration) -> Duration {
+ if request_timeout > REST_RESPONSE_GRACE {
+ request_timeout - REST_RESPONSE_GRACE
+ } else {
+ request_timeout
+ }
+}
+
const DEFAULT_REST_MAX_REQUEST_BYTES: ByteSize = ByteSize::new(32 * 1024 * 1024);
const DEFAULT_METRICS_ENABLED: bool = true;
const DEFAULT_METRICS_LISTEN: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9095);
@@ -254,15 +270,16 @@ const DEFAULT_SHUTDOWN_DRAIN_TIMEOUT: ConfigDuration = ConfigDuration::from_secs
const DEFAULT_CLUSTER_ID: &str = "default";
const DEFAULT_BOOTSTRAP_SERVERS: &str = "127.0.0.1:9123";
+const DEFAULT_CONNECTION_IDLE_TIMEOUT: ConfigDuration = ConfigDuration::from_secs(10 * 60);
const DEFAULT_TRUSTED_HEADER_NAME: &str = "x-forwarded-user";
const CLUSTER_BOOTSTRAP_SERVERS_KEY: &str = "bootstrap.servers";
const CLUSTER_CONNECT_TIMEOUT_KEY: &str = "connect-timeout";
+const CLUSTER_CONNECTION_IDLE_TIMEOUT_KEY: &str = "connection.idle-timeout";
+const CLUSTER_SECURITY_PROTOCOL_KEY: &str = "connection.security.protocol";
const CLUSTER_SERVICE_ACCOUNT_KEY: &str = "connection.service.account";
const CLUSTER_SERVICE_SECRET_KEY: &str = "connection.service.secret";
const CLUSTER_IDENTITY_MODE_KEY: &str = "connection.identity-mode";
-const CLUSTER_CONNECTION_MAX_KEY: &str = "connection.max";
-const CLUSTER_CONNECTION_IDLE_TIMEOUT_KEY: &str = "connection.idle-timeout";
type ApplyConfigValue<C> = fn(&mut C, &Value) -> Result<(), String>;
@@ -358,6 +375,16 @@ impl FromConfigValue for IdentityMode {
}
}
+impl FromConfigValue for ConnectionSecurityProtocol {
+ fn from_config_value(value: &Value) -> Result<Self, String> {
+ match String::from_config_value(value)?.as_str() {
+ "plaintext" => Ok(Self::Plaintext),
+ "sasl" => Ok(Self::Sasl),
+ _ => Err("expected plaintext or sasl".to_string()),
+ }
+ }
+}
+
impl FromConfigValue for Secret {
fn from_config_value(value: &Value) -> Result<Self, String> {
String::from_config_value(value).map(Self::new)
@@ -422,6 +449,11 @@ const CONFIG_ENTRIES: &[GatewayConfigEntry] = &[
REST_MAX_REQUEST_BYTES_KEY,
server.rest.max_body_bytes
),
+ typed_entry!(
+ GatewayConfigEntry,
+ REST_METADATA_MAX_CONCURRENT_REQUESTS_KEY,
+ server.rest.metadata_max_concurrent_requests
+ ),
typed_entry!(
GatewayConfigEntry,
REST_WRITE_MAX_ROWS_KEY,
@@ -507,7 +539,9 @@ const CONFIG_ENTRIES: &[GatewayConfigEntry] = &[
typed_entry!(GatewayConfigEntry, SECURITY_TRUSTED_HEADER_NAME_KEY, optional security.trusted_header_name),
];
-// `request-timeout` is omitted until fluss-rust exposes a general per-RPC timeout API.
+// TODO: Expose `request-timeout` after fluss-rust independently bounds a complete bootstrap
+// attempt (TCP connect, API version negotiation, SASL, and metadata), matching the Java client,
+// and provides a general per-RPC timeout. `connect-timeout` must remain TCP-connect-only.
const CLUSTER_ENTRIES: &[ClusterConfigEntry] = &[
typed_entry!(
ClusterConfigEntry,
@@ -519,11 +553,19 @@ const CLUSTER_ENTRIES: &[ClusterConfigEntry] = &[
CLUSTER_CONNECT_TIMEOUT_KEY,
connect_timeout
),
+ typed_entry!(
+ ClusterConfigEntry,
+ CLUSTER_CONNECTION_IDLE_TIMEOUT_KEY,
+ connection_idle_timeout
+ ),
+ typed_entry!(
+ ClusterConfigEntry,
+ CLUSTER_SECURITY_PROTOCOL_KEY,
+ security_protocol
+ ),
typed_entry!(ClusterConfigEntry, CLUSTER_SERVICE_ACCOUNT_KEY, optional service_account),
typed_entry!(ClusterConfigEntry, CLUSTER_SERVICE_SECRET_KEY, optional service_secret),
typed_entry!(ClusterConfigEntry, CLUSTER_IDENTITY_MODE_KEY, identity_mode),
- typed_entry!(ClusterConfigEntry, CLUSTER_CONNECTION_MAX_KEY, optional connection_max),
- typed_entry!(ClusterConfigEntry, CLUSTER_CONNECTION_IDLE_TIMEOUT_KEY, optional connection_idle_timeout),
];
/// Gateway listeners and instance identity.
@@ -548,6 +590,8 @@ pub struct RestServerConfig {
pub request_timeout: ConfigDuration,
/// Maximum accepted request body size. Exceeding it yields 413.
pub max_body_bytes: ByteSize,
+ /// Maximum metadata requests concurrently accessing Fluss. Exceeding it yields 429.
+ pub metadata_max_concurrent_requests: u32,
}
impl Default for RestServerConfig {
@@ -557,6 +601,7 @@ impl Default for RestServerConfig {
header_read_timeout: DEFAULT_REST_HEADER_READ_TIMEOUT,
request_timeout: DEFAULT_REST_REQUEST_TIMEOUT,
max_body_bytes: DEFAULT_REST_MAX_REQUEST_BYTES,
+ metadata_max_concurrent_requests: DEFAULT_REST_METADATA_MAX_CONCURRENT_REQUESTS,
}
}
}
@@ -578,6 +623,11 @@ impl RestServerConfig {
"{REST_MAX_REQUEST_BYTES_KEY} must be greater than zero"
));
}
+ if self.metadata_max_concurrent_requests == 0 {
+ problems.push(format!(
+ "{REST_METADATA_MAX_CONCURRENT_REQUESTS_KEY} must be greater than zero"
+ ));
+ }
}
}
@@ -604,13 +654,24 @@ impl Default for MetricsServerConfig {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)]
#[serde(rename_all = "lowercase")]
pub enum IdentityMode {
- /// One shared connection authenticated as the configured service account.
+ /// One shared connection, authenticated as the service account when the protocol supports it.
#[default]
Service,
/// Authenticate as the service account and carry the request's principal as the authorization ID.
User,
}
+/// Transport authentication used by the native Fluss client.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)]
+#[serde(rename_all = "lowercase")]
+pub enum ConnectionSecurityProtocol {
+ /// Connect without authenticating a Fluss user.
+ #[default]
+ Plaintext,
+ /// Authenticate the configured service account with SASL/PLAIN.
+ Sasl,
+}
+
/// Connection settings for one Fluss cluster.
#[derive(Debug, Clone, PartialEq, Deserialize)]
#[serde(deny_unknown_fields, default)]
@@ -618,14 +679,13 @@ pub struct ClusterConfig {
/// Comma-separated bootstrap addresses passed to the native Fluss client.
pub bootstrap_servers: String,
pub connect_timeout: ConfigDuration,
+ /// How long a connection can go without being acquired before the cleaner releases it.
+ pub connection_idle_timeout: ConfigDuration,
+ pub security_protocol: ConnectionSecurityProtocol,
/// Account used to authenticate to Fluss.
pub service_account: Option<String>,
pub service_secret: Option<Secret>,
pub identity_mode: IdentityMode,
- /// Cap on pooled per-user connections. User identity mode only.
- pub connection_max: Option<u32>,
- /// Idle reclamation for pooled per-user connections. User identity mode only.
- pub connection_idle_timeout: Option<ConfigDuration>,
}
impl Default for ClusterConfig {
@@ -633,11 +693,11 @@ impl Default for ClusterConfig {
Self {
bootstrap_servers: DEFAULT_BOOTSTRAP_SERVERS.to_string(),
connect_timeout: ConfigDuration::from_secs(10),
+ connection_idle_timeout: DEFAULT_CONNECTION_IDLE_TIMEOUT,
+ security_protocol: ConnectionSecurityProtocol::Plaintext,
service_account: None,
service_secret: None,
identity_mode: IdentityMode::Service,
- connection_max: None,
- connection_idle_timeout: None,
}
}
}
@@ -659,11 +719,20 @@ impl ClusterConfig {
.expect("bounded configuration durations fit u64 milliseconds"),
..NativeClientConfig::default()
};
- if let (Some(account), Some(secret)) = (&self.service_account, &self.service_secret) {
- native.security_protocol = "sasl".to_string();
- native.security_sasl_mechanism = "PLAIN".to_string();
- native.security_sasl_username = account.clone();
- native.security_sasl_password = secret.expose().to_string();
+ match self.security_protocol {
+ ConnectionSecurityProtocol::Plaintext => {
+ native.security_protocol = "PLAINTEXT".to_string();
+ }
+ ConnectionSecurityProtocol::Sasl => {
+ native.security_protocol = "sasl".to_string();
+ native.security_sasl_mechanism = "PLAIN".to_string();
+ if let Some(account) = &self.service_account {
+ native.security_sasl_username = account.clone();
+ }
+ if let Some(secret) = &self.service_secret {
+ native.security_sasl_password = secret.expose().to_string();
+ }
+ }
}
native
}
@@ -866,6 +935,7 @@ impl GatewayConfig {
if self.clusters.is_empty() {
problems.push(format!("{CLUSTERS_KEY} must declare at least one cluster"));
}
+ let handler_timeout = rest_handler_timeout(self.server.rest.request_timeout.get());
for (id, cluster) in &self.clusters {
if !valid_cluster_id(id) {
problems.push(format!(
@@ -878,29 +948,39 @@ impl GatewayConfig {
cluster_key(id, CLUSTER_BOOTSTRAP_SERVERS_KEY)
));
}
+ let connect_timeout = cluster.connect_timeout.get();
validate_duration(
&cluster_key(id, CLUSTER_CONNECT_TIMEOUT_KEY),
- cluster.connect_timeout.get(),
+ connect_timeout,
problems,
);
- if let Some(idle_timeout) = cluster.connection_idle_timeout {
- validate_duration(
- &cluster_key(id, CLUSTER_CONNECTION_IDLE_TIMEOUT_KEY),
- idle_timeout.get(),
- problems,
- );
- }
- if cluster.connection_max == Some(0) {
+ let idle_timeout = cluster.connection_idle_timeout.get();
+ validate_duration(
+ &cluster_key(id, CLUSTER_CONNECTION_IDLE_TIMEOUT_KEY),
+ idle_timeout,
+ problems,
+ );
+ if !idle_timeout.is_zero()
+ && idle_timeout <= MAX_CONFIG_DURATION
+ && !handler_timeout.is_zero()
+ && handler_timeout <= MAX_CONFIG_DURATION
+ && idle_timeout <= handler_timeout
+ {
problems.push(format!(
- "{} must be greater than zero",
- cluster_key(id, CLUSTER_CONNECTION_MAX_KEY)
+ "{} must be greater than the REST handler timeout {handler_timeout:?}",
+ cluster_key(id, CLUSTER_CONNECTION_IDLE_TIMEOUT_KEY)
));
}
- self.validate_credentials(id, cluster, problems);
+ self.validate_connection_security(id, cluster, problems);
}
}
- fn validate_credentials(&self, id: &str, cluster: &ClusterConfig, problems: &mut Vec<String>) {
+ fn validate_connection_security(
+ &self,
+ id: &str,
+ cluster: &ClusterConfig,
+ problems: &mut Vec<String>,
+ ) {
let account = cluster.service_account();
let secret = cluster.service_secret();
for (key, value) in [
@@ -912,31 +992,45 @@ impl GatewayConfig {
}
}
- let usable = |value: Option<&str>| value.is_some_and(|value| !value.trim().is_empty());
- let credentials_usable = usable(account) && usable(secret);
- if account.is_some() != secret.is_some() {
- problems.push(format!(
- "{} and {} must be set together",
- cluster_key(id, CLUSTER_SERVICE_ACCOUNT_KEY),
- cluster_key(id, CLUSTER_SERVICE_SECRET_KEY)
- ));
+ match cluster.security_protocol {
+ ConnectionSecurityProtocol::Plaintext => {
+ if account.is_some() || secret.is_some() {
+ problems.push(format!(
+ "{} and {} must be omitted when {} is plaintext",
+ cluster_key(id, CLUSTER_SERVICE_ACCOUNT_KEY),
+ cluster_key(id, CLUSTER_SERVICE_SECRET_KEY),
+ cluster_key(id, CLUSTER_SECURITY_PROTOCOL_KEY)
+ ));
+ }
+ }
+ ConnectionSecurityProtocol::Sasl => {
+ if account.is_none() && secret.is_none() {
+ problems.push(format!(
+ "{} and {} must be configured when {} is sasl",
+ cluster_key(id, CLUSTER_SERVICE_ACCOUNT_KEY),
+ cluster_key(id, CLUSTER_SERVICE_SECRET_KEY),
+ cluster_key(id, CLUSTER_SECURITY_PROTOCOL_KEY)
+ ));
+ } else if account.is_some() != secret.is_some() {
+ problems.push(format!(
+ "{} and {} must be set together when {} is sasl",
+ cluster_key(id, CLUSTER_SERVICE_ACCOUNT_KEY),
+ cluster_key(id, CLUSTER_SERVICE_SECRET_KEY),
+ cluster_key(id, CLUSTER_SECURITY_PROTOCOL_KEY)
+ ));
+ }
+ }
}
if cluster.identity_mode == IdentityMode::User {
- if self.security.authentication == AuthenticationMode::Trust {
- problems.push(format!(
- "{} user requires verified client identities; set {SECURITY_AUTHENTICATION_KEY} to password, token, or trusted-header",
- cluster_key(id, CLUSTER_IDENTITY_MODE_KEY)
- ));
- }
- if !credentials_usable {
- problems.push(format!(
- "{} user requires {} and {}",
- cluster_key(id, CLUSTER_IDENTITY_MODE_KEY),
- cluster_key(id, CLUSTER_SERVICE_ACCOUNT_KEY),
- cluster_key(id, CLUSTER_SERVICE_SECRET_KEY)
- ));
- }
+ // Refused rather than silently downgraded to the shared service account: turning
+ // "isolated per caller" into "everyone shares one identity" is a security problem, not a
+ // degradation (FIP-49: configuring user mode fails at startup until Fluss ships act-as).
+ problems.push(format!(
+ "{} user is not supported yet: fluss-rust cannot send a SASL authorization ID, and \
+ client authentication is not implemented",
+ cluster_key(id, CLUSTER_IDENTITY_MODE_KEY)
+ ));
}
}
@@ -1031,18 +1125,6 @@ impl GatewayConfig {
self.security.trusted_header_name()
));
}
- for (id, cluster) in &self.clusters {
- if cluster.identity_mode == IdentityMode::Service
- && (cluster.connection_max.is_some() || cluster.connection_idle_timeout.is_some())
- {
- warnings.push(format!(
- "{} and {} are ignored because {} is service",
- cluster_key(id, CLUSTER_CONNECTION_MAX_KEY),
- cluster_key(id, CLUSTER_CONNECTION_IDLE_TIMEOUT_KEY),
- cluster_key(id, CLUSTER_IDENTITY_MODE_KEY)
- ));
- }
- }
warnings
}
@@ -1052,7 +1134,7 @@ impl GatewayConfig {
}
}
-fn valid_cluster_id(id: &str) -> bool {
+pub(crate) fn valid_cluster_id(id: &str) -> bool {
if id.len() > 63 {
return false;
}
@@ -1631,6 +1713,7 @@ mod tests {
"127.0.0.1:8080".parse().unwrap()
);
assert_eq!(config.server.rest.max_body_bytes.bytes(), 32 * 1024 * 1024);
+ assert_eq!(config.server.rest.metadata_max_concurrent_requests, 16);
assert_eq!(
config.server.rest.request_timeout.get(),
Duration::from_secs(30)
@@ -1646,10 +1729,26 @@ mod tests {
cluster(&config, DEFAULT_CLUSTER_ID).bootstrap_servers,
DEFAULT_BOOTSTRAP_SERVERS
);
+ assert_eq!(
+ cluster(&config, DEFAULT_CLUSTER_ID)
+ .connection_idle_timeout
+ .get(),
+ Duration::from_secs(10 * 60)
+ );
assert_eq!(
cluster(&config, DEFAULT_CLUSTER_ID).identity_mode,
IdentityMode::Service
);
+ assert_eq!(
+ cluster(&config, DEFAULT_CLUSTER_ID).security_protocol,
+ ConnectionSecurityProtocol::Plaintext
+ );
+ assert_eq!(
+ cluster(&config, DEFAULT_CLUSTER_ID)
+ .native_client_config()
+ .security_protocol,
+ "PLAINTEXT"
+ );
assert_eq!(config.security.authentication, AuthenticationMode::Trust);
assert_eq!(config.request_limits, RequestLimitsConfig::default());
assert!(config.warnings().is_empty());
@@ -1662,7 +1761,8 @@ mod tests {
gateway.instance-id: gateway-1
gateway.rest.listen: 0.0.0.0:8080
gateway.rest.write.max-request-bytes: 32MiB
- gateway.rest.write.request-timeout: 30s
+ gateway.rest.request-timeout: 30s
+ gateway.rest.metadata.max-concurrent-requests: 16
gateway.metrics.enabled: true
gateway.metrics.exporters: prometheus
gateway.metrics.exporter.prometheus.listen: 0.0.0.0:9095
@@ -1677,6 +1777,7 @@ mod tests {
"0.0.0.0:8080".parse().unwrap()
);
assert_eq!(config.server.rest.max_body_bytes.bytes(), 32 * 1024 * 1024);
+ assert_eq!(config.server.rest.metadata_max_concurrent_requests, 16);
assert_eq!(
config.server.rest.request_timeout.get(),
Duration::from_secs(30)
@@ -1702,6 +1803,7 @@ mod tests {
"gateway.cluster.default.bootstrap.serverz: fluss:9123\n",
"gateway.cluster.default.request-timeout: 30s\n",
"gateway.cluster.default.connection.identity: user\n",
+ "gateway.cluster.default.connection.max: 512\n",
"gateway.cluster.default.client.: 1\n",
"gateway.cluster.default: fluss:9123\n",
] {
@@ -1746,15 +1848,16 @@ mod tests {
assert!(!config.server.metrics.enabled);
// Semantic errors in losing sources are masked, but the winning value is still parsed normally.
- let file = write_temp_config("gateway.cluster.default.connection.max: not-a-number\n");
- let env = BTreeMap::from([(
- "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECTION__MAX".to_string(),
- "10".to_string(),
- )]);
+ let file = write_temp_config("gateway.cluster.default.connect-timeout: not-a-duration\n");
+ let mut env = no_env();
+ env.insert(
+ "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT".to_string(),
+ "10s".to_string(),
+ );
let config = load(Some(file.path()), &env, &CliOverrides::default()).unwrap();
assert_eq!(
- cluster(&config, DEFAULT_CLUSTER_ID).connection_max,
- Some(10)
+ cluster(&config, DEFAULT_CLUSTER_ID).connect_timeout.get(),
+ Duration::from_secs(10)
);
}
@@ -1797,16 +1900,16 @@ mod tests {
assert!(error.to_string().contains("scalar"), "{error}");
let file =
- write_temp_config("gateway.cluster.default.connection.max:\n unexpected: mapping\n");
+ write_temp_config("gateway.cluster.default.connect-timeout:\n unexpected: mapping\n");
let env = BTreeMap::from([(
- "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECTION__MAX".to_string(),
- "10".to_string(),
+ "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT".to_string(),
+ "10s".to_string(),
)]);
let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err();
assert!(
error
.to_string()
- .contains("gateway.cluster.default.connection.max"),
+ .contains("gateway.cluster.default.connect-timeout"),
"{error}"
);
assert!(error.to_string().contains("scalar"), "{error}");
@@ -1856,16 +1959,21 @@ mod tests {
#[test]
fn public_environment_options_are_loaded_by_type() {
- let env = BTreeMap::from([
+ let mut env = no_env();
+ env.extend([
("FLUSS_GATEWAY__INSTANCE_ID".to_string(), "123".to_string()),
(
"FLUSS_GATEWAY__REST__LISTEN".to_string(),
"127.0.0.1:18080".to_string(),
),
(
- "FLUSS_GATEWAY__REST__WRITE__REQUEST_TIMEOUT".to_string(),
+ "FLUSS_GATEWAY__REST__REQUEST_TIMEOUT".to_string(),
"5s".to_string(),
),
+ (
+ "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT".to_string(),
+ "3s".to_string(),
+ ),
(
"FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES".to_string(),
"2MiB".to_string(),
@@ -1898,6 +2006,10 @@ mod tests {
config.server.rest.request_timeout.get(),
Duration::from_secs(5)
);
+ assert_eq!(
+ cluster(&config, DEFAULT_CLUSTER_ID).connect_timeout.get(),
+ Duration::from_secs(3)
+ );
assert_eq!(config.server.rest.max_body_bytes.bytes(), 2 * 1024 * 1024);
assert!(!config.server.metrics.enabled);
assert_eq!(
@@ -2002,20 +2114,115 @@ mod tests {
ConfigDuration::from_secs(MAX_CONFIG_DURATION.as_secs() + 1);
let errors = problems(config.validate().unwrap_err());
+ assert!(
+ errors
+ .iter()
+ .any(|error| { error == "gateway.rest.request-timeout must be greater than zero" }),
+ "got: {errors:?}"
+ );
assert!(
errors.iter().any(|error| {
- error == "gateway.rest.write.request-timeout must be greater than zero"
+ error == "gateway.shutdown.drain-timeout must not exceed 31536000 seconds"
}),
"got: {errors:?}"
);
+ }
+
+ #[test]
+ fn metadata_concurrency_limit_must_be_positive() {
+ let mut config = GatewayConfig::default();
+ config.server.rest.metadata_max_concurrent_requests = 0;
+
+ let errors = problems(config.validate().unwrap_err());
assert!(
errors.iter().any(|error| {
- error == "gateway.shutdown.drain-timeout must not exceed 31536000 seconds"
+ error == "gateway.rest.metadata.max-concurrent-requests must be greater than zero"
}),
"got: {errors:?}"
);
}
+ /// `user` identity mode is refused at startup instead of running as the shared service account,
+ /// and the message names the two upstream gaps that have to close first.
+ #[test]
+ fn user_identity_mode_is_refused_at_startup() {
+ let mut config = GatewayConfig::default();
+ let cluster = config.clusters.get_mut("default").expect("default cluster");
+ cluster.identity_mode = IdentityMode::User;
+
+ let errors = problems(config.validate().unwrap_err());
+ let refusal = errors
+ .iter()
+ .find(|error| error.contains("connection.identity-mode"))
+ .unwrap_or_else(|| panic!("{errors:?}"));
+ assert!(refusal.contains("authorization ID"), "{refusal}");
+ assert!(refusal.contains("client authentication"), "{refusal}");
+
+ assert!(GatewayConfig::default().validate().is_ok());
+ }
+
+ #[test]
+ fn connection_security_protocol_validates_credentials() {
+ assert!(GatewayConfig::default().validate().is_ok());
+ assert!(
+ load_file(
+ "gateway.cluster.default.connection.security.protocol: sasl\n\
+ gateway.cluster.default.connection.service.account: gateway_svc\n\
+ gateway.cluster.default.connection.service.secret: secret\n"
+ )
+ .is_ok()
+ );
+
+ for contents in [
+ "gateway.cluster.default.connection.security.protocol: sasl\n",
+ "gateway.cluster.default.connection.security.protocol: sasl\n\
+ gateway.cluster.default.connection.service.account: gateway_svc\n",
+ "gateway.cluster.default.connection.security.protocol: plaintext\n\
+ gateway.cluster.default.connection.service.account: gateway_svc\n\
+ gateway.cluster.default.connection.service.secret: secret\n",
+ ] {
+ let error = load_file(contents).unwrap_err().to_string();
+ assert!(error.contains("connection.security.protocol"), "{error}");
+ }
+ }
+
+ #[test]
+ fn backend_connect_timeout_is_independent_of_a_request_deadline() {
+ let mut config = GatewayConfig::default();
+ config.server.rest.request_timeout = ConfigDuration::from_secs(1);
+ config
+ .clusters
+ .get_mut(DEFAULT_CLUSTER_ID)
+ .unwrap()
+ .connect_timeout = ConfigDuration::from_secs(10);
+
+ assert!(config.validate().is_ok());
+ }
+
+ #[test]
+ fn connection_idle_timeout_must_outlive_the_handler_budget() {
+ let mut config = GatewayConfig::default();
+ config.server.rest.request_timeout = ConfigDuration::from_secs(5);
+ config
+ .clusters
+ .get_mut(DEFAULT_CLUSTER_ID)
+ .unwrap()
+ .connection_idle_timeout = ConfigDuration::from_secs(4);
+
+ let errors = problems(config.validate().unwrap_err());
+ assert!(errors.iter().any(|error| {
+ error.contains("gateway.cluster.default.connection.idle-timeout")
+ && error.contains("REST handler timeout 4s")
+ }));
+
+ config
+ .clusters
+ .get_mut(DEFAULT_CLUSTER_ID)
+ .unwrap()
+ .connection_idle_timeout = ConfigDuration::from_millis(4001);
+ assert!(config.validate().is_ok());
+ }
+
#[test]
fn programmatically_constructed_zero_byte_limits_are_validated() {
let mut config = GatewayConfig::default();
@@ -2233,11 +2440,11 @@ mod tests {
let config = load_file(
"gateway.clusters: default, analytics\n\
gateway.cluster.default.bootstrap.servers: fluss-1:9123,fluss-2:9123\n\
- gateway.cluster.default.connection.identity-mode: user\n\
+ gateway.cluster.default.connection.identity-mode: service\n\
+ gateway.cluster.default.connection.security.protocol: sasl\n\
gateway.cluster.default.connection.service.account: gateway_svc\n\
gateway.cluster.default.connection.service.secret: gw-pass\n\
- gateway.cluster.default.connection.max: 512\n\
- gateway.cluster.default.connection.idle-timeout: 10m\n\
+ gateway.cluster.default.connection.idle-timeout: 12m\n\
gateway.cluster.analytics.bootstrap.servers: analytics:9123,analytics-2:9123\n\
gateway.cluster.analytics.connect-timeout: 5s\n\
gateway.security.authentication: password\n\
@@ -2253,9 +2460,14 @@ mod tests {
let default = cluster(&config, "default");
assert_eq!(default.bootstrap_servers, "fluss-1:9123,fluss-2:9123");
- assert_eq!(default.identity_mode, IdentityMode::User);
+ assert_eq!(default.identity_mode, IdentityMode::Service);
+ assert_eq!(default.security_protocol, ConnectionSecurityProtocol::Sasl);
assert_eq!(default.service_account(), Some("gateway_svc"));
assert_eq!(default.service_secret(), Some("gw-pass"));
+ assert_eq!(
+ default.connection_idle_timeout.get(),
+ Duration::from_secs(12 * 60)
+ );
let native = default.native_client_config();
assert_eq!(native.bootstrap_servers, "fluss-1:9123,fluss-2:9123");
assert_eq!(native.connect_timeout_ms, 10_000);
@@ -2263,11 +2475,6 @@ mod tests {
assert_eq!(native.security_sasl_mechanism, "PLAIN");
assert_eq!(native.security_sasl_username, "gateway_svc");
assert_eq!(native.security_sasl_password, "gw-pass");
- assert_eq!(default.connection_max, Some(512));
- assert_eq!(
- default.connection_idle_timeout.map(ConfigDuration::get),
- Some(Duration::from_secs(600))
- );
assert_eq!(
cluster(&config, "analytics").bootstrap_servers,
"analytics:9123,analytics-2:9123"
@@ -2280,6 +2487,7 @@ mod tests {
let analytics_native = cluster(&config, "analytics").native_client_config();
assert_eq!(analytics_native.connect_timeout_ms, 5_000);
assert_eq!(analytics_native.security_protocol, "PLAINTEXT");
+ assert!(analytics_native.security_sasl_username.is_empty());
assert_eq!(config.request_limits.write_max_rows, 500);
assert!(config.request_limits.write_rate_limit_enabled);
assert_eq!(
@@ -2310,10 +2518,11 @@ mod tests {
"a:9123, b:9123, [::1]:9123"
);
- let env = BTreeMap::from([(
+ let mut env = no_env();
+ env.insert(
"FLUSS_GATEWAY__CLUSTER__DEFAULT__BOOTSTRAP__SERVERS".to_string(),
"env-a:9123, env-b:9123".to_string(),
- )]);
+ );
let from_env = load(None, &env, &CliOverrides::default()).unwrap();
assert_eq!(
cluster(&from_env, "default").bootstrap_servers,
@@ -2524,7 +2733,6 @@ mod tests {
// An account without its secret, and the reverse.
"gateway.cluster.default.connection.service.account: gateway_svc\n",
"gateway.cluster.default.connection.service.secret: gw-pass\n",
- "gateway.cluster.default.connection.max: 0\n",
"gateway.cluster.default.bootstrap.servers: \" \"\n",
"gateway.rest.write.rate-limit.requests-per-second: 0\n",
// The mode's credential table is missing.
@@ -2534,6 +2742,10 @@ mod tests {
gateway.security.trusted-header.name: \"bad header\"\n",
"gateway.rest.lookup.max-keys: 0\n",
"gateway.rest.prefix-lookup.max-prefixes: 0\n",
+ // User identity mode, however it is spelled out, cannot be served yet.
+ "gateway.security.authentication: password\n\
+ gateway.security.users: alice:secret\n\
+ gateway.cluster.default.connection.identity-mode: user\n",
] {
assert!(load_file(contents).is_err(), "accepted: {contents}");
}
@@ -2546,35 +2758,13 @@ mod tests {
.iter()
.any(|error| error == "gateway.clusters must declare at least one cluster")
);
-
- let mut config = GatewayConfig::default();
- config
- .clusters
- .get_mut(DEFAULT_CLUSTER_ID)
- .expect("default cluster")
- .identity_mode = IdentityMode::User;
- assert!(
- problems(config.validate().unwrap_err())
- .iter()
- .any(|error| error.contains("identity-mode user requires"))
- );
-
- assert!(
- load_file(
- "gateway.security.authentication: password\n\
- gateway.security.users: alice:secret\n\
- gateway.cluster.default.connection.identity-mode: user\n\
- gateway.cluster.default.connection.service.account: gateway_svc\n\
- gateway.cluster.default.connection.service.secret: gw-pass\n"
- )
- .is_ok()
- );
}
#[test]
fn diagnostics_redact_every_credential_and_keep_the_identities() {
let config = load_file(
- "gateway.cluster.default.connection.service.account: canonical-user\n\
+ "gateway.cluster.default.connection.security.protocol: sasl\n\
+ gateway.cluster.default.connection.service.account: canonical-user\n\
gateway.cluster.default.connection.service.secret: canonical-secret\n\
gateway.security.authentication: password\n\
gateway.security.users: alice:user-secret\n\
@@ -2644,8 +2834,8 @@ mod tests {
for entry in CONFIG_ENTRIES {
if entry.key == METRICS_EXPORTERS_KEY {
let file = write_temp_config(&format!("{}: otlp\n", entry.key));
- let env =
- BTreeMap::from([(environment_variable(entry.key), "prometheus".to_string())]);
+ let mut env = no_env();
+ env.insert(environment_variable(entry.key), "prometheus".to_string());
let from_env = load(Some(file.path()), &env, &CliOverrides::default())
.unwrap_or_else(|error| panic!("{}: {error}", entry.key));
let only_env = load(None, &env, &CliOverrides::default())
@@ -2657,6 +2847,7 @@ mod tests {
let (file_value, env_value) = match entry.key {
METRICS_ENABLED_KEY | REST_WRITE_RATE_LIMIT_ENABLED_KEY => ("true", "false"),
REST_WRITE_MAX_ROWS_KEY
+ | REST_METADATA_MAX_CONCURRENT_REQUESTS_KEY
| REST_WRITE_MAX_CONCURRENT_REQUESTS_KEY
| REST_WRITE_RATE_LIMIT_REQUESTS_PER_SECOND_KEY
| REST_LOOKUP_MAX_KEYS_KEY
@@ -2669,9 +2860,8 @@ mod tests {
| REST_LOOKUP_MAX_KEY_BYTES_KEY => ("1MiB", "2MiB"),
REST_LISTEN_KEY => ("127.0.0.1:11111", "127.0.0.1:22222"),
METRICS_LISTEN_KEY => ("127.0.0.1:11112", "127.0.0.1:22223"),
- REST_HEADER_READ_TIMEOUT_KEY
- | REST_REQUEST_TIMEOUT_KEY
- | SHUTDOWN_DRAIN_TIMEOUT_KEY => ("11s", "22s"),
+ REST_REQUEST_TIMEOUT_KEY => ("12s", "22s"),
+ REST_HEADER_READ_TIMEOUT_KEY | SHUTDOWN_DRAIN_TIMEOUT_KEY => ("11s", "22s"),
// Both modes must be valid on their own: the file value is loaded without the
// environment override, and password and token modes need a credential table.
SECURITY_AUTHENTICATION_KEY => ("trusted-header", "trust"),
@@ -2681,7 +2871,8 @@ mod tests {
let file = write_temp_config(&format!("{}: \"{file_value}\"\n", entry.key));
let from_file = load(Some(file.path()), &no_env(), &CliOverrides::default())
.unwrap_or_else(|error| panic!("{}: {error}", entry.key));
- let env = BTreeMap::from([(environment_variable(entry.key), env_value.to_string())]);
+ let mut env = no_env();
+ env.insert(environment_variable(entry.key), env_value.to_string());
let from_env = load(Some(file.path()), &env, &CliOverrides::default())
.unwrap_or_else(|error| panic!("{}: {error}", entry.key));
diff --git a/fluss-gateway/src/error.rs b/fluss-gateway/src/error.rs
index 02b0ef5d7..b9214853c 100644
--- a/fluss-gateway/src/error.rs
+++ b/fluss-gateway/src/error.rs
@@ -223,6 +223,9 @@ impl PartialSchema for ErrorCode {
impl ToSchema for ErrorCode {}
+/// The result of any gateway operation that can fail with a client-visible condition.
+pub type GatewayResult<T> = Result<T, GatewayError>;
+
/// Gateway-internal error: a condition kind plus a client-safe message.
///
/// Messages must never contain stack traces, internal addresses, or wire payloads. Operational detail belongs
diff --git a/fluss-gateway/src/lib.rs b/fluss-gateway/src/lib.rs
index bd9c0d40b..a83aea7ac 100644
--- a/fluss-gateway/src/lib.rs
+++ b/fluss-gateway/src/lib.rs
@@ -28,6 +28,7 @@
//! from the request plus current cluster state, so any instance can serve any request and instances can be added
//! or removed freely behind a plain load balancer.
+pub mod backend;
pub mod config;
pub mod error;
pub mod lifecycle;
diff --git a/fluss-gateway/src/lifecycle.rs b/fluss-gateway/src/lifecycle.rs
index cb5b611c5..48c7512ef 100644
--- a/fluss-gateway/src/lifecycle.rs
+++ b/fluss-gateway/src/lifecycle.rs
@@ -17,12 +17,16 @@
//! Process lifecycle for listeners and graceful shutdown.
//!
-//! Listener binding and process readiness are independent from Fluss availability.
+//! Listener binding and process readiness are independent from Fluss availability: the cluster registry is
+//! built from configuration and opens no connection, so the process serves while a cluster is down and the
+//! first request to it establishes the connection.
//!
-//! Shutdown drains in-flight requests. Because the gateway holds no request-spanning
-//! state, there is nothing to hand over, flush, or migrate: a terminated instance leaves no work that another
-//! instance would have to pick up.
+//! Shutdown drains in-flight requests and then releases the backend connections. Because the gateway holds
+//! no request-spanning state, there is nothing to hand over, flush, or migrate: a terminated instance leaves
+//! no work that another instance would have to pick up.
+use crate::backend::client::NativeFlussBackend;
+use crate::backend::connection::CLEANUP_INTERVAL;
use crate::config::GatewayConfig;
use crate::error::{GatewayError, panic_message};
use crate::observability;
@@ -164,6 +168,9 @@ impl Readiness {
pub struct RunningGateway {
local_addr: std::net::SocketAddr,
metrics_addr: Option<std::net::SocketAddr>,
+ /// Held as the concrete type because shutdown is not part of the backend contract a protocol
+ /// adapter sees.
+ backend: Arc<NativeFlussBackend>,
readiness: Arc<Readiness>,
drain_timeout: Duration,
shutdown: CancellationToken,
@@ -187,7 +194,7 @@ impl RunningGateway {
}
/// Stops accepting, drains in-flight requests within the configured drain timeout, then closes the
- /// background tasks. Consumes the gateway.
+ /// background tasks and the backend connections. Consumes the gateway.
pub async fn shutdown(self) -> Result<(), RunError> {
self.finish(None).await
}
@@ -195,13 +202,17 @@ impl RunningGateway {
async fn finish(mut self, unexpected_exit: Option<String>) -> Result<(), RunError> {
let shutdown_started = Instant::now();
self.readiness.begin_quiescing();
- // Draining gets the whole configured budget: the gateway holds no request-spanning state, so there is
- // no cleanup step after the tasks stop that would need a reserved tail. The deadline comes from the
- // timer's own clock, so it cannot skew against it.
+ // The drain gets the budget minus the tail that closing the backend connections needs. The
+ // deadline comes from the timer's own clock, so it cannot skew against it.
let deadline = tokio::time::Instant::now() + self.drain_timeout;
+ let task_deadline = deadline - cleanup_reserve(self.drain_timeout);
self.readiness.begin_draining();
self.shutdown.cancel();
- let cleanup_error = drain_tasks(&mut self.tasks, deadline).await;
+ let mut cleanup_error = drain_tasks(&mut self.tasks, task_deadline).await;
+ if let Err(error) = self.close_backend(deadline).await {
+ log::warn!("{error}");
+ cleanup_error = cleanup_error.or(Some(error));
+ }
self.readiness.set_stopped();
// The metrics listener is one of the drained tasks, so nothing recorded from here on could ever be
@@ -218,6 +229,22 @@ impl RunningGateway {
log::info!("fluss-gateway stopped after {elapsed:?}");
Ok(())
}
+
+ /// Closes every backend connection within whatever is left of the process deadline.
+ ///
+ /// The remaining budget can be zero if draining used all of it; the close is still attempted, bounded by
+ /// that budget, so shutdown cannot hang on an unresponsive cluster.
+ async fn close_backend(&self, deadline: tokio::time::Instant) -> Result<(), String> {
+ let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
+ let close_timeout = shutdown_work_budget(remaining);
+ match tokio::time::timeout(remaining, self.backend.close(close_timeout)).await {
+ Ok(Ok(())) => Ok(()),
+ Ok(Err(error)) => Err(format!("failed to close a Fluss backend: {error}")),
+ Err(_) => {
+ Err("the Fluss backends did not close before the process deadline".to_string())
+ }
+ }
+ }
}
/// Runs the gateway until a process shutdown signal or any process-owned task exits unexpectedly.
@@ -240,11 +267,15 @@ pub async fn run(config: GatewayConfig) -> Result<(), RunError> {
/// available.
pub async fn start(config: GatewayConfig) -> Result<RunningGateway, RunError> {
config.validate()?;
- start_internal(config).await
+ let backend = Arc::new(NativeFlussBackend::from_config(&config));
+ start_internal(config, backend).await
}
/// Binds the listeners, installs the router, and spawns every process-owned task.
-async fn start_internal(config: GatewayConfig) -> Result<RunningGateway, RunError> {
+async fn start_internal(
+ config: GatewayConfig,
+ backend: Arc<NativeFlussBackend>,
+) -> Result<RunningGateway, RunError> {
log::debug!("effective configuration: {}", config.redacted_debug());
for warning in config.warnings() {
log::warn!("{warning}");
@@ -268,11 +299,22 @@ async fn start_internal(config: GatewayConfig) -> Result<RunningGateway, RunErro
.map_err(|error| format!("failed to read the bound metrics listener address: {error}"))?;
let readiness = Arc::new(Readiness::new());
- let router = rest::build(&config.server.rest, &readiness, local_addr);
+ let router = rest::build(&config.server.rest, backend.clone(), &readiness, local_addr);
let header_read_timeout = config.server.rest.header_read_timeout.get();
let connection_drain = connection_drain_budget(config.shutdown.drain_timeout.get());
let shutdown = CancellationToken::new();
let mut tasks = JoinSet::new();
+ let cleaner_backend = backend.clone();
+ let cleaner_shutdown = shutdown.clone();
+ spawn_named(&mut tasks, "connection cleaner", async move {
+ let mut interval = tokio::time::interval(CLEANUP_INTERVAL);
+ loop {
+ tokio::select! {
+ () = cleaner_shutdown.cancelled() => return Ok(()),
+ _ = interval.tick() => cleaner_backend.clean_expired_connections().await,
+ }
+ }
+ });
spawn_named(
&mut tasks,
"REST listener",
@@ -318,6 +360,7 @@ async fn start_internal(config: GatewayConfig) -> Result<RunningGateway, RunErro
Ok(RunningGateway {
local_addr,
metrics_addr,
+ backend,
readiness,
drain_timeout: config.shutdown.drain_timeout.get(),
shutdown,
@@ -325,16 +368,28 @@ async fn start_internal(config: GatewayConfig) -> Result<RunningGateway, RunErro
})
}
-/// Upper bound on the tail `connection_drain_budget` reserves for cleanup.
+/// Upper bound on the tail [`cleanup_reserve`] keeps for post-drain cleanup.
const MAX_CLEANUP_HEADROOM: Duration = Duration::from_secs(2);
-/// Splits the configured shutdown budget into the connection drain and a cleanup tail.
+/// The tail of the shutdown budget reserved for what happens after the request drain: closing the backend
+/// connections, and joining whatever the connection drain did not finish.
///
-/// The drain has to end before the process deadline: otherwise `drain_tasks` aborts `serve()`
-/// mid-drain and the connection abort loses its join. The tail scales rather than being a fixed
-/// subtraction, which would leave short budgets with no drain at all.
+/// The tail scales rather than being a fixed subtraction, which would leave short budgets with no drain at
+/// all.
+fn cleanup_reserve(total: Duration) -> Duration {
+ std::cmp::min(total / 10, MAX_CLEANUP_HEADROOM)
+}
+
+/// Leaves a cleanup tail inside an outer shutdown budget.
+///
+/// Inner drains must finish before their outer deadline so forced closes and task joins can complete.
+fn shutdown_work_budget(total: Duration) -> Duration {
+ total.saturating_sub(cleanup_reserve(total))
+}
+
+/// Keeps the listener's graceful drain inside the process task deadline.
fn connection_drain_budget(total: Duration) -> Duration {
- total.saturating_sub(std::cmp::min(total / 10, MAX_CLEANUP_HEADROOM))
+ shutdown_work_budget(shutdown_work_budget(total))
}
/// Binds one configured HTTP listener and adds a contextual startup error.
@@ -558,6 +613,9 @@ async fn shutdown_signal() {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::backend::FlussBackend;
+ use crate::backend::context::RequestContext;
+ use crate::error::ErrorKind;
use std::sync::atomic::AtomicUsize;
struct DropGuard(Arc<AtomicUsize>);
@@ -670,11 +728,31 @@ mod tests {
assert!(
error
.to_string()
- .contains("gateway.rest.write.request-timeout must be greater than zero"),
+ .contains("gateway.rest.request-timeout must be greater than zero"),
"got: {error}"
);
}
+ /// Shutdown permanently closes the backend caches. The real-cluster E2E covers an installed native
+ /// connection; the cache test covers draining it.
+ #[tokio::test]
+ async fn shutdown_permanently_closes_backend_caches() {
+ let mut config = GatewayConfig::default();
+ config.server.rest.bind_address = "127.0.0.1:0".parse().expect("valid address");
+ config.server.metrics.enabled = false;
+ let backend = Arc::new(NativeFlussBackend::from_config(&config));
+
+ let gateway = start_internal(config, backend.clone())
+ .await
+ .expect("the gateway starts");
+ gateway.shutdown().await.expect("clean shutdown");
+
+ let context = RequestContext::for_test("default", Duration::from_secs(1));
+ let error = backend.list_databases(&context).await.unwrap_err();
+ assert_eq!(error.kind(), ErrorKind::Unavailable);
+ assert_eq!(error.message(), "the Fluss connection cache is closed");
+ }
+
/// Timed-out tasks are aborted and joined before cleanup returns.
#[tokio::test(start_paused = true)]
async fn timed_out_tasks_are_aborted_and_joined_at_the_absolute_deadline() {
@@ -747,23 +825,44 @@ mod tests {
assert!(tasks.is_empty());
}
- /// Every configured budget keeps a drain window and a cleanup tail, and the tail is capped.
+ /// Every nested shutdown layer finishes before the deadline that supervises it.
#[test]
- fn the_connection_drain_leaves_a_proportional_cleanup_tail() {
- for (total, drain) in [
- (Duration::from_secs(30), Duration::from_secs(28)),
- (Duration::from_secs(20), Duration::from_secs(18)),
- (Duration::from_secs(10), Duration::from_secs(9)),
- (Duration::from_millis(100), Duration::from_millis(90)),
+ fn shutdown_budgets_leave_each_outer_layer_headroom() {
+ for (total, task_drain, connection_drain) in [
+ (
+ Duration::from_secs(30),
+ Duration::from_secs(28),
+ Duration::from_secs(26),
+ ),
+ (
+ Duration::from_secs(20),
+ Duration::from_secs(18),
+ Duration::from_millis(16_200),
+ ),
+ (
+ Duration::from_secs(10),
+ Duration::from_secs(9),
+ Duration::from_millis(8_100),
+ ),
+ (
+ Duration::from_millis(100),
+ Duration::from_millis(90),
+ Duration::from_millis(81),
+ ),
] {
- assert_eq!(connection_drain_budget(total), drain, "total={total:?}");
+ assert_eq!(shutdown_work_budget(total), task_drain, "total={total:?}");
+ assert_eq!(
+ connection_drain_budget(total),
+ connection_drain,
+ "total={total:?}"
+ );
}
}
- /// A handler that outlives the connection drain budget is aborted and joined before `serve`
- /// returns, so no request work can continue once the process reports itself stopped.
+ /// A handler that outlives the connection drain is aborted and joined before the process task
+ /// deadline, so forced cleanup remains a successful shutdown rather than an outer timeout.
#[tokio::test]
- async fn a_handler_outlasting_the_drain_budget_is_dropped_before_serve_returns() {
+ async fn a_handler_outlasting_the_connection_drain_finishes_before_the_task_deadline() {
/// Reports that the handler future was dropped rather than left running.
struct DropFlag(Arc<std::sync::atomic::AtomicBool>);
impl Drop for DropFlag {
@@ -795,14 +894,20 @@ mod tests {
.await
.expect("bound");
let address = listener.local_addr().expect("address");
+ let shutdown_timeout = Duration::from_secs(1);
let shutdown = CancellationToken::new();
- let serving = tokio::spawn(serve(
- listener,
- router,
- Duration::from_secs(5),
- Duration::from_millis(200),
- shutdown.clone(),
- ));
+ let mut tasks = JoinSet::new();
+ spawn_named(
+ &mut tasks,
+ "test listener",
+ serve(
+ listener,
+ router,
+ Duration::from_secs(5),
+ connection_drain_budget(shutdown_timeout),
+ shutdown.clone(),
+ ),
+ );
// Send a request without reading the response, then wait until the handler is running.
let request = tokio::spawn(async move {
@@ -825,16 +930,15 @@ mod tests {
"the handler is still running before shutdown starts"
);
+ let task_deadline = tokio::time::Instant::now() + shutdown_work_budget(shutdown_timeout);
shutdown.cancel();
- let result = tokio::time::timeout(Duration::from_secs(10), serving)
- .await
- .expect("serve returns within the test timeout")
- .expect("serve does not panic");
+ let cleanup_error = drain_tasks(&mut tasks, task_deadline).await;
- assert!(result.is_ok(), "{result:?}");
+ assert!(cleanup_error.is_none(), "{cleanup_error:?}");
+ assert!(tasks.is_empty());
assert!(
dropped.load(Ordering::SeqCst),
- "the handler must be dropped before serve returns, not left running detached"
+ "the handler must be dropped before the task deadline, not left running detached"
);
request.abort();
}
diff --git a/fluss-gateway/src/observability.rs b/fluss-gateway/src/observability.rs
index c4bec8504..9b62dc500 100644
--- a/fluss-gateway/src/observability.rs
+++ b/fluss-gateway/src/observability.rs
@@ -105,6 +105,30 @@ pub const METRIC_DEFINITIONS: &[MetricDefinition] = &[
"REST request duration.",
&["cluster", "method", "operation"],
),
+ // FIP-49 connection families, reported per configured cluster by the backend runtime. The per-user
+ // act-as pool of the user identity mode reports into the same families when it arrives: they count the
+ // Fluss connections the gateway holds for a cluster, whoever they are opened for.
+ metric(
+ "fluss_gateway_connections_active",
+ MetricKind::Gauge,
+ None,
+ "Fluss connections the gateway currently holds for a configured cluster.",
+ &["cluster"],
+ ),
+ metric(
+ "fluss_gateway_connections_created_total",
+ MetricKind::Counter,
+ None,
+ "Fluss connections opened for a configured cluster.",
+ &["cluster"],
+ ),
+ metric(
+ "fluss_gateway_connections_closed_total",
+ MetricKind::Counter,
+ None,
+ "Fluss connections released for a configured cluster. `reason` is `idle` or `shutdown`.",
+ &["cluster", "reason"],
+ ),
// FIP-49 process and Tokio runtime families, sampled periodically by the runtime sampler.
metric(
"process_cpu_seconds_total",
@@ -225,6 +249,36 @@ pub fn metrics_handle() -> Option<PrometheusHandle> {
METRICS_HANDLE.get().cloned()
}
+/// Records one Fluss connection opened for a configured cluster.
+pub fn connection_created(cluster: &str) {
+ metrics::counter!(
+ "fluss_gateway_connections_created_total",
+ "cluster" => cluster.to_string()
+ )
+ .increment(1);
+}
+
+/// Records one Fluss connection released for a configured cluster.
+///
+/// `reason` comes from a fixed vocabulary, never from an error message, so the label stays bounded.
+pub fn connection_closed(cluster: &str, reason: &'static str) {
+ metrics::counter!(
+ "fluss_gateway_connections_closed_total",
+ "cluster" => cluster.to_string(),
+ "reason" => reason
+ )
+ .increment(1);
+}
+
+/// Sets how many Fluss connections the gateway currently holds for a configured cluster.
+pub fn connections_active(cluster: &str, active: usize) {
+ metrics::gauge!(
+ "fluss_gateway_connections_active",
+ "cluster" => cluster.to_string()
+ )
+ .set(active as f64);
+}
+
/// Samples the FIP-49 process and Tokio runtime gauges once.
///
/// Called periodically by the lifecycle's runtime sampler; each source that a platform cannot
diff --git a/fluss-gateway/src/protocol/mod.rs b/fluss-gateway/src/protocol/mod.rs
index 1f15dc3fa..7c11ee406 100644
--- a/fluss-gateway/src/protocol/mod.rs
+++ b/fluss-gateway/src/protocol/mod.rs
@@ -16,5 +16,8 @@
// under the License.
//! The gateway's REST API surface.
+//!
+//! The adapter is blind to how the backend reaches Fluss: it resolves a cluster, validates the request,
+//! calls a [`crate::backend::FlussBackend`] method, and maps the returned error to a status.
pub mod rest;
diff --git a/fluss-gateway/src/protocol/rest/clusters.rs b/fluss-gateway/src/protocol/rest/clusters.rs
new file mode 100644
index 000000000..02ff00c1d
--- /dev/null
+++ b/fluss-gateway/src/protocol/rest/clusters.rs
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Cluster discovery: `GET /v1/clusters`.
+
+use crate::error::ErrorEnvelope;
+use crate::protocol::rest::{RestState, json_response};
+use axum::extract::State;
+use axum::response::Response;
+use serde::Serialize;
+use utoipa::ToSchema;
+use utoipa_axum::router::OpenApiRouter;
+use utoipa_axum::routes;
+
+/// Response of `GET /v1/clusters`: the IDs used in the cluster-scoped paths of the other APIs.
+#[derive(Debug, Serialize, ToSchema)]
+pub struct ClustersResponse {
+ pub clusters: Vec<String>,
+}
+
+/// Cluster discovery routes, merged into the main router by [`crate::protocol::rest::build_router`].
+pub fn routes() -> OpenApiRouter<RestState> {
+ OpenApiRouter::new().routes(routes!(list_clusters))
+}
+
+/// Lists the configured clusters.
+///
+/// This is a configuration echo: it touches no connection, so it answers at the same speed whether
+/// Fluss is up or down, and a caller polling it cannot turn into a burst of connection attempts. It
+/// deliberately carries **no** reachability field — the gateway has no background probe, so any such
+/// field would report the outcome of some earlier request rather than the current state. Whether a
+/// cluster can be served shows up in the errors of its own requests and in the `connections_*` metrics.
+#[utoipa::path(
+ get,
+ path = "/v1/clusters",
+ operation_id = "listClusters",
+ tag = "metadata",
+ responses(
+ (status = 200, description = "Configured cluster IDs in lexical order", body = ClustersResponse),
+ (status = 400, description = "This operation accepts no query parameters", body = ErrorEnvelope),
+ (status = 405, description = "Wrong method for this route", body = ErrorEnvelope),
+ (status = 413, description = "Request body above the configured limit", body = ErrorEnvelope),
+ (status = 503, description = "Gateway starting or shutting down", body = ErrorEnvelope),
+ (status = 504, description = "Request deadline exceeded", body = ErrorEnvelope),
+ )
+)]
+pub(crate) async fn list_clusters(State(state): State<RestState>) -> Response {
+ let clusters = state
+ .backend
+ .clusters()
+ .iter()
+ .map(|id| id.as_str().to_string())
+ .collect();
+ json_response(&ClustersResponse { clusters }).expect("cluster discovery is serializable")
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::backend::fake::FakeFlussBackend;
+ use crate::protocol::rest::test_support;
+ use axum::body::Body;
+ use axum::http::{Method, Request, StatusCode};
+ use http_body_util::BodyExt;
+ use std::sync::Arc;
+ use tower::ServiceExt;
+
+ /// A serving router over two configured clusters.
+ fn app() -> axum::Router {
+ let state = test_support::state_with_backend(Arc::new(FakeFlussBackend::with_clusters(&[
+ "zeta", "alpha",
+ ])));
+ state.readiness.set_serving();
+ crate::protocol::rest::build_router(state, &test_support::test_options())
+ }
+
+ async fn request(
+ app: axum::Router,
+ method: Method,
+ path: &str,
+ ) -> (StatusCode, serde_json::Value) {
+ let response = app
+ .oneshot(
+ Request::builder()
+ .method(method)
+ .uri(path)
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let status = response.status();
+ let bytes = response.into_body().collect().await.unwrap().to_bytes();
+ (status, serde_json::from_slice(&bytes).unwrap())
+ }
+
+ /// Discovery lists every configured cluster ID in lexical order, and repeating the request returns
+ /// the identical body: the answer is a function of configuration, never of the caller or of Fluss.
+ #[tokio::test]
+ async fn discovery_is_ordered_and_repeatable() {
+ let app = app();
+ let (status, body) = request(app.clone(), Method::GET, "/v1/clusters").await;
+
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body, serde_json::json!({"clusters": ["alpha", "zeta"]}));
+
+ let (_, again) = request(app, Method::GET, "/v1/clusters").await;
+ assert_eq!(body, again);
+ }
+
+ /// The route defines no query parameters, so one is a bad request rather than being ignored.
+ #[tokio::test]
+ async fn discovery_rejects_query_parameters_and_other_methods() {
+ let (status, body) = request(app(), Method::GET, "/v1/clusters?max_results=1").await;
+ assert_eq!(status, StatusCode::BAD_REQUEST);
+ assert_eq!(body["error"]["code"], "invalid_argument");
+
+ let (status, body) = request(app(), Method::POST, "/v1/clusters").await;
+ assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED);
+ assert_eq!(body["error"]["code"], "method_not_allowed");
+ }
+
+ /// Discovery sits behind the acceptance guard: a draining gateway stops answering it, so a caller
+ /// cannot pick a cluster from an instance that is on its way out.
+ #[tokio::test]
+ async fn discovery_is_unavailable_while_draining() {
+ let state = test_support::test_state();
+ state.readiness.set_serving();
+ state.readiness.begin_quiescing();
+ let app = crate::protocol::rest::build_router(state, &test_support::test_options());
+
+ let (status, body) = request(app, Method::GET, "/v1/clusters").await;
+ assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
+ assert_eq!(body["error"]["code"], "unavailable");
+ }
+}
diff --git a/fluss-gateway/src/protocol/rest/health.rs b/fluss-gateway/src/protocol/rest/health.rs
index 9d1f4ffa7..c0e284dce 100644
--- a/fluss-gateway/src/protocol/rest/health.rs
+++ b/fluss-gateway/src/protocol/rest/health.rs
@@ -22,7 +22,7 @@
//! `GET /ready` is the readiness counterpart: 200 only while the gateway accepts traffic.
use crate::error::ErrorEnvelope;
-use crate::protocol::rest::{RequestId, RestState, error_response, json_response};
+use crate::protocol::rest::{RestState, error_response, json_response, request_id};
use axum::extract::{Request, State};
use axum::response::Response;
use serde::Serialize;
@@ -53,6 +53,7 @@ pub struct HealthResponse {
tag = "health",
responses(
(status = 200, description = "Gateway liveness and uptime", body = HealthResponse),
+ (status = 400, description = "This operation accepts no query parameters", body = ErrorEnvelope),
(status = 405, description = "Wrong method for this route", body = ErrorEnvelope),
)
)]
@@ -84,6 +85,7 @@ pub struct ReadyResponse {
tag = "health",
responses(
(status = 200, description = "The gateway accepts application traffic", body = ReadyResponse),
+ (status = 400, description = "This operation accepts no query parameters", body = ErrorEnvelope),
(status = 405, description = "Wrong method for this route", body = ErrorEnvelope),
(status = 503, description = "The gateway is starting or shutting down", body = ErrorEnvelope),
)
@@ -92,14 +94,7 @@ pub(crate) async fn ready(State(state): State<RestState>, request: Request) -> R
match state.readiness.ensure_accepting() {
Ok(()) => json_response(&ReadyResponse { status: "ready" })
.expect("the ready response is serializable"),
- Err(error) => {
- let request_id = request
- .extensions()
- .get::<RequestId>()
- .cloned()
- .unwrap_or_default();
- error_response(&error, &request_id)
- }
+ Err(error) => error_response(&error, &request_id(&request)),
}
}
@@ -173,6 +168,19 @@ mod tests {
assert_eq!(body_json(response).await["status"], "ok");
}
+ /// Endpoints without declared query parameters reject them consistently.
+ #[tokio::test]
+ async fn health_endpoints_reject_query_parameters() {
+ for path in ["/health?probe=deep", "/ready?probe=deep"] {
+ let response = get(app(), path).await;
+ assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}");
+ assert_eq!(
+ body_json(response).await["error"]["code"],
+ "invalid_argument"
+ );
+ }
+ }
+
/// `/ready` is 200 only while the gateway accepts traffic, so load balancers stop sending
/// traffic the moment shutdown starts instead of feeding requests the acceptance guard rejects.
#[tokio::test]
diff --git a/fluss-gateway/src/protocol/rest/metadata.rs b/fluss-gateway/src/protocol/rest/metadata.rs
new file mode 100644
index 000000000..26df5051b
--- /dev/null
+++ b/fluss-gateway/src/protocol/rest/metadata.rs
@@ -0,0 +1,370 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Metadata discovery: listing the databases of a cluster and the tables of a database.
+//!
+//! Both handlers do the same four things and nothing else: check that the cluster is one this gateway
+//! serves (404), validate the page parameters (400), call the backend, and cut the page out of the
+//! answer. Every other status — 429, 500, 503, 504 — is emitted by shared admission or middleware,
+//! or propagated from [`crate::backend::FlussBackend`], so this module knows nothing about
+//! connections, identity modes, or capacity.
+
+use crate::backend::context::RequestContext;
+use crate::backend::types::ClusterId;
+use crate::backend::{FlussBackend, unknown_cluster};
+use crate::error::{ErrorEnvelope, GatewayResult};
+use crate::protocol::rest::pagination::{Collection, Page};
+use crate::protocol::rest::{
+ RestState, error_response, json_response, request_context, request_id,
+};
+use axum::extract::{Path, Request, State};
+use axum::response::Response;
+use serde::Serialize;
+use std::sync::Arc;
+use utoipa::ToSchema;
+use utoipa_axum::router::OpenApiRouter;
+use utoipa_axum::routes;
+
+/// Response of `GET /v1/clusters/{cluster}/databases`.
+#[derive(Debug, Serialize, ToSchema)]
+pub struct DatabasesResponse {
+ pub databases: Vec<String>,
+ /// Present only while more entries follow.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub next_page_token: Option<String>,
+}
+
+/// Response of `GET /v1/clusters/{cluster}/databases/{database}/tables`.
+#[derive(Debug, Serialize, ToSchema)]
+pub struct TablesResponse {
+ pub tables: Vec<String>,
+ /// Present only while more entries follow.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub next_page_token: Option<String>,
+}
+
+/// Metadata routes, merged into the main router by [`crate::protocol::rest::build_router`].
+pub fn routes() -> OpenApiRouter<RestState> {
+ OpenApiRouter::new()
+ .routes(routes!(list_databases))
+ .routes(routes!(list_tables))
+}
+
+/// Lists the databases of one configured cluster.
+#[utoipa::path(
+ get,
+ path = "/v1/clusters/{cluster}/databases",
+ operation_id = "listDatabases",
+ tag = "metadata",
+ params(
+ ("cluster" = String, Path, description = "Configured cluster ID"),
+ ("max_results" = Option<usize>, Query,
+ description = "Maximum entries to return. Defaults to 100, capped at 1000.",
+ minimum = 1, maximum = 1000),
+ ("page_token" = Option<String>, Query,
+ description = "Opaque token from the `next_page_token` of the previous response."),
+ ),
+ responses(
+ (status = 200, description = "Databases in lexical order", body = DatabasesResponse),
+ (status = 400, description = "Invalid page parameter or page token", body = ErrorEnvelope),
+ (status = 404, description = "Unknown cluster", body = ErrorEnvelope),
+ (status = 429, description = "Metadata concurrency limit exceeded", body = ErrorEnvelope),
+ (status = 500, description = "Fluss backend failure", body = ErrorEnvelope),
+ (status = 503, description = "Fluss is unavailable, or the gateway is starting or shutting down", body = ErrorEnvelope),
+ (status = 504, description = "Request deadline exceeded", body = ErrorEnvelope),
+ )
+)]
+pub(crate) async fn list_databases(
+ State(state): State<RestState>,
+ Path(cluster): Path<String>,
+ request: Request,
+) -> Response {
+ let request_id = request_id(&request);
+ let prepared = prepare(&state, &request, &cluster, Collection::Databases, None);
+ let result = async {
+ let (backend, page, ctx) = prepared?;
+ let (databases, next_page_token) = page.apply(backend.list_databases(&ctx).await?);
+ json_response(&DatabasesResponse {
+ databases,
+ next_page_token,
+ })
+ }
+ .await;
+ result.unwrap_or_else(|error| error_response(&error, &request_id))
+}
+
+/// Lists the tables of one database.
+#[utoipa::path(
+ get,
+ path = "/v1/clusters/{cluster}/databases/{database}/tables",
+ operation_id = "listTables",
+ tag = "metadata",
+ params(
+ ("cluster" = String, Path, description = "Configured cluster ID"),
+ ("database" = String, Path, description = "Database name"),
+ ("max_results" = Option<usize>, Query,
+ description = "Maximum entries to return. Defaults to 100, capped at 1000.",
+ minimum = 1, maximum = 1000),
+ ("page_token" = Option<String>, Query,
+ description = "Opaque token from the `next_page_token` of the previous response."),
+ ),
+ responses(
+ (status = 200, description = "Tables in lexical order", body = TablesResponse),
+ (status = 400, description = "Invalid page parameter or page token", body = ErrorEnvelope),
+ (status = 404, description = "Unknown cluster or database", body = ErrorEnvelope),
+ (status = 429, description = "Metadata concurrency limit exceeded", body = ErrorEnvelope),
+ (status = 500, description = "Fluss backend failure", body = ErrorEnvelope),
+ (status = 503, description = "Fluss is unavailable, or the gateway is starting or shutting down", body = ErrorEnvelope),
+ (status = 504, description = "Request deadline exceeded", body = ErrorEnvelope),
+ )
+)]
+pub(crate) async fn list_tables(
+ State(state): State<RestState>,
+ Path((cluster, database)): Path<(String, String)>,
+ request: Request,
+) -> Response {
+ let request_id = request_id(&request);
+ let prepared = prepare(
+ &state,
+ &request,
+ &cluster,
+ Collection::Tables,
+ Some(&database),
+ );
+ let result = async {
+ let (backend, page, ctx) = prepared?;
+ let (tables, next_page_token) = page.apply(backend.list_tables(&ctx, &database).await?);
+ json_response(&TablesResponse {
+ tables,
+ next_page_token,
+ })
+ }
+ .await;
+ result.unwrap_or_else(|error| error_response(&error, &request_id))
+}
+
+/// Validates the cluster (404) and the page request (400), then builds the backend context.
+///
+/// Everything that reads the HTTP request happens here, before the backend is called: a malformed page
+/// token is therefore always a 400, never the 404 of a parent resource that happens not to exist. The
+/// cluster is checked first, because a path resource that does not exist outranks a bad query
+/// parameter.
+fn prepare(
+ state: &RestState,
+ request: &Request,
+ cluster: &str,
+ collection: Collection,
+ scope: Option<&str>,
+) -> GatewayResult<(Arc<dyn FlussBackend>, Page, RequestContext)> {
+ if !state.backend.has_cluster(cluster) {
+ return Err(unknown_cluster(cluster));
+ }
+ let page = Page::parse(request.uri(), cluster, collection, scope)?;
+ let cluster = ClusterId::try_from(cluster).expect("the backend serves this cluster ID");
+ let ctx = request_context(cluster, request);
+ Ok((state.backend.clone(), page, ctx))
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::backend::FlussBackend;
+ use crate::backend::fake::FakeFlussBackend;
+ use crate::error::GatewayError;
+ use crate::protocol::rest::test_support;
+ use axum::body::Body;
+ use axum::http::{Method, Request as HttpRequest, StatusCode};
+ use http_body_util::BodyExt;
+ use std::sync::Arc;
+ use std::time::Duration;
+ use tower::ServiceExt;
+
+ /// A serving router over one configured cluster backed by the fixture.
+ fn app(backend: Arc<FakeFlussBackend>) -> axum::Router {
+ app_with(backend, test_support::test_options())
+ }
+
+ fn app_with(
+ backend: Arc<FakeFlussBackend>,
+ options: crate::protocol::rest::RestOptions,
+ ) -> axum::Router {
+ let state = test_support::state_with_backend(backend as Arc<dyn FlussBackend>);
+ state.readiness.set_serving();
+ crate::protocol::rest::build_router(state, &options)
+ }
+
+ fn catalog() -> Arc<FakeFlussBackend> {
+ Arc::new(FakeFlussBackend::with_catalog(&[
+ ("sales", &["orders", "customers"]),
+ ("ops", &[]),
+ ]))
+ }
+
+ async fn get(app: &axum::Router, path: &str) -> (StatusCode, serde_json::Value) {
+ let response = app
+ .clone()
+ .oneshot(
+ HttpRequest::builder()
+ .uri(path)
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let status = response.status();
+ let bytes = response.into_body().collect().await.unwrap().to_bytes();
+ (status, serde_json::from_slice(&bytes).unwrap())
+ }
+
+ /// Both collections answer in lexical order and omit the token on a complete page.
+ #[tokio::test]
+ async fn the_collections_answer_in_order_without_a_token() {
+ let app = app(catalog());
+
+ let (status, body) = get(&app, "/v1/clusters/default/databases").await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body, serde_json::json!({"databases": ["ops", "sales"]}));
+
+ let (status, body) = get(&app, "/v1/clusters/default/databases/sales/tables").await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body, serde_json::json!({"tables": ["customers", "orders"]}));
+
+ // An empty database is an empty list, not a 404.
+ let (status, body) = get(&app, "/v1/clusters/default/databases/ops/tables").await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body, serde_json::json!({"tables": []}));
+ }
+
+ /// A page hands out the token that continues it, and the token is scoped to its endpoint.
+ #[tokio::test]
+ async fn a_partial_page_carries_a_token_scoped_to_its_endpoint() {
+ let app = app(catalog());
+
+ let (_, body) = get(&app, "/v1/clusters/default/databases?max_results=1").await;
+ assert_eq!(body["databases"], serde_json::json!(["ops"]));
+ let token = body["next_page_token"]
+ .as_str()
+ .expect("a partial page carries a token")
+ .to_string();
+
+ let (status, body) = get(
+ &app,
+ &format!("/v1/clusters/default/databases?page_token={token}"),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body, serde_json::json!({"databases": ["sales"]}));
+
+ // The same token on the tables endpoint is a bad request, never a reinterpretation.
+ let (status, body) = get(
+ &app,
+ &format!("/v1/clusters/default/databases/sales/tables?page_token={token}"),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST);
+ assert_eq!(body["error"]["code"], "invalid_argument");
+ }
+
+ /// An unconfigured or malformed cluster is `cluster_not_found`, and a page parameter is validated
+ /// before the parent resource is looked up so a bad token cannot answer 404.
+ #[tokio::test]
+ async fn unknown_clusters_and_bad_parameters_keep_their_own_status() {
+ let app = app(catalog());
+
+ for path in [
+ "/v1/clusters/other/databases",
+ "/v1/clusters/Not%20A%20Cluster/databases",
+ "/v1/clusters/other/databases/sales/tables",
+ ] {
+ let (status, body) = get(&app, path).await;
+ assert_eq!(status, StatusCode::NOT_FOUND, "{path}");
+ assert_eq!(body["error"]["code"], "cluster_not_found", "{path}");
+ }
+
+ for path in [
+ "/v1/clusters/default/databases?max_results=0",
+ "/v1/clusters/default/databases?page_token=nope!",
+ // The database does not exist, but the token is what is wrong.
+ "/v1/clusters/default/databases/missing/tables?max_results=99999",
+ ] {
+ let (status, body) = get(&app, path).await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{path}");
+ assert_eq!(body["error"]["code"], "invalid_argument", "{path}");
+ }
+
+ let (status, body) = get(&app, "/v1/clusters/default/databases/missing/tables").await;
+ assert_eq!(status, StatusCode::NOT_FOUND);
+ assert_eq!(body["error"]["code"], "database_not_found");
+ }
+
+ /// Every status beyond the adapter's own 400 and 404 is the backend error, mapped mechanically.
+ #[tokio::test]
+ async fn backend_failures_are_mapped_without_the_adapter_deciding() {
+ for (failure, status, code) in [
+ (
+ GatewayError::unavailable("Fluss is unavailable"),
+ StatusCode::SERVICE_UNAVAILABLE,
+ "unavailable",
+ ),
+ (
+ GatewayError::backend("Fluss denied the gateway"),
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "backend",
+ ),
+ ] {
+ let backend = catalog();
+ backend.fail_with(failure);
+ let (answered, body) = get(&app(backend), "/v1/clusters/default/databases").await;
+ assert_eq!(answered, status);
+ assert_eq!(body["error"]["code"], code);
+ }
+ }
+
+ /// A backend that outruns the request budget answers 504 from the handler's own deadline.
+ #[tokio::test]
+ async fn a_slow_backend_answers_the_request_timeout() {
+ let backend = catalog();
+ backend.set_latency(Duration::from_secs(30));
+ let options = crate::protocol::rest::RestOptions {
+ request_timeout: Duration::from_millis(200),
+ ..test_support::test_options()
+ };
+
+ let (status, body) = get(
+ &app_with(backend, options),
+ "/v1/clusters/default/databases",
+ )
+ .await;
+ assert_eq!(status, StatusCode::GATEWAY_TIMEOUT);
+ assert_eq!(body["error"]["code"], "timeout");
+ }
+
+ #[tokio::test]
+ async fn the_collections_are_read_only() {
+ let app = app(catalog());
+ let response = app
+ .oneshot(
+ HttpRequest::builder()
+ .method(Method::DELETE)
+ .uri("/v1/clusters/default/databases")
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED);
+ }
+}
diff --git a/fluss-gateway/src/protocol/rest/mod.rs b/fluss-gateway/src/protocol/rest/mod.rs
index 6e472cf58..0a92e5674 100644
--- a/fluss-gateway/src/protocol/rest/mod.rs
+++ b/fluss-gateway/src/protocol/rest/mod.rs
@@ -22,12 +22,21 @@
//! here.
//!
//! The middleware applies two per-request input-validation bounds — a maximum body size (413) and the
-//! per-request deadline (504).
+//! per-request deadline (504) — and assigns the request identity, deadline, and cancellation signal that
+//! [`RequestContext`] carries into the backend.
+pub mod clusters;
pub mod health;
+pub mod metadata;
pub mod openapi;
+pub mod pagination;
-use crate::config::RestServerConfig;
+use crate::backend::FlussBackend;
+use crate::backend::context::RequestContext;
+use crate::backend::types::ClusterId;
+#[cfg(test)]
+use crate::config::REST_RESPONSE_GRACE;
+use crate::config::{RestServerConfig, rest_handler_timeout};
use crate::error::{ErrorEnvelope, ErrorKind, GatewayError, panic_message};
use crate::lifecycle::Readiness;
use crate::observability;
@@ -44,6 +53,8 @@ use std::net::SocketAddr;
use std::panic::AssertUnwindSafe;
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
+use tokio::sync::Semaphore;
+use tokio_util::sync::CancellationToken;
use utoipa_axum::router::OpenApiRouter;
/// The deadline handed to a request whose configured timeout cannot be added to the current instant.
@@ -51,17 +62,15 @@ use utoipa_axum::router::OpenApiRouter;
/// Only reachable if a duration slips past configuration validation; one hour keeps such a request bounded.
const MAX_REQUEST_DEADLINE: Duration = Duration::from_secs(3600);
-/// Reserved between a request's deadline and the middleware timeout that backstops it, so a
-/// response completing at the deadline still gets out instead of racing the timeout. The timeout
-/// keeps the configured value; the deadline ends this much earlier.
-const RESPONSE_GRACE: Duration = Duration::from_secs(1);
-
/// Shared state for REST handlers.
///
/// Everything here is either immutable configuration or a shared process service. Nothing is scoped to a
/// request, a session, or a client.
#[derive(Clone)]
pub struct RestState {
+ /// The gateway's single backend dependency: it answers cluster discovery, validates the cluster a
+ /// request names, and serves every cluster-scoped operation.
+ pub backend: Arc<dyn FlussBackend>,
pub readiness: Arc<Readiness>,
pub bind_address: SocketAddr,
pub started_at: Instant,
@@ -100,11 +109,20 @@ impl RequestDeadline {
}
}
+/// Cancellation signal for the work of one request.
+///
+/// The middleware that inserts it holds the token's drop guard, so a request future that is dropped —
+/// the caller disconnected, the deadline layer gave up, shutdown aborted the connection — cancels the
+/// backend work instead of leaving it to notice on its own.
+#[derive(Clone, Debug)]
+pub struct RequestCancellation(CancellationToken);
+
/// Middleware limits, extracted from `[server.rest]`.
#[derive(Debug, Clone)]
pub struct RestOptions {
pub request_timeout: Duration,
pub max_body_bytes: u64,
+ pub metadata_max_concurrent_requests: u32,
}
impl From<&RestServerConfig> for RestOptions {
@@ -112,6 +130,7 @@ impl From<&RestServerConfig> for RestOptions {
Self {
request_timeout: config.request_timeout.get(),
max_body_bytes: config.max_body_bytes.bytes(),
+ metadata_max_concurrent_requests: config.metadata_max_concurrent_requests,
}
}
}
@@ -197,14 +216,38 @@ pub(crate) fn validate_json_content_type(headers: &HeaderMap) -> Result<(), Gate
}
}
-/// Rejects any query string on endpoints that define no query parameters.
-pub fn ensure_no_query(uri: &Uri) -> Result<(), GatewayError> {
- if uri.query().is_some() {
- return Err(GatewayError::invalid_argument(
- "this operation does not accept query parameters",
- ));
- }
- Ok(())
+/// The request ID the outermost middleware assigned, for a handler rendering its own envelope.
+pub(crate) fn request_id(request: &Request) -> RequestId {
+ request
+ .extensions()
+ .get::<RequestId>()
+ .cloned()
+ .unwrap_or_default()
+}
+
+/// Builds the backend context of one cluster-scoped request from the middleware-assigned metadata.
+///
+/// The caller is anonymous until the authenticator lands; the deadline and the cancellation signal are
+/// the ones the middleware also enforces, so a backend call cannot outlive its request.
+pub(crate) fn request_context(cluster_id: ClusterId, request: &Request) -> RequestContext {
+ let deadline = request
+ .extensions()
+ .get::<RequestDeadline>()
+ .copied()
+ .unwrap_or_else(|| deadline_from_now(MAX_REQUEST_DEADLINE));
+ let cancellation = request
+ .extensions()
+ .get::<RequestCancellation>()
+ .map_or_else(CancellationToken::new, |cancellation| {
+ cancellation.0.clone()
+ });
+ RequestContext::new(
+ Arc::from(request_id(request).as_str()),
+ cluster_id,
+ deadline.instant(),
+ cancellation,
+ None,
+ )
}
/// Marks a response as final so the error-normalising middleware does not rewrite its body. Use it for handler
@@ -222,10 +265,12 @@ pub fn shaped(mut response: Response) -> Response {
/// configured one when the port is 0.
pub fn build(
rest_config: &RestServerConfig,
+ backend: Arc<dyn FlussBackend>,
readiness: &Arc<Readiness>,
local_addr: SocketAddr,
) -> Router {
let state = RestState {
+ backend,
readiness: readiness.clone(),
bind_address: local_addr,
started_at: Instant::now(),
@@ -234,34 +279,76 @@ pub fn build(
build_router(state, &RestOptions::from(rest_config))
}
-/// Builds the P1 router: the health routes (`GET /health` for liveness, `GET /ready` for
-/// readiness) sit outside both the acceptance guard and the body/deadline budget so they answer
-/// while draining and never carry a body; every guarded route (today `/v1/openapi.json`, later all
-/// data/control routes) carries the request-body size and deadline budget, with the acceptance
-/// guard outermost so draining answers 503 without consuming bodies.
+/// Builds the router: the health routes (`GET /health` for liveness, `GET /ready` for readiness) sit
+/// outside both the acceptance guard and the body/deadline budget so they answer while draining and
+/// never carry a body; every guarded route (`/v1/openapi.json`, `/v1/clusters`, the metadata routes,
+/// and later all data/control routes) carries the request-body size and deadline budget, with the
+/// acceptance guard outermost so draining answers 503 without consuming bodies.
pub fn build_router(state: RestState, options: &RestOptions) -> Router {
- let (guarded_router, guarded_api) = OpenApiRouter::new()
+ let (queryless_guarded, mut guarded_api) = OpenApiRouter::new()
.merge(openapi::routes())
+ .merge(clusters::routes())
+ .split_for_parts();
+ let (metadata_router, metadata_api) = OpenApiRouter::new()
+ .merge(metadata::routes())
.split_for_parts();
let (open_router, open_api) = OpenApiRouter::new()
.merge(health::routes())
.split_for_parts();
- let mut api = guarded_api;
- api.merge(open_api);
- let _ = state.openapi.set(openapi::finalize(api));
+ guarded_api.merge(metadata_api);
+ guarded_api.merge(open_api);
+ let _ = state.openapi.set(openapi::finalize(guarded_api));
+ let backend = state.backend.clone();
+ let metadata_router =
+ apply_metadata_concurrency_limit(metadata_router, options.metadata_max_concurrent_requests);
+ let guarded_router = queryless_guarded
+ .route_layer(middleware::from_fn(reject_query_parameters))
+ .merge(metadata_router);
// `apply_data_limits` also stamps the request deadline, so the guarded side needs no separate
// `assign_request_deadline` layer.
let guarded = apply_data_limits(guarded_router.with_state(state.clone()), options);
let guarded = apply_acceptance_guard(guarded, state.readiness.clone());
let open = open_router
+ .route_layer(middleware::from_fn(reject_query_parameters))
.fallback(unknown_route)
.with_state(state)
.layer(middleware::from_fn(assign_request_deadline(
options.request_timeout,
)));
- apply_common_middleware(open.merge(guarded))
+ apply_common_middleware(open.merge(guarded), Some(backend))
+}
+
+/// Rejects query strings only on the route groups whose OpenAPI operations declare none.
+async fn reject_query_parameters(request: Request, next: Next) -> Response {
+ if request.uri().query().is_some() {
+ return error_response(
+ &GatewayError::invalid_argument("this operation does not accept query parameters"),
+ &request_id(&request),
+ );
+ }
+ next.run(request).await
+}
+
+/// Bounds metadata work without queueing requests behind an overloaded backend.
+fn apply_metadata_concurrency_limit<S>(router: Router<S>, max_requests: u32) -> Router<S>
+where
+ S: Clone + Send + Sync + 'static,
+{
+ let permits = Arc::new(Semaphore::new(max_requests as usize));
+ router.route_layer(middleware::from_fn(move |request: Request, next: Next| {
+ let permits = permits.clone();
+ async move {
+ let Ok(_permit) = permits.try_acquire_owned() else {
+ return error_response(
+ &GatewayError::resource_exhausted("too many concurrent metadata requests"),
+ &request_id(&request),
+ );
+ };
+ next.run(request).await
+ }
+ }))
}
/// Rejects new application work after graceful draining starts while keeping `GET /health` available.
@@ -291,20 +378,17 @@ fn apply_acceptance_guard(router: Router, readiness: Arc<Readiness>) -> Router {
/// Order (outermost first): request-id assignment and error normalisation, then access logging, then the
/// body size and deadline limits.
pub fn apply_middleware(router: Router, options: &RestOptions) -> Router {
- apply_common_middleware(apply_data_limits(router, options))
+ apply_common_middleware(apply_data_limits(router, options), None)
}
-/// The deadline of a request starting now, ending [`RESPONSE_GRACE`] before the middleware timeout.
+/// The deadline of a request starting now, ending [`REST_RESPONSE_GRACE`] before the middleware
+/// timeout.
///
/// The checked additions are defence in depth — configuration caps every duration well below the
/// instant-arithmetic limit — and, like Envoy's `grpc_timeout_offset`, the grace applies only when
/// the timeout is longer than it.
fn deadline_from_now(request_timeout: Duration) -> RequestDeadline {
- let handler_budget = if request_timeout > RESPONSE_GRACE {
- request_timeout - RESPONSE_GRACE
- } else {
- request_timeout
- };
+ let handler_budget = rest_handler_timeout(request_timeout);
let now = Instant::now();
RequestDeadline(
now.checked_add(handler_budget)
@@ -340,6 +424,13 @@ fn apply_data_limits(router: Router, options: &RestOptions) -> Router {
request
.extensions_mut()
.insert(deadline_from_now(request_timeout));
+ // The guard lives in this future, so whatever drops the request — the timeout below, a
+ // disconnected caller, an aborted connection during shutdown — cancels the handler's work.
+ let cancellation = CancellationToken::new();
+ request
+ .extensions_mut()
+ .insert(RequestCancellation(cancellation.clone()));
+ let _cancel_on_drop = cancellation.drop_guard();
let oversized = declared_content_length(&request).filter(|length| *length > max_body_bytes);
if let Some(length) = oversized {
@@ -383,11 +474,11 @@ fn apply_data_limits(router: Router, options: &RestOptions) -> Router {
.layer(middleware::from_fn(limits))
}
-fn apply_common_middleware(router: Router) -> Router {
+fn apply_common_middleware(router: Router, backend: Option<Arc<dyn FlussBackend>>) -> Router {
router
.layer(middleware::from_fn(catch_panic))
- .layer(middleware::from_fn(request_log))
- .layer(middleware::from_fn(request_context))
+ .layer(middleware::from_fn(request_log(backend)))
+ .layer(middleware::from_fn(request_context_id))
}
/// Turns a panic below this layer into the error envelope a client is promised.
@@ -414,31 +505,64 @@ async fn catch_panic(request: Request, next: Next) -> Response {
}
}
-async fn request_log(request: Request, next: Next) -> Response {
- let started = Instant::now();
- let method = request.method().clone();
- let route = request
- .extensions()
- .get::<MatchedPath>()
- .map(MatchedPath::as_str)
- .unwrap_or("<unmatched>")
- .to_string();
- let request_id = request
- .extensions()
- .get::<RequestId>()
- .cloned()
- .unwrap_or_default();
- let response = next.run(request).await;
- let elapsed = started.elapsed();
- let status = response.status();
- // No route carries a `{cluster}` segment, so every request gets the static `none` label;
- // caller-supplied values never become label values.
- observability::http_request("none", method.as_str(), &route, status.as_u16(), elapsed);
- log::info!(
- "{}",
- format_request_log(&method, &route, &request_id, status, elapsed.as_millis())
- );
- response
+/// Logs and counts one completed request.
+///
+/// The `cluster` label stays bounded: a route without a `{cluster}` segment reports `none`, and a
+/// segment that names no configured cluster reports `unknown`, so a caller-supplied value can never
+/// become a label value.
+fn request_log(
+ backend: Option<Arc<dyn FlussBackend>>,
+) -> impl Fn(Request, Next) -> std::pin::Pin<Box<dyn Future<Output = Response> + Send>> + Clone {
+ move |request: Request, next: Next| {
+ let backend = backend.clone();
+ Box::pin(async move {
+ let started = Instant::now();
+ let method = request.method().clone();
+ let route = request
+ .extensions()
+ .get::<MatchedPath>()
+ .map(MatchedPath::as_str)
+ .unwrap_or("<unmatched>")
+ .to_string();
+ let cluster =
+ cluster_label(backend.as_deref(), &route, request.uri().path()).to_string();
+ let request_id = request
+ .extensions()
+ .get::<RequestId>()
+ .cloned()
+ .unwrap_or_default();
+ let response = next.run(request).await;
+ let elapsed = started.elapsed();
+ let status = response.status();
+ observability::http_request(
+ &cluster,
+ method.as_str(),
+ &route,
+ status.as_u16(),
+ elapsed,
+ );
+ log::info!(
+ "{}",
+ format_request_log(&method, &route, &request_id, status, elapsed.as_millis())
+ );
+ response
+ })
+ }
+}
+
+/// The bounded `cluster` label of one request: a configured ID, `unknown`, or `none`.
+fn cluster_label<'a>(backend: Option<&dyn FlussBackend>, route: &str, path: &'a str) -> &'a str {
+ let Some(position) = route
+ .split('/')
+ .position(|segment| segment == "{cluster}")
+ .filter(|_| backend.is_some())
+ else {
+ return "none";
+ };
+ path.split('/')
+ .nth(position)
+ .filter(|candidate| backend.is_some_and(|backend| backend.has_cluster(candidate)))
+ .unwrap_or("unknown")
}
fn format_request_log(
@@ -455,7 +579,7 @@ fn format_request_log(
)
}
-async fn request_context(mut request: Request, next: Next) -> Response {
+async fn request_context_id(mut request: Request, next: Next) -> Response {
let request_id = RequestId(Arc::from(uuid::Uuid::new_v4().to_string()));
request.extensions_mut().insert(request_id.clone());
@@ -542,12 +666,19 @@ pub mod test_support {
RestOptions {
request_timeout: Duration::from_secs(5),
max_body_bytes: 1024,
+ metadata_max_concurrent_requests: 16,
}
}
/// Builds handler state over a fresh readiness gate, which starts neither serving nor draining.
pub fn test_state() -> RestState {
+ state_with_backend(Arc::new(crate::backend::fake::FakeFlussBackend::new()))
+ }
+
+ /// Builds handler state over one backend fixture.
+ pub fn state_with_backend(backend: Arc<dyn FlussBackend>) -> RestState {
RestState {
+ backend,
readiness: Arc::new(Readiness::new()),
bind_address: "127.0.0.1:0".parse().expect("valid"),
started_at: Instant::now(),
@@ -562,6 +693,7 @@ mod tests {
use axum::routing::{get, post};
use http_body_util::BodyExt;
use serde::Deserialize;
+ use std::sync::atomic::{AtomicUsize, Ordering};
use tower::ServiceExt;
#[derive(Debug, Deserialize, PartialEq)]
@@ -612,26 +744,29 @@ mod tests {
#[tokio::test]
async fn unknown_route_yields_404_envelope() {
let app = build_router(test_support::test_state(), &test_support::test_options());
- let response = app
- .oneshot(Request::builder().uri("/nope").body(Body::empty()).unwrap())
- .await
- .unwrap();
-
- assert_eq!(response.status(), StatusCode::NOT_FOUND);
- let header_id = response
- .headers()
- .get("x-request-id")
- .and_then(|v| v.to_str().ok())
- .map(str::to_string)
- .expect("x-request-id header");
-
- let json = body_json(response).await;
- assert_eq!(json["error"]["code"], "not_found");
- assert_eq!(json["error"]["request_id"], header_id.as_str());
- assert!(
- json["error"]["message"].as_str().unwrap().contains("/nope"),
- "message names the missing route: {json}"
- );
+ for path in ["/nope", "/nope?unknown=value"] {
+ let response = app
+ .clone()
+ .oneshot(Request::builder().uri(path).body(Body::empty()).unwrap())
+ .await
+ .unwrap();
+
+ assert_eq!(response.status(), StatusCode::NOT_FOUND, "{path}");
+ let header_id = response
+ .headers()
+ .get("x-request-id")
+ .and_then(|v| v.to_str().ok())
+ .map(str::to_string)
+ .expect("x-request-id header");
+
+ let json = body_json(response).await;
+ assert_eq!(json["error"]["code"], "not_found", "{path}");
+ assert_eq!(json["error"]["request_id"], header_id.as_str(), "{path}");
+ assert!(
+ json["error"]["message"].as_str().unwrap().contains("/nope"),
+ "message names the missing route: {json}"
+ );
+ }
}
#[tokio::test]
@@ -668,6 +803,7 @@ mod tests {
let options = RestOptions {
request_timeout: Duration::from_millis(50),
max_body_bytes: 1024,
+ metadata_max_concurrent_requests: 16,
};
let app = apply_middleware(Router::new().route("/slow", get(slow)), &options);
@@ -687,9 +823,9 @@ mod tests {
for (timeout, budget) in [
(
Duration::from_secs(30),
- Duration::from_secs(30) - RESPONSE_GRACE,
+ Duration::from_secs(30) - REST_RESPONSE_GRACE,
),
- (RESPONSE_GRACE, RESPONSE_GRACE),
+ (REST_RESPONSE_GRACE, REST_RESPONSE_GRACE),
(Duration::from_millis(200), Duration::from_millis(200)),
] {
let before = Instant::now();
@@ -718,8 +854,9 @@ mod tests {
"ok"
}
let options = RestOptions {
- request_timeout: RESPONSE_GRACE + Duration::from_millis(400),
+ request_timeout: REST_RESPONSE_GRACE + Duration::from_millis(400),
max_body_bytes: 1024,
+ metadata_max_concurrent_requests: 16,
};
let app = apply_middleware(
Router::new().route("/deadline", get(finishes_at_deadline)),
@@ -764,7 +901,7 @@ mod tests {
}
#[tokio::test]
- async fn concurrent_requests_are_never_rejected_with_429() {
+ async fn generic_middleware_does_not_impose_a_global_concurrency_limit() {
/// Holds a request open long enough for several to overlap.
async fn slow() -> &'static str {
tokio::time::sleep(Duration::from_millis(50)).await;
@@ -784,6 +921,60 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn metadata_concurrency_rejects_without_queueing_and_releases_its_permit() {
+ let entered = Arc::new(tokio::sync::Notify::new());
+ let release = Arc::new(tokio::sync::Notify::new());
+ let calls = Arc::new(AtomicUsize::new(0));
+ let handler = {
+ let entered = entered.clone();
+ let release = release.clone();
+ let calls = calls.clone();
+ move || {
+ let entered = entered.clone();
+ let release = release.clone();
+ let calls = calls.clone();
+ async move {
+ if calls.fetch_add(1, Ordering::SeqCst) == 0 {
+ entered.notify_one();
+ release.notified().await;
+ }
+ "ok"
+ }
+ }
+ };
+ let app = apply_common_middleware(
+ apply_metadata_concurrency_limit(Router::new().route("/metadata", get(handler)), 1),
+ None,
+ );
+ let request = || {
+ Request::builder()
+ .uri("/metadata")
+ .body(Body::empty())
+ .unwrap()
+ };
+
+ let first = tokio::spawn(app.clone().oneshot(request()));
+ entered.notified().await;
+
+ let rejected = app.clone().oneshot(request()).await.unwrap();
+ assert_eq!(rejected.status(), StatusCode::TOO_MANY_REQUESTS);
+ assert_eq!(rejected.headers()[header::RETRY_AFTER], "1");
+ assert_eq!(
+ body_json(rejected).await["error"]["code"],
+ "resource_exhausted"
+ );
+ assert_eq!(calls.load(Ordering::SeqCst), 1);
+
+ release.notify_one();
+ assert_eq!(first.await.unwrap().unwrap().status(), StatusCode::OK);
+ assert_eq!(
+ app.oneshot(request()).await.unwrap().status(),
+ StatusCode::OK
+ );
+ assert_eq!(calls.load(Ordering::SeqCst), 2);
+ }
+
#[tokio::test]
async fn success_responses_carry_request_id_header() {
let app = build_router(test_support::test_state(), &test_support::test_options());
@@ -836,6 +1027,38 @@ mod tests {
);
}
+ /// The `cluster` metric label stays inside its bounded vocabulary: a configured ID, `unknown` for
+ /// anything else a caller can put in the path, and `none` for a route without a cluster segment.
+ #[test]
+ fn the_cluster_metric_label_never_carries_a_caller_supplied_value() {
+ let backend = crate::backend::fake::FakeFlussBackend::new();
+ let route = "/v1/clusters/{cluster}/databases";
+
+ assert_eq!(
+ cluster_label(Some(&backend), route, "/v1/clusters/default/databases"),
+ "default"
+ );
+ for path in [
+ "/v1/clusters/other/databases",
+ "/v1/clusters/Not%20A%20Cluster/databases",
+ "/v1/clusters",
+ ] {
+ assert_eq!(
+ cluster_label(Some(&backend), route, path),
+ "unknown",
+ "{path}"
+ );
+ }
+ assert_eq!(
+ cluster_label(Some(&backend), "/v1/clusters", "/v1/clusters"),
+ "none"
+ );
+ assert_eq!(
+ cluster_label(None, route, "/v1/clusters/default/databases"),
+ "none"
+ );
+ }
+
#[test]
fn request_log_contains_protocol_context() {
let message = format_request_log(
diff --git a/fluss-gateway/src/protocol/rest/openapi.rs b/fluss-gateway/src/protocol/rest/openapi.rs
index 09a4f0c9f..cfb223521 100644
--- a/fluss-gateway/src/protocol/rest/openapi.rs
+++ b/fluss-gateway/src/protocol/rest/openapi.rs
@@ -80,6 +80,7 @@ pub(crate) fn finalize(api: utoipa::openapi::OpenApi) -> Value {
tag = "metadata",
responses(
(status = 200, description = "OpenAPI 3.1 document"),
+ (status = 400, description = "This operation accepts no query parameters", body = ErrorEnvelope),
(status = 405, description = "Wrong method for this route", body = ErrorEnvelope),
(status = 413, description = "Request body above the configured limit", body = ErrorEnvelope),
(status = 503, description = "Gateway starting or shutting down", body = ErrorEnvelope),
@@ -193,6 +194,28 @@ mod tests {
);
}
+ /// The document route follows the same strict query policy as every other endpoint.
+ #[tokio::test]
+ async fn document_route_rejects_query_parameters() {
+ let state = test_support::test_state();
+ state.readiness.set_serving();
+ let app = crate::protocol::rest::build_router(state, &test_support::test_options());
+ let response = app
+ .oneshot(
+ Request::builder()
+ .uri("/v1/openapi.json?format=yaml")
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+ let bytes = response.into_body().collect().await.unwrap().to_bytes();
+ let body: Value = serde_json::from_slice(&bytes).unwrap();
+ assert_eq!(body["error"]["code"], "invalid_argument");
+ }
+
/// The published `ErrorCode` vocabulary is generated from the taxonomy, so adding an [`ErrorKind`]
/// without regenerating the document fails here rather than shipping a stale contract.
#[tokio::test]
diff --git a/fluss-gateway/src/protocol/rest/pagination.rs b/fluss-gateway/src/protocol/rest/pagination.rs
new file mode 100644
index 000000000..2e7de790f
--- /dev/null
+++ b/fluss-gateway/src/protocol/rest/pagination.rs
@@ -0,0 +1,349 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Keyset pagination for the FIP-49 list endpoints.
+//!
+//! The page token is self-describing: it carries the cluster and collection it belongs to, its scope,
+//! and the last name of the page it follows, base64url-encoded. Nothing is stored server-side, so any
+//! instance can serve any page, and keying on the last name rather than on an offset means concurrent
+//! DDL cannot make a page skip or repeat an entry.
+//!
+//! Every part of a token's scope is validated on decode. The cluster is part of it: a cursor from one
+//! cluster replayed against another would answer with a page silently missing every entry up to that
+//! cursor, which a caller cannot detect.
+//!
+//! Fluss has no server-side pagination, so a page is cut from the full list of names. That is
+//! acceptable at catalog scale and is replaced the day the RPC offers a cursor.
+
+use crate::error::{GatewayError, GatewayResult};
+use axum::http::Uri;
+use base64::Engine;
+use base64::engine::general_purpose::URL_SAFE_NO_PAD;
+use serde::{Deserialize, Serialize};
+use std::borrow::Cow;
+
+/// Page size when the request does not ask for one (FIP-49).
+const DEFAULT_MAX_RESULTS: usize = 100;
+/// Largest page the gateway serves (FIP-49). Not configurable: it bounds one response body.
+const MAX_MAX_RESULTS: usize = 1000;
+/// Current token layout. A token from another version is rejected, never reinterpreted.
+const TOKEN_VERSION: u32 = 1;
+
+/// The paginated collection a token belongs to.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Collection {
+ Databases,
+ Tables,
+}
+
+impl Collection {
+ fn as_str(self) -> &'static str {
+ match self {
+ Self::Databases => "databases",
+ Self::Tables => "tables",
+ }
+ }
+}
+
+/// One validated page request: how many entries, and which name to continue after.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Page {
+ max_results: usize,
+ after: Option<String>,
+ /// The cluster this page belongs to, which the token has to agree with.
+ cluster: String,
+ collection: Collection,
+ /// The parent resource of a scoped collection, which the token has to agree with.
+ scope: Option<String>,
+}
+
+impl Page {
+ /// Validates the query string of a list request.
+ ///
+ /// Called before the request reaches the backend, so a malformed token is always 400 and never
+ /// turns into the 404 of a parent resource that happens not to exist.
+ pub fn parse(
+ uri: &Uri,
+ cluster: &str,
+ collection: Collection,
+ scope: Option<&str>,
+ ) -> GatewayResult<Self> {
+ let mut max_results = DEFAULT_MAX_RESULTS;
+ let mut after = None;
+ for (name, value) in query_pairs(uri)? {
+ match name {
+ "max_results" => {
+ max_results = value
+ .parse()
+ .ok()
+ .filter(is_supported_page_size)
+ .ok_or_else(|| {
+ GatewayError::invalid_argument(format!(
+ "max_results must be between 1 and {MAX_MAX_RESULTS}"
+ ))
+ })?;
+ }
+ "page_token" => {
+ after = Some(decode_token(value, cluster, collection, scope)?);
+ }
+ _ => {
+ return Err(GatewayError::invalid_argument(format!(
+ "unsupported query parameter `{name}`"
+ )));
+ }
+ }
+ }
+ Ok(Self {
+ max_results,
+ after,
+ cluster: cluster.to_string(),
+ collection,
+ scope: scope.map(str::to_string),
+ })
+ }
+
+ /// Cuts one page out of `names` and renders the token of the next one.
+ ///
+ /// The gateway sorts by UTF-8 byte order, because Fluss does not promise an order and keyset
+ /// pagination needs one.
+ pub fn apply(&self, mut names: Vec<String>) -> (Vec<String>, Option<String>) {
+ names.sort_unstable();
+ let start = match &self.after {
+ Some(after) => names.partition_point(|name| name.as_str() <= after.as_str()),
+ None => 0,
+ };
+ let remaining = &names[start..];
+ let has_more = remaining.len() > self.max_results;
+ let page: Vec<String> = remaining.iter().take(self.max_results).cloned().collect();
+ let next = page
+ .last()
+ .filter(|_| has_more)
+ .map(|last| encode_token(&self.cluster, self.collection, self.scope.as_deref(), last));
+ (page, next)
+ }
+}
+
+fn is_supported_page_size(value: &usize) -> bool {
+ (1..=MAX_MAX_RESULTS).contains(value)
+}
+
+/// The token payload. Compact field names keep the encoded token short.
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct TokenPayload<'a> {
+ v: u32,
+ #[serde(borrow)]
+ c: Cow<'a, str>,
+ #[serde(borrow)]
+ k: Cow<'a, str>,
+ #[serde(borrow, skip_serializing_if = "Option::is_none")]
+ db: Option<Cow<'a, str>>,
+ #[serde(borrow)]
+ after: Cow<'a, str>,
+}
+
+fn encode_token(cluster: &str, collection: Collection, scope: Option<&str>, after: &str) -> String {
+ let payload = TokenPayload {
+ v: TOKEN_VERSION,
+ c: Cow::Borrowed(cluster),
+ k: Cow::Borrowed(collection.as_str()),
+ db: scope.map(Cow::Borrowed),
+ after: Cow::Borrowed(after),
+ };
+ URL_SAFE_NO_PAD.encode(serde_json::to_vec(&payload).expect("the page token is serializable"))
+}
+
+/// Decodes a token and refuses one minted for another cluster or endpoint, rather than reinterpreting
+/// it against the collection at hand.
+fn decode_token(
+ token: &str,
+ cluster: &str,
+ collection: Collection,
+ scope: Option<&str>,
+) -> GatewayResult<String> {
+ let bytes = URL_SAFE_NO_PAD.decode(token).map_err(|_| bad_token())?;
+ let payload: TokenPayload<'_> = serde_json::from_slice(&bytes).map_err(|_| bad_token())?;
+ if payload.v != TOKEN_VERSION
+ || payload.c != cluster
+ || payload.k != collection.as_str()
+ || payload.db.as_deref() != scope
+ {
+ return Err(bad_token());
+ }
+ Ok(payload.after.into_owned())
+}
+
+fn bad_token() -> GatewayError {
+ GatewayError::invalid_argument("page_token is not a token this endpoint issued")
+}
+
+/// Splits the query string into its pairs, rejecting a repeated or valueless parameter.
+///
+/// Parsed here rather than with a form decoder: every parameter the gateway defines is either a number
+/// or a base64url token, so no percent-decoding is involved, and a strict reading turns a typo into a
+/// 400 instead of a silently ignored parameter.
+fn query_pairs(uri: &Uri) -> GatewayResult<Vec<(&str, &str)>> {
+ let mut pairs: Vec<(&str, &str)> = Vec::new();
+ for pair in uri.query().unwrap_or_default().split('&') {
+ if pair.is_empty() {
+ continue;
+ }
+ let (name, value) = pair.split_once('=').ok_or_else(|| {
+ GatewayError::invalid_argument(format!("query parameter `{pair}` has no value"))
+ })?;
+ if pairs.iter().any(|(seen, _)| *seen == name) {
+ return Err(GatewayError::invalid_argument(format!(
+ "query parameter `{name}` is repeated"
+ )));
+ }
+ pairs.push((name, value));
+ }
+ Ok(pairs)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::error::ErrorKind;
+
+ fn uri(query: &str) -> Uri {
+ format!("/v1/clusters/default/databases?{query}")
+ .parse()
+ .expect("valid test URI")
+ }
+
+ fn names(count: usize) -> Vec<String> {
+ (0..count).map(|index| format!("name_{index:03}")).collect()
+ }
+
+ /// Walking every page returns each name exactly once, in order, and the last page carries no token.
+ #[test]
+ fn a_token_walks_the_collection_once_and_in_order() {
+ let mut collected: Vec<String> = Vec::new();
+ let mut token: Option<String> = None;
+ for _ in 0..10 {
+ let query = match &token {
+ Some(token) => format!("max_results=3&page_token={token}"),
+ None => "max_results=3".to_string(),
+ };
+ let page = Page::parse(&uri(&query), "default", Collection::Databases, None).unwrap();
+ let (entries, next) = page.apply(names(7));
+ collected.extend(entries);
+ token = next;
+ if token.is_none() {
+ break;
+ }
+ }
+ assert_eq!(collected, names(7));
+ assert_eq!(token, None, "the last page carries no token");
+ }
+
+ /// A page cut from a collection that changed keeps its place: keyset pagination continues after the
+ /// last name returned, so an insert before it cannot repeat an entry and a delete cannot skip one.
+ #[test]
+ fn concurrent_changes_neither_repeat_nor_skip_an_entry() {
+ let (first, token) = Page::parse(
+ &uri("max_results=2"),
+ "default",
+ Collection::Databases,
+ None,
+ )
+ .unwrap()
+ .apply(vec!["b".into(), "d".into(), "f".into()]);
+ assert_eq!(first, ["b", "d"]);
+
+ let query = format!("max_results=2&page_token={}", token.expect("a next token"));
+ let (second, _) = Page::parse(&uri(&query), "default", Collection::Databases, None)
+ .unwrap()
+ // `a` was created and `d` dropped while the caller paged.
+ .apply(vec!["a".into(), "b".into(), "f".into()]);
+ assert_eq!(second, ["f"]);
+ }
+
+ /// A token only continues the endpoint that issued it, on the cluster that issued it.
+ #[test]
+ fn a_token_from_another_cluster_endpoint_or_version_is_refused() {
+ let token = encode_token("alpha", Collection::Tables, Some("sales"), "orders");
+ for (cluster, collection, scope) in [
+ ("zeta", Collection::Tables, Some("sales")),
+ ("alpha", Collection::Databases, None),
+ ("alpha", Collection::Tables, None),
+ ("alpha", Collection::Tables, Some("other")),
+ ] {
+ let error = decode_token(&token, cluster, collection, scope).unwrap_err();
+ assert_eq!(error.kind(), ErrorKind::InvalidArgument, "{cluster}");
+ }
+ assert_eq!(
+ decode_token(&token, "alpha", Collection::Tables, Some("sales")).unwrap(),
+ "orders"
+ );
+
+ // A future layout is rejected instead of being read with today's meaning.
+ let future =
+ URL_SAFE_NO_PAD.encode(br#"{"v":2,"c":"default","k":"databases","after":"a"}"#);
+ assert!(decode_token(&future, "default", Collection::Databases, None).is_err());
+ }
+
+ #[test]
+ fn a_malformed_token_is_rejected() {
+ for token in [
+ "not-base64!!",
+ // Valid base64 that is not the payload.
+ &URL_SAFE_NO_PAD.encode("{}"),
+ // The cluster the payload has to carry is missing.
+ &URL_SAFE_NO_PAD.encode(br#"{"v":1,"k":"databases","after":"a"}"#),
+ &URL_SAFE_NO_PAD
+ .encode(br#"{"v":1,"c":"default","k":"databases","after":"a","extra":1}"#),
+ "",
+ ] {
+ assert!(
+ decode_token(token, "default", Collection::Databases, None).is_err(),
+ "accepted {token:?}"
+ );
+ }
+ }
+
+ /// The FIP-49 page-size contract, and the parameters the endpoints do not define.
+ #[test]
+ fn the_page_size_is_bounded_and_unknown_parameters_are_refused() {
+ let parse = |query: &str| Page::parse(&uri(query), "default", Collection::Databases, None);
+ assert_eq!(parse("").unwrap().max_results, DEFAULT_MAX_RESULTS);
+ assert_eq!(parse("max_results=1").unwrap().max_results, 1);
+ assert_eq!(
+ parse(&format!("max_results={MAX_MAX_RESULTS}"))
+ .unwrap()
+ .max_results,
+ MAX_MAX_RESULTS
+ );
+ for rejected in [
+ "max_results=0",
+ "max_results=1001",
+ "max_results=-1",
+ "max_results=all",
+ "max_results=",
+ "max_results",
+ "max_results=1&max_results=2",
+ "maxResults=10",
+ "limit=10",
+ ] {
+ let error = parse(rejected)
+ .err()
+ .unwrap_or_else(|| panic!("{rejected}"));
+ assert_eq!(error.kind(), ErrorKind::InvalidArgument, "{rejected}");
+ }
+ }
+}
diff --git a/fluss-gateway/tests/e2e_cluster.rs b/fluss-gateway/tests/e2e_cluster.rs
new file mode 100644
index 000000000..fc5653fe8
--- /dev/null
+++ b/fluss-gateway/tests/e2e_cluster.rs
@@ -0,0 +1,283 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! End-to-end metadata API tests against a real Dockerized Fluss cluster.
+//!
+//! The test crosses every production boundary: a real HTTP client calls the compiled Gateway binary,
+//! which reaches the Docker cluster through the native `fluss-rs` backend. Run it with
+//! `just test-e2e`.
+
+mod support;
+
+use fluss::client::FlussConnection;
+use fluss::config::Config;
+use fluss::metadata::{DataTypes, Schema, TableDescriptor, TablePath};
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::time::Duration;
+use support::{Api, ChildGuard, await_http_ok, binary, free_port};
+
+const DATABASE: &str = "gateway_e2e";
+const TABLE: &str = "events";
+const SERVICE_ACCOUNT: &str = "admin";
+const SERVICE_SECRET: &str = "admin-secret";
+
+/// A detached Docker cluster managed through the same helper used by the other language bindings.
+///
+/// Drop is a best-effort backstop for assertions that panic; the happy path checks cleanup explicitly.
+struct FlussCluster {
+ helper: PathBuf,
+ name: String,
+ plaintext_bootstrap_servers: String,
+ sasl_bootstrap_servers: String,
+ stopped: bool,
+}
+
+impl FlussCluster {
+ fn start(port: u16) -> Self {
+ let helper = std::env::var_os("FLUSS_TEST_CLUSTER_BIN")
+ .map(PathBuf::from)
+ .unwrap_or_else(|| PathBuf::from("../fluss-rust/target/debug/fluss-test-cluster"));
+ assert!(
+ helper.is_file(),
+ "Fluss test cluster helper does not exist at {}; run `just test-e2e`",
+ helper.display()
+ );
+ let name = format!("gateway-e2e-{}-{port}", std::process::id());
+ // Construct the guard before invoking the helper, so partial startup is cleaned on any panic.
+ let mut cluster = Self {
+ helper,
+ name,
+ plaintext_bootstrap_servers: String::new(),
+ sasl_bootstrap_servers: String::new(),
+ stopped: false,
+ };
+ let output = Command::new(&cluster.helper)
+ .args(["start", "--name"])
+ .arg(&cluster.name)
+ .arg("--sasl")
+ .args(["--port", &port.to_string()])
+ .output()
+ .expect("start the Fluss test cluster helper");
+ assert!(
+ output.status.success(),
+ "Fluss test cluster failed to start:\nstdout:\n{}\nstderr:\n{}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = String::from_utf8(output.stdout).expect("cluster helper stdout is UTF-8");
+ let json = stdout
+ .lines()
+ .find_map(|line| line.strip_prefix("CLUSTER_JSON: "))
+ .expect("cluster helper returns CLUSTER_JSON");
+ let info: serde_json::Value = serde_json::from_str(json).expect("valid cluster JSON");
+ cluster.plaintext_bootstrap_servers = info["bootstrap_servers"]
+ .as_str()
+ .expect("cluster JSON contains bootstrap_servers")
+ .to_string();
+ cluster.sasl_bootstrap_servers = info["sasl_bootstrap_servers"]
+ .as_str()
+ .expect("SASL cluster JSON contains sasl_bootstrap_servers")
+ .to_string();
+ cluster
+ }
+
+ fn stop(mut self) {
+ let output =
+ stop_cluster(&self.helper, &self.name).expect("stop the Fluss test cluster helper");
+ assert!(
+ output.status.success(),
+ "Fluss test cluster failed to stop:\nstdout:\n{}\nstderr:\n{}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ self.stopped = true;
+ }
+}
+
+impl Drop for FlussCluster {
+ fn drop(&mut self) {
+ if !self.stopped {
+ let _ = stop_cluster(&self.helper, &self.name);
+ }
+ }
+}
+
+fn stop_cluster(helper: &Path, name: &str) -> std::io::Result<std::process::Output> {
+ Command::new(helper).args(["stop", "--name", name]).output()
+}
+
+/// Finds the SASL and plaintext host-port pairs for the coordinator and single tablet server.
+fn free_cluster_port_pair() -> u16 {
+ for _ in 0..100 {
+ let candidate = std::net::TcpListener::bind("127.0.0.1:0").expect("bind a candidate port");
+ let port = candidate.local_addr().expect("candidate address").port();
+ drop(candidate);
+ let Some(sasl_tablet) = port.checked_add(1) else {
+ continue;
+ };
+ let Some(plaintext_coordinator) = port.checked_add(100) else {
+ continue;
+ };
+ let Some(plaintext_tablet) = port.checked_add(101) else {
+ continue;
+ };
+ let mut reservations = Vec::new();
+ for candidate in [port, sasl_tablet, plaintext_coordinator, plaintext_tablet] {
+ match std::net::TcpListener::bind(("127.0.0.1", candidate)) {
+ Ok(listener) => reservations.push(listener),
+ Err(_) => break,
+ }
+ }
+ if reservations.len() == 4 {
+ return port;
+ }
+ }
+ panic!("failed to find the four ports for a SASL Fluss test cluster");
+}
+
+/// Writes the production Gateway configuration that points its default cluster at the test cluster.
+fn write_gateway_config(
+ directory: &tempfile::TempDir,
+ port: u16,
+ bootstrap_servers: &str,
+ security_protocol: &str,
+) -> std::path::PathBuf {
+ let path = directory.path().join("gateway.yaml");
+ let mut file = std::fs::File::create(&path).expect("create gateway config");
+ writeln!(file, "gateway.rest.listen: 127.0.0.1:{port}").expect("write REST listener");
+ writeln!(file, "gateway.metrics.enabled: false").expect("disable metrics listener");
+ writeln!(
+ file,
+ "gateway.cluster.default.bootstrap.servers: {bootstrap_servers}"
+ )
+ .expect("write Fluss bootstrap servers");
+ writeln!(
+ file,
+ "gateway.cluster.default.connection.security.protocol: {security_protocol}"
+ )
+ .expect("write Fluss security protocol");
+ if security_protocol == "sasl" {
+ writeln!(
+ file,
+ "gateway.cluster.default.connection.service.account: {SERVICE_ACCOUNT}"
+ )
+ .expect("write Gateway service account");
+ writeln!(
+ file,
+ "gateway.cluster.default.connection.service.secret: {SERVICE_SECRET}"
+ )
+ .expect("write Gateway service secret");
+ }
+ path
+}
+
+async fn assert_metadata_apis(bootstrap_servers: &str, security_protocol: &str) {
+ let gateway_port = free_port();
+ let directory = tempfile::tempdir().expect("temporary Gateway config directory");
+ let config = write_gateway_config(
+ &directory,
+ gateway_port,
+ bootstrap_servers,
+ security_protocol,
+ );
+ let child = binary()
+ .arg("--config")
+ .arg(config)
+ .spawn()
+ .expect("start the Gateway binary");
+ let mut gateway = ChildGuard(child);
+ let base = format!("http://127.0.0.1:{gateway_port}");
+ assert!(
+ await_http_ok(&format!("{base}/ready"), Duration::from_secs(15)).await,
+ "Gateway becomes ready"
+ );
+
+ let api = Api::new(base);
+ assert_eq!(
+ api.get_ok("/v1/clusters").await,
+ serde_json::json!({"clusters": ["default"]})
+ );
+
+ let databases = api.get_ok("/v1/clusters/default/databases").await;
+ assert!(
+ databases["databases"]
+ .as_array()
+ .expect("database array")
+ .iter()
+ .any(|database| database == DATABASE),
+ "created database is returned: {databases}"
+ );
+ assert_eq!(
+ api.get_ok(&format!("/v1/clusters/default/databases/{DATABASE}/tables"))
+ .await,
+ serde_json::json!({"tables": [TABLE]})
+ );
+
+ gateway.send_sigterm();
+ let status = gateway.wait_for_exit(Duration::from_secs(35)).await;
+ assert_eq!(status.code(), Some(0), "Gateway drains cleanly");
+}
+
+#[tokio::test]
+async fn metadata_apis_support_plaintext_and_sasl_fluss_clusters() {
+ let cluster = tokio::task::spawn_blocking(|| FlussCluster::start(free_cluster_port_pair()))
+ .await
+ .expect("cluster startup task");
+
+ let connection = FlussConnection::new(Config {
+ bootstrap_servers: cluster.sasl_bootstrap_servers.clone(),
+ security_protocol: "sasl".to_string(),
+ security_sasl_mechanism: "PLAIN".to_string(),
+ security_sasl_username: SERVICE_ACCOUNT.to_string(),
+ security_sasl_password: SERVICE_SECRET.to_string(),
+ ..Default::default()
+ })
+ .await
+ .expect("connect the catalog setup client");
+ let admin = connection.get_admin().expect("get Fluss admin client");
+ admin
+ .create_database(DATABASE, None, false)
+ .await
+ .expect("create the E2E database");
+ let descriptor = TableDescriptor::builder()
+ .schema(
+ Schema::builder()
+ .column("id", DataTypes::int())
+ .column("payload", DataTypes::string())
+ .build()
+ .expect("build the E2E schema"),
+ )
+ .build()
+ .expect("build the E2E table descriptor");
+ admin
+ .create_table(&TablePath::new(DATABASE, TABLE), &descriptor, false)
+ .await
+ .expect("create the E2E table");
+
+ assert_metadata_apis(&cluster.plaintext_bootstrap_servers, "plaintext").await;
+ assert_metadata_apis(&cluster.sasl_bootstrap_servers, "sasl").await;
+
+ connection
+ .close(Duration::from_secs(10))
+ .await
+ .expect("close the setup client");
+ tokio::task::spawn_blocking(|| cluster.stop())
+ .await
+ .expect("cluster cleanup task");
+}
diff --git a/fluss-gateway/tests/http_api.rs b/fluss-gateway/tests/http_api.rs
index e7ec86aa2..6f07c9f9f 100644
--- a/fluss-gateway/tests/http_api.rs
+++ b/fluss-gateway/tests/http_api.rs
@@ -148,6 +148,51 @@ async fn an_oversized_body_is_rejected_with_413_and_never_429() {
gateway.shutdown().await.expect("clean shutdown");
}
+/// The gateway serves while Fluss is unreachable: cluster discovery answers from configuration, the
+/// process reports itself ready, and only a request that actually needs the cluster fails — with 503,
+/// on the cold connection path.
+///
+/// This is the property that lets a gateway be deployed before, or independently of, its clusters.
+#[tokio::test]
+async fn metadata_discovery_serves_while_fluss_is_unreachable() {
+ use fluss_gateway::config::{ConfigDuration, GatewayConfig};
+
+ let mut config = GatewayConfig::default();
+ config.server.rest.bind_address = "127.0.0.1:0".parse().expect("valid");
+ config.server.metrics.enabled = false;
+ // Port 1 has no listener, so every connection attempt fails fast.
+ let cluster = config.clusters.get_mut("default").expect("default cluster");
+ cluster.bootstrap_servers = "127.0.0.1:1".to_string();
+ cluster.connect_timeout = ConfigDuration::from_millis(200);
+
+ let gateway = fluss_gateway::lifecycle::start(config)
+ .await
+ .expect("the gateway starts without Fluss");
+ let api = Api::new(format!("http://{}", gateway.local_addr()));
+
+ // Discovery is a configuration echo: the ID array carries no reachability field.
+ assert_eq!(
+ api.get_ok("/v1/clusters").await,
+ serde_json::json!({"clusters": ["default"]})
+ );
+ assert_eq!(api.get_ok("/ready").await["status"], "ready");
+
+ let response = api.get("/v1/clusters/default/databases").await;
+ assert_eq!(response.status(), 503);
+ let body: serde_json::Value = response.json().await.expect("JSON body");
+ assert_eq!(body["error"]["code"], "unavailable");
+ // The failure names the operation without leaking the bootstrap address.
+ let message = body["error"]["message"].as_str().expect("a message");
+ assert!(!message.contains("127.0.0.1"), "{message}");
+
+ // An unconfigured cluster is a 404 that never touches a connection.
+ assert_eq!(api.get("/v1/clusters/other/databases").await.status(), 404);
+ // The gateway is still serving after all of that.
+ assert_eq!(api.get_ok("/ready").await["status"], "ready");
+
+ gateway.shutdown().await.expect("clean shutdown");
+}
+
/// A gateway with a short header read timeout, for the connection-level tests.
async fn short_header_timeout_gateway() -> fluss_gateway::lifecycle::RunningGateway {
let mut config = fluss_gateway::config::GatewayConfig::default();
diff --git a/fluss-gateway/tests/process.rs b/fluss-gateway/tests/process.rs
index 01f68f429..8caf7af0d 100644
--- a/fluss-gateway/tests/process.rs
+++ b/fluss-gateway/tests/process.rs
@@ -77,7 +77,8 @@ async fn a_native_client_override_fails_before_binding_without_leaking_credentia
let path = dir.path().join("gateway.yaml");
std::fs::write(
&path,
- "gateway.cluster.default.connection.service.account: gateway-user\n\
+ "gateway.cluster.default.connection.security.protocol: sasl\n\
+ gateway.cluster.default.connection.service.account: gateway-user\n\
gateway.cluster.default.connection.service.secret: canonical-secret\n\
gateway.security.authentication: token\n\
gateway.security.tokens: token-secret:alice\n\
@@ -111,6 +112,7 @@ async fn canonical_credentials_are_redacted_from_startup_diagnostics() {
format!(
"gateway.rest.listen: 127.0.0.1:{port}\n\
gateway.metrics.enabled: false\n\
+ gateway.cluster.default.connection.security.protocol: sasl\n\
gateway.cluster.default.connection.service.account: canonical-user\n\
gateway.cluster.default.connection.service.secret: canonical-secret\n\
gateway.security.authentication: token\n\