You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/07/01 23:40:16 UTC
(iggy) 17/19: feat(server-ng): add HTTP admission control
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch http-listener
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 8285c851124eb6bd4829ebc67d8a51ea7f2fa004
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Thu Jul 2 00:30:21 2026 +0200
feat(server-ng): add HTTP admission control
HTTP handlers run on shard 0's single-threaded compio runtime, so
unbounded concurrent writes park unbounded handlers whose buffered
bodies and per-request CPU compete with the consensus pump. Three
bounds close that: the legacy DefaultBodyLimit (413 past
http.max_request_size) caps the per-request term, a per-session
in-flight cap (429) keeps one credential from monopolizing the shared
budget, and a global in-flight budget (503 server_busy) bounds the
multiplier, so budget x max_request_size caps worst-case buffered
bytes and budget x per-request CPU caps pump starvation. One RAII
guard owns both counter releases, so success, errors, the reply
timeout, and client disconnect all release admission through the same
Drop. ack=none produces stay outside admission: they never park a
handler, so their whole cost is the synchronous handler body.
---
core/server-ng/src/bootstrap.rs | 2 +-
core/server-ng/src/http.rs | 245 ++++++++++++++++++++++++++++++++++++++--
2 files changed, 237 insertions(+), 10 deletions(-)
diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs
index a3910964f..87ba615da 100644
--- a/core/server-ng/src/bootstrap.rs
+++ b/core/server-ng/src/bootstrap.rs
@@ -1946,7 +1946,7 @@ async fn start_tcp_runtime(
// caller of this function.
if config.http.enabled {
let http_addr = parse_socket_addr("http.address", &config.http.address)?;
- http::start(shard, http_addr, &config.http.jwt, &config.cluster).await?;
+ http::start(shard, http_addr, &config.http, &config.cluster).await?;
}
Ok(())
diff --git a/core/server-ng/src/http.rs b/core/server-ng/src/http.rs
index 596f00a94..e36f4b813 100644
--- a/core/server-ng/src/http.rs
+++ b/core/server-ng/src/http.rs
@@ -28,7 +28,7 @@ use std::net::{IpAddr, SocketAddr};
use std::rc::Rc;
use std::time::Duration;
-use axum::extract::{Path, Query, State};
+use axum::extract::{DefaultBodyLimit, Path, Query, State};
use axum::http::header::LOCATION;
use axum::http::{HeaderName, HeaderValue, StatusCode};
use axum::middleware::map_response;
@@ -37,7 +37,7 @@ use axum::routing::{delete, get, post, put};
use axum::{Json, Router};
use bytes::{Bytes, BytesMut};
use configs::cluster::{ClusterConfig, ClusterNodeConfig};
-use configs::http::HttpJwtConfig;
+use configs::http::HttpConfig;
use consensus::{MetadataHandle, VsrConsensus};
use iggy_binary_protocol::codes::{
GET_CONSUMER_GROUP_CODE, GET_CONSUMER_GROUPS_CODE, GET_STATS_CODE, GET_STREAM_CODE,
@@ -173,6 +173,25 @@ const FIRST_REQUEST_ID: u64 = 1;
/// caller's behalf.
const PARTITION_WRITE_REPLY_TIMEOUT: Duration = Duration::from_secs(10);
+/// Per-session cap on concurrently awaited partition writes (produce /
+/// consumer-offset). Bounds how much of [`MAX_IN_FLIGHT_WRITES_GLOBAL`] one
+/// credential can occupy, so a session that outruns its own commits saturates
+/// itself (429, its own backpressure signal) before it can starve every other
+/// session out of the shared budget.
+const MAX_IN_FLIGHT_WRITES_PER_SESSION: u32 = 32;
+
+/// Shard-0-global budget for concurrently awaited partition writes, across all
+/// sessions. Every admitted write parks a handler for up to
+/// [`PARTITION_WRITE_REPLY_TIMEOUT`] while pinning its buffered body, and its
+/// decode/encode/HS256 CPU runs on the same single-threaded core that pumps
+/// consensus. The budget therefore bounds both starvation terms: budget x
+/// `max_request_size` bounds the worst-case buffered bytes, and budget x
+/// per-request CPU bounds how far admitted HTTP work can delay the consensus
+/// pump. `?ack=none` produces sit outside the budget by design: they never
+/// park a handler, so their whole cost is the synchronous handler body itself,
+/// which admission could not shed anyway.
+const MAX_IN_FLIGHT_WRITES_GLOBAL: u32 = 128;
+
/// VSR client id stamped on HTTP data-plane reads (poll / consumer-offset).
/// HTTP reads never Register a VSR client, and shard-0 client ids are minted
/// from 1, so 0 can never name a live consumer-group member: a group-kind
@@ -234,6 +253,10 @@ struct HttpSession {
/// session eviction can tear the registry entry down fenced by the same
/// token.
registry_token: Cell<Option<InstanceToken>>,
+ /// Awaited partition writes currently in flight on this session, gated by
+ /// [`MAX_IN_FLIGHT_WRITES_PER_SESSION`]. Only [`InFlightWriteGuard`]
+ /// touches it, so every admission is paired with exactly one release.
+ in_flight_writes: Cell<u32>,
}
impl HttpSession {
@@ -309,6 +332,7 @@ impl IntoResponse for WriteError {
/// reply distinguishable only by header (see [`classify_partition_reply`]),
/// and an unanswered write is a distinct outcome the caller must treat as
/// unknown rather than failed.
+#[derive(Debug)]
enum PartitionWriteError {
/// Caller-side rejection (bad identifier, oversized batch, the interim
/// non-root denial) or a malformed reply frame, rendered through the
@@ -321,6 +345,15 @@ enum PartitionWriteError {
/// The in-process reply slot could not be installed. Transient server
/// condition -> the shared 503, retryable.
Unavailable,
+ /// This session is already at [`MAX_IN_FLIGHT_WRITES_PER_SESSION`]
+ /// awaited writes. 429: the caller's own concurrency is the problem, so
+ /// it must drain its outstanding writes before submitting more.
+ TooManyInFlight,
+ /// Shard 0 is already at [`MAX_IN_FLIGHT_WRITES_GLOBAL`] awaited writes
+ /// across all sessions. 503 with its own code (distinct from the shared
+ /// consensus-unavailable body) so an operator can tell admission shedding
+ /// from a consensus outage.
+ ServerBusy,
/// No committed reply within [`PARTITION_WRITE_REPLY_TIMEOUT`], or the
/// session's reply target was torn down mid-wait. 504: the commit may
/// still land (at-least-once), so this is a hard "outcome unknown", not a
@@ -335,6 +368,8 @@ impl IntoResponse for PartitionWriteError {
Self::Rejected(error) => CustomError::from(error).into_response(),
Self::NotFound => CustomError::ResourceNotFound.into_response(),
Self::Unavailable => service_unavailable(),
+ Self::TooManyInFlight => too_many_in_flight_response(),
+ Self::ServerBusy => server_busy_response(),
Self::Timeout(operation) => partition_write_timeout_response(operation),
}
}
@@ -389,6 +424,41 @@ fn service_unavailable() -> Response {
.into_response()
}
+/// 429 for a session at [`MAX_IN_FLIGHT_WRITES_PER_SESSION`] awaited partition
+/// writes. Shaped like every other HTTP error (`ErrorResponse`) so clients
+/// parse one error schema; the remedy is the caller's own: let outstanding
+/// writes finish, then retry.
+fn too_many_in_flight_response() -> Response {
+ (
+ StatusCode::TOO_MANY_REQUESTS,
+ Json(ErrorResponse {
+ id: StatusCode::TOO_MANY_REQUESTS.as_u16().into(),
+ code: "too_many_in_flight_writes".to_owned(),
+ reason: "session reached its in-flight write cap; await outstanding writes and retry"
+ .to_owned(),
+ field: None,
+ }),
+ )
+ .into_response()
+}
+
+/// 503 for shard 0 at [`MAX_IN_FLIGHT_WRITES_GLOBAL`] awaited partition writes
+/// across all sessions. A distinct `server_busy` code (unlike the shared
+/// consensus-unavailable 503) so admission shedding is tellable from a
+/// consensus outage; retry with backoff.
+fn server_busy_response() -> Response {
+ (
+ StatusCode::SERVICE_UNAVAILABLE,
+ Json(ErrorResponse {
+ id: StatusCode::SERVICE_UNAVAILABLE.as_u16().into(),
+ code: "server_busy".to_owned(),
+ reason: "shard is at its in-flight write budget; retry with backoff".to_owned(),
+ field: None,
+ }),
+ )
+ .into_response()
+}
+
/// Read consistency selected by the `?consistency=` query param.
///
/// `serializable` (the default) serves from this node's local metadata STM:
@@ -562,6 +632,10 @@ struct HttpInner {
/// across an `.await` (see [`HttpInner::resolve_session`]).
sessions: RefCell<HashMap<String, Rc<HttpSession>>>,
roster: ClusterRoster,
+ /// Awaited partition writes currently in flight across all sessions, gated
+ /// by [`MAX_IN_FLIGHT_WRITES_GLOBAL`]. Only [`InFlightWriteGuard`] touches
+ /// it, so every admission is paired with exactly one release.
+ in_flight_writes: Cell<u32>,
}
impl HttpInner {
@@ -696,6 +770,7 @@ impl HttpInner {
gate: Mutex::new(FIRST_REQUEST_ID),
data_request: Cell::new(FIRST_REQUEST_ID),
registry_token: Cell::new(None),
+ in_flight_writes: Cell::new(0),
}))
}
}
@@ -728,14 +803,14 @@ type HttpState = SendWrapper<Rc<HttpInner>>;
/// # Errors
///
/// Returns [`ServerNgError`] if the JWT manager cannot be built from
-/// `jwt_config` or the listener cannot bind to `addr`.
+/// `http_config.jwt` or the listener cannot bind to `addr`.
pub async fn start(
shard: &Rc<ServerNgShard>,
addr: SocketAddr,
- jwt_config: &HttpJwtConfig,
+ http_config: &HttpConfig,
cluster: &ClusterConfig,
) -> Result<(), ServerNgError> {
- let jwt = JwtManager::build(jwt_config)?;
+ let jwt = JwtManager::build(&http_config.jwt)?;
let (listener, bound_addr) = client_listener::tcp::bind(addr).await?;
info!(address = %bound_addr, "server-ng HTTP listener started");
@@ -749,8 +824,13 @@ pub async fn start(
nodes: cluster.nodes.clone(),
http_addr: bound_addr,
},
+ in_flight_writes: Cell::new(0),
}));
- let router = router(state);
+ // Saturating: a configured limit past the pointer width (32-bit target,
+ // >4 GiB value) clamps to the largest enforceable cap instead of wrapping.
+ let max_request_size =
+ usize::try_from(http_config.max_request_size.as_bytes_u64()).unwrap_or(usize::MAX);
+ let router = router(state, max_request_size);
let shutdown = shard.bus.token();
let handle = compio::runtime::spawn(async move {
@@ -768,7 +848,12 @@ pub async fn start(
/// Assemble the shard-0 router: unauthenticated health + login routes, plus
/// one authenticated probe that exercises the [`Authenticated`] extractor.
-fn router(state: HttpState) -> Router {
+///
+/// `max_request_size` becomes the router-wide `DefaultBodyLimit` (413 past
+/// it), exactly like the legacy server: it bounds the per-request term of the
+/// admission math - what one body may cost in bytes and decode CPU - while
+/// the in-flight caps bound the multiplier.
+fn router(state: HttpState, max_request_size: usize) -> Router {
// Cloned for the response layer so `X-Iggy-View` reads the live view per
// response; the original `state` is moved into `with_state` below.
let view_source = state.clone();
@@ -834,6 +919,7 @@ fn router(state: HttpState) -> Router {
.route("/clients", get(get_clients))
.route("/clients/{client_id}", get(get_client))
.with_state(state)
+ .layer(DefaultBodyLimit::max(max_request_size))
.layer(map_response(move |response: Response| {
let view_source = view_source.clone();
async move { insert_view_header(&view_source, response) }
@@ -2109,9 +2195,47 @@ const fn resync_required_polled_messages() -> PolledMessages {
}
}
+/// In-flight admission token for one awaited partition write. One guard owns
+/// both releases (session + global) so success, every error return, the reply
+/// timeout, and handler cancellation (the client hanging up mid-await drops
+/// this future) all decrement through the same `Drop`.
+struct InFlightWriteGuard<'a> {
+ session_in_flight: &'a Cell<u32>,
+ global_in_flight: &'a Cell<u32>,
+}
+
+impl Drop for InFlightWriteGuard<'_> {
+ fn drop(&mut self) {
+ self.session_in_flight.set(self.session_in_flight.get() - 1);
+ self.global_in_flight.set(self.global_in_flight.get() - 1);
+ }
+}
+
+/// Admit one awaited partition write against the per-session cap and the
+/// shard-0 global budget, incrementing both counters only when both pass. The
+/// session cap is checked first so a session that saturates itself reads as
+/// its own 429 rather than as server-wide pressure.
+fn admit_partition_write<'a>(
+ session_in_flight: &'a Cell<u32>,
+ global_in_flight: &'a Cell<u32>,
+) -> Result<InFlightWriteGuard<'a>, PartitionWriteError> {
+ if session_in_flight.get() >= MAX_IN_FLIGHT_WRITES_PER_SESSION {
+ return Err(PartitionWriteError::TooManyInFlight);
+ }
+ if global_in_flight.get() >= MAX_IN_FLIGHT_WRITES_GLOBAL {
+ return Err(PartitionWriteError::ServerBusy);
+ }
+ session_in_flight.set(session_in_flight.get() + 1);
+ global_in_flight.set(global_in_flight.get() + 1);
+ Ok(InFlightWriteGuard {
+ session_in_flight,
+ global_in_flight,
+ })
+}
+
/// Run one awaited partition write (produce / consumer-offset write) end to
-/// end: install the reply slot, dispatch into the partition plane, and wait
-/// (bounded) for the committed reply.
+/// end: admit against the in-flight caps, install the reply slot, dispatch
+/// into the partition plane, and wait (bounded) for the committed reply.
///
/// Slot-before-dispatch is load-bearing: every pre-dispatch gate failure
/// inside [`dispatch_partition_request`] replies through `send_to_client`,
@@ -2124,6 +2248,13 @@ async fn partition_write_replicated(
operation: Operation,
body: &[u8],
) -> Result<(), PartitionWriteError> {
+ // Admission sits here rather than before body decode: axum's extractors
+ // already buffered and deserialized the body (bounded by the router-wide
+ // `DefaultBodyLimit`) before the handler ran, so the caps gate what is
+ // actually unbounded - the slot install, the dispatch, and the parked
+ // reply await that pins this request's buffers for up to the reply
+ // timeout. Held across every exit below; released by `Drop`.
+ let _in_flight = admit_partition_write(&session.in_flight_writes, &state.in_flight_writes)?;
ensure_in_process_reply_target(state, session);
let request_id = session.next_data_request_id();
let message = build_request_message(
@@ -2982,4 +3113,100 @@ mod tests {
Err(PartitionWriteError::Rejected(IggyError::InvalidCommand))
));
}
+
+ #[test]
+ fn in_flight_write_guard_decrements_both_counters_on_drop() {
+ let session = Cell::new(0);
+ let global = Cell::new(0);
+ let guard = admit_partition_write(&session, &global).expect("below both caps");
+ assert_eq!(session.get(), 1);
+ assert_eq!(global.get(), 1);
+ drop(guard);
+ assert_eq!(session.get(), 0);
+ assert_eq!(global.get(), 0);
+ }
+
+ #[test]
+ fn admission_at_session_cap_rejects_with_too_many_in_flight() {
+ let session = Cell::new(MAX_IN_FLIGHT_WRITES_PER_SESSION);
+ let global = Cell::new(0);
+ assert!(matches!(
+ admit_partition_write(&session, &global),
+ Err(PartitionWriteError::TooManyInFlight)
+ ));
+ // A refusal must not leak a partial increment on either counter.
+ assert_eq!(session.get(), MAX_IN_FLIGHT_WRITES_PER_SESSION);
+ assert_eq!(global.get(), 0);
+ }
+
+ #[test]
+ fn admission_at_global_budget_rejects_with_server_busy() {
+ let session = Cell::new(0);
+ let global = Cell::new(MAX_IN_FLIGHT_WRITES_GLOBAL);
+ assert!(matches!(
+ admit_partition_write(&session, &global),
+ Err(PartitionWriteError::ServerBusy)
+ ));
+ assert_eq!(session.get(), 0);
+ assert_eq!(global.get(), MAX_IN_FLIGHT_WRITES_GLOBAL);
+ }
+
+ #[test]
+ fn interleaved_admission_reopens_exactly_released_session_slots() {
+ let session = Cell::new(0);
+ let global = Cell::new(0);
+ let mut guards = Vec::new();
+ for _ in 0..MAX_IN_FLIGHT_WRITES_PER_SESSION {
+ guards.push(admit_partition_write(&session, &global).expect("below both caps"));
+ }
+ assert!(matches!(
+ admit_partition_write(&session, &global),
+ Err(PartitionWriteError::TooManyInFlight)
+ ));
+ let released = 3;
+ guards.truncate((MAX_IN_FLIGHT_WRITES_PER_SESSION - released) as usize);
+ assert_eq!(session.get(), MAX_IN_FLIGHT_WRITES_PER_SESSION - released);
+ for _ in 0..released {
+ guards.push(admit_partition_write(&session, &global).expect("released slots"));
+ }
+ assert!(matches!(
+ admit_partition_write(&session, &global),
+ Err(PartitionWriteError::TooManyInFlight)
+ ));
+ drop(guards);
+ assert_eq!(session.get(), 0);
+ assert_eq!(global.get(), 0);
+ }
+
+ #[test]
+ fn global_budget_spans_sessions_and_reopens_after_release() {
+ let global = Cell::new(0);
+ let session_count =
+ MAX_IN_FLIGHT_WRITES_GLOBAL.div_ceil(MAX_IN_FLIGHT_WRITES_PER_SESSION) as usize;
+ let sessions: Vec<Cell<u32>> = (0..session_count).map(|_| Cell::new(0)).collect();
+ let mut guards = Vec::new();
+ 'fill: for session in &sessions {
+ for _ in 0..MAX_IN_FLIGHT_WRITES_PER_SESSION {
+ match admit_partition_write(session, &global) {
+ Ok(guard) => guards.push(guard),
+ Err(PartitionWriteError::ServerBusy) => break 'fill,
+ Err(other) => {
+ panic!("only the global budget may refuse this fill, got {other:?}")
+ }
+ }
+ }
+ }
+ assert_eq!(global.get(), MAX_IN_FLIGHT_WRITES_GLOBAL);
+ // A fresh session is refused on the shared budget, not its own cap.
+ let fresh = Cell::new(0);
+ assert!(matches!(
+ admit_partition_write(&fresh, &global),
+ Err(PartitionWriteError::ServerBusy)
+ ));
+ drop(guards.pop());
+ let readmitted = admit_partition_write(&fresh, &global).expect("budget slot released");
+ assert_eq!(fresh.get(), 1);
+ assert_eq!(global.get(), MAX_IN_FLIGHT_WRITES_GLOBAL);
+ drop(readmitted);
+ }
}