You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by gk...@apache.org on 2026/06/02 12:17:42 UTC
(iggy) 01/01: fix clients
This is an automated email from the ASF dual-hosted git repository.
numinnex pushed a commit to branch fix_clients
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit b733845d536e51c8853e6fe1f8232fab63e14062
Author: Grzegorz Koszyk <g....@gmail.com>
AuthorDate: Tue Jun 2 14:17:31 2026 +0200
fix clients
---
Cargo.toml | 1 -
core/integration/tests/sdk/clients.rs | 86 ++++++++++++
core/integration/tests/sdk/mod.rs | 2 +
core/server-ng/src/bootstrap.rs | 258 ++++++++++++++++++++++++----------
core/server-ng/src/session_manager.rs | 82 +++++++++--
core/shard/src/builder.rs | 8 +-
core/shard/src/lib.rs | 99 ++++++++++++-
core/shard/src/router.rs | 7 +
8 files changed, 458 insertions(+), 85 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index 6e74b0087..0f9ffd242 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -123,7 +123,6 @@ compio = { version = "=0.19.0", features = [
"fs",
] }
compio-buf = "0.8.2"
-# Pin compio-driver >= 0.11.2 to fix musl compilation (compio-rs/compio#668)
compio-driver = "0.12.1"
compio-quic = "=0.8.0"
compio-ws = "=0.4.0"
diff --git a/core/integration/tests/sdk/clients.rs b/core/integration/tests/sdk/clients.rs
new file mode 100644
index 000000000..0b1139ca4
--- /dev/null
+++ b/core/integration/tests/sdk/clients.rs
@@ -0,0 +1,86 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+//! VSR connected-client reads: `get_me` (own connection, from the local
+//! `SessionManager`) and `get_clients` (cluster-wide, scatter-gathered
+//! across all shards). Run on a 3-node cluster so the gather genuinely
+//! crosses shards.
+//!
+//! TODO(remove-when-scenarios-vsr): these are a focused stopgap. The
+//! canonical coverage lives in `server::scenarios::authentication_scenario`
+//! (`get_clients`/`get_me`/`get_client`), but that whole suite is
+//! `#[cfg(not(feature = "vsr"))]` because it also does real
+//! `send_messages` / `poll_messages`, which are not wired under VSR yet
+//! (data plane / partition reconciliation). Once the data plane lands and
+//! the `server` scenario suite is un-gated under VSR, it subsumes these
+//! tests -- delete this file then.
+
+#![cfg(feature = "vsr")]
+
+use iggy::prelude::*;
+use integration::iggy_harness;
+
+#[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic])]
+async fn get_me_returns_own_connection(harness: &TestHarness) {
+ let client = harness.new_client().await.unwrap();
+ client
+ .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+
+ let me = client.get_me().await.expect("get_me");
+ // Sourced from the per-shard SessionManager: authenticated user id +
+ // a real transport label + a non-empty peer address.
+ assert_eq!(me.user_id, Some(0), "root is user id 0 in server-ng");
+ assert!(!me.transport.is_empty(), "transport label must be set");
+ assert!(!me.address.is_empty(), "peer address must be set");
+
+ client.logout_user().await.unwrap();
+}
+
+#[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic])]
+async fn get_clients_gathers_all_connections(harness: &TestHarness) {
+ // Two extra clients alongside the harness client; on a 3-node cluster
+ // their connections are spread across shards, so a complete list
+ // exercises the cross-shard gather.
+ let a = harness.new_client().await.unwrap();
+ a.login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+ let b = harness.new_client().await.unwrap();
+ b.login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD)
+ .await
+ .unwrap();
+
+ let clients = a.get_clients().await.expect("get_clients");
+ // At least the two we just logged in (the gather may also include
+ // other harness/bookkeeping connections).
+ assert!(
+ clients.len() >= 2,
+ "expected >= 2 connected clients, got {}",
+ clients.len()
+ );
+ // Records are real, not the old empty stub: transport + address set.
+ for info in &clients {
+ assert!(!info.transport.is_empty(), "transport label must be set");
+ assert!(!info.address.is_empty(), "peer address must be set");
+ }
+
+ a.logout_user().await.unwrap();
+ b.logout_user().await.unwrap();
+}
diff --git a/core/integration/tests/sdk/mod.rs b/core/integration/tests/sdk/mod.rs
index 1d5633f67..eb66b3936 100644
--- a/core/integration/tests/sdk/mod.rs
+++ b/core/integration/tests/sdk/mod.rs
@@ -16,6 +16,8 @@
* under the License.
*/
+#[cfg(feature = "vsr")]
+mod clients;
mod hello_world;
#[cfg(not(feature = "vsr"))]
mod producer;
diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs
index 044e73ece..f2030af07 100644
--- a/core/server-ng/src/bootstrap.rs
+++ b/core/server-ng/src/bootstrap.rs
@@ -20,7 +20,7 @@
use crate::config_writer::write_current_config;
use crate::login_register::LoginRegisterError;
use crate::server_error::{ServerNgError, ShardJoinFailure, ShardJoinFailureKind};
-use crate::session_manager::SessionManager;
+use crate::session_manager::{ClientRecord, SessionManager};
use bytes::Bytes;
use configs::server_ng::ServerNgConfig;
use configs::sharding::{
@@ -32,8 +32,9 @@ use consensus::{LocalPipeline, MetadataHandle, PartitionsHandle, Sequencer, VsrC
// non-blocking variants for cancel-safe shutdown polling.
use crossfire::{AsyncRxTrait, AsyncTxTrait};
use iggy_binary_protocol::codes::{
- GET_CLUSTER_METADATA_CODE, GET_ME_CODE, GET_STATS_CODE, GET_STREAM_CODE, GET_STREAMS_CODE,
- GET_TOPIC_CODE, GET_TOPICS_CODE, GET_USER_CODE, GET_USERS_CODE, PING_CODE, POLL_MESSAGES_CODE,
+ GET_CLIENT_CODE, GET_CLIENTS_CODE, GET_CLUSTER_METADATA_CODE, GET_ME_CODE, GET_STATS_CODE,
+ GET_STREAM_CODE, GET_STREAMS_CODE, GET_TOPIC_CODE, GET_TOPICS_CODE, GET_USER_CODE,
+ GET_USERS_CODE, PING_CODE, POLL_MESSAGES_CODE,
};
use iggy_binary_protocol::requests::consumer_offsets::{
DeleteConsumerOffset2Request, DeleteConsumerOffsetRequest, StoreConsumerOffset2Request,
@@ -46,12 +47,14 @@ use iggy_binary_protocol::requests::personal_access_tokens::{
};
use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest;
use iggy_binary_protocol::requests::streams::{GetStreamRequest, GetStreamsRequest};
+use iggy_binary_protocol::requests::system::get_client::GetClientRequest;
use iggy_binary_protocol::requests::topics::{GetTopicRequest, GetTopicsRequest};
use iggy_binary_protocol::requests::users::{
GetUserRequest, LoginRegisterRequest, LoginRegisterWithPatRequest,
};
use iggy_binary_protocol::responses::clients::client_response::ClientResponse;
use iggy_binary_protocol::responses::clients::get_client::ClientDetailsResponse;
+use iggy_binary_protocol::responses::clients::get_clients::GetClientsResponse;
use iggy_binary_protocol::responses::personal_access_tokens::RawPersonalAccessTokenResponse;
use iggy_binary_protocol::responses::streams::StreamResponse;
use iggy_binary_protocol::responses::streams::get_stream::{
@@ -123,8 +126,9 @@ use shard::builder::IggyShardBuilder;
use shard::metrics::ShardMetrics;
use shard::shards_table::{PapayaShardsTable, calculate_shard_assignment};
use shard::{
- CoordinatorConfig, IggyShard, PartitionConsensusConfig, Receiver as ShardReceiver, ShardFrame,
- ShardIdentity, TaggedSender, channel, shard_mesh_channels,
+ ConnectedClientInfo, CoordinatorConfig, IggyShard, ListClientsHandler,
+ PartitionConsensusConfig, Receiver as ShardReceiver, ShardFrame, ShardIdentity, TaggedSender,
+ channel, shard_mesh_channels,
};
use std::cell::RefCell;
use std::collections::{HashMap, HashSet, VecDeque};
@@ -814,7 +818,7 @@ async fn shard_main(
);
let shard_metrics = ShardMetrics::for_shard();
- let shard = build_shard_for_thread(
+ let (shard, sessions) = build_shard_for_thread(
shard_id,
total_shards,
config,
@@ -873,7 +877,7 @@ async fn shard_main(
let coord = shard
.coordinator()
.expect("shard 0 always has a coordinator attached by the builder");
- let on_client_request = make_client_request_handler(&shard);
+ let on_client_request = make_client_request_handler(&shard, &sessions);
let accepted_replica = make_delegating_replica_accept_fn(Rc::clone(&coord));
let accepted_client = make_shard_zero_client_accept_fns(coord, &bus, on_client_request);
@@ -1028,7 +1032,7 @@ async fn build_shard_for_thread(
senders: Vec<TaggedSender>,
inbox: ShardReceiver<ShardFrame>,
metrics: ShardMetrics,
-) -> Result<Rc<ServerNgShard>, ServerNgError> {
+) -> Result<(Rc<ServerNgShard>, Rc<RefCell<SessionManager>>), ServerNgError> {
let shard_local_id = ShardId::new(shard_id);
let total_partitions = metadata.mux_stm.streams().read(|inner| {
inner
@@ -1123,9 +1127,14 @@ async fn build_shard_for_thread(
}
let shard_handle = Rc::new(RefCell::new(None));
+ // One per-shard SessionManager, shared by the client-request handler
+ // (binds sessions) and the get_clients handler (reads them). Created
+ // here so both wirings reference the same instance.
+ let sessions = Rc::new(RefCell::new(SessionManager::new()));
let on_replica_message = make_deferred_replica_message_handler(&shard_handle);
- let on_client_request = make_deferred_client_request_handler(&bus, &shard_handle);
+ let on_client_request = make_deferred_client_request_handler(&bus, &shard_handle, &sessions);
let on_metadata_submit = make_metadata_submit_handler(&shard_handle);
+ let on_list_clients = make_list_clients_handler(&sessions);
let shard_name = format!("server-ng-shard-{shard_id}");
let built = IggyShardBuilder::new(
ShardIdentity::new(shard_id, shard_name),
@@ -1133,6 +1142,7 @@ async fn build_shard_for_thread(
on_replica_message,
on_client_request,
on_metadata_submit,
+ on_list_clients,
metadata,
partitions,
senders,
@@ -1147,7 +1157,7 @@ async fn build_shard_for_thread(
let shard = Rc::new(built.shard);
*shard_handle.borrow_mut() = Some(Rc::downgrade(&shard));
- Ok(shard)
+ Ok((shard, sessions))
}
fn restore_metadata_consensus(
@@ -2079,9 +2089,12 @@ async fn start_manual_runtime(
type ClientRequestQueues = Rc<RefCell<HashMap<u128, VecDeque<Message<GenericHeader>>>>>;
type ActiveClientRequests = Rc<RefCell<HashSet<u128>>>;
-fn make_client_request_handler(shard: &Rc<ServerNgShard>) -> RequestHandler {
+fn make_client_request_handler(
+ shard: &Rc<ServerNgShard>,
+ sessions: &Rc<RefCell<SessionManager>>,
+) -> RequestHandler {
let shard = Rc::clone(shard);
- let sessions = Rc::new(RefCell::new(SessionManager::new()));
+ let sessions = Rc::clone(sessions);
let queues: ClientRequestQueues = Rc::new(RefCell::new(HashMap::new()));
let active: ActiveClientRequests = Rc::new(RefCell::new(HashSet::new()));
let sessions_for_disconnect = Rc::clone(&sessions);
@@ -2139,6 +2152,30 @@ fn make_client_request_handler(shard: &Rc<ServerNgShard>) -> RequestHandler {
})
}
+/// Build the per-shard [`ListClientsHandler`]: on a `ListClients`
+/// broadcast, serialize this shard's locally-homed connected clients from
+/// its `SessionManager` and push them back over the reply sender. The
+/// aggregation across all shards happens in
+/// [`shard::IggyShard::list_all_clients`].
+fn make_list_clients_handler(sessions: &Rc<RefCell<SessionManager>>) -> ListClientsHandler {
+ let sessions = Rc::clone(sessions);
+ Rc::new(move |reply| {
+ let clients: Vec<ConnectedClientInfo> = sessions
+ .borrow()
+ .iter_clients()
+ .map(|record| ConnectedClientInfo {
+ client_id: record.client_id,
+ user_id: record.user_id,
+ transport: record.transport,
+ address: record.address,
+ })
+ .collect();
+ // Best-effort: the gather side bounds itself by count + timeout, so
+ // a dropped reply (receiver gone) just means this shard is omitted.
+ let _ = reply.try_send(clients);
+ })
+}
+
fn make_deferred_replica_message_handler(shard_handle: &ServerNgShardHandle) -> MessageHandler {
let shard_handle = Rc::clone(shard_handle);
Rc::new(move |_replica_id, message| {
@@ -2151,9 +2188,10 @@ fn make_deferred_replica_message_handler(shard_handle: &ServerNgShardHandle) ->
fn make_deferred_client_request_handler(
bus: &Rc<IggyMessageBus>,
shard_handle: &ServerNgShardHandle,
+ sessions: &Rc<RefCell<SessionManager>>,
) -> RequestHandler {
let shard_handle = Rc::clone(shard_handle);
- let sessions = Rc::new(RefCell::new(SessionManager::new()));
+ let sessions = Rc::clone(sessions);
let queues: ClientRequestQueues = Rc::new(RefCell::new(HashMap::new()));
let active: ActiveClientRequests = Rc::new(RefCell::new(HashSet::new()));
let sessions_for_disconnect = Rc::clone(&sessions);
@@ -2531,24 +2569,55 @@ async fn handle_non_replicated_request(
}
GET_ME_CODE => {
// `get_me` reports the requesting connection's own identity,
- // which lives in the session manager + transport meta (not in
- // `IggyMetadata`), so it is built here rather than in
+ // sourced from this shard's `SessionManager` (not `IggyMetadata`),
+ // so it is built here rather than in
// `build_non_replicated_response`.
- let response = build_get_me_response(shard, sessions, transport_client_id);
- let commit = current_metadata_commit(shard);
- let reply = NonReplicatedResponse::Bytes(response.to_bytes()).into_reply(
- request.header(),
- request.header().client,
- request.header().session,
- commit,
- );
- if let Err(error) = shard
- .bus
- .send_to_client(transport_client_id, reply.into_generic().into_frozen())
- .await
- {
- warn!(transport_client_id, error = %error, "failed to send get_me reply");
- }
+ let response = build_get_me_response(sessions, transport_client_id);
+ send_non_replicated_bytes(
+ shard,
+ &request,
+ transport_client_id,
+ response.to_bytes(),
+ "get_me",
+ )
+ .await;
+ }
+ GET_CLIENTS_CODE => {
+ // Shared-nothing: each shard knows only its own connections, so
+ // gather across all shards (scatter-gather over the mesh).
+ let infos = shard.list_all_clients().await;
+ let response = GetClientsResponse {
+ clients: infos.iter().map(connected_client_to_response).collect(),
+ };
+ send_non_replicated_bytes(
+ shard,
+ &request,
+ transport_client_id,
+ response.to_bytes(),
+ "get_clients",
+ )
+ .await;
+ }
+ GET_CLIENT_CODE => {
+ // No reverse map from the wire u32 id to a u128 transport id /
+ // home shard (the u32 is just the seq tail), so gather all and
+ // filter -- same fan-out as `get_clients`.
+ let target = GetClientRequest::decode_from(request_body(&request))
+ .ok()
+ .map(|req| req.client_id);
+ let infos = shard.list_all_clients().await;
+ #[allow(clippy::cast_possible_truncation)]
+ let found = target.and_then(|id| infos.iter().find(|info| info.client_id as u32 == id));
+ // The SDK decodes an empty body as `None` (client not found).
+ let bytes = found.map_or_else(Bytes::new, |info| {
+ ClientDetailsResponse {
+ client: connected_client_to_response(info),
+ consumer_groups: Vec::new(),
+ }
+ .to_bytes()
+ });
+ send_non_replicated_bytes(shard, &request, transport_client_id, bytes, "get_client")
+ .await;
}
_ => match build_non_replicated_response(shard, code, request_body(&request)) {
Ok(response) => {
@@ -2584,56 +2653,103 @@ async fn handle_non_replicated_request(
}
}
-/// Build the `get_me` reply for the requesting connection from its
-/// session (user id) and transport meta (transport kind + peer address).
-/// `consumer_groups` is empty: server-ng does not yet track per-client
-/// consumer-group membership.
-///
-/// TODO(clients-table): the transport kind + peer address are read from
-/// the message-bus `client_meta` here, which couples this metadata read
-/// to the bus's connection bookkeeping. The client identity (id, user,
-/// transport, address, consumer-group membership) should instead live in
-/// a first-class clients table in `IggyMetadata`, populated at Register
-/// time, so `get_me` / `get_client` / `get_clients` all read from one
-/// authoritative source via `frontend()` rather than reaching into the
-/// bus. Rework this API once that table exists.
+/// Build the `get_me` reply for the requesting connection, sourced
+/// entirely from the per-shard [`SessionManager`] (`user_id`, transport
+/// kind, peer address) -- no message-bus lookup. `consumer_groups` is
+/// empty (see [`client_record_to_response`]).
fn build_get_me_response(
- shard: &Rc<ServerNgShard>,
sessions: &Rc<RefCell<SessionManager>>,
transport_client_id: u128,
) -> ClientDetailsResponse {
- // `get_me` is auth-gated, so a bound session's user id is normally
- // present. Fall back to the wire "no user" sentinel (`u32::MAX`)
- // rather than `0`, which is a valid id (server-ng is 0-based, so
- // root is user id 0) and would otherwise impersonate root.
- let user_id = sessions
+ let client = sessions
.borrow()
- .get_user_id(transport_client_id)
- .unwrap_or(u32::MAX);
- let (transport, address) = shard.bus.client_meta(transport_client_id).map_or_else(
- || (1u8, String::new()),
- |meta| {
- (
- transport_kind_to_wire(meta.transport),
- meta.peer_addr.to_string(),
- )
- },
- );
+ .client_record(transport_client_id)
+ .map_or_else(
+ || {
+ // No session record (shouldn't happen on an auth-gated
+ // read). Report the connection id with the "no user"
+ // sentinel + TCP default rather than impersonating root
+ // (user id 0 is a real user; server-ng is 0-based).
+ #[allow(clippy::cast_possible_truncation)]
+ ClientResponse {
+ client_id: transport_client_id as u32,
+ user_id: u32::MAX,
+ transport: 1,
+ address: String::new(),
+ consumer_groups_count: 0,
+ }
+ },
+ |record| client_record_to_response(&record),
+ );
ClientDetailsResponse {
- client: ClientResponse {
- // The transport client id is a u128 `(shard << 112) | seq`; the
- // legacy wire `client_id` is the u32 seq tail.
- #[allow(clippy::cast_possible_truncation)]
- client_id: transport_client_id as u32,
- user_id,
- transport,
- address,
- consumer_groups_count: 0,
- },
+ client,
consumer_groups: Vec::new(),
}
}
+/// Convert a [`ClientRecord`] (per-shard session state) into the wire
+/// [`ClientResponse`]. Shared by `get_me` and the `get_clients` gather.
+///
+/// TODO(consumer-group-membership): `consumer_groups_count` is always 0
+/// -- server-ng does not yet track which consumer groups a connection
+/// joined. Populate once the partition-reconciliation / consumer-group
+/// rebalancing work lands; until then `get_me` / `get_clients` report no
+/// memberships.
+fn client_record_to_response(record: &ClientRecord) -> ClientResponse {
+ // The transport client id is a u128 `(shard << 112) | seq`; the wire
+ // `client_id` is the u32 seq tail.
+ #[allow(clippy::cast_possible_truncation)]
+ ClientResponse {
+ client_id: record.client_id as u32,
+ user_id: record.user_id.unwrap_or(u32::MAX),
+ transport: transport_kind_to_wire(record.transport),
+ address: record.address.to_string(),
+ consumer_groups_count: 0,
+ }
+}
+
+/// Convert a gathered [`ConnectedClientInfo`] (one shard's view of a
+/// connected client) into the wire [`ClientResponse`] for `get_clients` /
+/// `get_client`. Same `consumer_groups_count: 0` caveat as
+/// [`client_record_to_response`].
+fn connected_client_to_response(info: &ConnectedClientInfo) -> ClientResponse {
+ #[allow(clippy::cast_possible_truncation)]
+ ClientResponse {
+ client_id: info.client_id as u32,
+ user_id: info.user_id.unwrap_or(u32::MAX),
+ transport: transport_kind_to_wire(info.transport),
+ address: info.address.to_string(),
+ consumer_groups_count: 0,
+ }
+}
+
+/// Send a non-replicated reply body to a client, stamping the current
+/// metadata commit. Shared by the `get_me` / `get_clients` / `get_client`
+/// arms.
+#[allow(clippy::future_not_send)]
+async fn send_non_replicated_bytes(
+ shard: &Rc<ServerNgShard>,
+ request: &Message<RequestHeader>,
+ transport_client_id: u128,
+ bytes: Bytes,
+ label: &'static str,
+) {
+ let commit = current_metadata_commit(shard);
+ let reply = NonReplicatedResponse::Bytes(bytes).into_reply(
+ request.header(),
+ request.header().client,
+ request.header().session,
+ commit,
+ );
+ if let Err(error) = shard
+ .bus
+ .send_to_client(transport_client_id, reply.into_generic().into_frozen())
+ .await
+ {
+ warn!(transport_client_id, label, error = %error, "failed to send non-replicated reply");
+ }
+}
+
/// Map the transport kind to the legacy wire discriminant
/// (`1=TCP, 2=QUIC, 4=WebSocket`); TLS variants report their base
/// transport. `ClientTransportKind` is `#[non_exhaustive]`, so any other
@@ -2787,7 +2903,7 @@ fn ensure_transport_connection(
};
sessions
.borrow_mut()
- .ensure_connection(transport_client_id, meta.peer_addr);
+ .ensure_connection(transport_client_id, meta.peer_addr, meta.transport);
}
fn maybe_rewrite_pat_request(
diff --git a/core/server-ng/src/session_manager.rs b/core/server-ng/src/session_manager.rs
index 1e783141a..4a39bde2b 100644
--- a/core/server-ng/src/session_manager.rs
+++ b/core/server-ng/src/session_manager.rs
@@ -27,6 +27,7 @@
//! in the consensus layer. This module tracks the binding between a
//! transport connection and the consensus-level `(client_id, session)` pair.
+use message_bus::installer::conn_info::ClientTransportKind;
use std::collections::HashMap;
use std::net::SocketAddr;
@@ -60,9 +61,28 @@ pub enum ConnectionState {
#[derive(Debug, Clone)]
pub struct Connection {
pub address: SocketAddr,
+ pub transport: ClientTransportKind,
pub state: ConnectionState,
}
+/// Flattened view of one connected client for the `get_me` /
+/// `get_clients` reads.
+///
+/// Holds everything those responses need from a single per-shard source,
+/// so the caller no longer also consults the message bus. `consumer_groups`
+/// membership is not modelled yet (see the TODO in the `get_clients`
+/// response builder); it ships empty until the partition-reconciliation /
+/// consumer-group-rebalancing work lands.
+#[derive(Debug, Clone)]
+pub struct ClientRecord {
+ /// Transport (coordinator-minted) client id; top 16 bits are the home
+ /// shard. The wire `client_id` is the `u32` seq tail.
+ pub client_id: u128,
+ pub user_id: Option<u32>,
+ pub transport: ClientTransportKind,
+ pub address: SocketAddr,
+}
+
/// Bridges transport connections to consensus sessions.
///
/// NOT thread-safe: each shard owns one `SessionManager` on its
@@ -92,9 +112,15 @@ impl SessionManager {
}
}
- pub fn ensure_connection(&mut self, connection_id: u128, address: SocketAddr) {
+ pub fn ensure_connection(
+ &mut self,
+ connection_id: u128,
+ address: SocketAddr,
+ transport: ClientTransportKind,
+ ) {
self.connections.entry(connection_id).or_insert(Connection {
address,
+ transport,
state: ConnectionState::Connected,
});
}
@@ -251,6 +277,26 @@ impl SessionManager {
self.connections.get(&connection_id)
}
+ /// Flatten one connection into a [`ClientRecord`] for `get_me`.
+ ///
+ /// This is the single per-shard source for the client-info reads:
+ /// `user_id`, `transport`, and `address` all come from the local
+ /// `SessionManager`, so the caller no longer consults the message
+ /// bus's `client_meta`.
+ #[must_use]
+ pub fn client_record(&self, connection_id: u128) -> Option<ClientRecord> {
+ let conn = self.connections.get(&connection_id)?;
+ Some(record_from(connection_id, conn))
+ }
+
+ /// Iterate every locally-homed connected client as a [`ClientRecord`].
+ /// The per-shard half of the `get_clients` scatter-gather.
+ pub fn iter_clients(&self) -> impl Iterator<Item = ClientRecord> + '_ {
+ self.connections
+ .iter()
+ .map(|(&id, conn)| record_from(id, conn))
+ }
+
/// Number of active connections.
#[must_use]
pub fn connection_count(&self) -> usize {
@@ -298,6 +344,22 @@ impl std::fmt::Display for SessionError {
impl std::error::Error for SessionError {}
+/// Flatten a connection + its id into a [`ClientRecord`].
+const fn record_from(connection_id: u128, conn: &Connection) -> ClientRecord {
+ let user_id = match conn.state {
+ ConnectionState::Authenticated { user_id } | ConnectionState::Bound { user_id, .. } => {
+ Some(user_id)
+ }
+ ConnectionState::Connected => None,
+ };
+ ClientRecord {
+ client_id: connection_id,
+ user_id,
+ transport: conn.transport,
+ address: conn.address,
+ }
+}
+
const fn state_name(state: &ConnectionState) -> &'static str {
match state {
ConnectionState::Connected => "Connected",
@@ -320,7 +382,7 @@ mod tests {
let mut mgr = SessionManager::new();
let conn = 1;
- mgr.ensure_connection(conn, addr(5000));
+ mgr.ensure_connection(conn, addr(5000), ClientTransportKind::Tcp);
assert_eq!(mgr.connection_count(), 1);
assert!(mgr.get_session(conn).is_none());
@@ -348,7 +410,7 @@ mod tests {
fn login_requires_connected_state() {
let mut mgr = SessionManager::new();
let conn = 1;
- mgr.ensure_connection(conn, addr(5000));
+ mgr.ensure_connection(conn, addr(5000), ClientTransportKind::Tcp);
mgr.login(conn, 1).unwrap();
// Double login should fail. Already Authenticated.
@@ -359,7 +421,7 @@ mod tests {
fn bind_requires_authenticated_state() {
let mut mgr = SessionManager::new();
let conn = 1;
- mgr.ensure_connection(conn, addr(5000));
+ mgr.ensure_connection(conn, addr(5000), ClientTransportKind::Tcp);
// Bind without login should fail.
assert!(mgr.bind_session(conn, 1, 1).is_err());
@@ -371,14 +433,14 @@ mod tests {
// First connection binds to client_id 99.
let conn1 = 1;
- mgr.ensure_connection(conn1, addr(5000));
+ mgr.ensure_connection(conn1, addr(5000), ClientTransportKind::Tcp);
mgr.login(conn1, 1).unwrap();
mgr.bind_session(conn1, 99, 10).unwrap();
assert_eq!(mgr.connection_for_client(99), Some(conn1));
// Second connection binds to same client_id. Evicts conn1.
let conn2 = 2;
- mgr.ensure_connection(conn2, addr(5001));
+ mgr.ensure_connection(conn2, addr(5001), ClientTransportKind::Tcp);
mgr.login(conn2, 1).unwrap();
mgr.bind_session(conn2, 99, 20).unwrap();
@@ -403,7 +465,7 @@ mod tests {
fn reset_to_connected_from_authenticated() {
let mut mgr = SessionManager::new();
let conn = 1;
- mgr.ensure_connection(conn, addr(5000));
+ mgr.ensure_connection(conn, addr(5000), ClientTransportKind::Tcp);
mgr.login(conn, 1).unwrap();
mgr.reset_to_connected(conn).unwrap();
// Back to Connected. Can login again.
@@ -414,7 +476,7 @@ mod tests {
fn reset_to_connected_rejects_wrong_state() {
let mut mgr = SessionManager::new();
let conn = 1;
- mgr.ensure_connection(conn, addr(5000));
+ mgr.ensure_connection(conn, addr(5000), ClientTransportKind::Tcp);
// Connected - reset should fail.
assert!(mgr.reset_to_connected(conn).is_err());
}
@@ -425,8 +487,8 @@ mod tests {
let c1 = 1;
let c2 = 2;
- mgr.ensure_connection(c1, addr(5000));
- mgr.ensure_connection(c2, addr(5001));
+ mgr.ensure_connection(c1, addr(5000), ClientTransportKind::Tcp);
+ mgr.ensure_connection(c2, addr(5001), ClientTransportKind::Tcp);
mgr.login(c1, 1).unwrap();
mgr.login(c2, 2).unwrap();
mgr.bind_session(c1, 100, 10).unwrap();
diff --git a/core/shard/src/builder.rs b/core/shard/src/builder.rs
index 52203db46..af77bafb4 100644
--- a/core/shard/src/builder.rs
+++ b/core/shard/src/builder.rs
@@ -30,8 +30,8 @@
use crate::coordinator::{ShardZeroCoordinator, classify_try_send_err};
use crate::metrics::{ShardMetrics, frame_drop_variant};
use crate::{
- CoordinatorConfig, IggyShard, LifecycleFrame, MetadataSubmitHandler, PartitionConsensusConfig,
- Receiver, ShardCtorError, ShardFrame, ShardIdentity, TaggedSender,
+ CoordinatorConfig, IggyShard, LifecycleFrame, ListClientsHandler, MetadataSubmitHandler,
+ PartitionConsensusConfig, Receiver, ShardCtorError, ShardFrame, ShardIdentity, TaggedSender,
};
use consensus::VsrConsensus;
use journal::JournalHandle;
@@ -65,6 +65,7 @@ where
on_replica_message: MessageHandler,
on_client_request: RequestHandler,
on_metadata_submit: MetadataSubmitHandler,
+ on_list_clients: ListClientsHandler,
metadata: IggyMetadata<VsrConsensus<B>, MJ, S, M>,
partitions: IggyPartitions<B>,
senders: Vec<TaggedSender>,
@@ -92,6 +93,7 @@ where
on_replica_message: MessageHandler,
on_client_request: RequestHandler,
on_metadata_submit: MetadataSubmitHandler,
+ on_list_clients: ListClientsHandler,
metadata: IggyMetadata<VsrConsensus<B>, MJ, S, M>,
partitions: IggyPartitions<B>,
senders: Vec<TaggedSender>,
@@ -107,6 +109,7 @@ where
on_replica_message,
on_client_request,
on_metadata_submit,
+ on_list_clients,
metadata,
partitions,
senders,
@@ -212,6 +215,7 @@ where
self.on_replica_message,
self.on_client_request,
self.on_metadata_submit,
+ self.on_list_clients,
self.metadata,
self.partitions,
self.senders,
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index cc1fbd09b..ffb0020bf 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -38,7 +38,7 @@ use journal::{Journal, JournalHandle};
use message_bus::MessageBus;
use message_bus::client_listener::RequestHandler;
use message_bus::fd_transfer::DupedFd;
-use message_bus::installer::conn_info::ClientConnMeta;
+use message_bus::installer::conn_info::{ClientConnMeta, ClientTransportKind};
use message_bus::replica::listener::MessageHandler;
use metadata::IggyMetadata;
use metadata::impls::metadata::StreamsFrontend;
@@ -140,6 +140,31 @@ pub enum MetadataSubmit {
/// must never receive this frame.
pub type MetadataSubmitHandler = Rc<dyn Fn(MetadataSubmit)>;
+/// One connected client's identity, as seen by the shard that homes it.
+///
+/// Gathered from every shard for `get_clients` (shared-nothing: each shard
+/// knows only its own connections, so the full list requires a broadcast
+/// -- see [`IggyShard::list_all_clients`]).
+#[derive(Debug, Clone)]
+pub struct ConnectedClientInfo {
+ /// Transport (coordinator-minted) client id; top 16 bits are the home
+ /// shard. The wire `client_id` is the `u32` seq tail.
+ pub client_id: u128,
+ pub user_id: Option<u32>,
+ pub transport: ClientTransportKind,
+ pub address: std::net::SocketAddr,
+}
+
+/// Handler each shard runs for an inbound [`LifecycleFrame::ListClients`].
+/// server-ng wires it to read the shard's `SessionManager` and push its
+/// connected clients back over the carried reply sender.
+pub type ListClientsHandler = Rc<dyn Fn(Sender<Vec<ConnectedClientInfo>>)>;
+
+/// Per-shard reply budget for the `list_all_clients` gather. A shard that
+/// doesn't answer within this window is skipped (partial result) so one
+/// wedged shard can't hang the read.
+const LIST_CLIENTS_GATHER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3);
+
/// Create a bounded inter-shard channel whose sender is tagged with the
/// owning shard.
///
@@ -336,6 +361,14 @@ pub enum LifecycleFrame {
/// the `reply` sender carried in [`MetadataSubmit`]. Always addressed to
/// shard 0; processing it on a peer is a routing bug.
MetadataSubmit(MetadataSubmit),
+ /// Broadcast query for `get_clients`: every shard replies with the
+ /// clients whose connections it homes, over `reply`. Unlike
+ /// [`MetadataSubmit`] this is sent to ALL shards (shared-nothing: each
+ /// shard knows only its own connections). See
+ /// [`IggyShard::list_all_clients`].
+ ListClients {
+ reply: Sender<Vec<ConnectedClientInfo>>,
+ },
}
/// Inter-shard channel envelope.
@@ -415,6 +448,12 @@ where
/// simulator stub ctor.
on_metadata_submit: MetadataSubmitHandler,
+ /// Handler for inbound [`LifecycleFrame::ListClients`] broadcast
+ /// queries. Every shard receives these (not just shard 0); server-ng
+ /// wires it to its per-shard `SessionManager`. Defaults to a no-op for
+ /// the simulator stub ctor.
+ on_list_clients: ListClientsHandler,
+
/// Channel senders to every shard, indexed by shard id.
/// Includes a sender to self so that local routing goes through the
/// same channel path as remote routing.
@@ -486,6 +525,7 @@ where
on_replica_message: MessageHandler,
on_client_request: RequestHandler,
on_metadata_submit: MetadataSubmitHandler,
+ on_list_clients: ListClientsHandler,
metadata: IggyMetadata<VsrConsensus<B>, MJ, S, M>,
partitions: IggyPartitions<B>,
senders: Vec<TaggedSender>,
@@ -510,6 +550,7 @@ where
on_replica_message,
on_client_request,
on_metadata_submit,
+ on_list_clients,
senders,
shard_count,
inbox,
@@ -542,6 +583,61 @@ where
}
}
+ /// Gather every shard's connected clients (the `get_clients`
+ /// scatter-gather). Broadcasts [`LifecycleFrame::ListClients`] to all
+ /// shards -- including self, so the local shard answers over the same
+ /// channel path -- and collects their replies.
+ ///
+ /// Bounded: a shard that doesn't reply within
+ /// [`LIST_CLIENTS_GATHER_TIMEOUT`] is skipped and the partial result is
+ /// logged, so one wedged shard cannot hang the read. Callers should
+ /// treat the result as best-effort-complete.
+ #[allow(clippy::future_not_send)]
+ pub async fn list_all_clients(&self) -> Vec<ConnectedClientInfo> {
+ let shard_count = self.shard_count as usize;
+ let (reply_tx, reply_rx) = channel::<Vec<ConnectedClientInfo>>(shard_count.max(1));
+ let mut expected = 0usize;
+ for sender in &self.senders {
+ let frame = ShardFrame::lifecycle(LifecycleFrame::ListClients {
+ reply: reply_tx.clone(),
+ });
+ if let Err(error) = sender.try_send(frame) {
+ tracing::warn!(
+ shard = self.id,
+ target = sender.shard_id(),
+ "list_all_clients: inbox rejected ListClients frame: {error:?}"
+ );
+ } else {
+ expected += 1;
+ }
+ }
+ // Drop the local handle so `recv` returns `Err` once every shard's
+ // reply sender is dropped (defensive; we also bound by count).
+ drop(reply_tx);
+
+ let mut clients = Vec::new();
+ let mut received = 0usize;
+ while received < expected {
+ match compio::time::timeout(LIST_CLIENTS_GATHER_TIMEOUT, reply_rx.recv()).await {
+ Ok(Ok(batch)) => {
+ clients.extend(batch);
+ received += 1;
+ }
+ Ok(Err(_)) => break, // all reply senders dropped
+ Err(_) => {
+ tracing::warn!(
+ shard = self.id,
+ received,
+ expected,
+ "list_all_clients: gather timed out; returning partial result"
+ );
+ break;
+ }
+ }
+ }
+ clients
+ }
+
/// Return a clone of the shard-0 coordinator handle, if attached.
/// Bootstrap uses this to wire the listener accept callbacks
/// (replica + client) to coordinator-driven fd-delegation instead
@@ -579,6 +675,7 @@ where
on_replica_message: std::rc::Rc::new(|_, _| {}),
on_client_request: std::rc::Rc::new(|_, _| {}),
on_metadata_submit: std::rc::Rc::new(|_| {}),
+ on_list_clients: std::rc::Rc::new(|_| {}),
plane,
coordinator: None,
senders: Vec::new(),
diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs
index f964a7c50..33bffc86c 100644
--- a/core/shard/src/router.rs
+++ b/core/shard/src/router.rs
@@ -411,6 +411,13 @@ where
);
(self.on_metadata_submit)(submit);
}
+ LifecycleFrame::ListClients { reply } => {
+ // Every shard handles this (not shard-0-only): each replies
+ // with the clients whose connections it homes. The handler
+ // (wired by server-ng) reads this shard's `SessionManager`
+ // and pushes the list over `reply`.
+ (self.on_list_clients)(reply);
+ }
}
}
}