You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2026/10/19 16:50:10 UTC
(pulsar) branch master updated: [feat][broker] Add scalable topic consumer coordination metrics (#26982)
This is an automated email from the ASF dual-hosted git repository.
codelipenghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new b786de3988ef [feat][broker] Add scalable topic consumer coordination metrics (#26982)
b786de3988ef is described below
commit b786de3988ef8da2451e709339588876971b1ed7
Author: Penghui Li <pe...@apache.org>
AuthorDate: Mon Oct 19 09:50:04 2026 -0700
[feat][broker] Add scalable topic consumer coordination metrics (#26982)
---
.../pulsar/broker/service/BrokerService.java | 3 +
.../scalable/ScalableTopicConsumerMetrics.java | 260 ++++++++++
.../service/scalable/ScalableTopicController.java | 27 +
.../service/scalable/ScalableTopicService.java | 42 +-
.../service/scalable/SubscriptionCoordinator.java | 124 ++++-
.../apache/pulsar/broker/PulsarServiceTest.java | 15 +
.../scalable/ScalableTopicConsumerMetricsTest.java | 543 +++++++++++++++++++++
.../ScalableTopicControllerFailoverTest.java | 4 +
.../service/scalable/ScalableTopicServiceTest.java | 92 +++-
.../SubscriptionCoordinatorMetricsTest.java | 446 +++++++++++++++++
10 files changed, 1546 insertions(+), 10 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 229fe3ac7409..f9c2bc45d3f8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -1074,6 +1074,9 @@ public class BrokerService implements Closeable {
log.info("Continuing to second phase in shutdown.");
List<CompletableFuture<Void>> asyncCloseFutures = new ArrayList<>();
+ if (scalableTopicService != null) {
+ scalableTopicService.closeMetrics();
+ }
listenChannels.forEach(ch -> {
if (ch.isOpen()) {
asyncCloseFutures.add(closeChannel(ch));
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicConsumerMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicConsumerMetrics.java
new file mode 100644
index 000000000000..38a0b7b368d5
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicConsumerMetrics.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.scalable;
+
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.BatchCallback;
+import io.opentelemetry.api.metrics.Meter;
+import io.prometheus.client.Collector;
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.CounterMetricFamily;
+import io.prometheus.client.GaugeMetricFamily;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLongArray;
+import java.util.function.Supplier;
+import lombok.CustomLog;
+import org.apache.pulsar.broker.service.scalable.SubscriptionCoordinator.ConsumerStats;
+import org.apache.pulsar.broker.service.scalable.SubscriptionCoordinator.RebalanceReason;
+
+/** Bounded-cardinality broker metrics, sharing the same state on both export paths. */
+@CustomLog
+final class ScalableTopicConsumerMetrics implements AutoCloseable {
+ static final String CONSUMER_COUNT = "pulsar.broker.scalable_topic.consumer.count";
+ static final String BLOCKED_COUNT = "pulsar.broker.scalable_topic.segment.blocked.count";
+ static final String REBALANCE_COUNT = "pulsar.broker.scalable_topic.rebalance.count";
+ private static final String PROM_CONSUMER_COUNT = "pulsar_scalable_topic_consumer_count";
+ private static final String PROM_BLOCKED_COUNT = "pulsar_scalable_topic_segment_blocked_count";
+ private static final String PROM_REBALANCE = "pulsar_scalable_topic_rebalance";
+ private static final String CONSUMER_DESCRIPTION = "Consumer sessions coordinated by this broker.";
+ private static final String BLOCKED_DESCRIPTION = "Segment/subscription pairs waiting for ancestors to drain, "
+ + "summed across subscriptions served by this broker.";
+ private static final String REBALANCE_DESCRIPTION = "Consumer assignment recomputations.";
+ private static final AttributeKey<String> CONSUMER_STATE =
+ AttributeKey.stringKey("pulsar.scalable_topic.consumer.state");
+ private static final AttributeKey<String> REBALANCE_REASON =
+ AttributeKey.stringKey("pulsar.scalable_topic.rebalance.reason");
+ private static final RebalanceReason[] REASONS = RebalanceReason.values();
+ private static final List<String> STATES = List.of("active", "idle", "disconnected");
+ private static final List<Attributes> STATE_ATTRIBUTES = STATES.stream()
+ .map(state -> Attributes.of(CONSUMER_STATE, state)).toList();
+ private static final List<String> REASON_NAMES = List.of(REASONS).stream()
+ .map(reason -> reason.name().toLowerCase(Locale.ROOT)).toList();
+ private static final List<Attributes> REASON_ATTRIBUTES = REASON_NAMES.stream()
+ .map(reason -> Attributes.of(REBALANCE_REASON, reason)).toList();
+
+ // CollectorRegistry is JVM-global in production. Share a registration when tests or embedded
+ // deployments run several brokers in one JVM, aggregating their state per cluster on that path.
+ private static final Map<CollectorRegistry, PrometheusCollector> COLLECTORS = new HashMap<>();
+
+ private final Supplier<ConsumerStats> statsSupplier;
+ private final AtomicLongArray rebalances = new AtomicLongArray(REASONS.length);
+ private final String cluster;
+ private final CollectorRegistry registry;
+ private final BatchCallback callback;
+ private boolean closed;
+
+ ScalableTopicConsumerMetrics(Meter meter, CollectorRegistry registry, String cluster,
+ Supplier<ConsumerStats> statsSupplier) {
+ this.registry = registry;
+ this.cluster = cluster;
+ this.statsSupplier = statsSupplier;
+ var consumers = meter.upDownCounterBuilder(CONSUMER_COUNT)
+ .setUnit("{consumer}").setDescription(CONSUMER_DESCRIPTION).buildObserver();
+ var blocked = meter.upDownCounterBuilder(BLOCKED_COUNT)
+ .setUnit("{segment}").setDescription(BLOCKED_DESCRIPTION).buildObserver();
+ var rebalance = meter.counterBuilder(REBALANCE_COUNT)
+ .setUnit("{rebalance}").setDescription(REBALANCE_DESCRIPTION).buildObserver();
+ callback = meter.batchCallback(() -> {
+ Snapshot snapshot = snapshot();
+ if (snapshot.stats != null) {
+ long[] counts = snapshot.consumerCounts();
+ for (int i = 0; i < counts.length; i++) {
+ consumers.record(counts[i], STATE_ATTRIBUTES.get(i));
+ }
+ blocked.record(snapshot.stats.blocked());
+ }
+ for (int i = 0; i < REASONS.length; i++) {
+ rebalance.record(snapshot.rebalances[i], REASON_ATTRIBUTES.get(i));
+ }
+ }, consumers, blocked, rebalance);
+ try {
+ synchronized (COLLECTORS) {
+ COLLECTORS.computeIfAbsent(registry, key -> {
+ PrometheusCollector collector = new PrometheusCollector();
+ key.register(collector);
+ return collector;
+ }).addSource(this);
+ }
+ } catch (RuntimeException e) {
+ // A failed registration must not leave a callback in the broker's shared meter.
+ callback.close();
+ throw e;
+ }
+ }
+
+ synchronized void recordRebalance(RebalanceReason reason) {
+ if (!closed) {
+ rebalances.incrementAndGet(reason.ordinal());
+ }
+ }
+
+ private Snapshot snapshot() {
+ long[] counts = new long[REASONS.length];
+ for (int i = 0; i < counts.length; i++) {
+ counts[i] = rebalances.get(i);
+ }
+ ConsumerStats stats = null;
+ try {
+ stats = statsSupplier.get();
+ } catch (RuntimeException e) {
+ // Gauges can be unavailable without removing this source's known cumulative counters.
+ log.debug().attr("cluster", cluster).exceptionMessage(e)
+ .log("Skipping failed scalable-topic consumer gauge snapshot");
+ }
+ return new Snapshot(stats, counts);
+ }
+
+ @Override
+ public synchronized void close() {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ callback.close();
+ synchronized (COLLECTORS) {
+ PrometheusCollector collector = COLLECTORS.get(registry);
+ if (collector.retireSource(this)) {
+ registry.unregister(collector);
+ COLLECTORS.remove(registry);
+ }
+ }
+ }
+
+ /** Null stats omit unavailable gauges without suppressing cumulative counters. */
+ private record Snapshot(ConsumerStats stats, long[] rebalances) {
+ long[] consumerCounts() {
+ return new long[]{stats.active(), stats.idle(), stats.disconnected()};
+ }
+
+ Snapshot plus(Snapshot other) {
+ long[] combined = rebalances.clone();
+ for (int i = 0; i < combined.length; i++) {
+ combined[i] += other.rebalances[i];
+ }
+ ConsumerStats combinedStats = stats == null ? other.stats
+ : other.stats == null ? stats : stats.plus(other.stats);
+ return new Snapshot(combinedStats, combined);
+ }
+ }
+
+ private static final class PrometheusCollector extends Collector implements Collector.Describable {
+ private record SourcesState(List<ScalableTopicConsumerMetrics> sources, Map<String, long[]> retiredRebalances) {
+ private static final SourcesState EMPTY = new SourcesState(List.of(), Map.of());
+ }
+
+ // Registration/retirement is serialized by COLLECTORS. A scrape captures one immutable
+ // state so a retiring source and its retained totals can never both be counted.
+ private volatile SourcesState sourcesState = SourcesState.EMPTY;
+
+ private void addSource(ScalableTopicConsumerMetrics source) {
+ SourcesState current = sourcesState;
+ List<ScalableTopicConsumerMetrics> sources = new ArrayList<>(current.sources);
+ sources.add(source);
+ sourcesState = new SourcesState(List.copyOf(sources), current.retiredRebalances);
+ }
+
+ /** Returns true once the last source retires and all retained totals can be discarded. */
+ private boolean retireSource(ScalableTopicConsumerMetrics source) {
+ SourcesState current = sourcesState;
+ List<ScalableTopicConsumerMetrics> remaining = new ArrayList<>(current.sources);
+ remaining.remove(source);
+ if (remaining.isEmpty()) {
+ sourcesState = SourcesState.EMPTY;
+ return true;
+ }
+ Map<String, long[]> retired = new HashMap<>(current.retiredRebalances);
+ long[] counts = retired.containsKey(source.cluster)
+ ? retired.get(source.cluster).clone() : new long[REASONS.length];
+ for (int i = 0; i < counts.length; i++) {
+ counts[i] += source.rebalances.get(i);
+ }
+ retired.put(source.cluster, counts);
+ // Retain only scalars, not the closed broker, gauge supplier, or OTel callback.
+ sourcesState = new SourcesState(List.copyOf(remaining), Map.copyOf(retired));
+ return false;
+ }
+
+ @Override
+ public List<MetricFamilySamples> describe() {
+ return List.of(new GaugeMetricFamily(PROM_CONSUMER_COUNT, CONSUMER_DESCRIPTION,
+ List.of("cluster", "consumer_state")),
+ new GaugeMetricFamily(PROM_BLOCKED_COUNT, BLOCKED_DESCRIPTION, List.of("cluster")),
+ new CounterMetricFamily(PROM_REBALANCE, REBALANCE_DESCRIPTION,
+ List.of("cluster", "rebalance_reason")));
+ }
+
+ @Override
+ public List<MetricFamilySamples> collect() {
+ var consumers = new GaugeMetricFamily(PROM_CONSUMER_COUNT, CONSUMER_DESCRIPTION,
+ List.of("cluster", "consumer_state"));
+ var blocked = new GaugeMetricFamily(PROM_BLOCKED_COUNT, BLOCKED_DESCRIPTION, List.of("cluster"));
+ var rebalance = new CounterMetricFamily(PROM_REBALANCE, REBALANCE_DESCRIPTION,
+ List.of("cluster", "rebalance_reason"));
+ SourcesState current = sourcesState;
+ Map<String, Snapshot> clusters = new HashMap<>();
+ current.retiredRebalances.forEach((cluster, counts) ->
+ clusters.put(cluster, new Snapshot(ConsumerStats.EMPTY, counts)));
+ Set<String> activeClusters = new HashSet<>();
+ for (ScalableTopicConsumerMetrics source : current.sources) {
+ try {
+ Snapshot snapshot = source.snapshot();
+ clusters.merge(source.cluster, snapshot, Snapshot::plus);
+ if (snapshot.stats != null) {
+ activeClusters.add(source.cluster);
+ }
+ } catch (RuntimeException e) {
+ // This registry also exports existing broker metrics. A failed new source
+ // must not abort their scrape, or prevent other brokers from being collected.
+ log.debug().attr("cluster", source.cluster).exceptionMessage(e)
+ .log("Skipping failed scalable-topic consumer metrics snapshot");
+ }
+ }
+ clusters.forEach((cluster, snapshot) -> {
+ if (activeClusters.contains(cluster)) {
+ long[] counts = snapshot.consumerCounts();
+ for (int i = 0; i < counts.length; i++) {
+ consumers.addMetric(List.of(cluster, STATES.get(i)), counts[i]);
+ }
+ blocked.addMetric(List.of(cluster), snapshot.stats.blocked());
+ }
+ for (int i = 0; i < REASONS.length; i++) {
+ rebalance.addMetric(List.of(cluster, REASON_NAMES.get(i)), snapshot.rebalances[i]);
+ }
+ });
+ return new ArrayList<>(List.of(consumers, blocked, rebalance));
+ }
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java
index 9231702fce4d..20c3d8017a0b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java
@@ -36,6 +36,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
import lombok.Getter;
import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.ServiceConfiguration;
@@ -96,6 +97,8 @@ public class ScalableTopicController {
private volatile SegmentLayout currentLayout;
+ private Consumer<SubscriptionCoordinator.RebalanceReason> rebalanceListener = __ -> { };
+
/** Per-subscription consumer tracking. */
private final ConcurrentHashMap<String, SubscriptionCoordinator> subscriptions = new ConcurrentHashMap<>();
@@ -950,6 +953,30 @@ public class ScalableTopicController {
private SubscriptionCoordinator createCoordinator(String subscription,
ScalableConsumerType consumerType) {
+ SubscriptionCoordinator coordinator = newCoordinator(subscription, consumerType);
+ coordinator.setRebalanceListener(rebalanceListener);
+ return coordinator;
+ }
+
+ /** Set before initialization, so restored and newly registered coordinators share the broker counter. */
+ void setRebalanceListener(Consumer<SubscriptionCoordinator.RebalanceReason> listener) {
+ rebalanceListener = listener;
+ }
+
+ SubscriptionCoordinator.ConsumerStats consumerStats() {
+ if (closed || !isLeader()) {
+ return SubscriptionCoordinator.ConsumerStats.EMPTY;
+ }
+ SubscriptionCoordinator.ConsumerStats stats = SubscriptionCoordinator.ConsumerStats.EMPTY;
+ for (SubscriptionCoordinator coordinator : subscriptions.values()) {
+ stats = stats.plus(coordinator.consumerStats());
+ }
+ // Leadership may have ended while the lock-free snapshots were being read.
+ return closed || !isLeader() ? SubscriptionCoordinator.ConsumerStats.EMPTY : stats;
+ }
+
+ private SubscriptionCoordinator newCoordinator(String subscription,
+ ScalableConsumerType consumerType) {
// Parent-drain ordering matters only for STREAM consumers (Exclusive per-segment
// subscription with broker-tracked cursors → preserving per-key order across a
// split requires waiting for the parent to drain before handing out children).
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
index 786478c99f35..39ac9767ee50 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
@@ -18,6 +18,7 @@
*/
package org.apache.pulsar.broker.service.scalable;
+import io.prometheus.client.CollectorRegistry;
import java.time.Duration;
import java.util.List;
import java.util.Map;
@@ -84,13 +85,34 @@ public class ScalableTopicService {
this.coordinationService = coordinationService;
}
+ private volatile ScalableTopicConsumerMetrics consumerMetrics;
+
// --- Lifecycle ---
- public void start() {
+ public synchronized void start() {
+ if (consumerMetrics != null) {
+ return;
+ }
+ try {
+ consumerMetrics = new ScalableTopicConsumerMetrics(
+ brokerService.getPulsar().getOpenTelemetry().getMeter(), CollectorRegistry.defaultRegistry,
+ brokerService.getPulsar().getConfiguration().getClusterName(), this::consumerStats);
+ } catch (RuntimeException e) {
+ log.warn().exceptionMessage(e)
+ .log("Failed to initialize scalable-topic consumer metrics; continuing without them");
+ }
log.info("ScalableTopicService started");
}
+ /** Release only this service's new instruments, without changing controller teardown. */
+ public void closeMetrics() {
+ if (consumerMetrics != null) {
+ consumerMetrics.close();
+ }
+ }
+
public void close() {
+ closeMetrics();
log.info().attr("controllerCount", controllers.size())
.log("Closing ScalableTopicService, releasing controllers");
List<CompletableFuture<Void>> closeFutures = controllers.values().stream()
@@ -105,6 +127,21 @@ public class ScalableTopicService {
controllers.clear();
}
+ SubscriptionCoordinator.ConsumerStats consumerStats() {
+ SubscriptionCoordinator.ConsumerStats stats = SubscriptionCoordinator.ConsumerStats.EMPTY;
+ for (CompletableFuture<ScalableTopicController> future : controllers.values()) {
+ // Collection never waits for controller initialization or a failed takeover.
+ if (!future.isDone() || future.isCompletedExceptionally()) {
+ continue;
+ }
+ ScalableTopicController controller = future.getNow(null);
+ if (controller != null) {
+ stats = stats.plus(controller.consumerStats());
+ }
+ }
+ return stats;
+ }
+
// --- Controller management ---
/**
@@ -128,6 +165,9 @@ public class ScalableTopicService {
private CompletableFuture<ScalableTopicController> newController(String key, TopicName topic) {
ScalableTopicController controller = new ScalableTopicController(
topic, resources, brokerService, coordinationService);
+ if (consumerMetrics != null) {
+ controller.setRebalanceListener(consumerMetrics::recordRebalance);
+ }
CompletableFuture<ScalableTopicController> future = new CompletableFuture<>();
// Released once its topic is deleted: the admin delete stops the controller on the
// broker serving it, not the ones other brokers hold for the topic. Closed before it is
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinator.java
index 186216140dec..6a3badf883b0 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinator.java
@@ -36,6 +36,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
import lombok.Getter;
import org.apache.pulsar.broker.resources.ScalableTopicResources;
import org.apache.pulsar.broker.service.TransportCnx;
@@ -67,6 +68,29 @@ import org.apache.pulsar.common.util.Backoff;
*/
public class SubscriptionCoordinator {
+ enum RebalanceReason {
+ CONSUMER_JOIN,
+ CONSUMER_LEAVE,
+ CONSUMER_EVICT,
+ LAYOUT_CHANGE,
+ SEGMENT_DRAINED
+ }
+
+ record ConsumerStats(long active, long idle, long disconnected, long blocked) {
+ static final ConsumerStats EMPTY = new ConsumerStats(0, 0, 0, 0);
+
+ ConsumerStats plus(ConsumerStats other) {
+ return new ConsumerStats(active + other.active, idle + other.idle,
+ disconnected + other.disconnected, blocked + other.blocked);
+ }
+ }
+
+ private Consumer<RebalanceReason> rebalanceListener = __ -> { };
+ // Updated under this coordinator's monitor; exporters only read the immutable volatile snapshot.
+ private volatile ConsumerStats consumerStatsSnapshot = ConsumerStats.EMPTY;
+ private Set<String> assignedConsumerNames = Set.of();
+ private long blockedSegmentCount;
+
private static final Logger LOG = Logger.get(SubscriptionCoordinator.class);
private final Logger log;
@@ -208,6 +232,18 @@ public class SubscriptionCoordinator {
Map<ConsumerSession, ConsumerAssignment> current =
computeAssignment(currentLayout, sessions.values());
ConsumerAssignment assignment = current.get(existing);
+ if (assignment != null) {
+ // Only this consumer receives an assignment on reconnect. A pending eviction
+ // may change the computed assignments of peers that have not been notified yet.
+ Set<String> assigned = new HashSet<>(assignedConsumerNames);
+ if (assignment.assignedSegments().isEmpty()) {
+ assigned.remove(consumerName);
+ } else {
+ assigned.add(consumerName);
+ }
+ assignedConsumerNames = assigned;
+ }
+ publishConsumerStats();
if (assignment != null) {
existing.sendAssignmentUpdate(assignment);
}
@@ -223,8 +259,9 @@ public class SubscriptionCoordinator {
// The registration is persisted: the next leader restores it.
throw notServing();
}
- sessions.put(consumerName, session);
- Map<ConsumerSession, ConsumerAssignment> result = rebalanceAndNotify();
+ ConsumerSession previous = sessions.put(consumerName, session);
+ Map<ConsumerSession, ConsumerAssignment> result = rebalanceAndNotify(
+ previous == null ? RebalanceReason.CONSUMER_JOIN : null);
// First consumer (or rejoining one) — kick off drain checks at the
// shortest delay rather than whatever long backoff we'd accumulated
// while idle.
@@ -281,11 +318,18 @@ public class SubscriptionCoordinator {
if (removed != null) {
removed.cancelGraceTimer();
}
+ RebalanceReason reason = removed == null ? null : RebalanceReason.CONSUMER_LEAVE;
if (sessions.isEmpty()) {
segmentAssignments.clear();
+ updateConsumerStats(Map.of());
+ if (reason != null) {
+ rebalanceListener.accept(reason);
+ }
return Map.of();
}
- return rebalanceAndNotify();
+ // Preserve the existing notification/index refresh even if another in-flight
+ // removal already removed this session; only the event count is suppressed.
+ return rebalanceAndNotify(reason);
}
});
}
@@ -302,6 +346,7 @@ public class SubscriptionCoordinator {
return;
}
session.markDisconnected();
+ publishConsumerStats();
}
/**
@@ -330,6 +375,7 @@ public class SubscriptionCoordinator {
Map<ConsumerSession, ConsumerAssignment> result =
computeAssignment(currentLayout, sessions.values());
updateSegmentAssignmentIndex(result);
+ updateConsumerStats(result);
return result;
}
@@ -346,9 +392,10 @@ public class SubscriptionCoordinator {
this.currentLayout = newLayout;
if (sessions.isEmpty()) {
segmentAssignments.clear();
+ updateConsumerStats(Map.of());
return CompletableFuture.completedFuture(Map.of());
}
- Map<ConsumerSession, ConsumerAssignment> result = rebalanceAndNotify();
+ Map<ConsumerSession, ConsumerAssignment> result = rebalanceAndNotify(RebalanceReason.LAYOUT_CHANGE);
// New sealed segments may have appeared — restart drain checks from the
// initial delay rather than continuing whatever long backoff we'd settled into.
resetAndRearmDrainPoll();
@@ -368,7 +415,8 @@ public class SubscriptionCoordinator {
this.drainChecker = checker;
log.info().log("Drain checker installed on existing coordinator; rebalancing");
if (!sessions.isEmpty()) {
- rebalanceAndNotify();
+ // Restoring STREAM ordering on reconnect is not a membership or layout change.
+ rebalanceAndNotify(null);
}
resetAndRearmDrainPoll();
}
@@ -402,6 +450,8 @@ public class SubscriptionCoordinator {
*/
public synchronized void close() {
closed = true;
+ consumerStatsSnapshot = ConsumerStats.EMPTY;
+ assignedConsumerNames = Set.of();
if (drainPollTask != null) {
drainPollTask.cancel(false);
drainPollTask = null;
@@ -435,6 +485,10 @@ public class SubscriptionCoordinator {
if (!segment.isActive() && drainedSegmentIds.contains(segment.segmentId())) {
return false;
}
+ return ancestorsDrained(segment, layout);
+ }
+
+ private boolean ancestorsDrained(SegmentInfo segment, SegmentLayout layout) {
Map<Long, SegmentInfo> segments = layout.getAllSegments();
Set<Long> visited = new HashSet<>();
Deque<Long> ancestors = new ArrayDeque<>(segment.parentIds());
@@ -473,7 +527,7 @@ public class SubscriptionCoordinator {
// on the same DAG branch often follow shortly.
if (!sessions.isEmpty()) {
log.info().attr("drained", segmentIds).log("Sealed segments drained, rebalancing");
- rebalanceAndNotify();
+ rebalanceAndNotify(RebalanceReason.SEGMENT_DRAINED);
}
resetAndRearmDrainPoll();
}
@@ -583,6 +637,53 @@ public class SubscriptionCoordinator {
return Set.copyOf(sessions.values());
}
+ /** Installed before this coordinator is published. The listener must not acquire controller locks. */
+ synchronized void setRebalanceListener(Consumer<RebalanceReason> listener) {
+ rebalanceListener = listener;
+ }
+
+ /** Collection never acquires coordination locks, computes assignments, or performs IO. */
+ ConsumerStats consumerStats() {
+ return consumerStatsSnapshot;
+ }
+
+ private void updateConsumerStats(Map<ConsumerSession, ConsumerAssignment> assignments) {
+ Set<String> assigned = new HashSet<>();
+ assignments.forEach((session, assignment) -> {
+ if (!assignment.assignedSegments().isEmpty()) {
+ assigned.add(session.getConsumerName());
+ }
+ });
+ assignedConsumerNames = assigned;
+ // Compute ancestor blocking on state transitions, never on a shared metrics scrape.
+ blockedSegmentCount = drainChecker == null || sessions.isEmpty() ? 0
+ : currentLayout.getAllSegments().values().stream()
+ .filter(segment -> segment.isActive() || !drainedSegmentIds.contains(segment.segmentId()))
+ .filter(segment -> !ancestorsDrained(segment, currentLayout))
+ .count();
+ publishConsumerStats();
+ }
+
+ private void publishConsumerStats() {
+ if (closed || sessions.isEmpty()) {
+ consumerStatsSnapshot = ConsumerStats.EMPTY;
+ return;
+ }
+ long active = 0;
+ long idle = 0;
+ long disconnected = 0;
+ for (ConsumerSession session : sessions.values()) {
+ if (!session.isConnected()) {
+ disconnected++;
+ } else if (assignedConsumerNames.contains(session.getConsumerName())) {
+ active++;
+ } else {
+ idle++;
+ }
+ }
+ consumerStatsSnapshot = new ConsumerStats(active, idle, disconnected, blockedSegmentCount);
+ }
+
// --- Internals ---
/**
@@ -608,6 +709,7 @@ public class SubscriptionCoordinator {
return;
}
sessions.remove(consumerName);
+ publishConsumerStats();
log.info().attr("consumer", consumerName)
.log("Consumer evicted after grace period");
}
@@ -624,9 +726,11 @@ public class SubscriptionCoordinator {
return;
}
if (!sessions.isEmpty()) {
- rebalanceAndNotify();
+ rebalanceAndNotify(RebalanceReason.CONSUMER_EVICT);
} else {
segmentAssignments.clear();
+ updateConsumerStats(Map.of());
+ rebalanceListener.accept(RebalanceReason.CONSUMER_EVICT);
}
}
});
@@ -760,10 +864,14 @@ public class SubscriptionCoordinator {
return result;
}
- private Map<ConsumerSession, ConsumerAssignment> rebalanceAndNotify() {
+ private Map<ConsumerSession, ConsumerAssignment> rebalanceAndNotify(RebalanceReason reason) {
Map<ConsumerSession, ConsumerAssignment> assignments =
computeAssignment(currentLayout, sessions.values());
updateSegmentAssignmentIndex(assignments);
+ updateConsumerStats(assignments);
+ if (reason != null) {
+ rebalanceListener.accept(reason);
+ }
for (var entry : assignments.entrySet()) {
entry.getKey().sendAssignmentUpdate(entry.getValue());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java
index c607ae11927c..1cfe9138602e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java
@@ -26,6 +26,7 @@ import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import static org.testng.AssertJUnit.assertSame;
+import io.prometheus.client.CollectorRegistry;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -183,6 +184,20 @@ public class PulsarServiceTest extends MockedPulsarServiceBaseTest {
assertEquals(thrownCnt, 6);
}
+ @Test
+ public void testScalableConsumerMetricsReleasedOnBrokerShutdown() throws Exception {
+ restartBroker(config -> config.setScalableTopicsEnabled(true));
+ assertThat(pulsar.getBrokerService().getScalableTopicService()).isNotNull();
+ String name = "pulsar_scalable_topic_consumer_count";
+ String[] labels = {"cluster", "consumer_state"};
+ String[] values = {conf.getClusterName(), "active"};
+ assertThat(CollectorRegistry.defaultRegistry.getSampleValue(name, labels, values)).isEqualTo(0.0);
+
+ pulsar.closeAsync().get(30, TimeUnit.SECONDS);
+
+ assertThat(CollectorRegistry.defaultRegistry.getSampleValue(name, labels, values)).isNull();
+ }
+
@Test
public void testAdvertisedAddress() throws Exception {
cleanup();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicConsumerMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicConsumerMetricsTest.java
new file mode 100644
index 000000000000..d13b33a28a25
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicConsumerMetricsTest.java
@@ -0,0 +1,543 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.scalable;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.data.MetricData;
+import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader;
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.Counter;
+import io.prometheus.client.Gauge;
+import io.prometheus.client.Histogram;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.service.scalable.SubscriptionCoordinator.ConsumerStats;
+import org.apache.pulsar.broker.service.scalable.SubscriptionCoordinator.RebalanceReason;
+import org.apache.pulsar.broker.stats.OpenTelemetryReplicatedSubscriptionStats;
+import org.apache.pulsar.broker.stats.PulsarBrokerOpenTelemetry;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class ScalableTopicConsumerMetricsTest {
+ @Test
+ public void testBothExportPathsReadTheSameState() throws Exception {
+ var registry = new CollectorRegistry();
+ var state = new AtomicReference<>(new ConsumerStats(3, 2, 1, 4));
+ try (var reader = InMemoryMetricReader.create();
+ var provider = SdkMeterProvider.builder().registerMetricReader(reader).build();
+ var metrics = new ScalableTopicConsumerMetrics(provider.get("test"), registry, "test", state::get)) {
+ metrics.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ metrics.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ metrics.recordRebalance(RebalanceReason.LAYOUT_CHANGE);
+ assertExports(reader, registry, state.get(), 2, 1);
+
+ state.set(new ConsumerStats(0, 1, 5, 0));
+ metrics.recordRebalance(RebalanceReason.LAYOUT_CHANGE);
+ assertExports(reader, registry, state.get(), 2, 2);
+ // Scraping must not itself change counters.
+ assertExports(reader, registry, state.get(), 2, 2);
+ }
+ assertThat(registry.metricFamilySamples().hasMoreElements()).isFalse();
+ }
+
+ @Test
+ public void testPrometheusWorksWithDisabledOpenTelemetryAndReleasesRegistration() {
+ var registry = new CollectorRegistry();
+ var meter = OpenTelemetry.noop().getMeter("test");
+ try (var first = new ScalableTopicConsumerMetrics(meter, registry, "test",
+ () -> new ConsumerStats(1, 0, 0, 2));
+ var second = new ScalableTopicConsumerMetrics(meter, registry, "test",
+ () -> new ConsumerStats(0, 1, 0, 3))) {
+ first.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ second.recordRebalance(RebalanceReason.CONSUMER_LEAVE);
+ assertThat(sample(registry, "consumer_count", "consumer_state", "active")).isEqualTo(1);
+ assertThat(sample(registry, "consumer_count", "consumer_state", "idle")).isEqualTo(1);
+ assertThat(sample(registry, "segment_blocked_count", null, null)).isEqualTo(5);
+ first.close();
+ first.close();
+ assertThat(sample(registry, "consumer_count", "consumer_state", "active")).isZero();
+ assertThat(sample(registry, "segment_blocked_count", null, null)).isEqualTo(3);
+ }
+ assertThat(registry.metricFamilySamples().hasMoreElements()).isFalse();
+ // A restarted broker can register the exact same metric names.
+ try (var metrics = new ScalableTopicConsumerMetrics(meter, registry, "test", () -> ConsumerStats.EMPTY)) {
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", "consumer_join")).isZero();
+ }
+ }
+
+ @Test
+ public void testSharedRegistryRetainsCountersWhenSourceClosesAndIsReplaced() {
+ var registry = new CollectorRegistry();
+ var meter = OpenTelemetry.noop().getMeter("test");
+ var retired = new AtomicBoolean();
+ try (var first = new ScalableTopicConsumerMetrics(meter, registry, "test", () -> {
+ assertThat(retired.get()).as("Retired gauge suppliers must not be collected").isFalse();
+ return new ConsumerStats(2, 0, 0, 2);
+ });
+ var second = new ScalableTopicConsumerMetrics(meter, registry, "test",
+ () -> new ConsumerStats(0, 1, 0, 3))) {
+ for (RebalanceReason reason : RebalanceReason.values()) {
+ first.recordRebalance(reason);
+ second.recordRebalance(reason);
+ }
+ assertReasonTotals(registry, 2);
+ first.close();
+ first.close();
+ retired.set(true);
+ for (RebalanceReason reason : RebalanceReason.values()) {
+ first.recordRebalance(reason); // Late events cannot alter this source's final retained totals.
+ }
+ assertReasonTotals(registry, 2);
+ assertThat(sample(registry, "consumer_count", "consumer_state", "active")).isZero();
+ assertThat(sample(registry, "consumer_count", "consumer_state", "idle")).isEqualTo(1);
+ assertThat(sample(registry, "segment_blocked_count", null, null)).isEqualTo(3);
+ for (RebalanceReason reason : RebalanceReason.values()) {
+ second.recordRebalance(reason);
+ }
+ assertReasonTotals(registry, 3);
+ try (var replacement = new ScalableTopicConsumerMetrics(meter, registry, "test",
+ () -> new ConsumerStats(1, 0, 0, 1))) {
+ assertReasonTotals(registry, 3);
+ for (RebalanceReason reason : RebalanceReason.values()) {
+ replacement.recordRebalance(reason);
+ }
+ assertReasonTotals(registry, 4);
+ assertThat(sample(registry, "consumer_count", "consumer_state", "active")).isEqualTo(1);
+ }
+ assertReasonTotals(registry, 4);
+ }
+ assertThat(registry.metricFamilySamples().hasMoreElements()).isFalse();
+ try (var restarted = new ScalableTopicConsumerMetrics(meter, registry, "test", () -> ConsumerStats.EMPTY)) {
+ assertReasonTotals(registry, 0);
+ }
+ }
+
+ @Test
+ public void testGaugeFailureDoesNotRemoveActiveRebalanceCountsAfterRetirement() {
+ var registry = new CollectorRegistry();
+ var meter = OpenTelemetry.noop().getMeter("test");
+ var fail = new AtomicBoolean();
+ try (var first = new ScalableTopicConsumerMetrics(meter, registry, "test", () -> ConsumerStats.EMPTY);
+ var second = new ScalableTopicConsumerMetrics(meter, registry, "test", () -> {
+ if (fail.get()) {
+ throw new IllegalStateException("Injected gauge source failure after retirement");
+ }
+ return new ConsumerStats(1, 0, 0, 0);
+ })) {
+ first.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ first.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ for (int i = 0; i < 3; i++) {
+ second.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ }
+ first.close();
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", "consumer_join")).isEqualTo(5);
+ fail.set(true);
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", "consumer_join")).isEqualTo(5);
+ assertThat(registry.getSampleValue("pulsar_scalable_topic_consumer_count",
+ new String[]{"cluster", "consumer_state"}, new String[]{"test", "active"})).isNull();
+ second.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", "consumer_join")).isEqualTo(6);
+ fail.set(false);
+ assertThat(sample(registry, "consumer_count", "consumer_state", "active")).isEqualTo(1);
+ }
+ }
+
+ @Test
+ public void testOpenTelemetryCountersSurviveGaugeFailureAndRetiredCallbacksStop() throws IOException {
+ var registry = new CollectorRegistry();
+ var firstCalls = new AtomicInteger();
+ var fail = new AtomicBoolean();
+ try (var reader = InMemoryMetricReader.create();
+ var provider = SdkMeterProvider.builder().registerMetricReader(reader).build();
+ var first = new ScalableTopicConsumerMetrics(provider.get("first"), registry, "test", () -> {
+ firstCalls.incrementAndGet();
+ return ConsumerStats.EMPTY;
+ });
+ var second = new ScalableTopicConsumerMetrics(provider.get("second"), registry, "test", () -> {
+ if (fail.get()) {
+ throw new IllegalStateException("Injected OpenTelemetry gauge failure");
+ }
+ return ConsumerStats.EMPTY;
+ })) {
+ first.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ second.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ reader.collectAllMetrics();
+ first.close();
+ int callsAtClose = firstCalls.get();
+ fail.set(true);
+ for (long expected = 1; expected <= 2; expected++) {
+ var metrics = reader.collectAllMetrics();
+ assertThat(metrics).extracting(MetricData::getName).containsOnly(
+ ScalableTopicConsumerMetrics.REBALANCE_COUNT);
+ long joins = metrics.stream().flatMap(metric -> metric.getLongSumData().getPoints().stream())
+ .filter(point -> "consumer_join".equals(point.getAttributes().get(
+ AttributeKey.stringKey("pulsar.scalable_topic.rebalance.reason"))))
+ .mapToLong(point -> point.getValue()).sum();
+ assertThat(joins).isEqualTo(expected);
+ assertThat(firstCalls.get()).as("Closed OTel callbacks must not be retained").isEqualTo(callsAtClose);
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", "consumer_join"))
+ .isEqualTo(expected + 1);
+ second.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ }
+ }
+ }
+
+ @Test
+ public void testRetiredClusterKeepsOnlyCountersUntilCollectorIsUnregistered() {
+ var registry = new CollectorRegistry();
+ var meter = OpenTelemetry.noop().getMeter("test");
+ try (var first = new ScalableTopicConsumerMetrics(meter, registry, "retired",
+ () -> new ConsumerStats(2, 0, 0, 4));
+ var second = new ScalableTopicConsumerMetrics(meter, registry, "test", () -> ConsumerStats.EMPTY)) {
+ first.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ second.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ second.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ first.close();
+ assertThat(registry.getSampleValue("pulsar_scalable_topic_rebalance_total",
+ new String[]{"cluster", "rebalance_reason"}, new String[]{"retired", "consumer_join"}))
+ .isEqualTo(1);
+ assertThat(registry.getSampleValue("pulsar_scalable_topic_consumer_count",
+ new String[]{"cluster", "consumer_state"}, new String[]{"retired", "active"})).isNull();
+ assertThat(registry.getSampleValue("pulsar_scalable_topic_segment_blocked_count",
+ new String[]{"cluster"}, new String[]{"retired"})).isNull();
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", "consumer_join")).isEqualTo(2);
+ try (var replacement = new ScalableTopicConsumerMetrics(meter, registry, "retired",
+ () -> new ConsumerStats(1, 0, 0, 0))) {
+ replacement.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ assertThat(registry.getSampleValue("pulsar_scalable_topic_rebalance_total",
+ new String[]{"cluster", "rebalance_reason"}, new String[]{"retired", "consumer_join"}))
+ .isEqualTo(2);
+ }
+ }
+ assertThat(registry.metricFamilySamples().hasMoreElements()).isFalse();
+ }
+
+ @Test
+ public void testCollectionConcurrentWithRetirementDoesNotDoubleCount() throws Exception {
+ var registry = new CollectorRegistry();
+ var meter = OpenTelemetry.noop().getMeter("test");
+ var block = new AtomicBoolean();
+ var collecting = new CompletableFuture<Void>();
+ var resume = new CompletableFuture<ConsumerStats>();
+ try (var first = new ScalableTopicConsumerMetrics(meter, registry, "test", () -> {
+ if (block.get()) {
+ collecting.complete(null);
+ return resume.join();
+ }
+ return ConsumerStats.EMPTY;
+ });
+ var second = new ScalableTopicConsumerMetrics(meter, registry, "test", () -> ConsumerStats.EMPTY);
+ var executor = Executors.newSingleThreadExecutor()) {
+ first.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ second.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ block.set(true);
+ try {
+ var sample = CompletableFuture.supplyAsync(() ->
+ sample(registry, "rebalance_total", "rebalance_reason", "consumer_join"), executor);
+ collecting.get(5, TimeUnit.SECONDS);
+ CompletableFuture.runAsync(first::close).get(5, TimeUnit.SECONDS);
+ resume.complete(ConsumerStats.EMPTY);
+ assertThat(sample.get(5, TimeUnit.SECONDS)).isEqualTo(2);
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", "consumer_join")).isEqualTo(2);
+ } finally {
+ resume.complete(ConsumerStats.EMPTY);
+ }
+ }
+ }
+
+ private static void assertReasonTotals(CollectorRegistry registry, long expected) {
+ for (String reason : List.of("consumer_join", "consumer_leave", "consumer_evict",
+ "layout_change", "segment_drained")) {
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", reason)).isEqualTo(expected);
+ }
+ }
+
+ @Test
+ public void testExistingPrometheusMetricsSurviveLifecycle() throws IOException {
+ var registry = new CollectorRegistry();
+ var counter = Counter.build("audit_existing_counter", "Existing counter").labelNames("owner")
+ .register(registry);
+ var gauge = Gauge.build("audit_existing_gauge", "Existing gauge").labelNames("owner")
+ .register(registry);
+ var histogram = Histogram.build("audit_existing_histogram", "Existing histogram")
+ .labelNames("owner").buckets(1, 2, 4).register(registry);
+ counter.labels("broker").inc(7);
+ gauge.labels("broker").set(3);
+ histogram.labels("broker").observe(1.5);
+ String baseline = existingPrometheusSnapshot(registry);
+ var meter = OpenTelemetry.noop().getMeter("test");
+ try (var metrics = new ScalableTopicConsumerMetrics(meter, registry, "test",
+ () -> new ConsumerStats(2, 1, 0, 3))) {
+ assertThat(existingPrometheusSnapshot(registry)).isEqualTo(baseline);
+ metrics.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", "consumer_join")).isEqualTo(1);
+ assertThat(existingPrometheusSnapshot(registry)).isEqualTo(baseline);
+ assertThat(existingPrometheusSnapshot(registry)).isEqualTo(baseline);
+ }
+ assertThat(existingPrometheusSnapshot(registry)).isEqualTo(baseline);
+ counter.labels("broker").inc();
+ gauge.labels("broker").set(5);
+ histogram.labels("broker").observe(3);
+ assertThat(registry.getSampleValue("audit_existing_counter_total", new String[]{"owner"},
+ new String[]{"broker"})).isEqualTo(8);
+ assertThat(registry.getSampleValue("audit_existing_gauge", new String[]{"owner"},
+ new String[]{"broker"})).isEqualTo(5);
+ assertThat(registry.getSampleValue("audit_existing_histogram_count", new String[]{"owner"},
+ new String[]{"broker"})).isEqualTo(2);
+ String updated = existingPrometheusSnapshot(registry);
+ assertThat(updated).isNotEqualTo(baseline);
+ try (var restarted = new ScalableTopicConsumerMetrics(meter, registry, "test", () -> ConsumerStats.EMPTY)) {
+ restarted.recordRebalance(RebalanceReason.LAYOUT_CHANGE);
+ assertThat(existingPrometheusSnapshot(registry)).isEqualTo(updated);
+ }
+ assertThat(existingPrometheusSnapshot(registry)).isEqualTo(updated);
+ }
+
+ @Test
+ public void testExistingOpenTelemetryMetricsSurviveLifecycle() throws IOException {
+ var registry = new CollectorRegistry();
+ var gaugeValue = new AtomicLong(9);
+ try (var reader = InMemoryMetricReader.create();
+ var provider = SdkMeterProvider.builder().registerMetricReader(reader).build()) {
+ var meter = provider.get("test");
+ PulsarService pulsar = mock(PulsarService.class);
+ PulsarBrokerOpenTelemetry telemetry = mock(PulsarBrokerOpenTelemetry.class);
+ when(pulsar.getOpenTelemetry()).thenReturn(telemetry);
+ when(telemetry.getMeter()).thenReturn(meter);
+ var existing = new OpenTelemetryReplicatedSubscriptionStats(pulsar);
+ try (var gauge = meter.gaugeBuilder("audit.existing.gauge").ofLongs().setUnit("{session}")
+ .setDescription("Existing asynchronous gauge")
+ .buildWithCallback(measurement -> measurement.record(gaugeValue.get(),
+ Attributes.of(AttributeKey.stringKey("owner"), "broker")))) {
+ existing.recordSnapshotStarted();
+ existing.recordSnapshotCompleted(125);
+ existing.recordSnapshotTimedOut(250);
+ Map<String, List<String>> baseline = existingOpenTelemetrySnapshot(reader);
+ assertThat(baseline).hasSize(3);
+ try (var metrics = new ScalableTopicConsumerMetrics(meter, registry, "test",
+ () -> new ConsumerStats(2, 1, 0, 3))) {
+ assertThat(existingOpenTelemetrySnapshot(reader)).isEqualTo(baseline);
+ metrics.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ assertThat(existingOpenTelemetrySnapshot(reader)).isEqualTo(baseline);
+ assertThat(existingOpenTelemetrySnapshot(reader)).isEqualTo(baseline);
+ }
+ assertThat(existingOpenTelemetrySnapshot(reader)).isEqualTo(baseline);
+ existing.recordSnapshotStarted();
+ existing.recordSnapshotCompleted(500);
+ gaugeValue.set(12);
+ Map<String, List<String>> updated = existingOpenTelemetrySnapshot(reader);
+ assertThat(updated).isNotEqualTo(baseline);
+ assertThat(updated).allSatisfy((name, snapshot) ->
+ assertThat(snapshot).isNotEqualTo(baseline.get(name)));
+ try (var restarted = new ScalableTopicConsumerMetrics(meter, registry, "test",
+ () -> ConsumerStats.EMPTY)) {
+ restarted.recordRebalance(RebalanceReason.LAYOUT_CHANGE);
+ assertThat(existingOpenTelemetrySnapshot(reader)).isEqualTo(updated);
+ }
+ assertThat(existingOpenTelemetrySnapshot(reader)).isEqualTo(updated);
+ }
+ }
+ }
+
+ @Test
+ public void testFailedSourceDoesNotBreakExistingMetricsOrHealthySource() throws IOException {
+ var registry = new CollectorRegistry();
+ Gauge.build("audit_existing_gauge", "Existing gauge").register(registry).set(17);
+ String baseline = existingPrometheusSnapshot(registry);
+ var fail = new AtomicBoolean();
+ var gaugeValue = new AtomicLong(19);
+ try (var reader = InMemoryMetricReader.create();
+ var provider = SdkMeterProvider.builder().registerMetricReader(reader).build();
+ var gauge = provider.get("test").gaugeBuilder("audit.existing.gauge").ofLongs()
+ .buildWithCallback(measurement -> measurement.record(gaugeValue.get()));
+ var healthy = new ScalableTopicConsumerMetrics(provider.get("healthy"), registry, "test",
+ () -> new ConsumerStats(2, 0, 0, 4));
+ var failing = new ScalableTopicConsumerMetrics(provider.get("failing"), registry, "test", () -> {
+ if (fail.get()) {
+ throw new IllegalStateException("Injected new metrics source failure");
+ }
+ return ConsumerStats.EMPTY;
+ })) {
+ healthy.recordRebalance(RebalanceReason.CONSUMER_JOIN);
+ Map<String, List<String>> telemetryBaseline = existingOpenTelemetrySnapshot(reader);
+ fail.set(true);
+ // Serialize the whole shared registry: checking only a healthy collector would hide propagation.
+ assertThat(existingPrometheusSnapshot(registry)).isEqualTo(baseline);
+ assertThat(sample(registry, "consumer_count", "consumer_state", "active")).isEqualTo(2);
+ assertThat(sample(registry, "segment_blocked_count", null, null)).isEqualTo(4);
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", "consumer_join")).isEqualTo(1);
+ assertThat(existingOpenTelemetrySnapshot(reader)).isEqualTo(telemetryBaseline);
+ gaugeValue.set(23);
+ assertThat(existingOpenTelemetrySnapshot(reader)).isNotEqualTo(telemetryBaseline);
+ }
+ assertThat(existingPrometheusSnapshot(registry)).isEqualTo(baseline);
+ }
+
+ @Test
+ public void testRegistrationCollisionDoesNotLeakOpenTelemetryCallback() throws IOException {
+ var registry = new CollectorRegistry();
+ var existing = Gauge.build("pulsar_scalable_topic_consumer_count", "Pre-existing collector")
+ .register(registry);
+ existing.set(29);
+ var supplierCalls = new AtomicInteger();
+ try (var reader = InMemoryMetricReader.create();
+ var provider = SdkMeterProvider.builder().registerMetricReader(reader).build();
+ var gauge = provider.get("test").gaugeBuilder("audit.existing.gauge").ofLongs()
+ .buildWithCallback(measurement -> measurement.record(31))) {
+ Map<String, List<String>> baseline = existingOpenTelemetrySnapshot(reader);
+ assertThatThrownBy(() -> new ScalableTopicConsumerMetrics(provider.get("test"), registry, "test", () -> {
+ supplierCalls.incrementAndGet();
+ return ConsumerStats.EMPTY;
+ })).isInstanceOf(IllegalArgumentException.class);
+ assertThat(existingOpenTelemetrySnapshot(reader)).isEqualTo(baseline);
+ assertThat(existingOpenTelemetrySnapshot(reader)).isEqualTo(baseline);
+ assertThat(supplierCalls.get()).as("Failed registration must remove its asynchronous callback").isZero();
+ assertThat(registry.getSampleValue("pulsar_scalable_topic_consumer_count")).isEqualTo(29);
+ existing.inc();
+ assertThat(registry.getSampleValue("pulsar_scalable_topic_consumer_count")).isEqualTo(30);
+ assertThat(serialize(registry)).contains("Pre-existing collector");
+ }
+ }
+
+ private static String serialize(CollectorRegistry registry) throws IOException {
+ var output = new StringBuilder();
+ var families = registry.metricFamilySamples();
+ while (families.hasMoreElements()) {
+ var family = families.nextElement();
+ output.append(family.name).append(" HELP ").append(family.help)
+ .append(" TYPE ").append(family.type).append('\n');
+ for (var sample : family.samples) {
+ output.append(sample.name).append(sample.labelNames).append(sample.labelValues)
+ .append('=').append(sample.value).append('\n');
+ }
+ }
+ return output.toString();
+ }
+
+ private static String existingPrometheusSnapshot(CollectorRegistry registry) throws IOException {
+ return serialize(registry).lines().filter(line -> line.contains("audit_existing_"))
+ .sorted().collect(Collectors.joining("\n"));
+ }
+
+ private static Map<String, List<String>> existingOpenTelemetrySnapshot(InMemoryMetricReader reader) {
+ Set<String> names = Set.of(OpenTelemetryReplicatedSubscriptionStats.SNAPSHOT_OPERATION_COUNT_METRIC_NAME,
+ OpenTelemetryReplicatedSubscriptionStats.SNAPSHOT_DURATION_METRIC_NAME, "audit.existing.gauge");
+ return reader.collectAllMetrics().stream().filter(metric -> names.contains(metric.getName()))
+ .collect(Collectors.toMap(MetricData::getName, metric -> {
+ List<String> snapshot = new ArrayList<>();
+ snapshot.add(metric.getDescription());
+ snapshot.add(metric.getUnit());
+ snapshot.add(metric.getType().name());
+ switch (metric.getType()) {
+ case LONG_SUM -> {
+ var sum = metric.getLongSumData();
+ snapshot.add(sum.getAggregationTemporality().name());
+ snapshot.add(Boolean.toString(sum.isMonotonic()));
+ sum.getPoints().forEach(point ->
+ snapshot.add(point.getAttributes() + "=" + point.getValue()));
+ }
+ case LONG_GAUGE -> metric.getLongGaugeData().getPoints()
+ .forEach(point -> snapshot.add(point.getAttributes() + "=" + point.getValue()));
+ case HISTOGRAM -> {
+ var histogram = metric.getHistogramData();
+ snapshot.add(histogram.getAggregationTemporality().name());
+ histogram.getPoints().forEach(point -> snapshot.add(point.getAttributes() + "="
+ + point.getCount() + ":" + point.getSum() + ":" + point.getMin() + ":"
+ + point.getMax() + ":" + point.getBoundaries() + ":" + point.getCounts()));
+ }
+ default -> throw new AssertionError("Unexpected existing instrument: " + metric.getType());
+ }
+ return snapshot;
+ }));
+ }
+
+ private static void assertExports(InMemoryMetricReader reader, CollectorRegistry registry,
+ ConsumerStats expected, long joins, long layouts) {
+ Map<String, MetricData> data = reader.collectAllMetrics().stream()
+ .collect(Collectors.toMap(MetricData::getName, metric -> metric));
+ assertThat(data).containsOnlyKeys(ScalableTopicConsumerMetrics.CONSUMER_COUNT,
+ ScalableTopicConsumerMetrics.BLOCKED_COUNT, ScalableTopicConsumerMetrics.REBALANCE_COUNT);
+ var consumers = data.get(ScalableTopicConsumerMetrics.CONSUMER_COUNT).getLongSumData();
+ assertThat(consumers.isMonotonic()).isFalse();
+ assertThat(consumers.getPoints()).hasSize(3);
+ Map<String, Long> expectedConsumers = Map.of("active", expected.active(), "idle", expected.idle(),
+ "disconnected", expected.disconnected());
+ consumers.getPoints().forEach(point -> {
+ assertThat(point.getAttributes().size()).isEqualTo(1);
+ String state = point.getAttributes().get(AttributeKey.stringKey("pulsar.scalable_topic.consumer.state"));
+ assertThat(point.getValue()).isEqualTo(expectedConsumers.get(state));
+ assertThat(sample(registry, "consumer_count", "consumer_state", state)).isEqualTo(point.getValue());
+ });
+ String blockedDescription = "Segment/subscription pairs waiting for ancestors to drain, "
+ + "summed across subscriptions served by this broker.";
+ assertThat(data.get(ScalableTopicConsumerMetrics.BLOCKED_COUNT).getDescription()).isEqualTo(blockedDescription);
+ assertThat(Collections.list(registry.metricFamilySamples()))
+ .filteredOn(family -> family.name.equals("pulsar_scalable_topic_segment_blocked_count"))
+ .singleElement().satisfies(family -> assertThat(family.help).isEqualTo(blockedDescription));
+ var blocked = data.get(ScalableTopicConsumerMetrics.BLOCKED_COUNT).getLongSumData();
+ assertThat(blocked.isMonotonic()).isFalse();
+ assertThat(blocked.getPoints()).singleElement().satisfies(point -> {
+ assertThat(point.getAttributes().isEmpty()).isTrue();
+ assertThat(point.getValue()).isEqualTo(expected.blocked());
+ assertThat(sample(registry, "segment_blocked_count", null, null)).isEqualTo(point.getValue());
+ });
+ var rebalances = data.get(ScalableTopicConsumerMetrics.REBALANCE_COUNT).getLongSumData();
+ assertThat(rebalances.isMonotonic()).isTrue();
+ assertThat(rebalances.getPoints()).hasSize(5);
+ rebalances.getPoints().forEach(point -> {
+ assertThat(point.getAttributes().size()).isEqualTo(1);
+ String reason = point.getAttributes().get(AttributeKey.stringKey("pulsar.scalable_topic.rebalance.reason"));
+ long count = switch (reason) {
+ case "consumer_join" -> joins;
+ case "layout_change" -> layouts;
+ default -> 0;
+ };
+ assertThat(point.getValue()).isEqualTo(count);
+ assertThat(sample(registry, "rebalance_total", "rebalance_reason", reason)).isEqualTo(count);
+ });
+ }
+
+ private static double sample(CollectorRegistry registry, String name, String label, String value) {
+ String[] labels = label == null ? new String[]{"cluster"} : new String[]{"cluster", label};
+ String[] values = label == null ? new String[]{"test"} : new String[]{"test", value};
+ Double sample = registry.getSampleValue("pulsar_scalable_topic_" + name, labels, values);
+ assertThat(sample).as("Prometheus sample %s with %s=%s", name, label, value).isNotNull();
+ return sample;
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerFailoverTest.java
index 3c5a5c994fbe..9dc64a37133a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerFailoverTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerFailoverTest.java
@@ -128,6 +128,8 @@ public class ScalableTopicControllerFailoverTest {
ScalableTopicController follower = broker2.newController();
follower.initialize().get();
assertThat(follower.isLeader()).as("broker 2, following broker 1").isFalse();
+ assertThat(leader.consumerStats()).isEqualTo(new SubscriptionCoordinator.ConsumerStats(1, 0, 0, 0));
+ assertThat(follower.consumerStats()).isEqualTo(SubscriptionCoordinator.ConsumerStats.EMPTY);
// The layout changes after the follower loaded it.
leader.splitSegment(1).get();
@@ -135,6 +137,8 @@ public class ScalableTopicControllerFailoverTest {
Awaitility.await().untilAsserted(() -> assertThat(follower.isLeader())
.as("broker 2, whose election took the controller lock over").isTrue());
+ assertThat(leader.consumerStats()).isEqualTo(SubscriptionCoordinator.ConsumerStats.EMPTY);
+ assertThat(follower.consumerStats()).isEqualTo(new SubscriptionCoordinator.ConsumerStats(0, 0, 1, 0));
assertThat(follower.getLayout().get().getAllSegments())
.as("segments of the layout the new leader serves")
.hasSize(SEGMENTS + 2);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java
index b3b25c0842bf..33230de6f970 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java
@@ -33,12 +33,15 @@ import static org.testng.Assert.assertNotSame;
import static org.testng.Assert.assertSame;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
+import io.opentelemetry.api.OpenTelemetry;
+import io.prometheus.client.CollectorRegistry;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@@ -53,6 +56,7 @@ import org.apache.pulsar.broker.resources.ScalableTopicResources;
import org.apache.pulsar.broker.resources.SubscriptionType;
import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.broker.service.TransportCnx;
+import org.apache.pulsar.broker.stats.PulsarBrokerOpenTelemetry;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.ScalableTopics;
import org.apache.pulsar.client.admin.Topics;
@@ -98,6 +102,7 @@ public class ScalableTopicServiceTest {
private ScalableTopics scalableTopicsAdmin;
private ScalableTopicService service;
+ private String clusterName;
@BeforeMethod
public void setUp() throws Exception {
@@ -115,7 +120,15 @@ public class ScalableTopicServiceTest {
scalableTopicsAdmin = mock(ScalableTopics.class);
when(brokerService.getPulsar()).thenReturn(pulsar);
- when(pulsar.getConfiguration()).thenReturn(new ServiceConfiguration());
+ ServiceConfiguration config = new ServiceConfiguration();
+ // Other tests can keep the shared collector alive, retaining this cluster's counters
+ // after teardown. Use a distinct label per test instead of assuming a fresh registry.
+ clusterName = "scalable-topic-service-test-" + UUID.randomUUID();
+ config.setClusterName(clusterName);
+ when(pulsar.getConfiguration()).thenReturn(config);
+ PulsarBrokerOpenTelemetry telemetry = mock(PulsarBrokerOpenTelemetry.class);
+ when(telemetry.getMeter()).thenReturn(OpenTelemetry.noop().getMeter("test"));
+ when(pulsar.getOpenTelemetry()).thenReturn(telemetry);
when(brokerService.getTopicIfExists(anyString()))
.thenReturn(CompletableFuture.completedFuture(Optional.empty()));
when(brokerService.deleteSchema(any(TopicName.class)))
@@ -580,6 +593,83 @@ public class ScalableTopicServiceTest {
assertEquals(c2.getTopicName(), tn2);
}
+ @Test
+ public void testMetricsInitializationFailureDoesNotPreventConsumerCoordination() throws Exception {
+ when(pulsar.getOpenTelemetry().getMeter())
+ .thenThrow(new IllegalArgumentException("Injected metrics initialization failure"));
+ ScalableTopicService withoutMetrics = new ScalableTopicService(brokerService, resources, coordinationService);
+ try {
+ withoutMetrics.start();
+ TopicName topic = scalableTopic("metrics-unavailable");
+ withoutMetrics.createScalableTopic(topic, 1).get();
+ ConsumerAssignment assignment = withoutMetrics.registerConsumer(topic, "checkpoint", "one", 1,
+ ScalableConsumerType.CHECKPOINT, mock(TransportCnx.class)).get();
+ assertEquals(assignment.assignedSegments().size(), 1);
+ assertTrue(withoutMetrics.getOrCreateController(topic).get().isLeader());
+ assertEquals(withoutMetrics.consumerStats(), new SubscriptionCoordinator.ConsumerStats(1, 0, 0, 0));
+ // The healthy service's existing registration and counters must remain untouched.
+ assertEquals(CollectorRegistry.defaultRegistry.getSampleValue("pulsar_scalable_topic_rebalance_total",
+ new String[]{"cluster", "rebalance_reason"}, new String[]{clusterName, "consumer_join"}), 0.0);
+ } finally {
+ withoutMetrics.close();
+ }
+ }
+
+ @Test
+ public void testConsumerMetricsAggregateLeadersAndRetainCountersAfterRelease() throws Exception {
+ TopicName first = scalableTopic("metrics-first");
+ TopicName second = scalableTopic("metrics-second");
+ service.createScalableTopic(first, 1).get();
+ service.createScalableTopic(second, 1).get();
+ service.registerConsumer(first, "stream", "one", 1, ScalableConsumerType.STREAM,
+ mock(TransportCnx.class)).get();
+ service.registerConsumer(second, "checkpoint", "two", 2, ScalableConsumerType.CHECKPOINT,
+ mock(TransportCnx.class)).get();
+ assertEquals(service.consumerStats(), new SubscriptionCoordinator.ConsumerStats(2, 0, 0, 0));
+ service.onConsumerDisconnect(first, "stream", "one");
+ assertEquals(service.consumerStats(), new SubscriptionCoordinator.ConsumerStats(1, 0, 1, 0));
+ service.releaseController(first).get();
+ assertEquals(service.consumerStats(), new SubscriptionCoordinator.ConsumerStats(1, 0, 0, 0));
+ service.releaseController(second).get();
+ assertEquals(service.consumerStats(), SubscriptionCoordinator.ConsumerStats.EMPTY);
+ assertEquals(CollectorRegistry.defaultRegistry.getSampleValue("pulsar_scalable_topic_rebalance_total",
+ new String[]{"cluster", "rebalance_reason"}, new String[]{clusterName, "consumer_join"}), 2.0);
+ service.start(); // Idempotent: must not reset counters or register another collector.
+ assertEquals(CollectorRegistry.defaultRegistry.getSampleValue("pulsar_scalable_topic_rebalance_total",
+ new String[]{"cluster", "rebalance_reason"}, new String[]{clusterName, "consumer_join"}), 2.0);
+ }
+
+ @Test
+ public void testClosingMetricsDoesNotReleaseControllersOrConsumerSessions() throws Exception {
+ TopicName topic = scalableTopic("metrics-only-cleanup");
+ service.createScalableTopic(topic, 1).get();
+ ScalableTopicController controller = service.getOrCreateController(topic).get();
+ service.registerConsumer(topic, "checkpoint", "one", 1, ScalableConsumerType.CHECKPOINT,
+ mock(TransportCnx.class)).get();
+ service.closeMetrics();
+ assertTrue(controller.isLeader());
+ assertEquals(resources.listConsumersAsync(topic, "checkpoint").get(), List.of("one"));
+ assertEquals(service.consumerStats(), new SubscriptionCoordinator.ConsumerStats(1, 0, 0, 0));
+ assertEquals(CollectorRegistry.defaultRegistry.getSampleValue("pulsar_scalable_topic_consumer_count",
+ new String[]{"cluster", "consumer_state"}, new String[]{clusterName, "active"}), null);
+ service.registerConsumer(topic, "checkpoint", "two", 2, ScalableConsumerType.CHECKPOINT,
+ mock(TransportCnx.class)).get();
+ assertEquals(service.consumerStats(), new SubscriptionCoordinator.ConsumerStats(1, 1, 0, 0));
+ }
+
+ @Test
+ public void testMetricCollectionDoesNotAcquireControllerMonitor() throws Exception {
+ TopicName topic = scalableTopic("metrics-no-controller-lock");
+ service.createScalableTopic(topic, 1).get();
+ ScalableTopicController controller = service.getOrCreateController(topic).get();
+ service.registerConsumer(topic, "checkpoint", "one", 1, ScalableConsumerType.CHECKPOINT,
+ mock(TransportCnx.class)).get();
+ synchronized (controller) {
+ assertEquals(CompletableFuture.supplyAsync(service::consumerStats, scheduler)
+ .get(5, TimeUnit.SECONDS), new SubscriptionCoordinator.ConsumerStats(1, 0, 0, 0));
+ }
+ }
+
/**
* Holds back the next delete of a path until it is released: {@link FaultInjectionMetadataStore}
* can fail an operation, but not delay it.
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinatorMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinatorMetricsTest.java
new file mode 100644
index 000000000000..294ff3d002b0
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinatorMetricsTest.java
@@ -0,0 +1,446 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.scalable;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import org.apache.pulsar.broker.resources.ScalableTopicResources;
+import org.apache.pulsar.broker.service.TransportCnx;
+import org.apache.pulsar.broker.service.scalable.SubscriptionCoordinator.ConsumerStats;
+import org.apache.pulsar.broker.service.scalable.SubscriptionCoordinator.RebalanceReason;
+import org.apache.pulsar.common.naming.TopicName;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class SubscriptionCoordinatorMetricsTest {
+ private ScalableTopicResources resources;
+ private ScheduledExecutorService scheduler;
+ private List<Runnable> scheduled;
+ private List<RebalanceReason> rebalances;
+ private List<SubscriptionCoordinator> coordinators;
+
+ @BeforeMethod
+ public void setup() {
+ resources = mock(ScalableTopicResources.class);
+ when(resources.registerConsumerAsync(any(), anyString(), anyString()))
+ .thenReturn(CompletableFuture.completedFuture(null));
+ when(resources.unregisterConsumerAsync(any(), anyString(), anyString()))
+ .thenReturn(CompletableFuture.completedFuture(null));
+ scheduler = mock(ScheduledExecutorService.class);
+ scheduled = new ArrayList<>();
+ rebalances = new ArrayList<>();
+ coordinators = new ArrayList<>();
+ when(scheduler.schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)))
+ .thenAnswer(invocation -> {
+ scheduled.add(invocation.getArgument(0));
+ return mock(ScheduledFuture.class);
+ });
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void cleanup() {
+ coordinators.forEach(SubscriptionCoordinator::close);
+ }
+
+ @Test
+ public void testSharedSegmentConsumersAndDisconnectedIdleMembers() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(4), true);
+ for (int i = 1; i <= 5; i++) {
+ register(coordinator, "c" + i, i);
+ }
+ assertStats(coordinator, 4, 1, 0, 0);
+ assertThat(coordinator.currentAssignment().values())
+ .as("all four owners of the shared segment count as active")
+ .filteredOn(assignment -> !assignment.assignedSegments().isEmpty())
+ .hasSize(4)
+ .allSatisfy(assignment -> {
+ assertThat(assignment.assignedSegments()).hasSize(1);
+ assertThat(assignment.assignedSegments().get(0).segmentId()).isZero();
+ assertThat(assignment.assignedSegments().get(0).bucketRanges()).hasSize(4);
+ });
+ assertThat(rebalances).containsOnly(RebalanceReason.CONSUMER_JOIN).hasSize(5);
+
+ Map<ConsumerSession, ConsumerAssignment> beforeDisconnect = coordinator.currentAssignment();
+ coordinator.onConsumerDisconnect("c1");
+ Runnable firstExpiry = scheduled.get(scheduled.size() - 1);
+ coordinator.onConsumerDisconnect("c5");
+ assertStats(coordinator, 3, 0, 2, 0);
+ coordinator.onConsumerDisconnect("c5");
+ assertStats(coordinator, 3, 0, 2, 0);
+
+ register(coordinator, "c1", 11);
+ register(coordinator, "c5", 15);
+ firstExpiry.run();
+ assertStats(coordinator, 4, 1, 0, 0);
+ assertThat(coordinator.currentAssignment()).as("reconnect preserves assignments")
+ .isEqualTo(beforeDisconnect);
+ assertThat(rebalances).as("disconnects, reconnects, stale grace tasks and snapshots do not rebalance")
+ .containsOnly(RebalanceReason.CONSUMER_JOIN).hasSize(5);
+ }
+
+ @Test
+ public void testCheckpointSurplusConsumersAreIdle() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(4), false);
+ coordinator.markCheckpointGroup();
+ for (int i = 1; i <= 5; i++) {
+ register(coordinator, "c" + i, i);
+ }
+ assertStats(coordinator, 1, 4, 0, 0);
+ coordinator.onLayoutChange(initialLayout(4).splitSegment(0, 0L)).join();
+ assertStats(coordinator, 3, 2, 0, 0);
+ }
+
+ @Test
+ public void testLayoutDrainAndLastLeaveRebalanceReasons() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), true);
+ register(coordinator, "c1", 1);
+ register(coordinator, "c2", 2);
+ coordinator.onLayoutChange(initialLayout(1).splitSegment(0, 0L)).join();
+ assertStats(coordinator, 1, 1, 0, 2);
+
+ coordinator.markSegmentsDrained(Set.of(0L));
+ assertStats(coordinator, 2, 0, 0, 0);
+ coordinator.markSegmentsDrained(Set.of(0L));
+ coordinator.unregisterConsumer("unknown", 99).join();
+ coordinator.unregisterConsumer("c1", 99).join();
+ coordinator.unregisterConsumer("c2", 2).join();
+ coordinator.unregisterConsumer("c1", 1).join();
+ assertStats(coordinator, 0, 0, 0, 0);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN,
+ RebalanceReason.CONSUMER_JOIN, RebalanceReason.LAYOUT_CHANGE,
+ RebalanceReason.SEGMENT_DRAINED, RebalanceReason.CONSUMER_LEAVE,
+ RebalanceReason.CONSUMER_LEAVE);
+ }
+
+ @Test
+ public void testBlockedSegmentsExcludeDrainedSealedSegmentsButCheckEveryAncestor() {
+ SegmentLayout layout = initialLayout(1).splitSegment(0, 0L).splitSegment(2, 0L);
+ SubscriptionCoordinator coordinator = coordinator(layout, true);
+ register(coordinator, "c1", 1);
+ assertStats(coordinator, 1, 0, 0, 4);
+
+ // Segment 2 is empty and drained, but its children still wait for grandparent 0.
+ coordinator.markSegmentsDrained(Set.of(2L));
+ assertStats(coordinator, 1, 0, 0, 3);
+ coordinator.markSegmentsDrained(Set.of(0L));
+ assertStats(coordinator, 1, 0, 0, 0);
+ assertThat(coordinator.currentAssignment().values().iterator().next().assignedSegments())
+ .as("drained parents retire rather than counting as blocked")
+ .extracting(ConsumerAssignment.AssignedSegment::segmentId)
+ .containsExactly(1L, 3L, 4L);
+ }
+
+ @Test
+ public void testRestoreAndDrainCheckerInstallationDoNotCountAsRebalances() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1).splitSegment(0, 0L), false);
+ coordinator.restoreConsumers(List.of("c1", "c2"));
+ assertStats(coordinator, 0, 0, 2, 0);
+ coordinator.installDrainChecker((segment, subscription) -> CompletableFuture.completedFuture(false));
+ assertStats(coordinator, 0, 0, 2, 2);
+ register(coordinator, "c1", 1);
+ assertStats(coordinator, 1, 0, 1, 2);
+ register(coordinator, "c2", 2);
+ assertStats(coordinator, 1, 1, 0, 2);
+ assertThat(rebalances).as("restore, checker recovery and reconnects are not new membership").isEmpty();
+ }
+
+ @Test
+ public void testGraceEvictionCountsOnceIncludingLastConsumer() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), false);
+ register(coordinator, "c1", 1);
+ register(coordinator, "c2", 2);
+ coordinator.onConsumerDisconnect("c1");
+ Runnable firstExpiry = scheduled.get(scheduled.size() - 1);
+ firstExpiry.run();
+ firstExpiry.run();
+ assertStats(coordinator, 1, 0, 0, 0);
+ coordinator.onConsumerDisconnect("c2");
+ scheduled.get(scheduled.size() - 1).run();
+ assertStats(coordinator, 0, 0, 0, 0);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN,
+ RebalanceReason.CONSUMER_JOIN, RebalanceReason.CONSUMER_EVICT,
+ RebalanceReason.CONSUMER_EVICT);
+ }
+
+ @Test
+ public void testMembershipRebalancesWaitForSuccessfulPersistence() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), false);
+ CompletableFuture<Void> registration = new CompletableFuture<>();
+ when(resources.registerConsumerAsync(any(), anyString(), anyString())).thenReturn(registration);
+ CompletableFuture<Map<ConsumerSession, ConsumerAssignment>> registering =
+ coordinator.registerConsumer("c1", 1, mock(TransportCnx.class));
+ assertThat(registering).isNotDone();
+ assertStats(coordinator, 0, 0, 0, 0);
+ assertThat(rebalances).isEmpty();
+
+ registration.complete(null);
+ assertThat(registering).isCompleted();
+ assertStats(coordinator, 1, 0, 0, 0);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN);
+
+ CompletableFuture<Void> deletion = new CompletableFuture<>();
+ when(resources.unregisterConsumerAsync(any(), anyString(), anyString())).thenReturn(deletion);
+ CompletableFuture<Map<ConsumerSession, ConsumerAssignment>> leaving =
+ coordinator.unregisterConsumer("c1", 1);
+ assertThat(leaving).isNotDone();
+ assertStats(coordinator, 1, 0, 0, 0);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN);
+
+ deletion.complete(null);
+ assertThat(leaving).isCompletedWithValue(Map.of());
+ assertStats(coordinator, 0, 0, 0, 0);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN, RebalanceReason.CONSUMER_LEAVE);
+ }
+
+ @Test
+ public void testLeaveCompletingAfterCloseDoesNotCount() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), false);
+ register(coordinator, "c1", 1);
+ CompletableFuture<Void> deletion = new CompletableFuture<>();
+ when(resources.unregisterConsumerAsync(any(), anyString(), anyString())).thenReturn(deletion);
+ CompletableFuture<Map<ConsumerSession, ConsumerAssignment>> leaving =
+ coordinator.unregisterConsumer("c1", 1);
+
+ coordinator.close();
+ deletion.complete(null);
+
+ assertThat(leaving).isCompletedWithValue(Map.of());
+ assertStats(coordinator, 0, 0, 0, 0);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN);
+ }
+
+ @Test
+ public void testEvictionSnapshotFollowsPublishedAssignments() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), false);
+ register(coordinator, "c1", 1);
+ TransportCnx remaining = mock(TransportCnx.class);
+ coordinator.registerConsumer("c2", 2, remaining).join();
+ clearInvocations(remaining);
+ CompletableFuture<Void> deletion = new CompletableFuture<>();
+ when(resources.unregisterConsumerAsync(any(), anyString(), anyString())).thenReturn(deletion);
+ coordinator.onConsumerDisconnect("c1");
+ scheduled.get(scheduled.size() - 1).run();
+ // Eviction has removed c1, but c2 is still idle until persistence completes and
+ // the original rebalance path actually publishes c2's replacement assignment.
+ assertStats(coordinator, 0, 1, 0, 0);
+ verify(remaining, times(0)).sendScalableConsumerAssignment(anyLong(), any());
+ deletion.complete(null);
+ assertStats(coordinator, 1, 0, 0, 0);
+ verify(remaining).sendScalableConsumerAssignment(anyLong(), any());
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN,
+ RebalanceReason.CONSUMER_JOIN, RebalanceReason.CONSUMER_EVICT);
+ }
+
+ @DataProvider
+ public Object[][] reconnectDuringEvictionScenarios() {
+ return new Object[][]{{true, false}, {true, true}, {false, false}, {false, true}};
+ }
+
+ @Test(dataProvider = "reconnectDuringEvictionScenarios")
+ public void testReconnectDuringEvictionOnlyUpdatesNotifiedConsumer(
+ boolean checkpoint, boolean reconnectAssignee) {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), !checkpoint);
+ if (checkpoint) {
+ coordinator.markCheckpointGroup();
+ }
+ register(coordinator, "c1", 1);
+ TransportCnx c2 = mock(TransportCnx.class);
+ TransportCnx c3 = mock(TransportCnx.class);
+ coordinator.registerConsumer("c2", 2, c2).join();
+ coordinator.registerConsumer("c3", 3, c3).join();
+ clearInvocations(c2, c3);
+ CompletableFuture<Void> deletion = new CompletableFuture<>();
+ when(resources.unregisterConsumerAsync(any(), anyString(), anyString())).thenReturn(deletion);
+ coordinator.onConsumerDisconnect("c1");
+ Runnable expiry = scheduled.get(scheduled.size() - 1);
+ String reconnectName = reconnectAssignee ? "c2" : "c3";
+ TransportCnx reconnected = reconnectAssignee ? c2 : c3;
+ TransportCnx peer = reconnectAssignee ? c3 : c2;
+ coordinator.onConsumerDisconnect(reconnectName);
+ expiry.run();
+ assertStats(coordinator, 0, 1, 1, 0);
+
+ coordinator.registerConsumer(reconnectName, 33, reconnected).join();
+
+ // c2's newly computed assignment must not count as active unless c2 itself was notified.
+ assertStats(coordinator, reconnectAssignee ? 1 : 0, reconnectAssignee ? 1 : 2, 0, 0);
+ verify(peer, never()).sendScalableConsumerAssignment(anyLong(), any());
+ verify(reconnected).sendScalableConsumerAssignment(eq(33L),
+ argThat(update -> update.getSegmentsCount() == (reconnectAssignee ? 1 : 0)));
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN,
+ RebalanceReason.CONSUMER_JOIN, RebalanceReason.CONSUMER_JOIN);
+ deletion.complete(null);
+ assertStats(coordinator, 1, 1, 0, 0);
+ verify(peer).sendScalableConsumerAssignment(anyLong(),
+ argThat(update -> update.getSegmentsCount() == (reconnectAssignee ? 0 : 1)));
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN,
+ RebalanceReason.CONSUMER_JOIN, RebalanceReason.CONSUMER_JOIN, RebalanceReason.CONSUMER_EVICT);
+ }
+
+ @Test
+ public void testEvictionCompletingAfterCloseDoesNotCount() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), false);
+ register(coordinator, "c1", 1);
+ CompletableFuture<Void> deletion = new CompletableFuture<>();
+ when(resources.unregisterConsumerAsync(any(), anyString(), anyString())).thenReturn(deletion);
+ coordinator.onConsumerDisconnect("c1");
+ scheduled.get(scheduled.size() - 1).run();
+ assertThat(rebalances).as("eviction waits for its persistence completion")
+ .containsExactly(RebalanceReason.CONSUMER_JOIN);
+
+ coordinator.close();
+ deletion.complete(null);
+
+ assertStats(coordinator, 0, 0, 0, 0);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN);
+ }
+
+ @Test
+ public void testFailedRegistrationAndLeaveDoNotCount() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), false);
+ when(resources.registerConsumerAsync(any(), anyString(), anyString()))
+ .thenReturn(CompletableFuture.failedFuture(new IllegalStateException("register failed")));
+ assertThat(coordinator.registerConsumer("c1", 1, mock(TransportCnx.class)))
+ .isCompletedExceptionally();
+ assertStats(coordinator, 0, 0, 0, 0);
+ assertThat(rebalances).isEmpty();
+
+ when(resources.registerConsumerAsync(any(), anyString(), anyString()))
+ .thenReturn(CompletableFuture.completedFuture(null));
+ register(coordinator, "c1", 1);
+ when(resources.unregisterConsumerAsync(any(), anyString(), anyString()))
+ .thenReturn(CompletableFuture.failedFuture(new IllegalStateException("unregister failed")));
+ assertThat(coordinator.unregisterConsumer("c1", 1)).isCompletedExceptionally();
+ assertStats(coordinator, 1, 0, 0, 0);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN);
+ }
+
+ @Test
+ public void testRegistrationCompletingAfterCloseDoesNotCount() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), false);
+ register(coordinator, "c1", 1);
+ CompletableFuture<Void> persistence = new CompletableFuture<>();
+ when(resources.registerConsumerAsync(any(), anyString(), anyString())).thenReturn(persistence);
+ CompletableFuture<Map<ConsumerSession, ConsumerAssignment>> registering =
+ coordinator.registerConsumer("c2", 2, mock(TransportCnx.class));
+ assertStats(coordinator, 1, 0, 0, 0);
+ coordinator.close();
+ persistence.complete(null);
+ assertThat(registering).isCompletedExceptionally();
+ assertStats(coordinator, 0, 0, 0, 0);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN);
+ }
+
+ @Test
+ public void testDrainPollCompletingAfterCloseDoesNotCountOrRearm() {
+ CompletableFuture<Boolean> drained = new CompletableFuture<>();
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1).splitSegment(0, 0L), false);
+ coordinator.installDrainChecker((segment, subscription) -> drained);
+ register(coordinator, "c1", 1);
+ scheduled.get(scheduled.size() - 1).run();
+ coordinator.close();
+ int scheduledBeforeCompletion = scheduled.size();
+ drained.complete(true);
+ assertStats(coordinator, 0, 0, 0, 0);
+ assertThat(scheduled).as("closed drain poll must not rearm").hasSize(scheduledBeforeCompletion);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN);
+ }
+
+ @Test
+ public void testMetricCollectionDoesNotAcquireCoordinatorMonitor() throws Exception {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), true);
+ register(coordinator, "c1", 1);
+ try (var executor = Executors.newSingleThreadExecutor()) {
+ synchronized (coordinator) {
+ assertThat(CompletableFuture.supplyAsync(coordinator::consumerStats, executor)
+ .get(5, TimeUnit.SECONDS)).isEqualTo(new ConsumerStats(1, 0, 0, 0));
+ }
+ }
+ }
+
+ @Test
+ public void testConcurrentDuplicateLeavePreservesAssignmentNotifications() {
+ SubscriptionCoordinator coordinator = coordinator(initialLayout(1), false);
+ register(coordinator, "c1", 1);
+ TransportCnx remaining = mock(TransportCnx.class);
+ coordinator.registerConsumer("c2", 2, remaining).join();
+ clearInvocations(remaining);
+ CompletableFuture<Void> firstDelete = new CompletableFuture<>();
+ CompletableFuture<Void> secondDelete = new CompletableFuture<>();
+ when(resources.unregisterConsumerAsync(any(), anyString(), anyString()))
+ .thenReturn(firstDelete, secondDelete);
+ var firstLeave = coordinator.unregisterConsumer("c1", 1);
+ var secondLeave = coordinator.unregisterConsumer("c1", 1);
+ firstDelete.complete(null);
+ secondDelete.complete(null);
+ assertThat(firstLeave).isCompleted();
+ assertThat(secondLeave).isCompleted();
+ verify(remaining, times(2)).sendScalableConsumerAssignment(anyLong(), any());
+ assertStats(coordinator, 1, 0, 0, 0);
+ assertThat(rebalances).containsExactly(RebalanceReason.CONSUMER_JOIN,
+ RebalanceReason.CONSUMER_JOIN, RebalanceReason.CONSUMER_LEAVE);
+ }
+
+ private SubscriptionCoordinator coordinator(SegmentLayout layout, boolean ordered) {
+ SegmentDrainChecker checker = ordered
+ ? (segment, subscription) -> CompletableFuture.completedFuture(false) : null;
+ SubscriptionCoordinator coordinator = new SubscriptionCoordinator("sub",
+ TopicName.get("topic://tenant/ns/metrics"), layout, resources, scheduler,
+ Duration.ofMinutes(1), checker, Duration.ofHours(1), Duration.ofHours(2));
+ coordinator.setRebalanceListener(rebalances::add);
+ coordinators.add(coordinator);
+ return coordinator;
+ }
+
+ private static SegmentLayout initialLayout(int bucketCount) {
+ return SegmentLayout.fromMetadata(ScalableTopicController.createInitialMetadata(1, bucketCount, Map.of()));
+ }
+
+ private static void register(SubscriptionCoordinator coordinator, String consumerName, long consumerId) {
+ coordinator.registerConsumer(consumerName, consumerId, mock(TransportCnx.class)).join();
+ }
+
+ private static void assertStats(SubscriptionCoordinator coordinator, long active, long idle,
+ long disconnected, long blocked) {
+ assertThat(coordinator.consumerStats()).as("coordinator consumer and ancestor-blocking snapshot")
+ .isEqualTo(new ConsumerStats(active, idle, disconnected, blocked));
+ }
+}