You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/13 04:41:43 UTC

(pulsar) branch branch-5.0 updated (ebd3b3b9f2a5 -> 9049f5a2c411)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a change to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git


    from ebd3b3b9f2a5 [fix][test] Give each geo-replicated cluster's bookies and ZooKeeper a hostname of their own (#26873)
     new 5b509a5c668e [fix][client] Avoid duplicate reconcile retries and queue layout stalls in V5 consumers (#26874)
     new 8f7884ebad33 [fix][broker] Keep the scalable-topics watch's initial listing from undoing newer events (#26876)
     new 67dd068083f3 [fix][test] Run the Oxia-backed tests against oxia/oxia:nightly-beta (#26880)
     new 740f8e4f20cb [fix][meta] Don't leak closed leader elections and their metadata store listeners (#26879)
     new 979d6ae890f1 [fix][broker] Stop scalable topic controllers from re-electing while the topic is deleted (#26875)
     new 9049f5a2c411 [fix][broker] Make the scalable topic controller follow its leader election (#26878)

The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../org/apache/pulsar/tests/OxiaTestImage.java     |  48 +++
 .../broker/resources/ScalableTopicMetadata.java    |   8 +
 .../broker/resources/ScalableTopicResources.java   |  11 +
 .../pulsar/broker/admin/v2/ScalableTopics.java     |  45 ++-
 .../apache/pulsar/broker/service/ServerCnx.java    |  20 +
 .../apache/pulsar/broker/service/TransportCnx.java |   6 +
 .../broker/service/scalable/ConsumerSession.java   |  11 +
 .../service/scalable/ScalableTopicController.java  | 433 +++++++++++++++++++--
 .../service/scalable/ScalableTopicService.java     |  71 +++-
 .../scalable/ScalableTopicsWatcherSession.java     | 108 +++--
 .../broker/service/scalable/SegmentLayout.java     |   5 +-
 .../service/scalable/SubscriptionCoordinator.java  |  44 ++-
 .../pulsar/broker/service/ServerCnxTest.java       |  18 +
 .../service/SharedMultiBrokerPulsarCluster.java    |  13 +-
 .../ScalableTopicControllerFailoverTest.java       | 387 ++++++++++++++++++
 .../scalable/ScalableTopicControllerTest.java      | 130 +++++++
 .../service/scalable/ScalableTopicServiceTest.java |  84 ++++
 ...ScalableTopicsWatcherSessionDeleteRaceTest.java | 269 +++++++++++--
 .../broker/service/scalable/SegmentLayoutTest.java |   2 +
 .../scalable/SubscriptionCoordinatorTest.java      |  41 ++
 .../v5/V5MultiBrokerControllerFailoverTest.java    | 147 +++++++
 .../v5/V5MultiBrokerScalableTopicDeleteTest.java   | 155 ++++++++
 .../client/api/v5/V5ScalableTopicDeleteTest.java   | 118 ++++++
 .../client/impl/v5/ScalableQueueConsumer.java      |  90 +++--
 .../client/impl/v5/ScalableStreamConsumer.java     |  75 +++-
 .../v5/ScalableQueueConsumerReconcileTest.java     | 291 ++++++++++++++
 .../v5/ScalableStreamConsumerReconcileTest.java    | 341 ++++++++++++++++
 .../api/coordination/CoordinationService.java      |  13 +-
 .../coordination/impl/CoordinationServiceImpl.java |  61 ++-
 .../coordination/impl/LeaderElectionImpl.java      |  81 +++-
 .../pulsar/metadata/BaseMetadataStoreTest.java     |   3 +-
 .../pulsar/metadata/OxiaPartitionKeyTest.java      |   3 +-
 .../pulsar/metadata/OxiaSequenceKeysTest.java      |   3 +-
 .../org/apache/pulsar/metadata/OxiaTestServer.java |  49 +++
 .../impl/CoordinationServiceImplTest.java          | 314 +++++++++++++++
 .../coordination/impl/LeaderElectionImplTest.java  | 146 ++++++-
 .../tests/integration/oxia/OxiaContainer.java      |  18 +-
 37 files changed, 3460 insertions(+), 202 deletions(-)
 create mode 100644 buildtools/src/main/java/org/apache/pulsar/tests/OxiaTestImage.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerFailoverTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5MultiBrokerControllerFailoverTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5MultiBrokerScalableTopicDeleteTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ScalableTopicDeleteTest.java
 create mode 100644 pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumerReconcileTest.java
 create mode 100644 pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumerReconcileTest.java
 create mode 100644 pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaTestServer.java
 create mode 100644 pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/CoordinationServiceImplTest.java


(pulsar) 05/06: [fix][broker] Stop scalable topic controllers from re-electing while the topic is deleted (#26875)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 979d6ae890f1785b0e9e0e54159327886d676cd3
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Oct 12 18:25:12 2026 -0700

    [fix][broker] Stop scalable topic controllers from re-electing while the topic is deleted (#26875)
    
    (cherry picked from commit 9f8e1c7637d19a13258e85f86bc8283db26e2209)
---
 .../broker/resources/ScalableTopicMetadata.java    |   8 ++
 .../broker/resources/ScalableTopicResources.java   |  11 ++
 .../pulsar/broker/admin/v2/ScalableTopics.java     |  45 +++++-
 .../service/scalable/ScalableTopicController.java  | 154 ++++++++++++++++++++
 .../service/scalable/ScalableTopicService.java     |  20 ++-
 .../broker/service/scalable/SegmentLayout.java     |   5 +-
 .../scalable/ScalableTopicControllerTest.java      |  50 +++++++
 .../service/scalable/ScalableTopicServiceTest.java |  56 ++++++++
 .../broker/service/scalable/SegmentLayoutTest.java |   2 +
 .../v5/V5MultiBrokerScalableTopicDeleteTest.java   | 155 +++++++++++++++++++++
 .../client/api/v5/V5ScalableTopicDeleteTest.java   | 118 ++++++++++++++++
 11 files changed, 615 insertions(+), 9 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicMetadata.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicMetadata.java
index 228e2c25e651..fec2ed76f3fe 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicMetadata.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicMetadata.java
@@ -58,4 +58,12 @@ public class ScalableTopicMetadata {
      * the namespace policy and then the broker configuration apply.
      */
     private AutoScalePolicyOverride autoScalePolicy;
+
+    /**
+     * Set while the topic is being deleted, like {@code PartitionedTopicMetadata.deleted}. No
+     * controller elects itself for a topic marked deleted, so the delete can remove the
+     * controller lock along with the rest of the topic's metadata. A delete that fails after
+     * marking the topic leaves it marked, until a retried delete removes it.
+     */
+    private boolean deleted;
 }
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicResources.java
index bec03c6029df..ae98693331d5 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicResources.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicResources.java
@@ -272,6 +272,17 @@ public class ScalableTopicResources extends BaseResources<ScalableTopicMetadata>
         return getStore().deleteRecursive(topicPath(tn));
     }
 
+    /**
+     * Mark the topic as being deleted, before its metadata is deleted. See
+     * {@link ScalableTopicMetadata#isDeleted()}.
+     */
+    public CompletableFuture<Void> markScalableTopicDeletedAsync(TopicName tn) {
+        return updateScalableTopicAsync(tn, md -> {
+            md.setDeleted(true);
+            return md;
+        });
+    }
+
     public CompletableFuture<Boolean> scalableTopicExistsAsync(TopicName tn) {
         return existsAsync(topicPath(tn));
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
index 84c8e0e09fa9..388eacd8e5a9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
@@ -647,6 +647,8 @@ public class ScalableTopics extends AdminResource {
     @Operation(summary = "Delete a scalable topic and all its segments.")
     @ApiResponses(value = {
             @ApiResponse(responseCode = "204", description = "Scalable topic deleted successfully"),
+            @ApiResponse(responseCode = "307",
+                    description = "Current broker is not the controller leader for this topic"),
             @ApiResponse(responseCode = "401",
                     description = "Don't have permission to administrate resources on this tenant"),
             @ApiResponse(responseCode = "403", description = "Don't have admin permission on the namespace"),
@@ -665,17 +667,30 @@ public class ScalableTopics extends AdminResource {
         validateNamespaceName(tenant, namespace);
         TopicName tn = TopicName.get(TopicDomain.topic.value(), namespaceName, encodedTopic);
 
+        // Served by the controller leader, so the broker holding the elected controller is the
+        // one that stops it below.
         validateNamespaceOperationAsync(namespaceName, NamespaceOperation.DELETE_TOPIC)
+                .thenCompose(__ -> redirectToControllerLeaderIfNeeded(tn))
                 .thenCompose(__ -> resources().getScalableTopicMetadataAsync(tn))
                 .thenCompose(optMd -> {
                     if (optMd.isEmpty()) {
                         throw new RestException(Response.Status.NOT_FOUND,
                                 "Scalable topic not found: " + tn);
                     }
-                    // Delete metadata first, then best-effort clean up segment topics and the schema
-                    // they share (deleting a segment topic leaves the schema in place)
-                    return resources().deleteScalableTopicAsync(tn)
-                            .thenCompose(__ -> deleteSegmentTopics(tn, optMd.get(), force))
+                    // Stop the controller before deleting the metadata: still running, its
+                    // leader election re-acquires the lock the delete removes, under the deleted
+                    // path, and it keeps serving consumers from the deleted layout. Mark the
+                    // topic deleted first, so the controllers following it on other brokers
+                    // don't take the lock over once it is released. Then delete metadata, then
+                    // best-effort clean up segment topics and the schema they share (deleting a
+                    // segment topic leaves the schema in place). The segments are read again
+                    // after the release, to include those of a split that was still in flight.
+                    return resources().markScalableTopicDeletedAsync(tn)
+                            .thenCompose(__ -> releaseController(tn))
+                            .thenCompose(__ -> resources().getScalableTopicMetadataAsync(tn, true))
+                            .thenCompose(latest -> resources().deleteScalableTopicAsync(tn)
+                                    .thenCompose(__ -> deleteSegmentTopics(tn, latest.orElse(optMd.get()),
+                                            force)))
                             .thenCompose(__ -> pulsar().getBrokerService().deleteSchema(tn)
                                     .exceptionally(ex -> {
                                         log.warn().attr("topic", tn).exceptionMessage(ex)
@@ -689,8 +704,10 @@ public class ScalableTopics extends AdminResource {
                     asyncResponse.resume(Response.noContent().build());
                 })
                 .exceptionally(ex -> {
-                    log.error().attr("clientAppId", clientAppId()).attr("topic", tn)
-                            .exception(ex).log("Failed to delete scalable topic");
+                    if (!isRedirectException(ex)) {
+                        log.error().attr("clientAppId", clientAppId()).attr("topic", tn)
+                                .exception(ex).log("Failed to delete scalable topic");
+                    }
                     resumeAsyncResponseExceptionally(asyncResponse, ex);
                     return null;
                 });
@@ -1221,6 +1238,22 @@ public class ScalableTopics extends AdminResource {
         }
     }
 
+    /**
+     * Stop this broker's controller for {@code tn}, if it holds one, closing its leader election.
+     * Best-effort: a failure is logged and the delete goes on.
+     */
+    private CompletableFuture<Void> releaseController(TopicName tn) {
+        ScalableTopicService svc = pulsar().getBrokerService().getScalableTopicService();
+        if (svc == null) {
+            return CompletableFuture.completedFuture(null);
+        }
+        return svc.releaseController(tn).exceptionally(ex -> {
+            log.warn().attr("topic", tn).exceptionMessage(ex)
+                    .log("Failed to release the controller of the scalable topic being deleted");
+            return null;
+        });
+    }
+
     /**
      * Best-effort delete the underlying topic for every segment in the DAG. Uses the
      * segment-aware admin endpoint, which routes to the segment-owning broker via the
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java
index fa80cc246636..9e8c9d67fef7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java
@@ -58,8 +58,11 @@ import org.apache.pulsar.common.policies.data.TopicPolicies;
 import org.apache.pulsar.common.scalable.HashRange;
 import org.apache.pulsar.common.scalable.SegmentInfo;
 import org.apache.pulsar.common.scalable.SegmentTopicName;
+import org.apache.pulsar.common.util.Backoff;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.compaction.Compactor;
+import org.apache.pulsar.metadata.api.Notification;
+import org.apache.pulsar.metadata.api.NotificationType;
 import org.apache.pulsar.metadata.api.coordination.CoordinationService;
 import org.apache.pulsar.metadata.api.coordination.LeaderElection;
 import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
@@ -125,6 +128,17 @@ public class ScalableTopicController {
 
     private volatile boolean closed = false;
 
+    /** Registered when this controller elects, see {@link #onControllerLockDeleted()}. */
+    private final ControllerLockListener controllerLockListener = new ControllerLockListener();
+    /** Registered when this controller elects, see {@link #onTopicRecordDeleted()}. */
+    private final TopicRecordListener topicRecordListener = new TopicRecordListener();
+
+    /** Set while re-electing after the controller lock was deleted, retries included. */
+    private final AtomicBoolean reElecting = new AtomicBoolean(false);
+
+    /** Run once this controller finds its topic deleted. Set by {@link ScalableTopicService}. */
+    private volatile Runnable topicDeletedAction = () -> { };
+
     ScalableTopicController(TopicName topicName,
                             ScalableTopicResources resources,
                             BrokerService brokerService,
@@ -176,6 +190,102 @@ public class ScalableTopicController {
         }
     }
 
+    /**
+     * The controller lock was deleted while this broker follows the leader: the leader released
+     * the controller or lost its metadata session, or the topic is being deleted. A follower's
+     * election is closed once it settles (see {@link #electLeader()}), so it doesn't take the lock
+     * over by itself, even for a topic being deleted, where it would re-create the lock under the
+     * record the delete is about to remove (failing that delete on stores that refuse to delete a
+     * node with children). Re-elect through {@link #initialize()} instead, unless the topic is gone
+     * or marked deleted, retrying with backoff while that fails.
+     */
+    private void onControllerLockDeleted() {
+        if (closed || leaderState != LeaderElectionState.Following
+                || !reElecting.compareAndSet(false, true)) {
+            return;
+        }
+        reElect(Backoff.builder()
+                .initialDelay(Duration.ofMillis(100))
+                .maxBackoff(Duration.ofSeconds(30))
+                .build());
+    }
+
+    private void reElect(Backoff backoff) {
+        resources.getScalableTopicMetadataAsync(topicName, true)
+                .thenCompose(optMd -> {
+                    if (closed || optMd.isEmpty() || optMd.get().isDeleted()) {
+                        log.debug().log("Controller lock released for a deleted topic, not re-electing");
+                        return CompletableFuture.completedFuture(null);
+                    }
+                    return initialize();
+                })
+                .whenComplete((__, ex) -> {
+                    if (ex == null || closed) {
+                        reElecting.set(false);
+                        return;
+                    }
+                    long delayMs = backoff.next().toMillis();
+                    log.warn().attr("retryInMs", delayMs).exceptionMessage(ex)
+                            .log("Failed to re-elect after the controller lock was released");
+                    scheduler().schedule(() -> reElect(backoff), delayMs, TimeUnit.MILLISECONDS);
+                });
+    }
+
+    /**
+     * The topic record was deleted. The admin delete stops the controller on the broker serving
+     * it, but other brokers can hold one for the topic too; left running, it keeps serving
+     * consumers from the deleted layout. The event can arrive after the topic was re-created under
+     * the same name, so act only if the record is still gone.
+     */
+    private void onTopicRecordDeleted() {
+        if (closed) {
+            return;
+        }
+        resources.getScalableTopicMetadataAsync(topicName, true)
+                .thenAccept(optMd -> {
+                    if (optMd.isEmpty() && !closed) {
+                        log.debug().log("Scalable topic deleted, releasing its controller");
+                        topicDeletedAction.run();
+                    }
+                })
+                .exceptionally(ex -> {
+                    log.warn().exceptionMessage(ex).log("Failed to check the record of a deleted topic");
+                    return null;
+                });
+    }
+
+    void setTopicDeletedAction(Runnable topicDeletedAction) {
+        this.topicDeletedAction = topicDeletedAction;
+    }
+
+    private final class ControllerLockListener implements ScalableTopicResources.MetadataPathListener {
+        @Override
+        public String getMetadataPath() {
+            return resources.controllerLockPath(topicName);
+        }
+
+        @Override
+        public void onNotification(Notification notification) {
+            if (notification.getType() == NotificationType.Deleted) {
+                onControllerLockDeleted();
+            }
+        }
+    }
+
+    private final class TopicRecordListener implements ScalableTopicResources.MetadataPathListener {
+        @Override
+        public String getMetadataPath() {
+            return resources.topicPath(topicName);
+        }
+
+        @Override
+        public void onNotification(Notification notification) {
+            if (notification.getType() == NotificationType.Deleted) {
+                onTopicRecordDeleted();
+            }
+        }
+    }
+
     /**
      * Initialize: load current layout from metadata store and attempt to become leader.
      *
@@ -192,9 +302,15 @@ public class ScalableTopicController {
                         return CompletableFuture.failedFuture(
                                 new IllegalStateException("Scalable topic not found: " + topicName));
                     }
+                    if (optMd.get().isDeleted()) {
+                        return CompletableFuture.failedFuture(
+                                new IllegalStateException("Scalable topic is being deleted: " + topicName));
+                    }
                     this.currentLayout = SegmentLayout.fromMetadata(optMd.get());
                     return electLeader();
                 })
+                .thenCompose(__ -> isLeader()
+                        ? releaseIfMarkedDeleted() : CompletableFuture.completedFuture(null))
                 .thenCompose(__ -> {
                     if (isLeader()) {
                         seedAutoScaleCooldownsFromLayout();
@@ -207,6 +323,26 @@ public class ScalableTopicController {
                 });
     }
 
+    /**
+     * The delete marks the topic before releasing its leader, but a controller that read the record
+     * before the mark can win the election after the release, re-creating the lock under the record
+     * the delete is about to remove. Read the record again once elected, and give the lock back if
+     * the topic is marked deleted or gone.
+     */
+    private CompletableFuture<Void> releaseIfMarkedDeleted() {
+        return resources.getScalableTopicMetadataAsync(topicName, true)
+                .thenCompose(optMd -> {
+                    if (optMd.isPresent() && !optMd.get().isDeleted()) {
+                        return CompletableFuture.completedFuture(null);
+                    }
+                    leaderState = LeaderElectionState.NoLeader;
+                    return leaderElection.asyncClose()
+                            .handle((__, ex) -> null)
+                            .thenCompose(__ -> CompletableFuture.failedFuture(new IllegalStateException(
+                                    "Scalable topic is being deleted: " + topicName)));
+                });
+    }
+
     /**
      * Recover the auto split/merge/rebucket cooldown clocks after winning leadership. The
      * timestamps are in-memory only, but the layout itself records when each segment was
@@ -751,8 +887,24 @@ public class ScalableTopicController {
         // NamespaceService.createLookupResult(brokerId, ...), matching the pattern used by
         // the cluster-leader redirection in NamespacesBase.
         String brokerId = brokerService.getPulsar().getBrokerId();
+        // Registered before electing, so that no deletion is missed once the election settles.
+        resources.registerPathListener(controllerLockListener);
+        resources.registerPathListener(topicRecordListener);
+        if (closed) {
+            // close() may have deregistered them before they were registered.
+            resources.deregisterPathListener(controllerLockListener);
+            resources.deregisterPathListener(topicRecordListener);
+            return CompletableFuture.failedFuture(
+                    new IllegalStateException("Controller closed for scalable topic: " + topicName));
+        }
         return leaderElection.elect(brokerId)
                 .thenAccept(state -> {
+                    if (state == LeaderElectionState.Following) {
+                        // A follower holds no lock. Closing its election keeps the election from
+                        // taking the lock over by itself once the lock is deleted, before this
+                        // controller can check the topic: onControllerLockDeleted() re-elects.
+                        leaderElection.asyncClose();
+                    }
                     this.leaderState = state;
                     log.info().attr("state", state)
                             .log("Leader election for scalable topic");
@@ -1646,6 +1798,8 @@ public class ScalableTopicController {
 
     public CompletableFuture<Void> close() {
         closed = true;
+        resources.deregisterPathListener(controllerLockListener);
+        resources.deregisterPathListener(topicRecordListener);
         cancelGcTask();
         cancelAutoScaleTask();
         // Stop each coordinator's drain poller before clearing — otherwise the scheduler
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
index 74b1b3859a43..76c4d1cafb5e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
@@ -113,7 +113,25 @@ public class ScalableTopicService {
         CompletableFuture<ScalableTopicController> stored = controllers.computeIfAbsent(key, k -> {
             ScalableTopicController controller = new ScalableTopicController(
                     topic, resources, brokerService, coordinationService);
-            return controller.initialize().thenApply(__ -> controller);
+            CompletableFuture<ScalableTopicController> future = new CompletableFuture<>();
+            // Released once its topic is deleted: the admin delete stops the controller on the
+            // broker serving it, not the ones other brokers hold for the topic.
+            controller.setTopicDeletedAction(() -> {
+                if (controllers.remove(key, future)) {
+                    controller.close();
+                }
+            });
+            controller.initialize().whenComplete((__, ex) -> {
+                if (ex == null) {
+                    future.complete(controller);
+                } else {
+                    // initialize() can fail after winning the election: close the controller, or
+                    // its election keeps the lock.
+                    controller.close();
+                    future.completeExceptionally(ex);
+                }
+            });
+            return future;
         });
         // Evict failed futures so subsequent callers can retry. This runs *outside*
         // computeIfAbsent, so modifying the map here is safe.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SegmentLayout.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SegmentLayout.java
index fc3a9ce1a462..a32b52f80aac 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SegmentLayout.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SegmentLayout.java
@@ -337,8 +337,8 @@ public class SegmentLayout {
 
     /**
      * Convert back to metadata for persistence, carrying over the non-layout fields
-     * (properties, per-topic auto-scale policy) from the record being replaced. Layout
-     * mutations must never silently drop fields they don't model.
+     * (properties, per-topic auto-scale policy, deleted mark) from the record being replaced.
+     * Layout mutations must never silently drop fields they don't model.
      */
     public ScalableTopicMetadata toMetadata(ScalableTopicMetadata original) {
         return ScalableTopicMetadata.builder()
@@ -347,6 +347,7 @@ public class SegmentLayout {
                 .segments(new LinkedHashMap<>(allSegments))
                 .properties(original.getProperties())
                 .autoScalePolicy(original.getAutoScalePolicy())
+                .deleted(original.isDeleted())
                 .build();
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerTest.java
index 261d58cc37cd..9df2b0348567 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerTest.java
@@ -48,6 +48,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.resources.ConsumerRegistration;
@@ -267,6 +268,55 @@ public class ScalableTopicControllerTest {
         }
     }
 
+    @Test
+    public void testInitializeFailsWhenTopicMarkedDeleted() throws Exception {
+        resources.markScalableTopicDeletedAsync(topicName).get();
+        try {
+            controller.initialize().get();
+            fail("expected IllegalStateException for a topic being deleted");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof IllegalStateException,
+                    "expected IllegalStateException, got " + e.getCause());
+            assertTrue(e.getCause().getMessage().contains("being deleted"));
+        }
+        assertFalse(store.exists(resources.controllerLockPath(topicName)).get(),
+                "no controller may be elected for a topic being deleted");
+    }
+
+    /**
+     * A controller that read the record before the delete marked it can still win the election
+     * after the delete released the leader. It must give the lock back once elected.
+     */
+    @Test
+    public void testInitializeGivesTheLockBackWhenTopicMarkedDeletedWhileElecting() throws Exception {
+        AtomicBoolean marked = new AtomicBoolean();
+        ScalableTopicResources markingResources = new ScalableTopicResources(store, 30) {
+            @Override
+            public CompletableFuture<Optional<ScalableTopicMetadata>> getScalableTopicMetadataAsync(
+                    TopicName tn, boolean refresh) {
+                // The first read sees the record unmarked, then the delete marks it.
+                return super.getScalableTopicMetadataAsync(tn, refresh)
+                        .thenCompose(md -> marked.compareAndSet(false, true)
+                                ? markScalableTopicDeletedAsync(tn).thenApply(__ -> md)
+                                : CompletableFuture.completedFuture(md));
+            }
+        };
+        ScalableTopicController racing = new ScalableTopicController(topicName, markingResources,
+                brokerService, coordinationService);
+        try {
+            racing.initialize().get();
+            fail("expected IllegalStateException for a topic marked deleted while electing");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause().getMessage().contains("being deleted"),
+                    "unexpected failure: " + e.getCause());
+        }
+        assertFalse(racing.isLeader());
+        assertFalse(store.get(resources.controllerLockPath(topicName)).get().isPresent(),
+                "the controller lock must be given back");
+        verify(scalableTopics, times(0)).createSegmentAsync(anyString(), any());
+        racing.close().join();
+    }
+
     @Test
     public void testLeaderBrokerIdPersisted() throws Exception {
         controller.initialize().get();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java
index 49b7117eb301..fbc35efe3cd8 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java
@@ -31,6 +31,8 @@ import static org.testng.Assert.assertNotSame;
 import static org.testng.Assert.assertSame;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
+import java.time.Duration;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
@@ -59,6 +61,7 @@ import org.apache.pulsar.metadata.api.coordination.CoordinationService;
 import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
 import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl;
 import org.apache.pulsar.metadata.impl.LocalMemoryMetadataStore;
+import org.awaitility.Awaitility;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -364,6 +367,59 @@ public class ScalableTopicServiceTest {
         verify(brokerService).deleteSchema(tn);
     }
 
+    @Test
+    public void testControllerReleasedWhenTopicRecordDeleted() throws Exception {
+        TopicName tn = scalableTopic("t-deleted-elsewhere");
+        service.createScalableTopic(tn, 1).get();
+        service.getOrCreateController(tn).get();
+        String lockPath = resources.controllerLockPath(tn);
+        assertTrue(store.exists(lockPath).get());
+
+        // Delete the metadata the way an admin delete served by another broker does, without
+        // releasing the controller held here first.
+        resources.deleteScalableTopicAsync(tn).get();
+
+        Awaitility.await().during(Duration.ofSeconds(2)).atMost(Duration.ofSeconds(10))
+                .untilAsserted(() -> assertFalse(store.exists(lockPath).get(),
+                        "the controller lock must not be re-acquired under the deleted topic"));
+        try {
+            service.getOrCreateController(tn).get();
+            fail("the deleted topic must not be served by its old controller");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause().getMessage().contains("Failed to initialize controller"),
+                    "got: " + e.getCause().getMessage());
+        }
+    }
+
+    /**
+     * A controller whose initialize() fails after winning the election is closed, or its election
+     * would keep the controller lock with no controller left to release it.
+     */
+    @Test
+    public void testControllerFailingAfterElectionGivesTheLockBack() throws Exception {
+        TopicName tn = scalableTopic("t-init-fails");
+        service.createScalableTopic(tn, 1).get();
+        ScalableTopicResources failingResources = new ScalableTopicResources(store, 30) {
+            @Override
+            public CompletableFuture<List<String>> listSubscriptionsAsync(TopicName topic) {
+                // Restoring the subscriptions runs once the controller won the election.
+                return CompletableFuture.failedFuture(new IllegalStateException("injected"));
+            }
+        };
+        ScalableTopicService failingService =
+                new ScalableTopicService(brokerService, failingResources, coordinationService);
+        try {
+            failingService.getOrCreateController(tn).get();
+            fail("expected the controller initialization to fail");
+        } catch (ExecutionException expected) {
+            // initialize() failed after the election
+        }
+        Awaitility.await().untilAsserted(() ->
+                assertFalse(store.get(resources.controllerLockPath(tn)).get().isPresent(),
+                        "the failed controller must give the lock back"));
+        failingService.close();
+    }
+
     // --- concurrent topic isolation ---
 
     @Test
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SegmentLayoutTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SegmentLayoutTest.java
index 2da9bad009cc..549152f3d3db 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SegmentLayoutTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SegmentLayoutTest.java
@@ -315,6 +315,7 @@ public class SegmentLayoutTest {
         // A layout mutation must round-trip every non-layout field, not just properties.
         metadata.setAutoScalePolicy(org.apache.pulsar.common.policies.data.AutoScalePolicyOverride
                 .builder().enabled(false).maxSegments(8).build());
+        metadata.setDeleted(true);
         SegmentLayout layout = SegmentLayout.fromMetadata(metadata);
         SegmentLayout afterSplit = layout.splitSegment(0, 0L);
 
@@ -325,6 +326,7 @@ public class SegmentLayoutTest {
         assertEquals(restored.getProperties().get("key"), "value");
         assertEquals(restored.getAutoScalePolicy(), metadata.getAutoScalePolicy(),
                 "split/merge must not drop the per-topic auto-scale policy");
+        assertTrue(restored.isDeleted(), "split/merge must not drop the deleted mark");
     }
 
     @Test
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5MultiBrokerScalableTopicDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5MultiBrokerScalableTopicDeleteTest.java
new file mode 100644
index 000000000000..df59e120e54e
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5MultiBrokerScalableTopicDeleteTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api.v5;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.resources.ScalableTopicResources;
+import org.apache.pulsar.broker.service.scalable.ScalableTopicService;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.ScalableSubscriptionType;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.apache.pulsar.metadata.api.GetResult;
+import org.awaitility.Awaitility;
+import org.testng.annotations.Test;
+
+/**
+ * Deleting a scalable topic must stop its controllers before the metadata delete removes the
+ * controller lock: the leader's, also when the delete goes through another broker, and the ones
+ * following it on the other brokers, which must not take the lock over.
+ */
+public class V5MultiBrokerScalableTopicDeleteTest extends V5MultiBrokerClientBaseTest {
+
+    @Test
+    public void deleteThroughNonLeaderBrokerStopsController() throws Exception {
+        String topic = newScalableTopic(1);
+        TopicName tn = TopicName.get(topic);
+        // Elect the controller on broker 0 only: no other broker holds one for the topic.
+        ScalableTopicService leaderService = brokers.get(0).getBrokerService().getScalableTopicService();
+        leaderService.getOrCreateController(tn).get(10, TimeUnit.SECONDS);
+        ScalableTopicResources resources = brokers.get(0).getPulsarResources().getScalableTopicResources();
+        String lockPath = resources.controllerLockPath(tn);
+        assertThat(resources.getStore().exists(lockPath).get())
+                .as("controller lock after the election").isTrue();
+
+        admins.get(1).scalableTopics().deleteScalableTopic(topic, true);
+
+        Awaitility.await().during(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(15))
+                .untilAsserted(() -> {
+                    assertThat(resources.getStore().exists(lockPath).get())
+                            .as("controller lock of the deleted topic").isFalse();
+                    assertThat(resources.getStore().exists(resources.topicPath(tn)).get())
+                            .as("record of the deleted topic").isFalse();
+                });
+        assertThat(admins.get(1).scalableTopics().listScalableTopics(getNamespace()))
+                .doesNotContain(topic);
+        assertThatThrownBy(() -> leaderService.getOrCreateController(tn).get(10, TimeUnit.SECONDS))
+                .hasMessageContaining("Failed to initialize controller");
+    }
+
+    /**
+     * With a controller on every broker, the delete removes the lock of the leader while the
+     * other brokers follow it. A follower's leader election takes the lock over right away, under
+     * the topic being deleted, so the record delete fails (NotEmpty on Oxia and ZooKeeper) or the
+     * lock outlives the record.
+     */
+    @Test
+    public void deleteThroughLeaderStopsFollowingControllers() throws Exception {
+        String topic = newScalableTopic(1);
+        // A subscription gives the metadata delete more to remove before it reaches the record.
+        admin.scalableTopics().createSubscription(topic, "sub", ScalableSubscriptionType.STREAM);
+        // Materializes the controller on every broker: the leader and two followers.
+        int leaderIndex = findControllerLeaderIndex(topic);
+
+        admins.get(leaderIndex).scalableTopics().deleteScalableTopic(topic, true);
+
+        assertTopicStaysDeleted(topic);
+    }
+
+    @Test
+    public void deleteThroughNonLeaderStopsFollowingControllers() throws Exception {
+        String topic = newScalableTopic(1);
+        admin.scalableTopics().createSubscription(topic, "sub", ScalableSubscriptionType.STREAM);
+        int leaderIndex = findControllerLeaderIndex(topic);
+
+        admins.get((leaderIndex + 1) % brokers.size()).scalableTopics().deleteScalableTopic(topic, true);
+
+        assertTopicStaysDeleted(topic);
+    }
+
+    /**
+     * Followers must still take over when the leader releases its controller while the topic is
+     * not being deleted (the leader broker unloads it or shuts down), and the new leader must
+     * serve as one.
+     */
+    @Test
+    public void followerTakesOverReleasedController() throws Exception {
+        String topic = newScalableTopic(1);
+        TopicName tn = TopicName.get(topic);
+        int leaderIndex = findControllerLeaderIndex(topic);
+
+        brokers.get(leaderIndex).getBrokerService().getScalableTopicService().releaseController(tn)
+                .get(10, TimeUnit.SECONDS);
+
+        ScalableTopicResources resources = brokers.get(0).getPulsarResources().getScalableTopicResources();
+        Awaitility.await().atMost(Duration.ofSeconds(15)).untilAsserted(() -> {
+            List<String> leaders = new ArrayList<>();
+            for (int i = 0; i < brokers.size(); i++) {
+                if (i != leaderIndex && brokers.get(i).getBrokerService().getScalableTopicService()
+                        .getOrCreateController(tn).get(10, TimeUnit.SECONDS).isLeader()) {
+                    leaders.add(brokers.get(i).getBrokerId());
+                }
+            }
+            assertThat(leaders).as("followers that took the controller over").hasSize(1);
+            Optional<GetResult> lock = resources.getStore().get(resources.controllerLockPath(tn)).get();
+            assertThat(lock).as("controller lock").isPresent();
+            assertThat(ObjectMapperFactory.getMapper().reader().readValue(lock.get().getValue(), String.class))
+                    .as("broker holding the controller lock").isEqualTo(leaders.get(0));
+        });
+    }
+
+    private void assertTopicStaysDeleted(String topic) throws Exception {
+        TopicName tn = TopicName.get(topic);
+        ScalableTopicResources resources = brokers.get(0).getPulsarResources().getScalableTopicResources();
+        Awaitility.await().during(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(15))
+                .untilAsserted(() -> {
+                    assertThat(resources.getStore().get(resources.controllerLockPath(tn)).get())
+                            .as("controller lock of the deleted topic").isEmpty();
+                    assertThat(resources.getStore().get(resources.topicPath(tn)).get())
+                            .as("record of the deleted topic").isEmpty();
+                });
+        assertThat(admin.scalableTopics().listScalableTopics(getNamespace())).doesNotContain(topic);
+        // No broker keeps a controller for the deleted topic: getting one creates a new controller,
+        // which fails to initialize.
+        Awaitility.await().atMost(Duration.ofSeconds(15)).untilAsserted(() -> {
+            for (PulsarService broker : brokers) {
+                assertThatThrownBy(() -> broker.getBrokerService().getScalableTopicService()
+                        .getOrCreateController(tn).get(10, TimeUnit.SECONDS))
+                        .as("controller of the deleted topic on " + broker.getBrokerId())
+                        .hasMessageContaining("Failed to initialize controller");
+            }
+        });
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ScalableTopicDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ScalableTopicDeleteTest.java
new file mode 100644
index 000000000000..04ed9c785382
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5ScalableTopicDeleteTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api.v5;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.resources.ScalableTopicResources;
+import org.apache.pulsar.client.api.v5.schema.Schema;
+import org.apache.pulsar.client.impl.ClientCnx;
+import org.apache.pulsar.client.impl.PulsarClientImpl;
+import org.apache.pulsar.common.api.proto.ScalableConsumerAssignment;
+import org.apache.pulsar.common.api.proto.ScalableConsumerType;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.protocol.Commands;
+import org.awaitility.Awaitility;
+import org.testng.annotations.Test;
+
+/**
+ * Deleting a scalable topic through the admin API must stop the topic's controller: once
+ * the delete returns, the controller must not re-elect itself under the deleted path, accept
+ * consumers against the deleted layout, or serve a topic re-created with the same name.
+ */
+public class V5ScalableTopicDeleteTest extends V5ClientBaseTest {
+
+    @Test
+    public void deleteReleasesElectedController() throws Exception {
+        String topic = newScalableTopic(1);
+        TopicName tn = TopicName.get(topic);
+        ScalableTopicResources resources = scalableTopicResources();
+        String lockPath = resources.controllerLockPath(tn);
+        electController(topic);
+        assertThat(resources.getStore().exists(lockPath).get())
+                .as("controller lock after the consumer registered").isTrue();
+
+        admin.scalableTopics().deleteScalableTopic(topic, true);
+
+        Awaitility.await().during(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(10))
+                .untilAsserted(() -> {
+                    assertThat(resources.getStore().exists(lockPath).get())
+                            .as("controller lock of the deleted topic").isFalse();
+                    assertThat(resources.getStore().exists(resources.topicPath(tn)).get())
+                            .as("record of the deleted topic").isFalse();
+                });
+    }
+
+    @Test
+    public void subscribeToDeletedTopicFails() throws Exception {
+        String topic = newScalableTopic(1);
+        TopicName tn = TopicName.get(topic);
+        electController(topic);
+
+        admin.scalableTopics().deleteScalableTopic(topic, true);
+
+        assertThatThrownBy(() -> subscribe(topic, "sub", "late-consumer"))
+                .hasMessageContaining("Failed to initialize controller");
+        ScalableTopicResources resources = scalableTopicResources();
+        assertThat(resources.getStore().exists(resources.consumerPath(tn, "sub", "late-consumer")).get())
+                .as("consumer registration under the deleted topic").isFalse();
+    }
+
+    @Test
+    public void recreatedTopicGetsFreshController() throws Exception {
+        String topic = newScalableTopic(1);
+        electController(topic);
+
+        admin.scalableTopics().deleteScalableTopic(topic, true);
+        admin.scalableTopics().createScalableTopic(topic, 2);
+
+        ScalableConsumerAssignment assignment = subscribe(topic, "sub", "consumer");
+        assertThat(assignment.getSegmentsCount())
+                .as("segments assigned from the re-created topic's layout").isEqualTo(2);
+    }
+
+    private ScalableTopicResources scalableTopicResources() {
+        return getPulsar().getPulsarResources().getScalableTopicResources();
+    }
+
+    /** A stream consumer registers with the topic's controller, which elects it on this broker. */
+    private void electController(String topic) throws Exception {
+        v5Client.newStreamConsumer(Schema.string())
+                .topic(topic)
+                .subscriptionName("elect")
+                .subscribe()
+                .close();
+    }
+
+    /** Send a raw scalable-topic subscribe to the broker and return the assignment it replies with. */
+    private ScalableConsumerAssignment subscribe(String topic, String subscription, String consumerName)
+            throws Exception {
+        @Cleanup
+        PulsarClientImpl client = (PulsarClientImpl) newPulsarClient();
+        ClientCnx cnx = client.getConnectionToServiceUrl().get(10, TimeUnit.SECONDS);
+        long requestId = client.newRequestId();
+        return cnx.<ScalableConsumerAssignment>sendScalableSessionRequest(
+                Commands.newScalableTopicSubscribe(requestId, topic, subscription, consumerName, 1L,
+                        ScalableConsumerType.STREAM),
+                requestId).get(10, TimeUnit.SECONDS);
+    }
+}


(pulsar) 04/06: [fix][meta] Don't leak closed leader elections and their metadata store listeners (#26879)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 740f8e4f20cb063606c39090a2c412db885c9627
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Oct 12 17:58:32 2026 -0700

    [fix][meta] Don't leak closed leader elections and their metadata store listeners (#26879)
    
    (cherry picked from commit 2b2fc043829f1d943750ed87d58ccb55ac6f9da8)
---
 .../api/coordination/CoordinationService.java      |  13 +-
 .../coordination/impl/CoordinationServiceImpl.java |  61 +++-
 .../coordination/impl/LeaderElectionImpl.java      |  81 ++++--
 .../impl/CoordinationServiceImplTest.java          | 314 +++++++++++++++++++++
 .../coordination/impl/LeaderElectionImplTest.java  | 146 +++++++++-
 5 files changed, 590 insertions(+), 25 deletions(-)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/CoordinationService.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/CoordinationService.java
index 708f91f4a33c..a371f551ed6d 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/CoordinationService.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/CoordinationService.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.metadata.api.coordination;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Consumer;
 import org.apache.pulsar.metadata.api.MetadataSerde;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
 
 /**
  * Interface for the coordination service. Provides abstraction for distributed locks and leader election.
@@ -28,7 +29,17 @@ import org.apache.pulsar.metadata.api.MetadataSerde;
 public interface CoordinationService extends AutoCloseable {
 
     /**
-     * Create a new {@link LeaderElection} controller.
+     * Get the {@link LeaderElection} controller of a path.
+     *
+     * <p>While an election for the path is open, it is returned and {@code stateChangesListener} is ignored: the
+     * election reports its state changes to the listener it was created with. Once it is closed, the next call
+     * creates a new election. The closed election can still be re-opened with {@link LeaderElection#elect(Object)},
+     * which fails with {@link MetadataStoreException.AlreadyClosedException} if a new election was created for the
+     * path in the meantime.
+     *
+     * <p>Since callers of the same path share its open election, a caller can get an election that another caller
+     * closes right after: re-opening it with {@link LeaderElection#elect(Object)} keeps the listener it was created
+     * with.
      *
      * @param clazz
      *            the class type to be used for serialization/deserialization
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/CoordinationServiceImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/CoordinationServiceImpl.java
index 89e96e4d748b..8917148be856 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/CoordinationServiceImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/CoordinationServiceImpl.java
@@ -18,9 +18,11 @@
  */
 package org.apache.pulsar.metadata.coordination.impl;
 
+import com.google.common.annotations.VisibleForTesting;
 import io.netty.util.concurrent.DefaultThreadFactory;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
@@ -37,12 +39,14 @@ import org.apache.bookkeeper.common.concurrent.FutureUtils;
 import org.apache.pulsar.common.util.GracefulExecutorServicesShutdown;
 import org.apache.pulsar.metadata.api.MetadataSerde;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.Notification;
 import org.apache.pulsar.metadata.api.coordination.CoordinationService;
 import org.apache.pulsar.metadata.api.coordination.LeaderElection;
 import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
 import org.apache.pulsar.metadata.api.coordination.LockManager;
 import org.apache.pulsar.metadata.api.extended.CreateOption;
 import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.api.extended.SessionEvent;
 
 @CustomLog
 @SuppressWarnings("unchecked")
@@ -55,7 +59,10 @@ public class CoordinationServiceImpl implements CoordinationService {
     private final MetadataStoreExtended store;
 
     private final Map<Object, LockManager<?>> lockManagers = new ConcurrentHashMap<>();
-    private final Map<String, LeaderElection<?>> leaderElections = new ConcurrentHashMap<>();
+    // The leader elections that are not closed, by path, to which the store notifications and session
+    // events are dispatched. A closed election is removed, and added back if it's re-opened.
+    private final Map<String, LeaderElectionImpl<?>> leaderElections = new ConcurrentHashMap<>();
+    private boolean leaderElectionListenersRegistered;
 
     private final ScheduledExecutorService executor;
 
@@ -161,7 +168,57 @@ public class CoordinationServiceImpl implements CoordinationService {
     public <T> LeaderElection<T> getLeaderElection(Class<T> clazz, String path,
             Consumer<LeaderElectionState> stateChangesListener) {
 
+        registerLeaderElectionListeners();
         return (LeaderElection<T>) leaderElections.computeIfAbsent(path,
-                key -> new LeaderElectionImpl<T>(store, clazz, path, stateChangesListener, executor));
+                key -> new LeaderElectionImpl<T>(this, store, clazz, path, stateChangesListener, executor));
+    }
+
+    /**
+     * Register the store listener and the session listener of the elections, once: the store can't unregister
+     * a listener, so the elections share them, and they dispatch to the elections that are not closed. The store
+     * calls its listeners in registration order, so every election sees a notification where the first election
+     * was created: after the listeners registered before it, like the one of the broker's metadata resources, and
+     * before the ones registered after it. That's why they are registered with the first election, rather than
+     * when this service is created.
+     */
+    private synchronized void registerLeaderElectionListeners() {
+        if (!leaderElectionListenersRegistered) {
+            store.registerListener(this::handleLeaderElectionNotification);
+            store.registerSessionListener(this::handleLeaderElectionSessionEvent);
+            leaderElectionListenersRegistered = true;
+        }
+    }
+
+    private void handleLeaderElectionNotification(Notification notification) {
+        LeaderElectionImpl<?> leaderElection = leaderElections.get(notification.getPath());
+        if (leaderElection != null) {
+            leaderElection.handlePathNotification(notification);
+        }
+    }
+
+    private void handleLeaderElectionSessionEvent(SessionEvent event) {
+        leaderElections.values().forEach(leaderElection -> leaderElection.handleSessionNotification(event));
+    }
+
+    /**
+     * Stop dispatching the store notifications and session events to a leader election that was closed.
+     */
+    void removeLeaderElection(String path, LeaderElectionImpl<?> leaderElection) {
+        leaderElections.remove(path, leaderElection);
+    }
+
+    /**
+     * Dispatch the store notifications and session events again to a closed leader election that is
+     * re-opened, unless another election was created for its path since it was closed.
+     *
+     * @return whether the election was added back
+     */
+    boolean readdLeaderElection(String path, LeaderElectionImpl<?> leaderElection) {
+        return leaderElections.putIfAbsent(path, leaderElection) == null;
+    }
+
+    @VisibleForTesting
+    Map<String, LeaderElectionImpl<?>> getLeaderElections() {
+        return Collections.unmodifiableMap(leaderElections);
     }
 }
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
index 302d18b574cd..d1e6cef0b943 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
@@ -47,6 +47,8 @@ import org.apache.pulsar.metadata.cache.impl.JSONMetadataSerdeSimpleType;
 
 @CustomLog
 class LeaderElectionImpl<T> implements LeaderElection<T> {
+    // Dispatches the store notifications and session events to this election while it's not closed
+    private final CoordinationServiceImpl coordinationService;
     private final String path;
     private final MetadataSerde<T> serde;
     private final MetadataStoreExtended store;
@@ -83,9 +85,10 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
         this.leaderElectionCompletionTimeoutSeconds = leaderElectionCompletionTimeoutSeconds;
     }
 
-    LeaderElectionImpl(MetadataStoreExtended store, Class<T> clazz, String path,
-            Consumer<LeaderElectionState> stateChangesListener,
+    LeaderElectionImpl(CoordinationServiceImpl coordinationService, MetadataStoreExtended store, Class<T> clazz,
+            String path, Consumer<LeaderElectionState> stateChangesListener,
                        ScheduledExecutorService executor) {
+        this.coordinationService = coordinationService;
         this.path = path;
         this.serde = new JSONMetadataSerdeSimpleType<>(TypeFactory.defaultInstance().constructSimpleType(clazz, null));
         this.store = store;
@@ -94,8 +97,6 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
         this.stateChangesListener = stateChangesListener;
         this.executor = executor;
         this.sequencer = FutureUtil.Sequencer.create();
-        store.registerListener(this::handlePathNotification);
-        store.registerSessionListener(this::handleSessionNotification);
     }
 
     /**
@@ -125,6 +126,12 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
         if (internalState == InternalState.Closed) {
             // Reopened after close() (e.g. the broker's LeaderElectionService is close()d and then
             // start()ed again): reset so a fresh election cycle runs and readers wait for it.
+            if (!coordinationService.readdLeaderElection(path, this)) {
+                // Both elections would compete for the same lock: the new one keeps the path
+                return FutureUtil.failedFuture(new AlreadyClosedException(
+                        "The leader election was closed and a new one was created on " + path));
+            }
+            internalState = InternalState.Init;
             leaderElectionState = LeaderElectionState.NoLeader;
             currentLeaderFuture = new CompletableFuture<>();
         }
@@ -137,6 +144,10 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
     }
 
     private synchronized CompletableFuture<LeaderElectionState> elect() {
+        if (internalState == InternalState.Closed) {
+            // Closed while a retry or a revalidation was pending: only elect(T) re-opens the election
+            return FutureUtil.failedFuture(new AlreadyClosedException("The leader election was already closed"));
+        }
         // First check if there's already a leader elected
         internalState = InternalState.ElectionInProgress;
         return store.get(path).thenCompose(optLock -> {
@@ -149,6 +160,11 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
     }
 
     private synchronized CompletableFuture<LeaderElectionState> handleExistingLeaderValue(GetResult res) {
+        if (internalState == InternalState.Closed) {
+            // Closed while the lock was being read: the value read must not bring the election back, nor make it
+            // delete the lock
+            return FutureUtil.failedFuture(new AlreadyClosedException("The leader election was already closed"));
+        }
         T existingValue;
         try {
             existingValue = serde.deserialize(path, res.getValue(), res.getStat());
@@ -206,6 +222,11 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
     }
 
     private synchronized CompletableFuture<LeaderElectionState> tryToBecomeLeader() {
+        if (internalState == InternalState.Closed) {
+            // Closed while the lock was read or deleted: don't take it. A close while the lock is written is
+            // handled once the write completes.
+            return FutureUtil.failedFuture(new AlreadyClosedException("The leader election was already closed"));
+        }
         T value = proposedValue.get();
         byte[] payload;
         try {
@@ -284,6 +305,7 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
         }
 
         internalState = InternalState.Closed;
+        coordinationService.removeLeaderElection(path, this);
         // A closed election reports "no leader" rather than waiting or failing: callers like the
         // extensible load manager's handleNoChannelOwnerError() key off the resulting
         // "no channel owner" condition to restart the election.
@@ -371,14 +393,29 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
         return future.isDone() && !future.isCompletedExceptionally() ? future.join() : Optional.empty();
     }
 
-    private void handleSessionNotification(SessionEvent event) {
+    void handleSessionNotification(SessionEvent event) {
         // Ensure we're only processing one session event at a time.
         sequencer.sequential(() -> FutureUtil.composeAsync(() -> {
             if (event == SessionEvent.Reconnected || event == SessionEvent.SessionReestablished) {
-                log.info().attr("path", path).attr("event", event).log("Revalidating leadership");
-                return elect().thenAccept(leaderState -> {
+                CompletableFuture<LeaderElectionState> revalidation;
+                synchronized (LeaderElectionImpl.this) {
+                    if (internalState == InternalState.Init || internalState == InternalState.Closed) {
+                        // Not taking part in the election: elect(T) was never called, or the election was closed
+                        return CompletableFuture.completedFuture(null);
+                    }
+                    log.info().attr("path", path).attr("event", event).log("Revalidating leadership");
+                    revalidation = elect();
+                }
+                return revalidation.thenAccept(leaderState -> {
                     log.info().attr("path", path).attr("state", leaderState).log("Resynced leadership");
                 }).exceptionally(ex -> {
+                    synchronized (LeaderElectionImpl.this) {
+                        if (internalState == InternalState.Closed) {
+                            log.debug().attr("path", path)
+                                    .log("Leader election closed while revalidating leadership");
+                            return null;
+                        }
+                    }
                     log.warn().exception(ex).log("Failure when processing session event");
                     return null;
                 });
@@ -387,12 +424,8 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
         }, executor));
     }
 
-    private void handlePathNotification(Notification notification) {
-        if (!path.equals(notification.getPath())) {
-            // Ignore notifications we don't care about
-            return;
-        }
-
+    // Called for the notifications on the path of this election
+    void handlePathNotification(Notification notification) {
         synchronized (this) {
             if (internalState != InternalState.LeaderIsPresent) {
                 // Ignoring notification since we're not trying to become leader
@@ -413,20 +446,28 @@ class LeaderElectionImpl<T> implements LeaderElection<T> {
                 if (proposedValue.isPresent()) {
                     elect()
                             .exceptionally(ex -> {
-                                log.warn().attr("path", path).exception(ex).log("Leader election has failed");
                                 synchronized (LeaderElectionImpl.this) {
+                                    if (internalState == InternalState.Closed) {
+                                        // Closed while re-electing: there's nothing to report nor to retry
+                                        log.debug().attr("path", path)
+                                                .log("Leader election closed while re-electing");
+                                        return null;
+                                    }
+                                    log.warn().attr("path", path).exception(ex).log("Leader election has failed");
                                     try {
                                         stateChangesListener.accept(leaderElectionState);
                                     } catch (Throwable t) {
                                         log.warn().exception(t).log("Exception in state change listener");
                                     }
 
-                                    if (internalState != InternalState.Closed) {
-                                        executor.schedule(() -> {
-                                            log.info().attr("path", path).log("Retrying Leader election");
-                                            elect();
-                                        }, LEADER_ELECTION_RETRY_DELAY_SECONDS, TimeUnit.SECONDS);
-                                    }
+                                    executor.schedule(() -> {
+                                        synchronized (LeaderElectionImpl.this) {
+                                            if (internalState != InternalState.Closed) {
+                                                log.info().attr("path", path).log("Retrying Leader election");
+                                                elect();
+                                            }
+                                        }
+                                    }, LEADER_ELECTION_RETRY_DELAY_SECONDS, TimeUnit.SECONDS);
                                 }
                                 return null;
                             });
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/CoordinationServiceImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/CoordinationServiceImplTest.java
new file mode 100644
index 000000000000..4fd0f0762a4d
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/CoordinationServiceImplTest.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata.coordination.impl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+import lombok.Cleanup;
+import org.apache.pulsar.metadata.api.GetResult;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreException.AlreadyClosedException;
+import org.apache.pulsar.metadata.api.Notification;
+import org.apache.pulsar.metadata.api.Option;
+import org.apache.pulsar.metadata.api.Stat;
+import org.apache.pulsar.metadata.api.coordination.LeaderElection;
+import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
+import org.apache.pulsar.metadata.api.extended.SessionEvent;
+import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore;
+import org.apache.pulsar.metadata.impl.LocalMemoryMetadataStore;
+import org.awaitility.Awaitility;
+import org.testng.annotations.Test;
+
+/**
+ * The leader elections of {@link CoordinationServiceImpl}: how they are tracked, and how a closed one stays out of
+ * the election.
+ */
+public class CoordinationServiceImplTest {
+
+    private static final String PATH = "/leader";
+
+    @Test(timeOut = 30000)
+    public void closedElectionIsDropped() throws Exception {
+        @Cleanup
+        TestStore store = new TestStore();
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
+
+        // Like the controllers of scalable topics that are created and deleted: one election per topic
+        for (int i = 0; i < 3; i++) {
+            String path = PATH + "-" + i;
+            LeaderElection<String> election = cs.getLeaderElection(String.class, path, __ -> {
+            });
+            assertThat(election.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+            assertThat(cs.getLeaderElections().get(path)).isSameAs(election);
+
+            election.close();
+            assertThat(cs.getLeaderElections()).as("elections after closing %s", path).doesNotContainKey(path);
+        }
+
+        // The store can't unregister a listener: one listener of each kind dispatches to the open elections,
+        // so the closed ones are no longer reachable from the store
+        assertThat(store.listenerCount).as("store listeners").hasValue(1);
+        assertThat(store.sessionListenerCount).as("session listeners").hasValue(1);
+    }
+
+    @Test(timeOut = 30000)
+    public void electionCreatedAfterACloseReportsToItsOwnListener() throws Exception {
+        @Cleanup
+        TestStore store = new TestStore();
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
+
+        List<LeaderElectionState> firstStates = new CopyOnWriteArrayList<>();
+        LeaderElection<String> first = cs.getLeaderElection(String.class, PATH, firstStates::add);
+        assertThat(first.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+        first.close();
+
+        // Like the controller of a scalable topic created again after the previous one was closed
+        List<LeaderElectionState> secondStates = new CopyOnWriteArrayList<>();
+        LeaderElection<String> second = cs.getLeaderElection(String.class, PATH, secondStates::add);
+        assertThat(second).isNotSameAs(first);
+        assertThat(second.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+        // The deletion of the first lock can reach the second election after it took the lock, and make it
+        // revalidate its leadership: it may report Leading more than once
+        assertThat(secondStates).as("states reported to the second listener").containsOnly(
+                LeaderElectionState.Leading);
+        assertThat(firstStates).as("states reported to the first listener").containsExactly(
+                LeaderElectionState.Leading);
+    }
+
+    @Test(timeOut = 30000)
+    public void reopenedElectionKeepsReceivingNotifications() throws Exception {
+        @Cleanup
+        TestStore store = new TestStore();
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
+
+        LeaderElection<String> election = cs.getLeaderElection(String.class, PATH, __ -> {
+        });
+        assertThat(election.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+        election.close();
+
+        // Like the broker's LeaderElectionService, closed and started again
+        assertThat(election.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+        assertThat(cs.getLeaderElections().get(PATH)).isSameAs(election);
+
+        // It still sees its lock deleted, and takes it again
+        store.delete(PATH, Optional.empty()).join();
+        Awaitility.await().untilAsserted(() ->
+                assertThat(store.get(PATH).join()).as("lock taken again").isPresent());
+
+        // And closing the coordination service closes it, which releases the lock
+        cs.close();
+        assertThat(election.getState()).isEqualTo(LeaderElectionState.NoLeader);
+        assertThat(store.get(PATH).join()).as("lock after closing the coordination service").isEmpty();
+    }
+
+    @Test(timeOut = 30000)
+    public void closedElectionIsNotReopenedOverANewerOne() throws Exception {
+        @Cleanup
+        TestStore store = new TestStore();
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
+
+        LeaderElection<String> first = cs.getLeaderElection(String.class, PATH, __ -> {
+        });
+        assertThat(first.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+        first.close();
+        LeaderElection<String> second = cs.getLeaderElection(String.class, PATH, __ -> {
+        });
+        assertThat(second.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+
+        // Both would compete for the same lock: the newer election keeps the path
+        assertThatThrownBy(() -> first.elect("broker-1").join()).hasCauseInstanceOf(AlreadyClosedException.class);
+        assertThat(first.getState()).isEqualTo(LeaderElectionState.NoLeader);
+        assertThat(cs.getLeaderElections().get(PATH)).isSameAs(second);
+        assertThat(second.getState()).isEqualTo(LeaderElectionState.Leading);
+    }
+
+    @Test(timeOut = 30000)
+    public void closedElectionIgnoresSessionEvents() throws Exception {
+        @Cleanup
+        TestStore store = new TestStore();
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
+
+        List<LeaderElectionState> states = new CopyOnWriteArrayList<>();
+        LeaderElection<String> election = cs.getLeaderElection(String.class, PATH, states::add);
+        assertThat(election.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+        election.close();
+
+        store.triggerSessionEvent(SessionEvent.SessionReestablished);
+        awaitQueuedTasks(election);
+
+        assertThat(store.get(PATH).join()).as("lock of the closed election").isEmpty();
+        assertThat(election.getState()).isEqualTo(LeaderElectionState.NoLeader);
+        assertThat(states).as("reported states").containsExactly(LeaderElectionState.Leading);
+    }
+
+    @Test(timeOut = 30000)
+    public void electionClosedWithASessionEventPendingIgnoresIt() throws Exception {
+        @Cleanup
+        TestStore store = new TestStore();
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
+
+        List<LeaderElectionState> states = new CopyOnWriteArrayList<>();
+        LeaderElection<String> election = cs.getLeaderElection(String.class, PATH, states::add);
+        assertThat(election.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+
+        // Hold the executor, so that the session event reaches the election while it's open, and is processed
+        // once it's closed
+        CountDownLatch release = new CountDownLatch(1);
+        ((LeaderElectionImpl<?>) election).getSchedulerExecutor().execute(() -> {
+            try {
+                release.await();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        });
+        try {
+            store.triggerSessionEvent(SessionEvent.SessionReestablished);
+            election.close();
+        } finally {
+            release.countDown();
+        }
+        awaitQueuedTasks(election);
+
+        assertThat(store.get(PATH).join()).as("lock of the closed election").isEmpty();
+        assertThat(election.getState()).isEqualTo(LeaderElectionState.NoLeader);
+        assertThat(states).as("reported states").containsExactly(LeaderElectionState.Leading);
+    }
+
+    @Test(timeOut = 30000)
+    public void neverElectedElectionIgnoresSessionEvents() throws Exception {
+        @Cleanup
+        TestStore store = new TestStore();
+        @Cleanup
+        CoordinationServiceImpl leaderCs = new CoordinationServiceImpl(store);
+        @Cleanup
+        CoordinationServiceImpl observerCs = new CoordinationServiceImpl(store);
+
+        LeaderElection<String> leader = leaderCs.getLeaderElection(String.class, PATH, __ -> {
+        });
+        assertThat(leader.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+
+        // Only reads the leader, like BookKeeper's tools reading the current auditor
+        LeaderElectionImpl<String> observer = (LeaderElectionImpl<String>) observerCs.getLeaderElection(
+                String.class, PATH, __ -> {
+                });
+        observer.setLeaderElectionCompletionTimeoutSeconds(1);
+
+        store.triggerSessionEvent(SessionEvent.SessionReestablished);
+        awaitQueuedTasks(observer);
+
+        // It doesn't take part in the election, so it keeps reading the leader from the store
+        assertThat(observer.getLeaderValue()).succeedsWithin(Duration.ofSeconds(5))
+                .isEqualTo(Optional.of("broker-1"));
+        assertThat(observer.getState()).isEqualTo(LeaderElectionState.NoLeader);
+    }
+
+    @Test(timeOut = 30000)
+    public void electionClosedWhileReelectingDoesNotTakeTheLock() throws Exception {
+        @Cleanup
+        TestStore store = new TestStore();
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
+
+        List<LeaderElectionState> states = new CopyOnWriteArrayList<>();
+        LeaderElection<String> election = cs.getLeaderElection(String.class, PATH, states::add);
+        assertThat(election.elect("broker-1").join()).isEqualTo(LeaderElectionState.Leading);
+
+        // The lock is deleted, and the election is closed while it reads the lock to elect again
+        CompletableFuture<Void> readGate = new CompletableFuture<>();
+        store.readGate = readGate;
+        store.delete(PATH, Optional.empty()).join();
+        Awaitility.await().untilAsserted(() -> assertThat(store.heldReads).hasValue(1));
+        election.close();
+        store.writes.clear();
+        store.readGate = null;
+        readGate.complete(null);
+
+        assertThat(store.writes).as("paths written after the close").doesNotContain(PATH);
+        assertThat(store.get(PATH).join()).as("lock of the closed election").isEmpty();
+        assertThat(election.getState()).isEqualTo(LeaderElectionState.NoLeader);
+        assertThat(states).as("reported states").containsExactly(LeaderElectionState.Leading);
+    }
+
+    // The elections do their work, e.g. processing a session event, on the coordination service's executor
+    private static void awaitQueuedTasks(LeaderElection<?> election) throws Exception {
+        ((LeaderElectionImpl<?>) election).getSchedulerExecutor().submit(() -> {
+        }).get();
+    }
+
+    /**
+     * An in-memory store that counts the listeners registered on it, can trigger session events, records the
+     * paths written, and can hold the reads until the test lets them go.
+     */
+    private static class TestStore extends FaultInjectionMetadataStore {
+        private final AtomicInteger listenerCount = new AtomicInteger();
+        private final AtomicInteger sessionListenerCount = new AtomicInteger();
+        private final List<String> writes = new CopyOnWriteArrayList<>();
+        private final AtomicInteger heldReads = new AtomicInteger();
+        // While set, the reads wait for it to complete
+        private volatile CompletableFuture<Void> readGate;
+
+        TestStore() throws Exception {
+            super(new LocalMemoryMetadataStore("memory:local", MetadataStoreConfig.builder().build()));
+        }
+
+        @Override
+        public CompletableFuture<Optional<GetResult>> get(String path, Set<Option> opts) {
+            CompletableFuture<Void> gate = readGate;
+            if (gate == null) {
+                return super.get(path, opts);
+            }
+            heldReads.incrementAndGet();
+            return gate.thenCompose(__ -> super.get(path, opts));
+        }
+
+        @Override
+        public CompletableFuture<Stat> put(String path, byte[] value, Optional<Long> expectedVersion,
+                                           Set<Option> opts) {
+            writes.add(path);
+            return super.put(path, value, expectedVersion, opts);
+        }
+
+        @Override
+        public void registerListener(Consumer<Notification> listener) {
+            listenerCount.incrementAndGet();
+            super.registerListener(listener);
+        }
+
+        @Override
+        public void registerSessionListener(Consumer<SessionEvent> listener) {
+            sessionListenerCount.incrementAndGet();
+            super.registerSessionListener(listener);
+        }
+    }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java
index 4827c4a197ad..0a64206023d5 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java
@@ -18,20 +18,45 @@
  */
 package org.apache.pulsar.metadata.coordination.impl;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
 import lombok.Cleanup;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.BaseMetadataStoreTest;
+import org.apache.pulsar.metadata.api.GetResult;
 import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.MetadataStoreException.AlreadyClosedException;
+import org.apache.pulsar.metadata.api.MetadataStoreException.BadVersionException;
+import org.apache.pulsar.metadata.api.Notification;
+import org.apache.pulsar.metadata.api.NotificationType;
+import org.apache.pulsar.metadata.api.Option;
+import org.apache.pulsar.metadata.api.Stat;
 import org.apache.pulsar.metadata.api.coordination.CoordinationService;
+import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
 import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.api.extended.SessionEvent;
+import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore;
+import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore.OperationType;
+import org.apache.pulsar.metadata.impl.LocalMemoryMetadataStore;
+import org.mockito.ArgumentCaptor;
 import org.testng.annotations.Test;
 
 public class LeaderElectionImplTest extends BaseMetadataStoreTest {
@@ -71,14 +96,16 @@ public class LeaderElectionImplTest extends BaseMetadataStoreTest {
     }
 
     @Test(timeOut = 20000)
-    public void getLeaderValueTimesOutWhenElectionNeverCompletes() {
+    public void getLeaderValueTimesOutWhenElectionNeverCompletes() throws Exception {
         MetadataStoreExtended store = mock(MetadataStoreExtended.class);
         // The store never answers, so the election never settles.
         when(store.get(anyString())).thenReturn(new CompletableFuture<>());
 
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
         @Cleanup("shutdownNow")
         ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
-        LeaderElectionImpl<String> le = new LeaderElectionImpl<>(store, String.class,
+        LeaderElectionImpl<String> le = new LeaderElectionImpl<>(cs, store, String.class,
                 "/getLeaderValueTimesOutWhenElectionNeverCompletes", __ -> {
         }, executor);
         le.setLeaderElectionCompletionTimeoutSeconds(1);
@@ -90,4 +117,119 @@ public class LeaderElectionImplTest extends BaseMetadataStoreTest {
                 .cause()
                 .hasMessageContaining("did not complete within");
     }
+
+    @Test(timeOut = 20000)
+    public void closedElectionDoesNotRetryTheElection() throws Exception {
+        String path = "/closedElectionDoesNotRetryTheElection";
+        @Cleanup
+        FaultInjectionMetadataStore store = new FaultInjectionMetadataStore(
+                new LocalMemoryMetadataStore("memory:local", MetadataStoreConfig.builder().build()));
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
+        // Lets the test run the retry the election schedules
+        ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
+        LeaderElectionImpl<String> le = new LeaderElectionImpl<>(cs, store, String.class, path, __ -> {
+        }, executor);
+        assertThat(le.elect("test-1").join()).isEqualTo(LeaderElectionState.Leading);
+
+        // The lock is deleted and the election fails to take it again: it schedules a retry
+        store.delete(path, Optional.empty()).join();
+        store.failConditional(new MetadataStoreException("injected"),
+                (op, p) -> op == OperationType.GET && p.equals(path));
+        le.handlePathNotification(new Notification(NotificationType.Deleted, path));
+        ArgumentCaptor<Runnable> retry = ArgumentCaptor.forClass(Runnable.class);
+        verify(executor).schedule(retry.capture(), anyLong(), any(TimeUnit.class));
+
+        le.close();
+        retry.getValue().run();
+
+        assertThat(store.get(path).join()).as("lock of the closed election").isEmpty();
+        assertThat(le.getState()).isEqualTo(LeaderElectionState.NoLeader);
+    }
+
+    @Test(timeOut = 20000)
+    public void closedElectionDoesNotRetryAfterAConflict() throws Exception {
+        String path = "/closedElectionDoesNotRetryAfterAConflict";
+        AtomicBoolean conflictOnNextPut = new AtomicBoolean();
+        CompletableFuture<Void> conflictReported = new CompletableFuture<>();
+        @Cleanup
+        FaultInjectionMetadataStore store = new FaultInjectionMetadataStore(
+                new LocalMemoryMetadataStore("memory:local", MetadataStoreConfig.builder().build())) {
+            @Override
+            public CompletableFuture<Stat> put(String p, byte[] value, Optional<Long> expectedVersion,
+                                               Set<Option> opts) {
+                if (p.equals(path) && conflictOnNextPut.compareAndSet(true, false)) {
+                    // Another participant took the lock first: the conflict is reported once the test
+                    // releases it
+                    return conflictReported.thenCompose(
+                            __ -> FutureUtil.failedFuture(new BadVersionException("injected")));
+                }
+                return super.put(p, value, expectedVersion, opts);
+            }
+        };
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
+        @Cleanup("shutdownNow")
+        ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+        BlockingQueue<LeaderElectionState> stateChanges = new LinkedBlockingQueue<>();
+        LeaderElectionImpl<String> le = new LeaderElectionImpl<>(cs, store, String.class, path, stateChanges::add,
+                executor);
+
+        conflictOnNextPut.set(true);
+        CompletableFuture<LeaderElectionState> election = le.elect("test-1");
+        // Closed while taking the lock, before the conflict is reported
+        le.close();
+        conflictReported.complete(null);
+
+        // The conflict doesn't run the closed election again
+        assertThatThrownBy(() -> election.get(5, TimeUnit.SECONDS))
+                .hasCauseInstanceOf(AlreadyClosedException.class);
+        assertThat(stateChanges).as("state changes").isEmpty();
+        assertThat(le.getState()).isEqualTo(LeaderElectionState.NoLeader);
+        assertThat(store.get(path).join()).as("lock of the closed election").isEmpty();
+    }
+
+    @Test(timeOut = 20000)
+    public void electionClosedWhileReadingTheLockStaysClosed() throws Exception {
+        String path = "/electionClosedWhileReadingTheLockStaysClosed";
+        AtomicBoolean holdNextRead = new AtomicBoolean();
+        CompletableFuture<Void> heldReadReleased = new CompletableFuture<>();
+        @Cleanup
+        FaultInjectionMetadataStore store = new FaultInjectionMetadataStore(
+                new LocalMemoryMetadataStore("memory:local", MetadataStoreConfig.builder().build())) {
+            @Override
+            public CompletableFuture<Optional<GetResult>> get(String p, Set<Option> opts) {
+                CompletableFuture<Optional<GetResult>> result = super.get(p, opts);
+                // The read completes once the test releases it
+                return p.equals(path) && holdNextRead.compareAndSet(true, false)
+                        ? result.thenCombine(heldReadReleased, (r, __) -> r)
+                        : result;
+            }
+        };
+        @Cleanup
+        CoordinationServiceImpl cs = new CoordinationServiceImpl(store);
+        @Cleanup("shutdownNow")
+        ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+        BlockingQueue<LeaderElectionState> stateChanges = new LinkedBlockingQueue<>();
+        LeaderElectionImpl<String> le = new LeaderElectionImpl<>(cs, store, String.class, path, stateChanges::add,
+                executor);
+        assertThat(le.elect("test-1").join()).isEqualTo(LeaderElectionState.Leading);
+        assertThat(stateChanges.poll()).isEqualTo(LeaderElectionState.Leading);
+
+        // A session event revalidates the leadership, and the election is closed while the revalidation
+        // reads the lock
+        holdNextRead.set(true);
+        le.handleSessionNotification(SessionEvent.SessionReestablished);
+        // Once this runs, the revalidation is waiting for the read
+        executor.submit(() -> {
+        }).get();
+        le.close();
+        // The read returns the lock as it was before close() deleted it
+        heldReadReleased.complete(null);
+
+        // That doesn't bring the closed election back
+        assertThat(stateChanges).as("state changes after close()").isEmpty();
+        assertThat(le.getState()).isEqualTo(LeaderElectionState.NoLeader);
+        assertThat(le.getLeaderValue().join()).isEmpty();
+    }
 }


(pulsar) 01/06: [fix][client] Avoid duplicate reconcile retries and queue layout stalls in V5 consumers (#26874)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 5b509a5c668e12b88d3625daf9c89238cee2ea55
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Oct 12 14:59:17 2026 -0700

    [fix][client] Avoid duplicate reconcile retries and queue layout stalls in V5 consumers (#26874)
    
    (cherry picked from commit 1ed5a81b43874a4d1411f57d208e242d6fabee06)
---
 .../client/impl/v5/ScalableQueueConsumer.java      |  90 ++++--
 .../client/impl/v5/ScalableStreamConsumer.java     |  75 ++++-
 .../v5/ScalableQueueConsumerReconcileTest.java     | 291 ++++++++++++++++++
 .../v5/ScalableStreamConsumerReconcileTest.java    | 341 +++++++++++++++++++++
 4 files changed, 756 insertions(+), 41 deletions(-)

diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumer.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumer.java
index 085dfdedec10..02e40c15078d 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumer.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumer.java
@@ -107,8 +107,17 @@ final class ScalableQueueConsumer<T> implements QueueConsumerImpl<T>, DagWatchCl
 
     /** Most recent layout target. Reconciles always converge toward this value. */
     private volatile ClientSegmentLayout latestLayout;
-    /** Coalesces concurrent reconcile attempts; only one runs at a time. */
+    /** Set when the segment consumers must be converged onto {@link #latestLayout} again. */
+    private final AtomicBoolean reconcileRequested = new AtomicBoolean(false);
+    /** Held while a layout is applied, so only one is applied at a time. */
     private final AtomicBoolean reconcileInProgress = new AtomicBoolean(false);
+    /** Whether a retry of the segments that failed to subscribe is scheduled. Guarded by {@code this}. */
+    private boolean retryScheduled = false;
+    /** Whether the backoff was reset after the pending retry was scheduled. Guarded by {@code this}. */
+    private boolean backoffResetSinceRetry = false;
+    /** Identifies the latest retry: a retry it replaced does nothing. Guarded by {@code this}. */
+    private long retryGeneration = 0;
+    /** Paces the retries. Guarded by {@code this}. */
     private final Backoff reconcileBackoff = Backoff.builder()
             .initialDelay(Duration.ofMillis(100))
             .maxBackoff(Duration.ofSeconds(30))
@@ -308,43 +317,68 @@ final class ScalableQueueConsumer<T> implements QueueConsumerImpl<T>, DagWatchCl
     @Override
     public void onLayoutChange(ClientSegmentLayout newLayout, ClientSegmentLayout oldLayout) {
         // Fully async: safe to run on the netty IO thread that delivered the update.
-        // Store the target and kick off a reconcile. If a reconcile is already running,
-        // it will re-read latestLayout once it finishes. On failure, the reconcile
-        // reschedules itself with exponential backoff until it succeeds (or we close).
         latestLayout = newLayout;
         reconcile();
     }
 
+    /**
+     * Converge the per-segment consumers onto {@link #latestLayout}. A layout is applied as soon as
+     * it arrives, not after the consumers of the previous one have subscribed: a subscribe that keeps
+     * reconnecting must not delay subscribing the segments a split creates. A consumer that fails to
+     * subscribe is evicted and retried with backoff.
+     */
     private void reconcile() {
-        if (closed) {
-            return;
-        }
-        if (!reconcileInProgress.compareAndSet(false, true)) {
-            // A reconcile is already in progress; it will observe the updated latestLayout
-            // when it finishes and re-run if needed.
-            return;
+        reconcileRequested.set(true);
+        // One thread applies at a time, and applies again if asked to meanwhile.
+        while (!closed && reconcileRequested.get() && reconcileInProgress.compareAndSet(false, true)) {
+            try {
+                reconcileRequested.set(false);
+                evictFailedSegmentConsumers();
+                subscribeSegments(latestLayout).whenComplete((__, ex) -> {
+                    if (ex == null) {
+                        synchronized (this) {
+                            reconcileBackoff.reset();
+                            backoffResetSinceRetry = retryScheduled;
+                        }
+                    } else if (!closed) {
+                        scheduleRetry(ex);
+                    }
+                });
+            } finally {
+                reconcileInProgress.set(false);
+            }
         }
-        ClientSegmentLayout target = latestLayout;
-        subscribeSegments(target).whenComplete((__, ex) -> {
-            reconcileInProgress.set(false);
-            if (closed) {
+    }
+
+    /**
+     * Retry the segments that failed to subscribe after a backoff, with at most one retry scheduled.
+     * A retry scheduled before the backoff was last reset is replaced, so a new failure doesn't wait
+     * out the longer delay of earlier ones.
+     */
+    private void scheduleRetry(Throwable ex) {
+        Duration delay;
+        long generation;
+        synchronized (this) {
+            if (retryScheduled && !backoffResetSinceRetry) {
                 return;
             }
-            if (ex == null) {
-                reconcileBackoff.reset();
-                // If a newer layout arrived during this reconcile, run again to converge.
-                if (latestLayout != target) {
-                    reconcile();
+            retryScheduled = true;
+            backoffResetSinceRetry = false;
+            generation = ++retryGeneration;
+            delay = reconcileBackoff.next();
+        }
+        log.warn().attr("delayMs", delay.toMillis()).exceptionMessage(ex)
+                .log("Failed to apply layout update, retrying after backoff");
+        scheduler().schedule(() -> {
+            synchronized (this) {
+                if (generation != retryGeneration) {
+                    // Replaced by a later retry.
+                    return;
                 }
-                return;
+                retryScheduled = false;
             }
-            // Evict failed subscribe futures so the next attempt can re-try them.
-            evictFailedSegmentConsumers();
-            Duration delay = reconcileBackoff.next();
-            log.warn().attr("delayMs", delay.toMillis()).exceptionMessage(ex)
-                    .log("Failed to apply layout update, retrying after backoff");
-            scheduler().schedule(this::reconcile, delay.toMillis(), TimeUnit.MILLISECONDS);
-        });
+            reconcile();
+        }, delay.toMillis(), TimeUnit.MILLISECONDS);
     }
 
     private void evictFailedSegmentConsumers() {
diff --git a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumer.java b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumer.java
index e48d901c3779..9764b27876c0 100644
--- a/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumer.java
+++ b/pulsar-client-v5/src/main/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumer.java
@@ -182,6 +182,13 @@ final class ScalableStreamConsumer<T>
     private volatile List<ActiveSegment> latestAssignment;
     /** Coalesces concurrent reconcile attempts; only one runs at a time. */
     private final AtomicBoolean reconcileInProgress = new AtomicBoolean(false);
+    /** Whether a reconcile retry is scheduled. Guarded by {@code this}. */
+    private boolean retryScheduled = false;
+    /** Whether the backoff was reset after the pending retry was scheduled. Guarded by {@code this}. */
+    private boolean backoffResetSinceRetry = false;
+    /** Identifies the latest retry: a retry it replaced does nothing. Guarded by {@code this}. */
+    private long retryGeneration = 0;
+    /** Paces the retries. Guarded by {@code this}. */
     private final Backoff reconcileBackoff = Backoff.builder()
             .initialDelay(Duration.ofMillis(100))
             .maxBackoff(Duration.ofSeconds(30))
@@ -542,7 +549,9 @@ final class ScalableStreamConsumer<T>
                                          CompletableFuture<Void> result) {
         subscribeAssigned(assigned).whenComplete((__, ex) -> {
             if (ex == null) {
-                reconcileBackoff.reset();
+                synchronized (this) {
+                    reconcileBackoff.reset();
+                }
                 result.complete(null);
                 return;
             }
@@ -556,7 +565,10 @@ final class ScalableStreamConsumer<T>
                 return;
             }
             evictFailedSegmentConsumers();
-            Duration delay = reconcileBackoff.next();
+            Duration delay;
+            synchronized (this) {
+                delay = reconcileBackoff.next();
+            }
             log.info().attr("delayMs", delay.toMillis()).exceptionMessage(ex)
                     .log("Initial subscribe rejected during rebalance, retrying after backoff");
             // Retry with the session's current assignment, not the one this attempt started with: the
@@ -578,11 +590,18 @@ final class ScalableStreamConsumer<T>
      * consumer may not have released a segment or bucket range yet, so our subscribe is rejected
      * ({@code ConsumerBusy} on Exclusive, {@code ConsumerAssignError} on overlapping STICKY
      * ranges) — and neither is retried at the v4 layer.
+     *
+     * <p>The next reconcile starts only once this one's releases have drained and its subscribes
+     * have settled. A segment's release state (drain bookkeeping, receive epoch, acked positions)
+     * is keyed by segment id, so applying a newer assignment meanwhile could re-subscribe, or
+     * release again, a segment that is still draining.
      */
     private void reconcile() {
         if (closed || !reconcileInProgress.compareAndSet(false, true)) {
             return;
         }
+        // Evict failed subscribe futures so this attempt re-tries them.
+        evictFailedSegmentConsumers();
         List<ActiveSegment> target = latestAssignment;
         subscribeAssigned(target).whenComplete((__, ex) -> {
             reconcileInProgress.set(false);
@@ -590,22 +609,52 @@ final class ScalableStreamConsumer<T>
                 return;
             }
             if (ex == null) {
-                reconcileBackoff.reset();
-                // If a newer assignment arrived during this reconcile, run again to converge.
-                if (latestAssignment != target) {
-                    reconcile();
+                synchronized (this) {
+                    reconcileBackoff.reset();
+                    backoffResetSinceRetry = retryScheduled;
                 }
-                return;
             }
-            // Evict failed subscribe futures so the next attempt can re-try them.
-            evictFailedSegmentConsumers();
-            Duration delay = reconcileBackoff.next();
-            log.warn().attr("delayMs", delay.toMillis()).exceptionMessage(ex)
-                    .log("Failed to apply assignment update, retrying after backoff");
-            scheduler().schedule(this::reconcile, delay.toMillis(), TimeUnit.MILLISECONDS);
+            // If a newer assignment arrived during this reconcile, run again to converge: that run also
+            // re-tries what failed in this one.
+            if (latestAssignment != target) {
+                reconcile();
+            } else if (ex != null) {
+                scheduleRetry(ex);
+            }
         });
     }
 
+    /**
+     * Retry the reconcile after a backoff, with at most one retry scheduled. A retry scheduled before
+     * the backoff was last reset is replaced, so a new failure doesn't wait out the longer delay of
+     * earlier ones.
+     */
+    private void scheduleRetry(Throwable ex) {
+        Duration delay;
+        long generation;
+        synchronized (this) {
+            if (retryScheduled && !backoffResetSinceRetry) {
+                return;
+            }
+            retryScheduled = true;
+            backoffResetSinceRetry = false;
+            generation = ++retryGeneration;
+            delay = reconcileBackoff.next();
+        }
+        log.warn().attr("delayMs", delay.toMillis()).exceptionMessage(ex)
+                .log("Failed to apply assignment update, retrying after backoff");
+        scheduler().schedule(() -> {
+            synchronized (this) {
+                if (generation != retryGeneration) {
+                    // Replaced by a later retry.
+                    return;
+                }
+                retryScheduled = false;
+            }
+            reconcile();
+        }, delay.toMillis(), TimeUnit.MILLISECONDS);
+    }
+
     private void evictFailedSegmentConsumers() {
         for (var entry : segmentConsumers.entrySet()) {
             var future = entry.getValue();
diff --git a/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumerReconcileTest.java b/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumerReconcileTest.java
new file mode 100644
index 000000000000..982a947dd1d5
--- /dev/null
+++ b/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/ScalableQueueConsumerReconcileTest.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.impl.v5;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import com.google.common.util.concurrent.MoreExecutors;
+import io.netty.util.Timer;
+import java.lang.reflect.Proxy;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiFunction;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.api.PulsarClientException.AlreadyClosedException;
+import org.apache.pulsar.client.api.v5.schema.Schema;
+import org.apache.pulsar.client.impl.PulsarClientImpl;
+import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData;
+import org.apache.pulsar.client.util.ExecutorProvider;
+import org.apache.pulsar.client.util.ScheduledExecutorProvider;
+import org.apache.pulsar.common.api.proto.ScalableTopicDAG;
+import org.apache.pulsar.common.api.proto.SegmentState;
+import org.apache.pulsar.common.naming.TopicName;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class ScalableQueueConsumerReconcileTest {
+
+    private static final TopicName TOPIC = TopicName.get("topic://public/default/reconcile-test");
+
+    /** Segment subscribe attempts per segment. */
+    private final Map<Long, Integer> attempts = new HashMap<>();
+    /** Scheduled retries, run by the test. */
+    private final ArrayDeque<Runnable> retries = new ArrayDeque<>();
+    /** The delay of each retry scheduled, in milliseconds. */
+    private final List<Long> retryDelays = new ArrayList<>();
+    private ExecutorService externalExecutor;
+    private PulsarClientImpl v4;
+    private PulsarClientV5 client;
+
+    @BeforeMethod
+    public void setup() {
+        attempts.clear();
+        retries.clear();
+        retryDelays.clear();
+        externalExecutor = MoreExecutors.newDirectExecutorService();
+        v4 = mock(PulsarClientImpl.class);
+        client = mock(PulsarClientV5.class);
+        ExecutorProvider external = mock(ExecutorProvider.class);
+        ScheduledExecutorProvider scheduledProvider = mock(ScheduledExecutorProvider.class);
+        ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class);
+        when(client.v4Client()).thenReturn(v4);
+        when(v4.externalExecutorProvider()).thenReturn(external);
+        when(external.getExecutor()).thenReturn(externalExecutor);
+        when(v4.timer()).thenReturn(mock(Timer.class));
+        when(v4.getScheduledExecutorProvider()).thenReturn(scheduledProvider);
+        when(scheduledProvider.getExecutor()).thenReturn(scheduler);
+        when(scheduler.schedule(any(Runnable.class), anyLong(), any())).thenAnswer(invocation -> {
+            retries.add(invocation.getArgument(0));
+            retryDelays.add(invocation.getArgument(1));
+            return null;
+        });
+    }
+
+    @AfterMethod(alwaysRun = true)
+    public void cleanup() {
+        externalExecutor.shutdown();
+    }
+
+    @Test
+    public void keepsAtMostOneRetryScheduled() throws Exception {
+        // Segment 1's subscribe keeps failing with a transient error.
+        subscribeSegments((segmentId, attempt) -> segmentId == 1
+                ? CompletableFuture.failedFuture(new PulsarClientException("transient"))
+                : CompletableFuture.completedFuture(consumer()));
+        DagWatchClient.LayoutChangeListener[] listener = new DagWatchClient.LayoutChangeListener[1];
+        ClientSegmentLayout initial = layout(0);
+        ScalableQueueConsumer<byte[]> consumer = create(initial, listener);
+        try {
+            ClientSegmentLayout split = layout(1);
+            listener[0].onLayoutChange(split, initial);
+            assertEquals(attempts.get(1L), 1);
+            assertEquals(retries.size(), 1, "the failed subscribe must be retried");
+
+            // The layout is pushed again while the retry waits: each push retries segment 1 right
+            // away, which fails again, without scheduling a second retry.
+            listener[0].onLayoutChange(split, split);
+            listener[0].onLayoutChange(split, split);
+            assertEquals(attempts.get(1L), 3);
+            assertEquals(retries.size(), 1, "at most one retry must be scheduled");
+
+            retries.poll().run();
+            assertEquals(attempts.get(1L), 4);
+            assertEquals(retries.size(), 1, "the retry must be rescheduled while the subscribe keeps failing");
+            assertEquals(attempts.get(0L), 1, "subscribed segments are not recreated");
+            assertEquals(attempts.get(2L), 1, "subscribed segments are not recreated");
+        } finally {
+            consumer.close();
+        }
+    }
+
+    @Test
+    public void retriesANewFailureWithoutWaitingOutAnEarlierBackoff() throws Exception {
+        // Segment 1's subscribe fails three times, then succeeds; segment 3's always fails.
+        subscribeSegments((segmentId, attempt) -> segmentId == 3 || (segmentId == 1 && attempt <= 3)
+                ? CompletableFuture.failedFuture(new PulsarClientException("transient"))
+                : CompletableFuture.completedFuture(consumer()));
+        DagWatchClient.LayoutChangeListener[] listener = new DagWatchClient.LayoutChangeListener[1];
+        ClientSegmentLayout initial = layout(0);
+        ScalableQueueConsumer<byte[]> consumer = create(initial, listener);
+        try {
+            // Each failed retry doubles the backoff.
+            ClientSegmentLayout split = layout(1);
+            listener[0].onLayoutChange(split, initial);
+            retries.poll().run();
+            retries.poll().run();
+            assertEquals(attempts.get(1L), 3);
+            assertTrue(retryDelays.get(2) >= 300, "the backoff must have grown: " + retryDelays);
+
+            // The layout is pushed again while that retry waits, and segment 1 subscribes. Then segment 2
+            // splits and segment 3 fails to subscribe: it's retried after the initial delay, not after the
+            // longer one of the retry still pending.
+            listener[0].onLayoutChange(split, split);
+            assertEquals(attempts.get(1L), 4);
+            listener[0].onLayoutChange(layout(2), split);
+            assertEquals(retries.size(), 2);
+            assertTrue(retryDelays.get(3) < 200, "a new failure must be retried soon: " + retryDelays);
+
+            retries.poll().run();
+            assertEquals(attempts.get(3L), 1, "a replaced retry must do nothing");
+            retries.poll().run();
+            assertEquals(attempts.get(3L), 2);
+            assertEquals(retries.size(), 1);
+        } finally {
+            consumer.close();
+        }
+    }
+
+    @Test
+    public void subscribesNewSegmentsWhileAnotherSubscribeIsPending() throws Exception {
+        // Segment 1's subscribe doesn't complete, as while it keeps reconnecting.
+        CompletableFuture<Consumer<byte[]>> segment1Subscribe = new CompletableFuture<>();
+        subscribeSegments((segmentId, attempt) -> segmentId == 1 ? segment1Subscribe
+                : CompletableFuture.completedFuture(consumer()));
+        DagWatchClient.LayoutChangeListener[] listener = new DagWatchClient.LayoutChangeListener[1];
+        ClientSegmentLayout initial = layout(0);
+        ScalableQueueConsumer<byte[]> consumer = create(initial, listener);
+        try {
+            // Segment 0 splits into 1 and 2, then segment 2 splits into 3 and 4 while segment 1's
+            // subscribe is still pending.
+            ClientSegmentLayout split = layout(1);
+            listener[0].onLayoutChange(split, initial);
+            listener[0].onLayoutChange(layout(2), split);
+            assertEquals(attempts.get(3L), 1, "a new segment must be subscribed right away");
+            assertEquals(attempts.get(4L), 1, "a new segment must be subscribed right away");
+            assertEquals(attempts.get(1L), 1, "an in-flight subscribe must be kept, not duplicated");
+        } finally {
+            segment1Subscribe.complete(consumer());
+            consumer.close();
+        }
+    }
+
+    @Test
+    public void closesASegmentThatLeftTheLayoutOnceItsSubscribeCompletes() throws Exception {
+        // Segment 2's subscribe doesn't complete until the test completes it.
+        CompletableFuture<Consumer<byte[]>> segment2Subscribe = new CompletableFuture<>();
+        subscribeSegments((segmentId, attempt) -> segmentId == 2 ? segment2Subscribe
+                : CompletableFuture.completedFuture(consumer()));
+        DagWatchClient.LayoutChangeListener[] listener = new DagWatchClient.LayoutChangeListener[1];
+        ClientSegmentLayout initial = layout(0);
+        ScalableQueueConsumer<byte[]> consumer = create(initial, listener);
+        try {
+            // Segment 0 splits into 1 and 2, and segment 2 leaves the layout while its subscribe is pending.
+            ClientSegmentLayout split = layout(1);
+            listener[0].onLayoutChange(split, initial);
+            listener[0].onLayoutChange(layout(3), split);
+            AtomicBoolean segment2Closed = new AtomicBoolean();
+            segment2Subscribe.complete(consumer(() -> segment2Closed.set(true)));
+            assertTrue(segment2Closed.get(), "a segment that left the layout must be closed once subscribed");
+            assertEquals(attempts.get(2L), 1);
+        } finally {
+            consumer.close();
+        }
+    }
+
+    /** Answer each segment subscribe with {@code subscribe}, given the segment and its attempt number. */
+    private void subscribeSegments(BiFunction<Long, Integer, CompletableFuture<Consumer<byte[]>>> subscribe) {
+        when(v4.<byte[]>subscribeSegmentAsync(any(), any())).thenAnswer(invocation -> {
+            ConsumerConfigurationData<byte[]> conf = invocation.getArgument(0);
+            String topic = conf.getTopicNames().iterator().next();
+            long segmentId = Long.parseLong(topic.substring(topic.lastIndexOf('-') + 1));
+            return subscribe.apply(segmentId, attempts.merge(segmentId, 1, Integer::sum));
+        });
+    }
+
+    private ScalableQueueConsumer<byte[]> create(ClientSegmentLayout initial,
+                                                 DagWatchClient.LayoutChangeListener[] listener) {
+        DagWatchClient watch = mock(DagWatchClient.class);
+        when(watch.topicName()).thenReturn(TOPIC);
+        doAnswer(invocation -> {
+            listener[0] = invocation.getArgument(0);
+            return null;
+        }).when(watch).setListener(any());
+        ConsumerConfigurationData<byte[]> conf = new ConsumerConfigurationData<>();
+        conf.setSubscriptionName("test");
+        return ScalableQueueConsumer.createAsyncImpl(client, Schema.bytes(), conf, watch, initial, null, null)
+                .join();
+    }
+
+    /**
+     * Segment 0; after the first split, sealed segment 0 with children 1 and 2; after the second,
+     * segment 2 sealed too, with children 3 and 4; once the sealed segments are garbage collected,
+     * only 1, 3 and 4.
+     */
+    private static ClientSegmentLayout layout(long epoch) {
+        ScalableTopicDAG dag = new ScalableTopicDAG().setEpoch(epoch);
+        if (epoch < 3) {
+            dag.addSegment().setSegmentId(0).setHashStart(0).setHashEnd(0xffff)
+                    .setState(epoch >= 1 ? SegmentState.SEALED : SegmentState.ACTIVE).setCreatedAtEpoch(0);
+        }
+        if (epoch >= 1) {
+            dag.addSegment().setSegmentId(1).setHashStart(0).setHashEnd(0x7fff)
+                    .setState(SegmentState.ACTIVE).setCreatedAtEpoch(1).addParentId(0);
+        }
+        if (epoch >= 1 && epoch < 3) {
+            dag.addSegment().setSegmentId(2).setHashStart(0x8000).setHashEnd(0xffff)
+                    .setState(epoch >= 2 ? SegmentState.SEALED : SegmentState.ACTIVE).setCreatedAtEpoch(1)
+                    .addParentId(0);
+        }
+        if (epoch >= 2) {
+            dag.addSegment().setSegmentId(3).setHashStart(0x8000).setHashEnd(0xbfff)
+                    .setState(SegmentState.ACTIVE).setCreatedAtEpoch(2).addParentId(2);
+            dag.addSegment().setSegmentId(4).setHashStart(0xc000).setHashEnd(0xffff)
+                    .setState(SegmentState.ACTIVE).setCreatedAtEpoch(2).addParentId(2);
+        }
+        return ClientSegmentLayout.fromProto(dag, TOPIC);
+    }
+
+    /** A segment consumer with nothing to receive: its pending receive never completes. */
+    private static Consumer<byte[]> consumer() {
+        return consumer(() -> {
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Consumer<byte[]> consumer(Runnable onClose) {
+        CompletableFuture<Object> receive = new CompletableFuture<>();
+        return (Consumer<byte[]>) Proxy.newProxyInstance(Consumer.class.getClassLoader(),
+                new Class<?>[]{Consumer.class}, (proxy, method, args) -> switch (method.getName()) {
+                    case "receiveAsync" -> receive;
+                    case "closeAsync" -> {
+                        onClose.run();
+                        receive.completeExceptionally(new AlreadyClosedException("closed"));
+                        yield CompletableFuture.completedFuture(null);
+                    }
+                    case "hashCode" -> System.identityHashCode(proxy);
+                    case "equals" -> proxy == args[0];
+                    case "toString" -> "consumer";
+                    default -> throw new UnsupportedOperationException(method.getName());
+                });
+    }
+}
diff --git a/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumerReconcileTest.java b/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumerReconcileTest.java
new file mode 100644
index 000000000000..a4b28ece7945
--- /dev/null
+++ b/pulsar-client-v5/src/test/java/org/apache/pulsar/client/impl/v5/ScalableStreamConsumerReconcileTest.java
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.impl.v5;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import com.google.common.util.concurrent.MoreExecutors;
+import io.netty.util.Timer;
+import java.lang.reflect.Proxy;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.function.BiFunction;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.PulsarClientException.AlreadyClosedException;
+import org.apache.pulsar.client.api.PulsarClientException.ConsumerBusyException;
+import org.apache.pulsar.client.api.v5.schema.Schema;
+import org.apache.pulsar.client.impl.PulsarClientImpl;
+import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
+import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData;
+import org.apache.pulsar.client.impl.v5.SegmentRouter.ActiveSegment;
+import org.apache.pulsar.client.util.ExecutorProvider;
+import org.apache.pulsar.client.util.ScheduledExecutorProvider;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.scalable.HashRange;
+import org.apache.pulsar.common.scalable.SegmentTopicName;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class ScalableStreamConsumerReconcileTest {
+
+    private static final TopicName TOPIC = TopicName.get("topic://public/default/reconcile-test");
+
+    /** Segment subscribe attempts per segment. */
+    private final Map<Long, Integer> attempts = new HashMap<>();
+    /** Scheduled reconcile retries, run by the test. */
+    private final ArrayDeque<Runnable> retries = new ArrayDeque<>();
+    /** The delay of each reconcile retry scheduled, in milliseconds. */
+    private final List<Long> retryDelays = new ArrayList<>();
+    /** Scheduled release drain polls, run by the test. */
+    private final ArrayDeque<Runnable> drainPolls = new ArrayDeque<>();
+    /** Messages handed to the application. */
+    private final List<MessageV5<byte[]>> delivered = new ArrayList<>();
+    private ExecutorService externalExecutor;
+    private PulsarClientImpl v4;
+    private PulsarClientV5 client;
+    private ScalableConsumerClient session;
+    private ScalableConsumerClient.AssignmentChangeListener listener;
+
+    @BeforeMethod
+    public void setup() {
+        attempts.clear();
+        retries.clear();
+        retryDelays.clear();
+        drainPolls.clear();
+        delivered.clear();
+        externalExecutor = MoreExecutors.newDirectExecutorService();
+        v4 = mock(PulsarClientImpl.class);
+        client = mock(PulsarClientV5.class);
+        ExecutorProvider external = mock(ExecutorProvider.class);
+        ScheduledExecutorProvider scheduledProvider = mock(ScheduledExecutorProvider.class);
+        ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class);
+        when(client.v4Client()).thenReturn(v4);
+        when(v4.externalExecutorProvider()).thenReturn(external);
+        when(external.getExecutor()).thenReturn(externalExecutor);
+        when(v4.timer()).thenReturn(mock(Timer.class));
+        when(v4.getConfiguration()).thenReturn(new ClientConfigurationData());
+        when(v4.getScheduledExecutorProvider()).thenReturn(scheduledProvider);
+        when(scheduledProvider.getExecutor()).thenReturn(scheduler);
+        // The reconcile retries are scheduled as Runnables, the release drain polls as Callables.
+        when(scheduler.schedule(any(Runnable.class), anyLong(), any())).thenAnswer(invocation -> {
+            retries.add(invocation.getArgument(0));
+            retryDelays.add(invocation.getArgument(1));
+            return null;
+        });
+        when(scheduler.schedule(any(Callable.class), anyLong(), any())).thenAnswer(invocation -> {
+            Callable<?> task = invocation.getArgument(0);
+            drainPolls.add(() -> {
+                try {
+                    task.call();
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            });
+            return null;
+        });
+        session = mock(ScalableConsumerClient.class);
+        doAnswer(invocation -> {
+            listener = invocation.getArgument(0);
+            return null;
+        }).when(session).setListener(any());
+    }
+
+    @AfterMethod(alwaysRun = true)
+    public void cleanup() {
+        externalExecutor.shutdown();
+    }
+
+    @Test
+    public void keepsAtMostOneRetryScheduled() throws Exception {
+        // Segment 1's subscribe keeps being rejected, as while its previous owner still holds it.
+        subscribeSegments((segmentId, attempt) -> segmentId == 1
+                ? CompletableFuture.failedFuture(new ConsumerBusyException("busy"))
+                : CompletableFuture.completedFuture(consumer(new CompletableFuture<>())));
+        ScalableStreamConsumer<byte[]> consumer = create(List.of(segment(0)));
+        try {
+            listener.onAssignmentChange(List.of(segment(0), segment(1)), List.of(segment(0)));
+            assertEquals(attempts.get(1L), 1);
+            assertEquals(retries.size(), 1, "the rejected subscribe must be retried");
+
+            // The assignment is pushed again while the retry waits: each push retries segment 1
+            // right away, which fails again, without scheduling a second retry.
+            listener.onAssignmentChange(List.of(segment(0), segment(1)), List.of(segment(0), segment(1)));
+            listener.onAssignmentChange(List.of(segment(0), segment(1)), List.of(segment(0), segment(1)));
+            assertEquals(attempts.get(1L), 3);
+            assertEquals(retries.size(), 1, "at most one retry must be scheduled");
+
+            retries.poll().run();
+            assertEquals(attempts.get(1L), 4);
+            assertEquals(retries.size(), 1, "the retry must be rescheduled while the subscribe keeps failing");
+        } finally {
+            consumer.close();
+        }
+    }
+
+    @Test
+    public void retriesANewFailureWithoutWaitingOutAnEarlierBackoff() throws Exception {
+        // Segment 1's subscribe is rejected three times, then accepted; segment 2's is always rejected.
+        subscribeSegments((segmentId, attempt) -> segmentId == 2 || (segmentId == 1 && attempt <= 3)
+                ? CompletableFuture.failedFuture(new ConsumerBusyException("busy"))
+                : CompletableFuture.completedFuture(consumer(new CompletableFuture<>())));
+        ScalableStreamConsumer<byte[]> consumer = create(List.of(segment(0)));
+        try {
+            // Each failed retry doubles the backoff.
+            listener.onAssignmentChange(List.of(segment(0), segment(1)), List.of(segment(0)));
+            retries.poll().run();
+            retries.poll().run();
+            assertEquals(attempts.get(1L), 3);
+            assertTrue(retryDelays.get(2) >= 300, "the backoff must have grown: " + retryDelays);
+
+            // The assignment is pushed again while that retry waits, and segment 1 is accepted. Then
+            // segment 2 is assigned and rejected: it's retried after the initial delay, not after the
+            // longer one of the retry still pending.
+            listener.onAssignmentChange(List.of(segment(0), segment(1)), List.of(segment(0), segment(1)));
+            assertEquals(attempts.get(1L), 4);
+            listener.onAssignmentChange(List.of(segment(0), segment(1), segment(2)),
+                    List.of(segment(0), segment(1)));
+            assertEquals(retries.size(), 2);
+            assertTrue(retryDelays.get(3) < 200, "a new failure must be retried soon: " + retryDelays);
+
+            retries.poll().run();
+            assertEquals(attempts.get(2L), 1, "a replaced retry must do nothing");
+            retries.poll().run();
+            assertEquals(attempts.get(2L), 2);
+            assertEquals(retries.size(), 1);
+        } finally {
+            consumer.close();
+        }
+    }
+
+    @Test
+    public void keepsTheRetryThatFiresWhileASegmentDrains() throws Exception {
+        // Segment 1's subscribe keeps being rejected; segment 0's consumer hands the application a message.
+        CompletableFuture<Message<byte[]>> segment0Receive = new CompletableFuture<>();
+        subscribeSegments((segmentId, attempt) -> segmentId == 1
+                ? CompletableFuture.failedFuture(new ConsumerBusyException("busy"))
+                : CompletableFuture.completedFuture(consumer(segmentId == 0 ? segment0Receive
+                        : new CompletableFuture<>())));
+        ScalableStreamConsumer<byte[]> consumer = create(List.of(segment(0)));
+        try {
+            segment0Receive.complete(message());
+            listener.onAssignmentChange(List.of(segment(0), segment(1)), List.of(segment(0)));
+            assertEquals(retries.size(), 1);
+
+            // Segment 0 moves to another member while the application holds its message, so the
+            // reconcile releasing it waits for the ack: the retry fires meanwhile, and does nothing.
+            listener.onAssignmentChange(List.of(segment(1)), List.of(segment(0), segment(1)));
+            retries.poll().run();
+            assertEquals(attempts.get(1L), 2);
+            assertEquals(retries.size(), 0);
+
+            // Once the release drained, the reconcile's own failure schedules the next retry.
+            consumer.acknowledgeCumulative(delivered.get(0).id());
+            runDrainPolls();
+            assertEquals(retries.size(), 1, "the rejected subscribe must still be retried");
+        } finally {
+            consumer.close();
+        }
+    }
+
+    @Test
+    public void appliesAnAssignmentThatArrivedDuringAFailedReconcileRightAway() throws Exception {
+        // Segment 1's subscribe keeps being rejected; segment 0's consumer hands the application a message.
+        CompletableFuture<Message<byte[]>> segment0Receive = new CompletableFuture<>();
+        subscribeSegments((segmentId, attempt) -> segmentId == 1
+                ? CompletableFuture.failedFuture(new ConsumerBusyException("busy"))
+                : CompletableFuture.completedFuture(consumer(segmentId == 0 ? segment0Receive
+                        : new CompletableFuture<>())));
+        ScalableStreamConsumer<byte[]> consumer = create(List.of(segment(0)));
+        try {
+            segment0Receive.complete(message());
+
+            // Segment 0 moves to another member while the application holds its message, and segment 1,
+            // assigned instead, is rejected: the reconcile completes once the release drained. Segment 2
+            // is assigned meanwhile.
+            listener.onAssignmentChange(List.of(segment(1)), List.of(segment(0)));
+            listener.onAssignmentChange(List.of(segment(1), segment(2)), List.of(segment(1)));
+            consumer.acknowledgeCumulative(delivered.get(0).id());
+            runDrainPolls();
+            assertEquals(attempts.get(2L), 1, "a newer assignment must be applied right after a failed reconcile");
+            assertEquals(retries.size(), 1, "the rejected subscribe must still be retried");
+        } finally {
+            consumer.close();
+        }
+    }
+
+    @Test
+    public void resubscribesASegmentOnlyAfterItsReleaseDrained() throws Exception {
+        // Segment 0's first consumer hands the application a message; its next one, another.
+        CompletableFuture<Message<byte[]>> firstReceive = new CompletableFuture<>();
+        CompletableFuture<Message<byte[]>> secondReceive = new CompletableFuture<>();
+        subscribeSegments((segmentId, attempt) -> CompletableFuture.completedFuture(consumer(
+                segmentId != 0 ? new CompletableFuture<>() : attempt == 1 ? firstReceive : secondReceive)));
+        ScalableStreamConsumer<byte[]> consumer = create(List.of(segment(0)));
+        try {
+            firstReceive.complete(message());
+            assertEquals(delivered.size(), 1);
+
+            // Segment 0 moves to another member and back while the application holds its message:
+            // the assignment that gives it back is applied once the release has drained.
+            listener.onAssignmentChange(List.of(segment(1)), List.of(segment(0)));
+            listener.onAssignmentChange(List.of(segment(0), segment(1)), List.of(segment(1)));
+            assertEquals(attempts.get(0L), 1, "a segment must not be subscribed again while it drains");
+
+            consumer.acknowledgeCumulative(delivered.get(0).id());
+            runDrainPolls();
+            assertEquals(attempts.get(0L), 2, "the segment must be subscribed again once its release drained");
+            secondReceive.complete(message());
+            assertEquals(delivered.size(), 2, "the new consumer of the segment must deliver");
+        } finally {
+            consumer.close();
+        }
+    }
+
+    /** Run the release drain polls, including those they schedule, until none is left. */
+    private void runDrainPolls() {
+        for (int i = 0; i < 100 && !drainPolls.isEmpty(); i++) {
+            drainPolls.poll().run();
+        }
+        assertEquals(drainPolls.size(), 0);
+    }
+
+    /** Answer each segment subscribe with {@code subscribe}, given the segment and its attempt number. */
+    private void subscribeSegments(BiFunction<Long, Integer, CompletableFuture<Consumer<byte[]>>> subscribe) {
+        when(v4.<byte[]>subscribeSegmentAsync(any(), any())).thenAnswer(invocation -> {
+            ConsumerConfigurationData<byte[]> conf = invocation.getArgument(0);
+            String topic = conf.getTopicNames().iterator().next();
+            long segmentId = Long.parseLong(topic.substring(topic.lastIndexOf('-') + 1));
+            return subscribe.apply(segmentId, attempts.merge(segmentId, 1, Integer::sum));
+        });
+    }
+
+    private ScalableStreamConsumer<byte[]> create(List<ActiveSegment> initialAssignment) {
+        ConsumerConfigurationData<byte[]> conf = new ConsumerConfigurationData<>();
+        conf.setSubscriptionName("test");
+        return ScalableStreamConsumer.createAsyncImpl(client, Schema.bytes(), conf, session, TOPIC.toString(),
+                initialAssignment, msg -> {
+                    delivered.add(msg);
+                    return CompletableFuture.completedFuture(null);
+                }).join();
+    }
+
+    /** A segment this consumer owns whole. */
+    private static ActiveSegment segment(long segmentId) {
+        HashRange range = HashRange.of(0, 0xffff);
+        return new ActiveSegment(segmentId, range,
+                SegmentTopicName.fromParent(TOPIC, range, segmentId).toString(), null, List.of(), List.of());
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Message<byte[]> message() {
+        Message<byte[]> message = mock(Message.class);
+        when(message.getMessageId()).thenReturn(MessageId.earliest);
+        return message;
+    }
+
+    /** A segment consumer whose first receive is {@code firstReceive}; later receives never complete. */
+    @SuppressWarnings("unchecked")
+    private static Consumer<byte[]> consumer(CompletableFuture<Message<byte[]>> firstReceive) {
+        List<CompletableFuture<Message<byte[]>>> receives = new ArrayList<>();
+        return (Consumer<byte[]>) Proxy.newProxyInstance(Consumer.class.getClassLoader(),
+                new Class<?>[]{Consumer.class}, (proxy, method, args) -> switch (method.getName()) {
+                    case "receiveAsync" -> {
+                        CompletableFuture<Message<byte[]>> receive =
+                                receives.isEmpty() ? firstReceive : new CompletableFuture<>();
+                        receives.add(receive);
+                        yield receive;
+                    }
+                    case "acknowledgeCumulativeAsync" -> CompletableFuture.completedFuture(null);
+                    case "closeAsync" -> {
+                        receives.forEach(receive -> receive.completeExceptionally(
+                                new AlreadyClosedException("closed")));
+                        yield CompletableFuture.completedFuture(null);
+                    }
+                    case "hashCode" -> System.identityHashCode(proxy);
+                    case "equals" -> proxy == args[0];
+                    case "toString" -> "consumer";
+                    default -> throw new UnsupportedOperationException(method.getName());
+                });
+    }
+}


(pulsar) 03/06: [fix][test] Run the Oxia-backed tests against oxia/oxia:nightly-beta (#26880)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 67dd068083f3fdc2be07b7fc19c3c1ea99fb47fe
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Oct 12 17:58:03 2026 -0700

    [fix][test] Run the Oxia-backed tests against oxia/oxia:nightly-beta (#26880)
    
    (cherry picked from commit 24bc77f0fd8529e06b952a2476122781eccc619e)
---
 .../org/apache/pulsar/tests/OxiaTestImage.java     | 48 +++++++++++++++++++++
 .../service/SharedMultiBrokerPulsarCluster.java    |  4 +-
 .../pulsar/metadata/BaseMetadataStoreTest.java     |  3 +-
 .../pulsar/metadata/OxiaPartitionKeyTest.java      |  3 +-
 .../pulsar/metadata/OxiaSequenceKeysTest.java      |  3 +-
 .../org/apache/pulsar/metadata/OxiaTestServer.java | 49 ++++++++++++++++++++++
 .../tests/integration/oxia/OxiaContainer.java      | 18 ++++++--
 7 files changed, 117 insertions(+), 11 deletions(-)

diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/OxiaTestImage.java b/buildtools/src/main/java/org/apache/pulsar/tests/OxiaTestImage.java
new file mode 100644
index 000000000000..b85998f15a13
--- /dev/null
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/OxiaTestImage.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests;
+
+/**
+ * The Oxia Docker image that the Oxia-backed tests run against.
+ */
+public final class OxiaTestImage {
+
+    /**
+     * Oxia's {@code nightly-beta} image, which Oxia's
+     * <a href="https://github.com/oxia-db/oxia/blob/main/.github/workflows/ci-nightly-images.yaml">nightly images
+     * workflow</a> rebuilds every day from Oxia's development branch. The tests run against it, rather than
+     * {@code nightly-stable}, which the workflow builds from the current release branch, so that an Oxia change that
+     * breaks Pulsar shows up before it reaches an Oxia release.
+     *
+     * <p>If an Oxia nightly breaks the Pulsar tests, pin an Oxia release tag here, such as {@code oxia/oxia:0.16.11}.
+     *
+     * <p>Testcontainers pulls an image only when it is missing, so a local run keeps the nightly it pulled first. Run
+     * {@code docker pull oxia/oxia:nightly-beta} before reproducing a CI failure. The tests log the Oxia commit they
+     * run against, from {@link #REVISION_LABEL}.
+     */
+    public static final String NAME = "oxia/oxia:nightly-beta";
+
+    /**
+     * The label of the Oxia image that holds the Oxia commit the image was built from.
+     */
+    public static final String REVISION_LABEL = "org.opencontainers.image.revision";
+
+    private OxiaTestImage() {
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedMultiBrokerPulsarCluster.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedMultiBrokerPulsarCluster.java
index 881bccd3d23f..9a75b1fc8122 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedMultiBrokerPulsarCluster.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedMultiBrokerPulsarCluster.java
@@ -33,6 +33,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfo;
+import org.apache.pulsar.metadata.OxiaTestServer;
 import org.apache.pulsar.metadata.bookkeeper.BKCluster;
 import org.apache.pulsar.tests.ThreadLeakDetectorListener;
 
@@ -134,8 +135,7 @@ public class SharedMultiBrokerPulsarCluster {
         // broker its own session and the proper ephemeral / CAS semantics. Container-based
         // because oxia ships no in-process server; tests skip cleanly when Docker isn't
         // available.
-        oxiaServer = new OxiaContainer(OxiaContainer.DEFAULT_IMAGE_NAME);
-        oxiaServer.start();
+        oxiaServer = OxiaTestServer.start(1);
         metadataStoreUrl = "oxia://" + oxiaServer.getServiceAddress();
 
         // Single shared bookie. Same minimal config as SharedPulsarCluster — write quorum stays
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
index fee0955744c7..4b1165d6c197 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java
@@ -159,8 +159,7 @@ public abstract class BaseMetadataStoreTest extends TestRetrySupport {
             return null;
         }
         if (oxiaServer == null) {
-            oxiaServer = new OxiaContainer(OxiaContainer.DEFAULT_IMAGE_NAME);
-            oxiaServer.start();
+            oxiaServer = OxiaTestServer.start(1);
         }
         return oxiaServer.getServiceAddress();
     }
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaPartitionKeyTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaPartitionKeyTest.java
index 6dbe6c363ddc..6400e59b586c 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaPartitionKeyTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaPartitionKeyTest.java
@@ -50,8 +50,7 @@ public class OxiaPartitionKeyTest {
 
     @BeforeClass
     public void start() {
-        oxiaServer = new OxiaContainer(OxiaContainer.DEFAULT_IMAGE_NAME).withShards(SHARDS);
-        oxiaServer.start();
+        oxiaServer = OxiaTestServer.start(SHARDS);
     }
 
     @AfterClass(alwaysRun = true)
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaSequenceKeysTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaSequenceKeysTest.java
index d0fcf694f43f..1145507ce0a3 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaSequenceKeysTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaSequenceKeysTest.java
@@ -51,8 +51,7 @@ public class OxiaSequenceKeysTest {
 
     @BeforeClass
     public void start() {
-        oxiaServer = new OxiaContainer(OxiaContainer.DEFAULT_IMAGE_NAME).withShards(SHARDS);
-        oxiaServer.start();
+        oxiaServer = OxiaTestServer.start(SHARDS);
     }
 
     @AfterClass(alwaysRun = true)
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaTestServer.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaTestServer.java
new file mode 100644
index 000000000000..629c09dbd20f
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaTestServer.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata;
+
+import io.oxia.testcontainers.OxiaContainer;
+import lombok.CustomLog;
+import org.apache.pulsar.tests.OxiaTestImage;
+import org.testcontainers.utility.DockerImageName;
+
+/**
+ * Starts the Oxia server of the Oxia-backed tests, from {@link OxiaTestImage#NAME}.
+ */
+@CustomLog
+public final class OxiaTestServer {
+
+    /**
+     * Starts an Oxia server with the given number of shards, and logs the Oxia commit it runs, since the image tag
+     * moves every day.
+     */
+    public static OxiaContainer start(int shards) {
+        OxiaContainer oxiaServer = new OxiaContainer(DockerImageName.parse(OxiaTestImage.NAME)).withShards(shards);
+        oxiaServer.start();
+        String revision = oxiaServer.getContainerInfo().getConfig().getLabels().get(OxiaTestImage.REVISION_LABEL);
+        log.info()
+                .attr("image", OxiaTestImage.NAME)
+                .attr("revision", revision)
+                .log("Oxia server started");
+        return oxiaServer;
+    }
+
+    private OxiaTestServer() {
+    }
+}
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/oxia/OxiaContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/oxia/OxiaContainer.java
index 12f599d779d3..7cfbb92cb261 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/oxia/OxiaContainer.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/oxia/OxiaContainer.java
@@ -19,11 +19,15 @@
 
 package org.apache.pulsar.tests.integration.oxia;
 
+import com.github.dockerjava.api.command.InspectContainerResponse;
 import java.time.Duration;
+import lombok.CustomLog;
+import org.apache.pulsar.tests.OxiaTestImage;
 import org.apache.pulsar.tests.integration.containers.ChaosContainer;
 import org.apache.pulsar.tests.integration.containers.PulsarContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
 
+@CustomLog
 public class OxiaContainer extends ChaosContainer<OxiaContainer> {
 
     public static final String NAME = "oxia";
@@ -32,10 +36,8 @@ public class OxiaContainer extends ChaosContainer<OxiaContainer> {
     public static final int METRICS_PORT = 8080;
     private static final int DEFAULT_SHARDS = 1;
 
-    private static final String DEFAULT_IMAGE_NAME = "oxia/oxia:main";
-
     public OxiaContainer(String clusterName) {
-        this(clusterName, DEFAULT_IMAGE_NAME, DEFAULT_SHARDS);
+        this(clusterName, OxiaTestImage.NAME, DEFAULT_SHARDS);
     }
 
     @SuppressWarnings("resource")
@@ -65,6 +67,16 @@ public class OxiaContainer extends ChaosContainer<OxiaContainer> {
         return OxiaContainer.NAME + ":" + OXIA_PORT;
     }
 
+    @Override
+    protected void containerIsStarted(InspectContainerResponse containerInfo) {
+        super.containerIsStarted(containerInfo);
+        // The image tag moves every day, so record the Oxia commit the test runs against
+        log.info()
+                .attr("image", getDockerImageName())
+                .attr("revision", containerInfo.getConfig().getLabels().get(OxiaTestImage.REVISION_LABEL))
+                .log("Oxia server started");
+    }
+
     @Override
     public String getContainerName() {
         return clusterName + "-oxia";


(pulsar) 02/06: [fix][broker] Keep the scalable-topics watch's initial listing from undoing newer events (#26876)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 8f7884ebad33f1e7985bd8edc891d2a7a694c7a1
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Oct 12 17:25:54 2026 -0700

    [fix][broker] Keep the scalable-topics watch's initial listing from undoing newer events (#26876)
    
    (cherry picked from commit 9939ea2b3dd1cf20b235b146481a8dfee3b07ed9)
---
 .../scalable/ScalableTopicsWatcherSession.java     | 108 +++++----
 ...ScalableTopicsWatcherSessionDeleteRaceTest.java | 269 +++++++++++++++++++--
 2 files changed, 308 insertions(+), 69 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicsWatcherSession.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicsWatcherSession.java
index ca8521c0c485..055be5adc6df 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicsWatcherSession.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicsWatcherSession.java
@@ -20,10 +20,12 @@ package org.apache.pulsar.broker.service.scalable;
 
 import io.github.merlimat.slog.Logger;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
@@ -98,15 +100,25 @@ public class ScalableTopicsWatcherSession implements ScalableTopicResources.Name
     private final Set<String> currentSet = Collections.synchronizedSet(new HashSet<>());
 
     /**
-     * Id of the metadata read started by each topic's latest Created / Modified event, while
-     * that read is in flight. Only that read may change the topic's membership: an earlier one
-     * — or one still in flight when the topic is deleted — may have seen an older record, and
-     * applying it would undo the later event, e.g. bring a deleted topic back. Guarded by
-     * {@code currentSet}'s monitor, together with the membership changes it gates.
+     * Id of each topic's latest metadata read, while that read is in flight: the read started by
+     * the topic's latest Created / Modified event, or by {@link #start()} for a topic that changed
+     * while the initial listing was in flight. Only that read may change the topic's membership:
+     * an earlier one — or one still in flight when the topic is deleted — may have seen an older
+     * record, and applying it would undo the later event, e.g. bring a deleted topic back. Guarded
+     * by {@code currentSet}'s monitor, together with the membership changes it gates.
      */
     private final Map<String, Long> inFlightReads = new HashMap<>();
     private long lastReadId;
 
+    /**
+     * Last event of each topic that changed while the initial listing was in flight, or
+     * {@code null} once the listing has been applied. The listing may have read the store
+     * before or after any of these events, so it can't be trusted for these topics:
+     * {@link #start()} settles them once the listing is in place. Guarded by
+     * {@code currentSet}'s monitor.
+     */
+    private Map<String, NotificationType> changedDuringListing = new HashMap<>();
+
     private final AtomicBoolean closed = new AtomicBoolean();
     private final AtomicBoolean snapshotEmitted = new AtomicBoolean();
 
@@ -150,38 +162,46 @@ public class ScalableTopicsWatcherSession implements ScalableTopicResources.Name
      * listener.
      */
     public CompletableFuture<Void> start() {
-        // Register BEFORE computing the initial set: any event that arrives mid-snapshot
-        // is captured by the listener and either (a) already in the initial set we're
-        // about to emit, in which case the redundant Add is a no-op on the client
-        // (set semantics), or (b) genuinely newer than the snapshot, in which case it
-        // correctly flows through as a Diff after the Snapshot frame.
+        // Register BEFORE computing the initial set, so that no event is lost. An event that
+        // arrives before the listing completes may or may not be reflected in it: it is only
+        // recorded in changedDuringListing, and settled once the listing is in place.
         resources.registerNamespaceListener(this);
 
         return resources.findScalableTopicsByPropertiesAsync(namespace, propertyFilters)
-                .thenAccept(initialTopics -> {
+                .thenAccept(listedTopics -> {
                     if (closed.get()) {
                         return;
                     }
-                    // Replace currentSet under sync to avoid races with onNotification.
+                    List<String> initialTopics;
+                    List<String> topicsToRead = new ArrayList<>();
                     synchronized (currentSet) {
-                        currentSet.clear();
-                        currentSet.addAll(initialTopics);
+                        currentSet.addAll(listedTopics);
+                        changedDuringListing.forEach((topic, lastEvent) -> {
+                            if (lastEvent == NotificationType.Deleted) {
+                                // Gone, unless created again — and then a Created event follows.
+                                currentSet.remove(topic);
+                            } else {
+                                // Created or Modified, possibly after the listing read the store.
+                                topicsToRead.add(topic);
+                            }
+                        });
+                        changedDuringListing = null;
+                        initialTopics = new ArrayList<>(currentSet);
                     }
-                    snapshotEmitted.set(true);
                     // Hash short-circuit: if the client tells us it already has this
                     // exact set (reconnect within an unchanged window), don't waste
                     // bytes on the wire. Future Diffs flow as usual.
-                    if (clientHash != null) {
-                        String serverHash = TopicList.calculateHash(initialTopics);
-                        if (clientHash.equals(serverHash)) {
-                            log.info().attr("topics", initialTopics.size())
-                                    .log("Reconnect hash matched; skipping snapshot");
-                            return;
-                        }
+                    if (clientHash != null && clientHash.equals(TopicList.calculateHash(initialTopics))) {
+                        log.info().attr("topics", initialTopics.size())
+                                .log("Reconnect hash matched; skipping snapshot");
+                    } else {
+                        log.info().attr("topics", initialTopics.size()).log("Initial snapshot");
+                        cnx.ctx().writeAndFlush(
+                                Commands.newWatchScalableTopicsSnapshot(watchId, initialTopics));
                     }
-                    log.info().attr("topics", initialTopics.size()).log("Initial snapshot");
-                    cnx.ctx().writeAndFlush(
-                            Commands.newWatchScalableTopicsSnapshot(watchId, initialTopics));
+                    // Diffs are computed against the snapshot's set: let them out only after it.
+                    snapshotEmitted.set(true);
+                    topicsToRead.forEach(this::evaluateTopic);
                 });
     }
 
@@ -209,17 +229,27 @@ public class ScalableTopicsWatcherSession implements ScalableTopicResources.Name
 
         String topicName = TopicName.get("topic", namespace, Codec.decode(rest)).toString();
 
-        if (notification.getType() == NotificationType.Deleted) {
-            synchronized (currentSet) {
+        synchronized (currentSet) {
+            if (changedDuringListing != null) {
+                // Settled by start() once the listing is in place.
+                changedDuringListing.put(topicName, notification.getType());
+                return;
+            }
+            if (notification.getType() == NotificationType.Deleted) {
                 inFlightReads.remove(topicName);
                 if (currentSet.remove(topicName)) {
                     enqueueRemoved(topicName);
                 }
+                return;
             }
-            return;
         }
 
         // Created or Modified — fetch the new value to evaluate the filter against.
+        evaluateTopic(topicName);
+    }
+
+    /** Read the topic's record and add it to, or remove it from, the matching set accordingly. */
+    private void evaluateTopic(String topicName) {
         long readId;
         synchronized (currentSet) {
             readId = ++lastReadId;
@@ -299,6 +329,13 @@ public class ScalableTopicsWatcherSession implements ScalableTopicResources.Name
         if (closed.get()) {
             return;
         }
+        // The pending changes are relative to the snapshot's set: send them only after the
+        // snapshot frame, and leave them pending until then, so that a later change can still
+        // cancel them out. flushScheduled stays set, so nothing else gets scheduled meanwhile.
+        if (!snapshotEmitted.get()) {
+            scheduler.schedule(this::flushPending, COALESCE_WINDOW.toMillis(), TimeUnit.MILLISECONDS);
+            return;
+        }
         Set<String> added;
         Set<String> removed;
         synchronized (coalesceLock) {
@@ -311,21 +348,6 @@ public class ScalableTopicsWatcherSession implements ScalableTopicResources.Name
         if (added.isEmpty() && removed.isEmpty()) {
             return;
         }
-        // Wait until the initial snapshot was emitted: any deltas that fire before the
-        // snapshot is sent have already been folded into currentSet via onNotification's
-        // wasInSet check (same set we built the snapshot from), so they're correctly
-        // represented. We just need to send them AFTER the snapshot frame.
-        if (!snapshotEmitted.get()) {
-            // Re-defer: the snapshot future is short-lived, retry shortly.
-            scheduler.schedule(this::flushPending, COALESCE_WINDOW.toMillis(), TimeUnit.MILLISECONDS);
-            // Re-enqueue what we drained, since the next flush will rebuild from pending.
-            synchronized (coalesceLock) {
-                pendingAdded.addAll(added);
-                pendingRemoved.addAll(removed);
-                flushScheduled = true;
-            }
-            return;
-        }
         log.info().attr("added", added.size()).attr("removed", removed.size()).log("Pushing diff");
         cnx.ctx().writeAndFlush(Commands.newWatchScalableTopicsDiff(watchId, added, removed));
     }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicsWatcherSessionDeleteRaceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicsWatcherSessionDeleteRaceTest.java
index 4fb36c78015d..9a0a3c3ee48e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicsWatcherSessionDeleteRaceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicsWatcherSessionDeleteRaceTest.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.broker.service.scalable;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 import io.netty.buffer.ByteBuf;
@@ -34,9 +35,10 @@ import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.pulsar.broker.resources.ScalableTopicMetadata;
 import org.apache.pulsar.broker.resources.ScalableTopicResources;
 import org.apache.pulsar.broker.service.ServerCnx;
@@ -44,6 +46,7 @@ import org.apache.pulsar.common.api.proto.BaseCommand;
 import org.apache.pulsar.common.api.proto.CommandWatchScalableTopicsUpdate;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.topics.TopicList;
 import org.apache.pulsar.metadata.api.MetadataStoreConfig;
 import org.apache.pulsar.metadata.api.Notification;
 import org.apache.pulsar.metadata.api.NotificationType;
@@ -57,11 +60,13 @@ import org.testng.annotations.Test;
 /**
  * A namespace watch session must report a deleted topic as removed even when a metadata read it
  * started for an earlier event of the same topic completes only after the deletion: that read
- * saw the record before it was deleted and must not bring the topic back.
+ * saw the record before it was deleted and must not bring the topic back. Likewise, the listing
+ * a session starts from must not undo the events it handled while the listing was in flight:
+ * the listing may have read the store before them.
  *
- * <p>Each test makes the race deterministic: the session's reads are held until the record is
- * deleted and the session has handled the {@code Deleted} notification, and the coalesced diff
- * is flushed by hand afterwards.
+ * <p>Each test makes the race deterministic: the session's reads — and, at start, its listing —
+ * are held until the record has changed and the session has handled the notification, and the
+ * coalesced diff is flushed by hand afterwards.
  */
 public class ScalableTopicsWatcherSessionDeleteRaceTest {
 
@@ -72,10 +77,17 @@ public class ScalableTopicsWatcherSessionDeleteRaceTest {
     private final Queue<Notification> handledNotifications = new ConcurrentLinkedQueue<>();
     /** The topic set a client applying the session's Snapshot / Diff frames would hold. */
     private final Set<String> clientView = ConcurrentHashMap.newKeySet();
+    private NamespaceName namespace;
+    private ServerCnx cnx;
+    private ChannelHandlerContext ctx;
+    private ScheduledExecutorService scheduler;
     private ScalableTopicsWatcherSession session;
 
     @BeforeMethod
     public void setUp() throws Exception {
+        scheduledFlushes.clear();
+        handledNotifications.clear();
+        clientView.clear();
         store = new LocalMemoryMetadataStore("memory:local", MetadataStoreConfig.builder().build());
         resources = new HeldReadsResources(store);
         // Registered after the resources' own listener, and notifications reach the listeners
@@ -83,21 +95,21 @@ public class ScalableTopicsWatcherSessionDeleteRaceTest {
         // has handled it.
         store.registerListener(handledNotifications::add);
 
-        ServerCnx cnx = mock(ServerCnx.class);
-        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        cnx = mock(ServerCnx.class);
+        ctx = mock(ChannelHandlerContext.class);
         when(cnx.ctx()).thenReturn(ctx);
         when(ctx.writeAndFlush(any())).thenAnswer(invocation -> {
             applyFrame(invocation.getArgument(0));
             return null;
         });
-        ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class);
+        scheduler = mock(ScheduledExecutorService.class);
         when(scheduler.schedule(any(Runnable.class), anyLong(), any(TimeUnit.class))).thenAnswer(invocation -> {
             scheduledFlushes.add(invocation.getArgument(0));
             return null;
         });
 
-        NamespaceName ns = NamespaceName.get("tenant/ns-" + UUID.randomUUID().toString().substring(0, 8));
-        session = new ScalableTopicsWatcherSession(1L, ns, Map.of(), null, cnx, resources, scheduler);
+        namespace = NamespaceName.get("tenant/ns-" + UUID.randomUUID().toString().substring(0, 8));
+        session = newSession(Map.of(), null);
     }
 
     @AfterMethod(alwaysRun = true)
@@ -160,15 +172,159 @@ public class ScalableTopicsWatcherSessionDeleteRaceTest {
         assertThat(clientView).as("topics after the delete").doesNotContain(tn.toString());
     }
 
+    @Test
+    public void topicDeletedWhileTheListingIsInFlightIsNotKept() throws Exception {
+        TopicName tn = newTopic();
+        CompletableFuture<Void> started = startWithListingHeld();
+
+        resources.deleteScalableTopicAsync(tn).get();
+        awaitHandled(NotificationType.Deleted, resources.topicPath(tn));
+        applyListing(started);
+
+        assertThat(clientView).as("topics after the listing").isEmpty();
+    }
+
+    @Test
+    public void topicDeletedWhileTheListingIsInFlightIsNotKeptOnReconnect() throws Exception {
+        TopicName tn = newTopic();
+        // A client reconnecting with the topic in its set: its hash matches the listing.
+        clientView.add(tn.toString());
+        session = newSession(Map.of(), TopicList.calculateHash(List.of(tn.toString())));
+        CompletableFuture<Void> started = startWithListingHeld();
+
+        resources.deleteScalableTopicAsync(tn).get();
+        awaitHandled(NotificationType.Deleted, resources.topicPath(tn));
+        applyListing(started);
+
+        assertThat(clientView).as("topics after the listing").isEmpty();
+    }
+
+    @Test
+    public void topicCreatedWhileTheListingIsInFlightIsRemovedOnDelete() throws Exception {
+        CompletableFuture<Void> started = startWithListingHeld();
+        TopicName tn = newTopic();
+        applyListing(started);
+        assertThat(clientView).as("topics after the listing").containsExactly(tn.toString());
+
+        resources.deleteScalableTopicAsync(tn).get();
+        awaitHandled(NotificationType.Deleted, resources.topicPath(tn));
+        flushDiffs();
+
+        assertThat(clientView).as("topics after the delete").isEmpty();
+    }
+
+    @Test
+    public void topicLeavingTheFilterWhileTheListingIsInFlightIsNotKept() throws Exception {
+        session = newSession(Map.of("owner", "alice"), null);
+        TopicName tn = newTopic(Map.of("owner", "alice"));
+        CompletableFuture<Void> started = startWithListingHeld();
+
+        setOwner(tn, "bob");
+        applyListing(started);
+
+        assertThat(clientView).as("topics after the listing").isEmpty();
+    }
+
+    @Test
+    public void topicJoiningTheFilterWhileTheListingIsInFlightIsRemovedWhenItLeaves() throws Exception {
+        session = newSession(Map.of("owner", "alice"), null);
+        TopicName tn = newTopic(Map.of("owner", "bob"));
+        CompletableFuture<Void> started = startWithListingHeld();
+
+        setOwner(tn, "alice");
+        applyListing(started);
+        assertThat(clientView).as("topics after the listing").containsExactly(tn.toString());
+
+        setOwner(tn, "bob");
+        resources.releaseReads();
+        flushDiffs();
+
+        assertThat(clientView).as("topics after leaving the filter").isEmpty();
+    }
+
+    @Test
+    public void diffScheduledBeforeTheSnapshotIsWrittenFollowsIt() throws Exception {
+        TopicName tn = newTopic();
+        CompletableFuture<Void> started = startWithListingHeld();
+        // Once the session has applied the listing, which has the topic, and right before it writes
+        // the snapshot: delete the topic and run the flush this schedules.
+        AtomicBoolean deleted = new AtomicBoolean();
+        when(cnx.ctx()).thenAnswer(invocation -> {
+            if (deleted.compareAndSet(false, true)) {
+                resources.deleteScalableTopicAsync(tn).get();
+                awaitHandled(NotificationType.Deleted, resources.topicPath(tn));
+                assertThat(scheduledFlushes).as("flush scheduled by the delete").isNotEmpty();
+                for (int due = scheduledFlushes.size(); due > 0; due--) {
+                    scheduledFlushes.remove().run();
+                }
+            }
+            return ctx;
+        });
+        applyListing(started);
+
+        assertThat(clientView).as("topics after the delete").isEmpty();
+    }
+
+    @Test
+    public void topicDeletedWhileAFlushWaitsForTheSnapshotIsNotKept() throws Exception {
+        CompletableFuture<Void> started = startWithListingHeld();
+        // Once the session has applied the listing, and right before it writes the snapshot: create a
+        // topic, so that its add is pending, and run the flush this schedules. The flush has to wait for
+        // the snapshot: delete the topic while it schedules itself again.
+        AtomicReference<TopicName> created = new AtomicReference<>();
+        AtomicBoolean deleteOnNextSchedule = new AtomicBoolean();
+        doAnswer(invocation -> {
+            if (deleteOnNextSchedule.compareAndSet(true, false)) {
+                resources.deleteScalableTopicAsync(created.get()).get();
+                awaitHandled(NotificationType.Deleted, resources.topicPath(created.get()));
+            }
+            scheduledFlushes.add(invocation.getArgument(0));
+            return null;
+        }).when(scheduler).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
+        when(cnx.ctx()).thenAnswer(invocation -> {
+            if (created.get() == null) {
+                created.set(newTopic());
+                resources.releaseReads();
+                assertThat(scheduledFlushes).as("flush scheduled by the create").isNotEmpty();
+                deleteOnNextSchedule.set(true);
+                for (int due = scheduledFlushes.size(); due > 0; due--) {
+                    scheduledFlushes.remove().run();
+                }
+            }
+            return ctx;
+        });
+        applyListing(started);
+
+        assertThat(deleteOnNextSchedule).as("topic deleted while the flush waited").isFalse();
+        assertThat(clientView).as("topics after the delete").isEmpty();
+    }
+
+    private ScalableTopicsWatcherSession newSession(Map<String, String> propertyFilters, String clientHash) {
+        return new ScalableTopicsWatcherSession(1L, namespace, propertyFilters, clientHash, cnx, resources,
+                scheduler);
+    }
+
     /** Create a topic record and wait until its {@code Created} notification was handled. */
     private TopicName newTopic() throws Exception {
-        TopicName tn = TopicName.get("topic://" + session.getNamespaceName() + "/t-"
-                + UUID.randomUUID().toString().substring(0, 8));
-        resources.createScalableTopicAsync(tn, ScalableTopicMetadata.builder().build()).get();
+        return newTopic(Map.of());
+    }
+
+    private TopicName newTopic(Map<String, String> properties) throws Exception {
+        TopicName tn = TopicName.get("topic://" + namespace + "/t-" + UUID.randomUUID().toString().substring(0, 8));
+        resources.createScalableTopicAsync(tn, ScalableTopicMetadata.builder().properties(properties).build()).get();
         awaitHandled(NotificationType.Created, resources.topicPath(tn));
         return tn;
     }
 
+    /** Set the topic's {@code owner} property and wait until the {@code Modified} notification was handled. */
+    private void setOwner(TopicName tn, String owner) throws Exception {
+        resources.updateScalableTopicAsync(tn, md -> {
+            md.setProperties(Map.of("owner", owner));
+            return md;
+        }).get();
+        awaitHandled(NotificationType.Modified, resources.topicPath(tn));
+    }
+
     /**
      * Delete the topic record while the reads the session started are held, wait until the
      * session has handled the {@code Deleted} notification, then let the reads complete and push
@@ -178,6 +334,34 @@ public class ScalableTopicsWatcherSessionDeleteRaceTest {
         resources.deleteScalableTopicAsync(tn).get();
         awaitHandled(NotificationType.Deleted, resources.topicPath(tn));
         resources.releaseReads();
+        flushDiffs();
+    }
+
+    /**
+     * Start the session with its reads and its initial listing held, and wait until the listing
+     * has read the store: it misses every change made from now on.
+     */
+    private CompletableFuture<Void> startWithListingHeld() throws Exception {
+        resources.holdReads();
+        resources.holdListing();
+        CompletableFuture<Void> started = session.start();
+        resources.awaitListingRead();
+        return started;
+    }
+
+    /**
+     * Let the reads the session started so far complete, then its listing, then the reads it
+     * started on applying the listing, and push the coalesced diff.
+     */
+    private void applyListing(CompletableFuture<Void> started) throws Exception {
+        resources.releaseReads();
+        resources.releaseListing();
+        started.get(10, TimeUnit.SECONDS);
+        resources.releaseReads();
+        flushDiffs();
+    }
+
+    private void flushDiffs() {
         for (Runnable flush; (flush = scheduledFlushes.poll()) != null; ) {
             flush.run();
         }
@@ -218,39 +402,72 @@ public class ScalableTopicsWatcherSessionDeleteRaceTest {
     }
 
     /**
-     * Holds back every topic-record read the session starts after {@link #holdReads()}: the
-     * read goes to the store right away, so it sees the record as it is at that moment, but its
-     * result reaches the session only on {@link #releaseReads()}, in the calling thread.
+     * Holds back the session's topic-record reads and its initial listing. A held read or listing
+     * goes to the store right away, so it sees the store as it is at that moment, but its result
+     * reaches the session only when released, in the releasing thread.
      */
     private static class HeldReadsResources extends ScalableTopicResources {
-        private final List<CompletableFuture<?>> heldReads = new CopyOnWriteArrayList<>();
-        private volatile CompletableFuture<Void> release;
+        private record HeldRead(CompletableFuture<?> result, CompletableFuture<Void> release) {
+        }
+
+        private final Queue<HeldRead> heldReads = new ConcurrentLinkedQueue<>();
+        private volatile boolean holdReads;
+        private volatile CompletableFuture<Void> listingRelease;
+        private volatile CompletableFuture<List<String>> heldListing;
 
         HeldReadsResources(MetadataStoreExtended store) {
             super(store, 30);
         }
 
+        /** From now on, hold each read until the next {@link #releaseReads()}. */
         void holdReads() {
-            release = new CompletableFuture<>();
+            holdReads = true;
         }
 
+        /** Let the reads held so far reach the session, in the order they were started. */
         void releaseReads() throws Exception {
-            for (CompletableFuture<?> read : heldReads) {
-                read.get(10, TimeUnit.SECONDS);
+            for (HeldRead read; (read = heldReads.poll()) != null; ) {
+                read.result().get(10, TimeUnit.SECONDS);
+                read.release().complete(null);
             }
-            release.complete(null);
+        }
+
+        /** Hold the next listing, until {@link #releaseListing()}. */
+        void holdListing() {
+            listingRelease = new CompletableFuture<>();
+        }
+
+        void awaitListingRead() throws Exception {
+            heldListing.get(10, TimeUnit.SECONDS);
+        }
+
+        void releaseListing() throws Exception {
+            awaitListingRead();
+            listingRelease.complete(null);
         }
 
         @Override
         public CompletableFuture<Optional<ScalableTopicMetadata>> getScalableTopicMetadataAsync(TopicName tn,
                                                                                                  boolean refresh) {
             CompletableFuture<Optional<ScalableTopicMetadata>> read = super.getScalableTopicMetadataAsync(tn, refresh);
-            CompletableFuture<Void> gate = release;
-            if (gate == null) {
+            if (!holdReads) {
                 return read;
             }
-            heldReads.add(read);
-            return gate.thenCompose(__ -> read);
+            CompletableFuture<Void> release = new CompletableFuture<>();
+            heldReads.add(new HeldRead(read, release));
+            return release.thenCompose(__ -> read);
+        }
+
+        @Override
+        public CompletableFuture<List<String>> findScalableTopicsByPropertiesAsync(NamespaceName ns,
+                                                                                   Map<String, String> filters) {
+            CompletableFuture<List<String>> listing = super.findScalableTopicsByPropertiesAsync(ns, filters);
+            CompletableFuture<Void> release = listingRelease;
+            if (release == null) {
+                return listing;
+            }
+            heldListing = listing;
+            return release.thenCompose(__ -> listing);
         }
     }
 }


(pulsar) 06/06: [fix][broker] Make the scalable topic controller follow its leader election (#26878)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 9049f5a2c411b5fa34bd9d5f0c19904f795f9f54
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Oct 12 21:10:23 2026 -0700

    [fix][broker] Make the scalable topic controller follow its leader election (#26878)
    
    (cherry picked from commit 93be85aee3cf3357d4f44c7e3e7300ca572f7564)
---
 .../apache/pulsar/broker/service/ServerCnx.java    |  20 ++
 .../apache/pulsar/broker/service/TransportCnx.java |   6 +
 .../broker/service/scalable/ConsumerSession.java   |  11 +
 .../service/scalable/ScalableTopicController.java  | 333 ++++++++++++++----
 .../service/scalable/ScalableTopicService.java     |  89 +++--
 .../service/scalable/SubscriptionCoordinator.java  |  44 ++-
 .../pulsar/broker/service/ServerCnxTest.java       |  18 +
 .../service/SharedMultiBrokerPulsarCluster.java    |   9 +
 .../ScalableTopicControllerFailoverTest.java       | 387 +++++++++++++++++++++
 .../scalable/ScalableTopicControllerTest.java      |  80 +++++
 .../service/scalable/ScalableTopicServiceTest.java |  28 ++
 .../scalable/SubscriptionCoordinatorTest.java      |  41 +++
 .../v5/V5MultiBrokerControllerFailoverTest.java    | 147 ++++++++
 13 files changed, 1120 insertions(+), 93 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index c851333a7286..4fa9a1520fde 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -1157,6 +1157,9 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
     private final java.util.concurrent.ConcurrentHashMap<Long, ScalableConsumerRegistrationRef>
             scalableConsumerRegistrations = new java.util.concurrent.ConcurrentHashMap<>();
 
+    /** Set once the connection is closed to make its scalable consumers subscribe again. */
+    private final AtomicBoolean closedForScalableConsumers = new AtomicBoolean();
+
     /**
      * A scalable-topic consumer registration made on this connection. {@code answered} records
      * whether the response to its subscribe, {@code requestId}, was written; see
@@ -1307,6 +1310,23 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
         }
     }
 
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The protocol has no command to close a single scalable consumer session, so this closes the
+     * connection: the client subscribes all its scalable consumer sessions again, with the controller
+     * leader it looks up, and reconnects its producers and consumers. Once for all the sessions of
+     * the connection: the channel stays active until the close runs on its event loop.
+     */
+    @Override
+    public void closeScalableConsumerSession(long consumerId) {
+        if (ctx.channel().isActive() && closedForScalableConsumers.compareAndSet(false, true)) {
+            log.info().attr("consumerId", consumerId)
+                    .log("Closing the connection of a scalable consumer to make it subscribe again");
+            ctx.close();
+        }
+    }
+
     @Override
     protected void handleCommandScalableTopicUnsubscribe(
             CommandScalableTopicUnsubscribe commandScalableTopicUnsubscribe) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
index 53c071a5780f..5fdf6bb59d49 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java
@@ -106,4 +106,10 @@ public interface TransportCnx {
      * otherwise.
      */
     void sendScalableConsumerAssignment(long consumerId, ScalableConsumerAssignment assignment);
+
+    /**
+     * Make a scalable-topic consumer subscribe again: the controller holding its session stopped
+     * serving as the leader, and the consumer must subscribe with the one that leads now.
+     */
+    void closeScalableConsumerSession(long consumerId);
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ConsumerSession.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ConsumerSession.java
index 71e62ef14ecf..dff0e3f1af68 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ConsumerSession.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ConsumerSession.java
@@ -167,6 +167,17 @@ public class ConsumerSession {
         }
     }
 
+    /**
+     * Make the consumer subscribe again, with whichever broker is the controller leader now: the
+     * controller holding this session stopped serving as the leader. No-op while disconnected.
+     */
+    public void resubscribe() {
+        TransportCnx localCnx = this.cnx;
+        if (localCnx != null && connected) {
+            localCnx.closeScalableConsumerSession(consumerId);
+        }
+    }
+
     /**
      * Push an assignment to this consumer, if currently connected. The connection sends it as the
      * response to the consumer's subscribe while that is unanswered, and as a
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java
index 9e8c9d67fef7..9231702fce4d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicController.java
@@ -61,6 +61,7 @@ import org.apache.pulsar.common.scalable.SegmentTopicName;
 import org.apache.pulsar.common.util.Backoff;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.compaction.Compactor;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.Notification;
 import org.apache.pulsar.metadata.api.NotificationType;
 import org.apache.pulsar.metadata.api.coordination.CoordinationService;
@@ -123,9 +124,41 @@ public class ScalableTopicController {
     private volatile long lastMergeAtMs = Long.MIN_VALUE;
     private volatile long lastRebucketAtMs = Long.MIN_VALUE;
 
+    /**
+     * {@link LeaderElectionState#Leading} while this controller serves as the leader, which starts once it
+     * has taken over (see {@link #takeOverLeadership()}), after its election reported Leading.
+     */
     @Getter
     private volatile LeaderElectionState leaderState = LeaderElectionState.NoLeader;
 
+    /** The state its leader election last reported to this controller. Guarded by {@code this}. */
+    private LeaderElectionState electionState = LeaderElectionState.NoLeader;
+
+    /**
+     * Incremented on every change of {@link #electionState}: a takeover started before the last change is
+     * stale, and must neither restore consumer sessions nor serve. Guarded by {@code this}.
+     */
+    private long term;
+
+    /**
+     * One per term the election makes this controller the leader in: completed once the controller
+     * serves, or once the term ends, across failed takeover attempts. Completed outside of a term.
+     * Guarded by {@code this}.
+     */
+    private CompletableFuture<Void> leadership = CompletableFuture.completedFuture(null);
+
+    /** The latest takeover attempt of the current term, completed if none. Guarded by {@code this}. */
+    private CompletableFuture<Void> takeOverAttempt = CompletableFuture.completedFuture(null);
+
+    /**
+     * Cadence of the retries of a failed takeover, e.g. on a metadata store error. Reset on every
+     * takeover. Guarded by {@code this}.
+     */
+    private final Backoff takeOverBackoff = Backoff.builder()
+            .initialDelay(Duration.ofSeconds(1))
+            .maxBackoff(Duration.ofMinutes(1))
+            .build();
+
     private volatile boolean closed = false;
 
     /** Registered when this controller elects, see {@link #onControllerLockDeleted()}. */
@@ -169,18 +202,23 @@ public class ScalableTopicController {
     }
 
     /**
-     * Reacts to leader election state transitions. On {@link LeaderElectionState#NoLeader}
+     * Reacts to leader election state transitions, including those of the elections the
+     * leader election runs by itself after the one {@link #initialize()} started: when the
+     * controller lock is deleted (its holder released it or lost its metadata session) and
+     * when the metadata session is re-established. Leading makes this controller take over
+     * as the leader; any other state makes it stop serving as the leader at once, so that a
+     * deposed leader never acts alongside the new one. On {@link LeaderElectionState#NoLeader}
      * we kick off another {@link #initialize()} so the cluster always converges toward
      * having a leader.
+     *
+     * <p>The election calls this while holding its monitor, one state change at a time.
      */
     private void onLeaderStateChange(LeaderElectionState state) {
         log.info().attr("state", state).log("Leader state change for scalable topic");
-        if (state != LeaderElectionState.Leading) {
-            // Stepped down (or never was leader). Stop the GC and auto-scale ticks so the
-            // deposed leader doesn't race the new one on layout writes / backing-topic
-            // deletes. The new leader's initialize() will reschedule.
-            cancelGcTask();
-            cancelAutoScaleTask();
+        if (state == LeaderElectionState.Leading) {
+            takeOverLeadership();
+        } else {
+            stopServingAsLeader(state);
         }
         if (state == LeaderElectionState.NoLeader && !closed) {
             initialize().exceptionally(ex -> {
@@ -193,15 +231,15 @@ public class ScalableTopicController {
     /**
      * The controller lock was deleted while this broker follows the leader: the leader released
      * the controller or lost its metadata session, or the topic is being deleted. A follower's
-     * election is closed once it settles (see {@link #electLeader()}), so it doesn't take the lock
-     * over by itself, even for a topic being deleted, where it would re-create the lock under the
-     * record the delete is about to remove (failing that delete on stores that refuse to delete a
-     * node with children). Re-elect through {@link #initialize()} instead, unless the topic is gone
-     * or marked deleted, retrying with backoff while that fails.
+     * election is closed once it settles (see {@link #electLeader()} and
+     * {@link #stopServingAsLeader(LeaderElectionState)}), so it doesn't take the lock over by
+     * itself, even for a topic being deleted, where it would re-create the lock under the record
+     * the delete is about to remove (failing that delete on stores that refuse to delete a node
+     * with children). Re-elect through {@link #initialize()} instead, unless the topic is gone or
+     * marked deleted, retrying with backoff while that fails.
      */
     private void onControllerLockDeleted() {
-        if (closed || leaderState != LeaderElectionState.Following
-                || !reElecting.compareAndSet(false, true)) {
+        if (closed || !isFollowing() || !reElecting.compareAndSet(false, true)) {
             return;
         }
         reElect(Backoff.builder()
@@ -211,7 +249,7 @@ public class ScalableTopicController {
     }
 
     private void reElect(Backoff backoff) {
-        resources.getScalableTopicMetadataAsync(topicName, true)
+        readTopicRecord()
                 .thenCompose(optMd -> {
                     if (closed || optMd.isEmpty() || optMd.get().isDeleted()) {
                         log.debug().log("Controller lock released for a deleted topic, not re-electing");
@@ -241,7 +279,7 @@ public class ScalableTopicController {
         if (closed) {
             return;
         }
-        resources.getScalableTopicMetadataAsync(topicName, true)
+        readTopicRecord()
                 .thenAccept(optMd -> {
                     if (optMd.isEmpty() && !closed) {
                         log.debug().log("Scalable topic deleted, releasing its controller");
@@ -309,40 +347,179 @@ public class ScalableTopicController {
                     this.currentLayout = SegmentLayout.fromMetadata(optMd.get());
                     return electLeader();
                 })
-                .thenCompose(__ -> isLeader()
-                        ? releaseIfMarkedDeleted() : CompletableFuture.completedFuture(null))
-                .thenCompose(__ -> {
-                    if (isLeader()) {
-                        seedAutoScaleCooldownsFromLayout();
-                        scheduleGcTask();
-                        scheduleAutoScaleTask();
-                        return ensureActiveSegmentsExist()
-                                .thenCompose(___ -> restoreSessionsFromStore());
+                // The election reported its outcome to onLeaderStateChange before completing:
+                // wait for the takeover attempt that started, if it made this controller the leader.
+                .thenCompose(__ -> currentTakeOverAttempt());
+    }
+
+    private synchronized CompletableFuture<Void> currentTakeOverAttempt() {
+        return takeOverAttempt;
+    }
+
+    /**
+     * The takeover in progress if the leader election made this controller the leader and it
+     * doesn't serve yet, completed otherwise: a request arriving meanwhile can wait for it rather
+     * than fail as reaching a broker that is not the leader.
+     */
+    synchronized CompletableFuture<Void> takeOverInProgress() {
+        return leadership;
+    }
+
+    /** Whether no state change of the leader election, nor a close, came after {@code term}. */
+    private synchronized boolean isCurrentTerm(long term) {
+        return !closed && term == this.term;
+    }
+
+    private synchronized boolean isFollowing() {
+        return electionState == LeaderElectionState.Following;
+    }
+
+    /**
+     * The leader election made this controller the leader: take over. The layout is reloaded,
+     * since the previous leader may have changed it after this controller loaded it, the
+     * cooldowns and the consumer sessions of the previous leader are recovered, and the active
+     * segments are healed. Only then does this controller serve as the leader: a consumer
+     * registering before the sessions are restored would be handed segments that a restored
+     * session owns. A failed takeover is retried, backing off, as long as the election keeps
+     * this controller the leader.
+     */
+    private void takeOverLeadership() {
+        long term;
+        synchronized (this) {
+            if (closed || electionState == LeaderElectionState.Leading) {
+                return;
+            }
+            electionState = LeaderElectionState.Leading;
+            term = ++this.term;
+            leadership = new CompletableFuture<>();
+            takeOverBackoff.reset();
+        }
+        startTakeOver(term);
+    }
+
+    private void startTakeOver(long term) {
+        if (!isCurrentTerm(term)) {
+            return;
+        }
+        // Not under the monitor: a takeover that finds the topic deleted closes this controller,
+        // and with it the leader election, whose monitor comes first.
+        CompletableFuture<Void> takeover = takeOver(term);
+        long retryDelayMs;
+        synchronized (this) {
+            if (term != this.term) {
+                return;
+            }
+            takeOverAttempt = takeover;
+            retryDelayMs = takeOverBackoff.next().toMillis();
+        }
+        takeover.exceptionally(ex -> {
+            if (!isCurrentTerm(term)) {
+                // Closed, e.g. for a deleted topic, or overtaken by a later state change.
+                return null;
+            }
+            log.warn().attr("retryDelayMs", retryDelayMs).exceptionMessage(ex)
+                    .log("Failed to take over as the scalable topic leader, retrying");
+            scheduler().schedule(() -> startTakeOver(term), retryDelayMs, TimeUnit.MILLISECONDS);
+            return null;
+        });
+    }
+
+    /**
+     * A topic deleted or being deleted has nothing to serve. The delete marks the topic before
+     * releasing its leader, but this controller may have read the record before the mark and won
+     * the election after the release, re-creating the lock under the record the delete is about
+     * to remove: it gives the lock back by closing, and the takeover fails. ScalableTopicService
+     * replaces a closed controller.
+     */
+    private CompletableFuture<Void> takeOver(long term) {
+        return readTopicRecord()
+                .thenCompose(optMd -> {
+                    if (!isCurrentTerm(term)) {
+                        return CompletableFuture.completedFuture(null);
                     }
-                    return CompletableFuture.completedFuture(null);
+                    if (optMd.isEmpty() || optMd.get().isDeleted()) {
+                        log.info().log("Scalable topic deleted while taking over as its leader, closing"
+                                + " the controller");
+                        return close().thenCompose(__ -> CompletableFuture.failedFuture(
+                                new IllegalStateException("Scalable topic is being deleted: " + topicName)));
+                    }
+                    this.currentLayout = SegmentLayout.fromMetadata(optMd.get());
+                    seedAutoScaleCooldownsFromLayout();
+                    return ensureActiveSegmentsExist()
+                            .thenCompose(__ -> restoreSessionsFromStore(term))
+                            .thenRun(() -> serveAsLeader(term));
                 });
     }
 
     /**
-     * The delete marks the topic before releasing its leader, but a controller that read the record
-     * before the mark can win the election after the release, re-creating the lock under the record
-     * the delete is about to remove. Read the record again once elected, and give the lock back if
-     * the topic is marked deleted or gone.
+     * The record of the topic, read from the store. Empty if it is missing or can't be read, like
+     * the empty one that a store re-creating missing parents leaves under the lock of a deleted
+     * topic.
      */
-    private CompletableFuture<Void> releaseIfMarkedDeleted() {
+    private CompletableFuture<Optional<ScalableTopicMetadata>> readTopicRecord() {
         return resources.getScalableTopicMetadataAsync(topicName, true)
-                .thenCompose(optMd -> {
-                    if (optMd.isPresent() && !optMd.get().isDeleted()) {
-                        return CompletableFuture.completedFuture(null);
+                .exceptionally(ex -> {
+                    if (FutureUtil.unwrapCompletionException(ex)
+                            instanceof MetadataStoreException.ContentDeserializationException) {
+                        return Optional.empty();
                     }
-                    leaderState = LeaderElectionState.NoLeader;
-                    return leaderElection.asyncClose()
-                            .handle((__, ex) -> null)
-                            .thenCompose(__ -> CompletableFuture.failedFuture(new IllegalStateException(
-                                    "Scalable topic is being deleted: " + topicName)));
+                    throw FutureUtil.wrapToCompletionException(ex);
                 });
     }
 
+    private void serveAsLeader(long term) {
+        CompletableFuture<Void> served;
+        synchronized (this) {
+            if (closed || term != this.term) {
+                return;
+            }
+            leaderState = LeaderElectionState.Leading;
+            scheduleGcTask();
+            scheduleAutoScaleTask();
+            served = leadership;
+        }
+        // Not under the monitor: the requests that waited for the takeover go on from here.
+        served.complete(null);
+    }
+
+    /**
+     * The leader election lost the controller lock, or reported that another broker holds it:
+     * stop serving as the leader at once. Besides the GC and auto split/merge ticks, the
+     * consumer sessions are dropped: a deposed leader must neither push assignments nor evict
+     * the consumers whose grace period runs out. The consumers connected here are made to
+     * subscribe again, with whichever broker leads now, which restores their sessions from the
+     * metadata store: nothing else would tell them to move.
+     */
+    private void stopServingAsLeader(LeaderElectionState state) {
+        CompletableFuture<Void> endedTerm = null;
+        synchronized (this) {
+            if (closed) {
+                return;
+            }
+            if (state != electionState) {
+                electionState = state;
+                term++;
+                leaderState = state;
+                endedTerm = leadership;
+                takeOverAttempt = CompletableFuture.completedFuture(null);
+                cancelGcTask();
+                cancelAutoScaleTask();
+                closeSubscriptions(true);
+            }
+        }
+        if (endedTerm != null) {
+            // The requests waiting for a takeover go on, and find that this controller doesn't serve.
+            endedTerm.complete(null);
+        }
+        if (state == LeaderElectionState.Following) {
+            // Like in electLeader(): a follower holds no lock, and closing its election keeps it
+            // from taking the lock over by itself once the lock is deleted, before this controller
+            // can check the topic. onControllerLockDeleted() re-elects. Not under the monitor: the
+            // election's comes first.
+            leaderElection.asyncClose();
+        }
+    }
+
     /**
      * Recover the auto split/merge/rebucket cooldown clocks after winning leadership. The
      * timestamps are in-memory only, but the layout itself records when each segment was
@@ -728,25 +905,31 @@ public class ScalableTopicController {
      * on successful leader election so the newly-elected leader can resume servicing
      * consumers that were registered under a previous leader.
      */
-    private CompletableFuture<Void> restoreSessionsFromStore() {
+    private CompletableFuture<Void> restoreSessionsFromStore(long term) {
         return resources.listSubscriptionsAsync(topicName)
                 .thenCompose(subNames -> {
                     if (subNames.isEmpty()) {
                         return CompletableFuture.completedFuture(null);
                     }
                     CompletableFuture<?>[] futures = subNames.stream()
-                            .map(this::restoreSubscription)
+                            .map(subscription -> restoreSubscription(subscription, term))
                             .toArray(CompletableFuture[]::new);
                     return CompletableFuture.allOf(futures);
                 });
     }
 
-    private CompletableFuture<Void> restoreSubscription(String subscription) {
+    private CompletableFuture<Void> restoreSubscription(String subscription, long term) {
         return resources.listConsumersAsync(topicName, subscription)
                 .thenAccept(consumerNames -> {
-                    SubscriptionCoordinator coordinator = subscriptions.computeIfAbsent(
-                            subscription, this::createCoordinator);
-                    coordinator.restoreConsumers(consumerNames);
+                    synchronized (this) {
+                        if (closed || term != this.term) {
+                            // Stopped serving meanwhile: the sessions are the next leader's.
+                            return;
+                        }
+                        SubscriptionCoordinator coordinator = subscriptions.computeIfAbsent(
+                                subscription, this::createCoordinator);
+                        coordinator.restoreConsumers(consumerNames);
+                    }
                     log.info().attr("subscription", subscription)
                             .attr("consumerCount", consumerNames.size())
                             .log("Restored subscription");
@@ -899,13 +1082,13 @@ public class ScalableTopicController {
         }
         return leaderElection.elect(brokerId)
                 .thenAccept(state -> {
-                    if (state == LeaderElectionState.Following) {
+                    if (state == LeaderElectionState.Following || closed) {
                         // A follower holds no lock. Closing its election keeps the election from
                         // taking the lock over by itself once the lock is deleted, before this
                         // controller can check the topic: onControllerLockDeleted() re-elects.
+                        // Closed while electing, this controller closes what elect() reopened.
                         leaderElection.asyncClose();
                     }
-                    this.leaderState = state;
                     log.info().attr("state", state)
                             .log("Leader election for scalable topic");
                 });
@@ -1116,9 +1299,13 @@ public class ScalableTopicController {
                                                                    ScalableConsumerType
                                                                            consumerType,
                                                                    TransportCnx cnx) {
-        checkLeader();
-        SubscriptionCoordinator coordinator = subscriptions.computeIfAbsent(
-                subscription, sub -> createCoordinator(sub, consumerType));
+        SubscriptionCoordinator coordinator;
+        synchronized (this) {
+            // Under the monitor: stopping to serve in between would miss the coordinator.
+            checkLeader();
+            coordinator = subscriptions.computeIfAbsent(
+                    subscription, sub -> createCoordinator(sub, consumerType));
+        }
         // The coordinator may have been created on the failover-restore path (consumer
         // type unknown then; we defaulted to "no parent-drain enforcement"). Now that we
         // know the type, upgrade if it's STREAM. installDrainChecker is a no-op if the
@@ -1796,17 +1983,45 @@ public class ScalableTopicController {
 
     // --- Lifecycle ---
 
+    /**
+     * Release the controller: its leader election is closed, which releases the controller lock
+     * if it holds it. The consumers connected here are not moved: on a broker shutdown their
+     * connections close anyway, and a deleted topic has nothing left to subscribe to.
+     */
     public CompletableFuture<Void> close() {
-        closed = true;
-        resources.deregisterPathListener(controllerLockListener);
-        resources.deregisterPathListener(topicRecordListener);
-        cancelGcTask();
-        cancelAutoScaleTask();
-        // Stop each coordinator's drain poller before clearing — otherwise the scheduler
-        // task keeps running after the controller goes away.
-        subscriptions.values().forEach(SubscriptionCoordinator::close);
+        // The election first, outside the monitor (the election calls onLeaderStateChange
+        // holding its own): ScalableTopicService replaces a controller seen closed, and the
+        // replacement gets an election of its own only once this one is closed.
+        CompletableFuture<Void> electionClosed = leaderElection.asyncClose();
+        CompletableFuture<Void> endedTerm;
+        synchronized (this) {
+            closed = true;
+            resources.deregisterPathListener(controllerLockListener);
+            resources.deregisterPathListener(topicRecordListener);
+            leaderState = LeaderElectionState.NoLeader;
+            endedTerm = leadership;
+            cancelGcTask();
+            cancelAutoScaleTask();
+            closeSubscriptions(false);
+        }
+        endedTerm.complete(null);
+        return electionClosed;
+    }
+
+    public boolean isClosed() {
+        return closed;
+    }
+
+    private void closeSubscriptions(boolean resubscribeConsumers) {
+        // Stop each coordinator's drain poller and grace timers before clearing — otherwise
+        // the scheduler tasks keep running after the controller stops serving.
+        subscriptions.values().forEach(coordinator -> {
+            coordinator.close();
+            if (resubscribeConsumers) {
+                coordinator.getConsumers().forEach(ConsumerSession::resubscribe);
+            }
+        });
         subscriptions.clear();
-        return leaderElection.asyncClose();
     }
 
     // --- Internal helpers ---
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
index 76c4d1cafb5e..efe523107211 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.service.scalable;
 
+import java.time.Duration;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
@@ -61,6 +62,9 @@ public class ScalableTopicService {
     public static final GetStatsOptions SEGMENT_STATS_OPTIONS =
             new GetStatsOptions(false, true, false, false, false);
 
+    /** How long a request waits for a controller taking over as the leader, before going on. */
+    private static final Duration TAKEOVER_WAIT = Duration.ofSeconds(10);
+
     private final BrokerService brokerService;
     private final ScalableTopicResources resources;
     private final CoordinationService coordinationService;
@@ -106,48 +110,77 @@ public class ScalableTopicService {
 
     /**
      * Get or create a controller for a scalable topic. The controller will attempt
-     * leader election; only the leader actively coordinates consumers.
+     * leader election; only the leader actively coordinates consumers. A controller that
+     * closed itself, having found its topic deleted while taking over, is replaced. A takeover
+     * in progress is waited for, bounded, so that a request arriving meanwhile doesn't fail.
      */
     public CompletableFuture<ScalableTopicController> getOrCreateController(TopicName topic) {
         String key = topic.toString();
-        CompletableFuture<ScalableTopicController> stored = controllers.computeIfAbsent(key, k -> {
-            ScalableTopicController controller = new ScalableTopicController(
-                    topic, resources, brokerService, coordinationService);
-            CompletableFuture<ScalableTopicController> future = new CompletableFuture<>();
-            // Released once its topic is deleted: the admin delete stops the controller on the
-            // broker serving it, not the ones other brokers hold for the topic.
-            controller.setTopicDeletedAction(() -> {
-                if (controllers.remove(key, future)) {
-                    controller.close();
-                }
-            });
-            controller.initialize().whenComplete((__, ex) -> {
-                if (ex == null) {
-                    future.complete(controller);
-                } else {
-                    // initialize() can fail after winning the election: close the controller, or
-                    // its election keeps the lock.
-                    controller.close();
-                    future.completeExceptionally(ex);
-                }
-            });
-            return future;
-        });
+        CompletableFuture<ScalableTopicController> stored = controllers.compute(key,
+                (k, existing) -> existing == null || isClosed(existing) ? newController(key, topic) : existing);
         // Evict failed futures so subsequent callers can retry. This runs *outside*
-        // computeIfAbsent, so modifying the map here is safe.
+        // compute, so modifying the map here is safe.
         return stored.exceptionally(ex -> {
             controllers.remove(key, stored);
             throw new RuntimeException("Failed to initialize controller for " + topic, ex);
+        }).thenCompose(this::awaitTakeOver);
+    }
+
+    private CompletableFuture<ScalableTopicController> newController(String key, TopicName topic) {
+        ScalableTopicController controller = new ScalableTopicController(
+                topic, resources, brokerService, coordinationService);
+        CompletableFuture<ScalableTopicController> future = new CompletableFuture<>();
+        // Released once its topic is deleted: the admin delete stops the controller on the
+        // broker serving it, not the ones other brokers hold for the topic. Closed before it is
+        // removed, as in releaseController().
+        controller.setTopicDeletedAction(() -> controller.close()
+                .whenComplete((__, ___) -> controllers.remove(key, future)));
+        controller.initialize().whenComplete((__, ex) -> {
+            if (ex == null) {
+                future.complete(controller);
+            } else {
+                // Release its leader election, and the lock if it was elected: the next
+                // controller of the topic gets an election of its own once this one is closed.
+                controller.close().exceptionally(closeEx -> {
+                    log.warn().attr("topic", topic).exceptionMessage(closeEx)
+                            .log("Failed to close a controller that failed to initialize");
+                    return null;
+                });
+                future.completeExceptionally(ex);
+            }
         });
+        return future;
+    }
+
+    private static boolean isClosed(CompletableFuture<ScalableTopicController> controller) {
+        return controller.isDone() && !controller.isCompletedExceptionally() && controller.join().isClosed();
+    }
+
+    private CompletableFuture<ScalableTopicController> awaitTakeOver(ScalableTopicController controller) {
+        CompletableFuture<Void> takeover = controller.takeOverInProgress();
+        if (takeover.isDone()) {
+            return CompletableFuture.completedFuture(controller);
+        }
+        // Timed out, or the term ended without serving: the request goes on and finds that the
+        // controller doesn't serve.
+        return FutureUtil.addTimeoutHandling(takeover.thenApply(__ -> controller), TAKEOVER_WAIT,
+                        brokerService.getPulsar().getExecutor(),
+                        () -> FutureUtil.createTimeoutException("The controller of " + controller.getTopicName()
+                                + " did not take over in time", ScalableTopicService.class, "awaitTakeOver"))
+                .handle((__, ___) -> controller);
     }
 
     /**
-     * Release the controller for a topic (e.g., on topic unload).
+     * Release the controller for a topic (e.g., on topic unload). It is closed before it is
+     * removed, so that a controller created for the topic afterwards gets a leader election of
+     * its own, rather than this one's.
      */
     public CompletableFuture<Void> releaseController(TopicName topic) {
-        CompletableFuture<ScalableTopicController> future = controllers.remove(topic.toString());
+        String key = topic.toString();
+        CompletableFuture<ScalableTopicController> future = controllers.get(key);
         if (future != null) {
-            return future.thenCompose(ScalableTopicController::close);
+            return future.thenCompose(ScalableTopicController::close)
+                    .whenComplete((__, ___) -> controllers.remove(key, future));
         }
         return CompletableFuture.completedFuture(null);
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinator.java
index aad851201567..186216140dec 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinator.java
@@ -197,6 +197,9 @@ public class SubscriptionCoordinator {
      */
     public synchronized CompletableFuture<Map<ConsumerSession, ConsumerAssignment>> registerConsumer(
             String consumerName, long consumerId, TransportCnx cnx) {
+        if (closed) {
+            return CompletableFuture.failedFuture(notServing());
+        }
         ConsumerSession existing = sessions.get(consumerName);
         if (existing != null) {
             // Reconnect: attach the new connection, cancel any grace timer, and push the
@@ -216,6 +219,10 @@ public class SubscriptionCoordinator {
         return resources.registerConsumerAsync(topicName, subscriptionName, consumerName)
                 .thenApply(__ -> {
                     synchronized (this) {
+                        if (closed) {
+                            // The registration is persisted: the next leader restores it.
+                            throw notServing();
+                        }
                         sessions.put(consumerName, session);
                         Map<ConsumerSession, ConsumerAssignment> result = rebalanceAndNotify();
                         // First consumer (or rejoining one) — kick off drain checks at the
@@ -250,6 +257,9 @@ public class SubscriptionCoordinator {
         return resources.unregisterConsumerAsync(topicName, subscriptionName, consumerName)
                 .thenApply(__ -> {
                     synchronized (this) {
+                        if (closed) {
+                            return Map.<ConsumerSession, ConsumerAssignment>of();
+                        }
                         ConsumerSession current = sessions.get(consumerName);
                         if (current != null && current.getConsumerId() != expectedConsumerId) {
                             // A same-name consumer rejoined while the delete was in flight
@@ -288,7 +298,7 @@ public class SubscriptionCoordinator {
      */
     public synchronized void onConsumerDisconnect(String consumerName) {
         ConsumerSession session = sessions.get(consumerName);
-        if (session == null || !session.isConnected()) {
+        if (closed || session == null || !session.isConnected()) {
             return;
         }
         session.markDisconnected();
@@ -301,6 +311,9 @@ public class SubscriptionCoordinator {
      */
     public synchronized Map<ConsumerSession, ConsumerAssignment> restoreConsumers(
             Collection<String> persistedConsumerNames) {
+        if (closed) {
+            return Map.of();
+        }
         for (String name : persistedConsumerNames) {
             if (sessions.containsKey(name)) {
                 continue;
@@ -327,6 +340,9 @@ public class SubscriptionCoordinator {
      */
     public synchronized CompletableFuture<Map<ConsumerSession, ConsumerAssignment>> onLayoutChange(
             SegmentLayout newLayout) {
+        if (closed) {
+            return CompletableFuture.completedFuture(Map.of());
+        }
         this.currentLayout = newLayout;
         if (sessions.isEmpty()) {
             segmentAssignments.clear();
@@ -346,7 +362,7 @@ public class SubscriptionCoordinator {
      * defaulted to "no enforcement"). No-op if a checker is already installed.
      */
     synchronized void installDrainChecker(SegmentDrainChecker checker) {
-        if (this.drainChecker != null || checker == null) {
+        if (closed || this.drainChecker != null || checker == null) {
             return;
         }
         this.drainChecker = checker;
@@ -376,9 +392,13 @@ public class SubscriptionCoordinator {
     }
 
     /**
-     * Stop the periodic drain-status poller. Called by the controller on close. Idempotent.
-     * Also flips a {@code closed} flag so any {@link #pollDrainStatus()} iteration that's
-     * mid-flight aborts its rearm step instead of leaking a task into the executor.
+     * Stop the periodic drain-status poller and the grace-period timers. Called by the
+     * controller on close and when it stops serving as the leader. Idempotent. Also flips a
+     * {@code closed} flag so any {@link #pollDrainStatus()} iteration that's mid-flight aborts
+     * its rearm step instead of leaking a task into the executor. Nothing that completes after
+     * it (a drain poll, a grace timer, a registration in flight) acts on the sessions or pushes
+     * an assignment, and new registrations fail: the sessions are left, persisted, to the next
+     * leader.
      */
     public synchronized void close() {
         closed = true;
@@ -386,6 +406,7 @@ public class SubscriptionCoordinator {
             drainPollTask.cancel(false);
             drainPollTask = null;
         }
+        sessions.values().forEach(ConsumerSession::cancelGraceTimer);
     }
 
     // --- Drain tracking ---
@@ -437,6 +458,9 @@ public class SubscriptionCoordinator {
      * goes through {@link #pollDrainStatus()}, which queries the broker.
      */
     synchronized void markSegmentsDrained(Set<Long> segmentIds) {
+        if (closed) {
+            return;
+        }
         boolean changed = false;
         for (long id : segmentIds) {
             if (drainedSegmentIds.add(id)) {
@@ -576,7 +600,7 @@ public class SubscriptionCoordinator {
     private void evictExpiredConsumer(String consumerName) {
         synchronized (this) {
             ConsumerSession session = sessions.get(consumerName);
-            if (session == null) {
+            if (closed || session == null) {
                 return;
             }
             if (session.isConnected()) {
@@ -596,6 +620,9 @@ public class SubscriptionCoordinator {
                 })
                 .thenRun(() -> {
                     synchronized (this) {
+                        if (closed) {
+                            return;
+                        }
                         if (!sessions.isEmpty()) {
                             rebalanceAndNotify();
                         } else {
@@ -605,6 +632,11 @@ public class SubscriptionCoordinator {
                 });
     }
 
+    private IllegalStateException notServing() {
+        return new IllegalStateException("The controller no longer serves subscription " + subscriptionName
+                + " of topic " + topicName);
+    }
+
     /**
      * Compute a balanced assignment of segments to consumers.
      *
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
index ffe390458f41..6722440e0ca7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
@@ -1671,6 +1671,24 @@ public class ServerCnxTest {
         channel.finish();
     }
 
+    /**
+     * A scalable topic controller that stops serving makes the consumers connected to it subscribe
+     * again. There's no command closing a single scalable consumer session: their connection is
+     * closed.
+     */
+    @Test(timeOut = 30000)
+    public void testCloseScalableConsumerSessionClosesTheConnection() throws Exception {
+        resetChannel();
+        channel.writeInbound(Commands.newConnect("none", "", null));
+        assertTrue(getResponse() instanceof CommandConnected);
+
+        serverCnx.closeScalableConsumerSession(7L);
+        serverCnx.closeScalableConsumerSession(8L);
+
+        assertFalse(channel.isActive());
+        channel.finish();
+    }
+
     /**
      * PIP-486 clean leave: unsubscribe for an unknown consumer id is an idempotent success; a
      * failed unregister answers an error and keeps the per-connection registration ref (so the
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedMultiBrokerPulsarCluster.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedMultiBrokerPulsarCluster.java
index 9a75b1fc8122..c09c647a28e6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedMultiBrokerPulsarCluster.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedMultiBrokerPulsarCluster.java
@@ -77,6 +77,7 @@ public class SharedMultiBrokerPulsarCluster {
     private final List<PulsarService> brokers = new ArrayList<>(NUM_BROKERS);
     private final List<PulsarAdmin> admins = new ArrayList<>(NUM_BROKERS);
     private final List<PulsarClient> clients = new ArrayList<>(NUM_BROKERS);
+    private int extraBrokers;
 
     /** Returns the singleton instance, starting the cluster on first invocation. */
     public static SharedMultiBrokerPulsarCluster get() throws Exception {
@@ -124,6 +125,14 @@ public class SharedMultiBrokerPulsarCluster {
         return clients.get(0);
     }
 
+    /**
+     * Start one more broker in the cluster, for a test that shuts a broker down, e.g. to fail over
+     * what it leads. It is not one of {@link #getBrokers()}: the caller closes it.
+     */
+    public synchronized PulsarService startExtraBroker() throws Exception {
+        return startBroker(NUM_BROKERS + extraBrokers++);
+    }
+
     @SuppressWarnings("deprecation")
     private void start() throws Exception {
         log.info().attr("brokers", NUM_BROKERS).log("Starting SharedMultiBrokerPulsarCluster");
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerFailoverTest.java
new file mode 100644
index 000000000000..3c5a5c994fbe
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerFailoverTest.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.scalable;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.resources.ScalableTopicResources;
+import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.broker.service.TransportCnx;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.ScalableTopics;
+import org.apache.pulsar.common.api.proto.ScalableConsumerType;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.apache.pulsar.metadata.TestZKServer;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.NotificationType;
+import org.apache.pulsar.metadata.api.coordination.CoordinationService;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.api.extended.SessionEvent;
+import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
+import org.apache.pulsar.metadata.impl.ZKMetadataStore;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Op;
+import org.apache.zookeeper.ZooDefs;
+import org.awaitility.Awaitility;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Controller-lock failover between brokers: the {@link ScalableTopicController} must follow its leader
+ * election when the election changes state by itself, after the initial election. Runs on a real
+ * ZooKeeper server so that every broker has its own metadata session, as the controller lock is an
+ * ephemeral node owned by the leader's session.
+ */
+public class ScalableTopicControllerFailoverTest {
+
+    private static final int SEGMENTS = 2;
+
+    private TestZKServer zks;
+    private TopicName topic;
+    private Broker broker1;
+    private Broker broker2;
+
+    @BeforeClass(alwaysRun = true)
+    public void startZooKeeper() throws Exception {
+        zks = new TestZKServer();
+    }
+
+    @AfterClass(alwaysRun = true)
+    public void stopZooKeeper() throws Exception {
+        if (zks != null) {
+            zks.close();
+        }
+    }
+
+    @BeforeMethod(alwaysRun = true)
+    public void setUp() throws Exception {
+        topic = TopicName.get("topic://tenant/ns/topic-" + UUID.randomUUID().toString().substring(0, 8));
+        broker1 = new Broker("broker-1");
+        broker2 = new Broker("broker-2");
+        broker1.resources.createScalableTopicAsync(topic,
+                ScalableTopicController.createInitialMetadata(SEGMENTS, 4, Map.of())).get();
+    }
+
+    @AfterMethod(alwaysRun = true)
+    public void tearDown() throws Exception {
+        if (broker1 != null) {
+            broker1.close();
+        }
+        if (broker2 != null) {
+            broker2.close();
+        }
+    }
+
+    /**
+     * The leader goes away, as when its broker shuts down or crashes, and the controller lock is released:
+     * the election of the follower takes the lock over by itself. The follower must then serve as the
+     * leader, from the layout and the consumer sessions the previous leader left behind.
+     */
+    @Test
+    public void testFollowerServesAsLeaderAfterTakingTheLockOver() throws Exception {
+        ScalableTopicController leader = broker1.newController();
+        leader.initialize().get();
+        leader.registerConsumer("sub", "consumer-1", 1L, ScalableConsumerType.STREAM,
+                mock(TransportCnx.class)).get();
+        ScalableTopicController follower = broker2.newController();
+        follower.initialize().get();
+        assertThat(follower.isLeader()).as("broker 2, following broker 1").isFalse();
+        // The layout changes after the follower loaded it.
+        leader.splitSegment(1).get();
+
+        leader.close().get();
+
+        Awaitility.await().untilAsserted(() -> assertThat(follower.isLeader())
+                .as("broker 2, whose election took the controller lock over").isTrue());
+        assertThat(follower.getLayout().get().getAllSegments())
+                .as("segments of the layout the new leader serves")
+                .hasSize(SEGMENTS + 2);
+        // consumer-1 is restored from its persisted session: it keeps one of the two segments it can be
+        // assigned until the sealed segment drains, and consumer-2 gets the other one.
+        ConsumerAssignment assignment = follower.registerConsumer("sub", "consumer-2", 2L,
+                ScalableConsumerType.STREAM, mock(TransportCnx.class)).get();
+        assertThat(assignment.assignedSegments())
+                .as("segments assigned to consumer-2 next to the restored consumer-1")
+                .hasSize(1);
+    }
+
+    /**
+     * The leader loses the controller lock to another broker without releasing it, as when its metadata
+     * session expires and the other broker takes the lock over before it reconnects: its election then
+     * follows the new leader, and it must stop serving as the leader.
+     */
+    @Test
+    public void testLeaderStopsServingWhenAnotherBrokerTakesTheLockOver() throws Exception {
+        ScalableTopicController first = broker1.newController();
+        first.initialize().get();
+        ScalableTopicController second = broker2.newController();
+        second.initialize().get();
+        assertThat(first.isLeader()).as("broker 1, elected first").isTrue();
+
+        broker2.takeOverControllerLock();
+
+        Awaitility.await().untilAsserted(() -> {
+            assertThat(second.isLeader()).as("broker 2, now holding the controller lock").isTrue();
+            assertThat(first.isLeader()).as("broker 1, which lost the controller lock").isFalse();
+        });
+        assertThatThrownBy(() -> first.registerConsumer("sub", "consumer-1", 1L,
+                ScalableConsumerType.STREAM, mock(TransportCnx.class)))
+                .hasMessageContaining("not the leader");
+    }
+
+    /**
+     * A leader deposed by its election closes the election, as a follower does once elected: the
+     * election would otherwise take the lock back by itself once the new leader releases it, and for
+     * a topic being deleted, re-create the lock under the record the delete is about to remove.
+     */
+    @Test
+    public void testDeposedLeaderLeavesTheLockOfATopicBeingDeleted() throws Exception {
+        ScalableTopicController first = broker1.newController();
+        first.initialize().get();
+        ScalableTopicController second = broker2.newController();
+        second.initialize().get();
+        broker2.takeOverControllerLock();
+        Awaitility.await().until(() -> second.isLeader()
+                && first.getLeaderBrokerId().get().equals(Optional.of(broker2.brokerId)));
+
+        String lockPath = broker1.resources.controllerLockPath(topic);
+        List<NotificationType> lockEvents = new CopyOnWriteArrayList<>();
+        broker1.store.registerListener(notification -> {
+            if (notification.getPath().equals(lockPath)) {
+                lockEvents.add(notification.getType());
+            }
+        });
+        broker1.resources.markScalableTopicDeletedAsync(topic).get();
+        second.close().get();
+
+        Awaitility.await().until(() -> lockEvents.contains(NotificationType.Deleted));
+        Awaitility.await().during(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(5))
+                .until(() -> !lockEvents.contains(NotificationType.Created));
+    }
+
+    /**
+     * A controller created again for a topic on a broker that released an earlier one, as when the earlier one
+     * failed to initialize, must follow its own leader election, not the released controller's.
+     */
+    @Test
+    public void testRecreatedControllerServesAsLeaderAfterTakingTheLockOver() throws Exception {
+        ScalableTopicController released = broker1.newController();
+        released.initialize().get();
+        released.close().get();
+        ScalableTopicController leader = broker2.newController();
+        leader.initialize().get();
+        ScalableTopicController recreated = broker1.newController();
+        recreated.initialize().get();
+        assertThat(recreated.isLeader()).as("broker 1, following broker 2").isFalse();
+
+        leader.close().get();
+
+        Awaitility.await().untilAsserted(() -> assertThat(recreated.isLeader())
+                .as("broker 1, whose election took the controller lock over").isTrue());
+    }
+
+    /**
+     * Nothing else tells the consumers connected to a leader that loses the lock to move: it makes
+     * them subscribe again, and the new leader gives them their session back and pushes them the
+     * layout changes it makes.
+     */
+    @Test
+    public void testConsumersOfADeposedLeaderSubscribeAgainWithTheNewOne() throws Exception {
+        ScalableTopicController first = broker1.newController();
+        first.initialize().get();
+        TransportCnx cnx1 = mock(TransportCnx.class);
+        first.registerConsumer("sub", "consumer-1", 1L, ScalableConsumerType.STREAM, cnx1).get();
+        ScalableTopicController second = broker2.newController();
+        second.initialize().get();
+
+        broker2.takeOverControllerLock();
+
+        Awaitility.await().untilAsserted(() -> verify(cnx1).closeScalableConsumerSession(1L));
+        Awaitility.await().until(second::isLeader);
+        TransportCnx cnx2 = mock(TransportCnx.class);
+        ConsumerAssignment assignment = second.registerConsumer("sub", "consumer-1", 2L,
+                ScalableConsumerType.STREAM, cnx2).get();
+        assertThat(assignment.assignedSegments()).as("segments assigned to consumer-1").hasSize(SEGMENTS);
+        assertThat(broker2.resources.listConsumersAsync(topic, "sub").get())
+                .as("registered consumers").containsExactly("consumer-1");
+        second.splitSegment(0).get();
+        verify(cnx2, atLeastOnce()).sendScalableConsumerAssignment(eq(2L),
+                argThat(update -> update.getLayoutEpoch() == 1));
+    }
+
+    /**
+     * A leader that loses the lock and gets it back made its connected consumers subscribe again
+     * meanwhile: they get their sessions back from it.
+     */
+    @Test
+    public void testConsumersSubscribeAgainAfterALeadershipBlip() throws Exception {
+        ScalableTopicController controller = broker1.newController();
+        controller.initialize().get();
+        TransportCnx cnx = mock(TransportCnx.class);
+        controller.registerConsumer("sub", "consumer-1", 1L, ScalableConsumerType.STREAM, cnx).get();
+        ScalableTopicController other = broker2.newController();
+        other.initialize().get();
+
+        broker2.takeOverControllerLock();
+        Awaitility.await().until(() -> !controller.isLeader());
+        broker1.takeOverControllerLock();
+        Awaitility.await().until(controller::isLeader);
+
+        verify(cnx).closeScalableConsumerSession(1L);
+        ConsumerAssignment assignment = controller.registerConsumer("sub", "consumer-1", 2L,
+                ScalableConsumerType.STREAM, mock(TransportCnx.class)).get();
+        assertThat(assignment.assignedSegments()).as("segments assigned to consumer-1").hasSize(SEGMENTS);
+        assertThat(broker1.resources.listConsumersAsync(topic, "sub").get())
+                .as("registered consumers").containsExactly("consumer-1");
+    }
+
+    /**
+     * A released controller's leader election takes no part in elections, not even when the
+     * metadata session reconnects: the controller created again on the broker leads.
+     */
+    @Test
+    public void testReleasedControllerTakesNoPartAfterReconnect() throws Exception {
+        ScalableTopicController released = broker1.newController();
+        released.initialize().get();
+        released.close().get();
+        BlockingQueue<SessionEvent> sessionEvents = new LinkedBlockingQueue<>();
+        broker1.store.registerSessionListener(sessionEvents::add);
+
+        zks.stop();
+        zks.start();
+
+        Awaitility.await().until(() -> sessionEvents.contains(SessionEvent.Reconnected));
+        String lockPath = broker1.resources.controllerLockPath(topic);
+        Awaitility.await().during(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(5))
+                .until(() -> !broker1.store.exists(lockPath).get());
+        ScalableTopicController recreated = broker1.newController();
+        recreated.initialize().get();
+        assertThat(recreated.isLeader()).as("the controller created again on broker 1").isTrue();
+    }
+
+    /**
+     * A follower seeing the lock released for a topic whose record can't be read, like the empty
+     * one that a store re-creating missing parents leaves under the lock of a deleted topic, has
+     * nothing to serve: it doesn't take the lock.
+     */
+    @Test
+    public void testFollowerLeavesTheLockOfAnUnreadableTopic() throws Exception {
+        ScalableTopicController leader = broker1.newController();
+        leader.initialize().get();
+        ScalableTopicController follower = broker2.newController();
+        follower.initialize().get();
+        broker1.store.put(broker1.resources.topicPath(topic), new byte[0], Optional.empty()).get();
+
+        leader.close().get();
+
+        String lockPath = broker2.resources.controllerLockPath(topic);
+        Awaitility.await().during(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(5))
+                .until(() -> !broker2.store.exists(lockPath).get());
+        assertThat(follower.isLeader()).as("broker 2's controller").isFalse();
+    }
+
+    /** A broker of the test: its own metadata session, coordination service and controllers. */
+    private final class Broker implements AutoCloseable {
+        private final String brokerId;
+        private final MetadataStoreExtended store;
+        private final CoordinationService coordinationService;
+        private final ScalableTopicResources resources;
+        private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
+        private final BrokerService brokerService = mock(BrokerService.class);
+        private final List<ScalableTopicController> controllers = new ArrayList<>();
+
+        Broker(String brokerId) throws Exception {
+            this.brokerId = brokerId;
+            store = MetadataStoreExtended.create(zks.getConnectionString(), MetadataStoreConfig.builder().build());
+            coordinationService = new CoordinationServiceImpl(store);
+            resources = new ScalableTopicResources(store, 30);
+
+            PulsarService pulsar = mock(PulsarService.class);
+            PulsarAdmin admin = mock(PulsarAdmin.class);
+            ScalableTopics scalableTopics = mock(ScalableTopics.class);
+            when(brokerService.getPulsar()).thenReturn(pulsar);
+            when(pulsar.getBrokerId()).thenReturn(brokerId);
+            when(pulsar.getExecutor()).thenReturn(scheduler);
+            when(pulsar.getAdminClient()).thenReturn(admin);
+            when(admin.scalableTopics()).thenReturn(scalableTopics);
+            when(scalableTopics.createSegmentAsync(anyString(), any()))
+                    .thenReturn(CompletableFuture.completedFuture(null));
+            when(scalableTopics.terminateSegmentAsync(anyString()))
+                    .thenReturn(CompletableFuture.completedFuture(null));
+            // No segment ever drains.
+            when(scalableTopics.getSegmentSubscriptionBacklogAsync(anyString(), anyString()))
+                    .thenReturn(CompletableFuture.completedFuture(1L));
+        }
+
+        ScalableTopicController newController() {
+            ScalableTopicController controller =
+                    new ScalableTopicController(topic, resources, brokerService, coordinationService);
+            controllers.add(controller);
+            return controller;
+        }
+
+        /**
+         * Replace the controller lock with one owned by this broker's session, in a single transaction: the
+         * other brokers' elections, re-electing when they see the lock deleted, find this broker holding it.
+         */
+        void takeOverControllerLock() throws Exception {
+            String lockPath = resources.controllerLockPath(topic);
+            byte[] value = ObjectMapperFactory.getMapper().writer().writeValueAsBytes(brokerId);
+            ((ZKMetadataStore) ((DualMetadataStore) store).getSourceStore()).getZkClient().multi(List.of(
+                    Op.delete(lockPath, -1),
+                    Op.create(lockPath, value, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL)));
+        }
+
+        @Override
+        public void close() throws Exception {
+            for (ScalableTopicController controller : controllers) {
+                controller.close().get();
+            }
+            coordinationService.close();
+            store.close();
+            scheduler.shutdownNow();
+        }
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerTest.java
index 9df2b0348567..2f2cc626693b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicControllerTest.java
@@ -49,6 +49,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.resources.ConsumerRegistration;
@@ -289,6 +290,9 @@ public class ScalableTopicControllerTest {
      */
     @Test
     public void testInitializeGivesTheLockBackWhenTopicMarkedDeletedWhileElecting() throws Exception {
+        // Replaced by the racing controller, like ScalableTopicService does: closed first, so that
+        // the racing one gets a leader election of its own.
+        controller.close().join();
         AtomicBoolean marked = new AtomicBoolean();
         ScalableTopicResources markingResources = new ScalableTopicResources(store, 30) {
             @Override
@@ -317,6 +321,82 @@ public class ScalableTopicControllerTest {
         racing.close().join();
     }
 
+    /**
+     * Like a record marked deleted, a record that can't be read once the controller won the
+     * election, like the empty one that a store re-creating missing parents leaves under the lock
+     * of a deleted topic, leaves nothing to serve: the controller gives the lock back rather than
+     * retrying the takeover forever.
+     */
+    @Test
+    public void testInitializeGivesTheLockBackWhenTopicUnreadableWhileElecting() throws Exception {
+        // Replaced by the racing controller, like ScalableTopicService does: closed first, so that
+        // the racing one gets a leader election of its own.
+        controller.close().join();
+        AtomicBoolean emptied = new AtomicBoolean();
+        ScalableTopicResources emptyingResources = new ScalableTopicResources(store, 30) {
+            @Override
+            public CompletableFuture<Optional<ScalableTopicMetadata>> getScalableTopicMetadataAsync(
+                    TopicName tn, boolean refresh) {
+                // The first read sees the record, then it is emptied.
+                return super.getScalableTopicMetadataAsync(tn, refresh)
+                        .thenCompose(md -> emptied.compareAndSet(false, true)
+                                ? store.put(topicPath(tn), new byte[0], Optional.empty()).thenApply(__ -> md)
+                                : CompletableFuture.completedFuture(md));
+            }
+        };
+        ScalableTopicController racing = new ScalableTopicController(topicName, emptyingResources,
+                brokerService, coordinationService);
+        try {
+            racing.initialize().get();
+            fail("expected a failure for a topic whose record can't be read");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause().getMessage().contains("being deleted"),
+                    "unexpected failure: " + e.getCause());
+        }
+        assertFalse(racing.isLeader());
+        assertFalse(store.get(resources.controllerLockPath(topicName)).get().isPresent(),
+                "the controller lock must be given back");
+        verify(scalableTopics, times(0)).createSegmentAsync(anyString(), any());
+        racing.close().join();
+    }
+
+    /**
+     * A failed takeover is retried, backing off, while the election keeps the controller the
+     * leader. A request reaching the controller in between waits for the retry, rather than
+     * failing at once as reaching a broker that is not the leader.
+     */
+    @Test
+    public void testFailedTakeOverIsRetriedWhileRequestsWait() throws Exception {
+        // Replaced by the failing controller, like ScalableTopicService does: closed first, so that
+        // the failing one gets a leader election of its own.
+        controller.close().join();
+        AtomicInteger listings = new AtomicInteger();
+        ScalableTopicResources failingOnceResources = new ScalableTopicResources(store, 30) {
+            @Override
+            public CompletableFuture<List<String>> listSubscriptionsAsync(TopicName tn) {
+                // Restoring the consumer sessions is the last step of a takeover.
+                return listings.getAndIncrement() == 0
+                        ? CompletableFuture.failedFuture(new IllegalStateException("injected"))
+                        : super.listSubscriptionsAsync(tn);
+            }
+        };
+        ScalableTopicController retrying = new ScalableTopicController(topicName, failingOnceResources,
+                brokerService, coordinationService);
+        try {
+            retrying.initialize().get();
+            fail("expected the first takeover attempt to fail");
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause().getMessage().contains("injected"), "unexpected failure: " + e.getCause());
+        }
+        CompletableFuture<Void> takeover = retrying.takeOverInProgress();
+        assertFalse(takeover.isDone(), "a request must wait for the retry");
+        assertFalse(retrying.isLeader());
+
+        takeover.get(10, TimeUnit.SECONDS);
+        assertTrue(retrying.isLeader());
+        retrying.close().join();
+    }
+
     @Test
     public void testLeaderBrokerIdPersisted() throws Exception {
         controller.initialize().get();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java
index fbc35efe3cd8..df1057b303d1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/ScalableTopicServiceTest.java
@@ -236,6 +236,34 @@ public class ScalableTopicServiceTest {
         }
     }
 
+    @Test
+    public void testControllerCreatedAfterFailedInitializationLeads() throws Exception {
+        TopicName tn = scalableTopic("t-late");
+        try {
+            service.getOrCreateController(tn).get();
+            fail("expected failure: the topic doesn't exist yet");
+        } catch (ExecutionException expected) {
+            // The failed controller must not leave its leader election to the next one.
+        }
+        service.createScalableTopic(tn, 1).get();
+
+        ScalableTopicController controller = service.getOrCreateController(tn).get();
+        assertTrue(controller.isLeader(), "the only controller of the topic must lead it");
+    }
+
+    @Test
+    public void testClosedControllerIsReplaced() throws Exception {
+        TopicName tn = scalableTopic("t-closed");
+        service.createScalableTopic(tn, 1).get();
+        // A controller closes itself when it finds its topic deleted while taking over.
+        ScalableTopicController closed = service.getOrCreateController(tn).get();
+        closed.close().get();
+
+        ScalableTopicController controller = service.getOrCreateController(tn).get();
+        assertNotSame(controller, closed);
+        assertTrue(controller.isLeader(), "the controller replacing the closed one must lead");
+    }
+
     @Test
     public void testReleaseControllerDrops() throws Exception {
         TopicName tn = scalableTopic("t-release");
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinatorTest.java
index 458759fbd9df..9a11ede592f0 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/scalable/SubscriptionCoordinatorTest.java
@@ -19,8 +19,13 @@
 package org.apache.pulsar.broker.service.scalable;
 
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.clearInvocations;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
@@ -473,6 +478,42 @@ public class SubscriptionCoordinatorTest {
         });
     }
 
+    @Test
+    public void testClosedCoordinatorNeitherRegistersNorPushes() {
+        TransportCnx cnx1 = mock(TransportCnx.class);
+        coordinator.registerConsumer("consumer-1", 1L, cnx1).join();
+        CompletableFuture<Void> persisted = new CompletableFuture<>();
+        when(resources.registerConsumerAsync(any(), anyString(), eq("consumer-2"))).thenReturn(persisted);
+        TransportCnx cnx2 = mock(TransportCnx.class);
+        CompletableFuture<Map<ConsumerSession, ConsumerAssignment>> inFlight =
+                coordinator.registerConsumer("consumer-2", 2L, cnx2);
+
+        // The controller stops serving as the leader while consumer-2 registers.
+        coordinator.close();
+        clearInvocations(cnx1);
+        persisted.complete(null);
+        coordinator.onLayoutChange(initialLayout.splitSegment(0, System.currentTimeMillis()));
+
+        assertTrue(inFlight.isCompletedExceptionally(), "the registration completing after close must fail");
+        assertTrue(coordinator.registerConsumer("consumer-3", 3L, mock(TransportCnx.class))
+                .isCompletedExceptionally(), "a registration after close must fail");
+        verify(cnx1, never()).sendScalableConsumerAssignment(anyLong(), any());
+        verify(cnx2, never()).sendScalableConsumerAssignment(anyLong(), any());
+    }
+
+    @Test
+    public void testClosedCoordinatorDoesNotEvictExpiredSession() {
+        coordinator.registerConsumer("consumer-1", 1L, mock(TransportCnx.class)).join();
+        coordinator.onConsumerDisconnect("consumer-1");
+
+        // The controller stops serving as the leader: the session is the next leader's to restore.
+        coordinator.close();
+
+        // Past the 200ms grace period, the persisted registration must still be there.
+        Awaitility.await().during(1, TimeUnit.SECONDS).atMost(2, TimeUnit.SECONDS).untilAsserted(() ->
+                verify(resources, never()).unregisterConsumerAsync(topicName, "test-sub", "consumer-1"));
+    }
+
     @Test
     public void testRestoreConsumersInstallsDisconnectedSessions() {
         Map<ConsumerSession, ConsumerAssignment> assignments =
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5MultiBrokerControllerFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5MultiBrokerControllerFailoverTest.java
new file mode 100644
index 000000000000..255d9bc0d4dd
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v5/V5MultiBrokerControllerFailoverTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api.v5;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import java.time.Duration;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.service.SharedMultiBrokerPulsarCluster;
+import org.apache.pulsar.broker.service.scalable.ScalableTopicController;
+import org.apache.pulsar.broker.service.scalable.ScalableTopicService;
+import org.apache.pulsar.client.api.v5.config.SubscriptionInitialPosition;
+import org.apache.pulsar.client.api.v5.schema.Schema;
+import org.apache.pulsar.common.naming.TopicName;
+import org.awaitility.Awaitility;
+import org.testng.annotations.Test;
+
+/**
+ * Controller-leader failover across brokers: every broker that handled a request for a scalable
+ * topic holds a controller for it, following the leader. When the leader goes away, the leader
+ * election of one of them takes the controller lock over, and that broker must serve the topic.
+ */
+public class V5MultiBrokerControllerFailoverTest extends V5MultiBrokerClientBaseTest {
+
+    @Test
+    public void testStreamConsumerSubscribesAfterControllerFailover() throws Exception {
+        String topic = newScalableTopic(2);
+        TopicName tn = TopicName.get(topic);
+        @Cleanup
+        Producer<String> producer = v5Client.newProducer(Schema.string())
+                .topic(topic)
+                .create();
+        int formerLeader = findControllerLeaderIndex(topic);
+        String formerLeaderId = brokers.get(formerLeader).getBrokerId();
+        ScalableTopicController follower = scalableTopicService((formerLeader + 1) % brokers.size())
+                .getOrCreateController(tn).get();
+
+        // The leader releases the controller lock, as its broker does when it shuts down: the
+        // election of one of the followers takes it over.
+        scalableTopicService(formerLeader).releaseController(tn).get();
+        Awaitility.await().untilAsserted(() -> assertThat(follower.getLeaderBrokerId().get())
+                .as("controller leader after the former leader released the lock")
+                .isPresent()
+                .isNotEqualTo(Optional.of(formerLeaderId)));
+
+        // The subscribe waits for the new leader to take over, if it hasn't yet.
+        @Cleanup
+        StreamConsumer<String> consumer = v5Client.newStreamConsumer(Schema.string())
+                .topic(topic)
+                .subscriptionName("sub")
+                .subscriptionInitialPosition(SubscriptionInitialPosition.EARLIEST)
+                .subscribe();
+        producer.newMessage().key("k").value("v").send();
+        Message<String> message = consumer.receive(Duration.ofSeconds(10));
+        assertThat(message).as("message received under the new controller leader").isNotNull();
+        assertThat(message.value()).isEqualTo("v");
+
+        // The former leader, holding a controller for the topic again, follows the new leader.
+        ScalableTopicController recreated = scalableTopicService(formerLeader).getOrCreateController(tn).get();
+        assertThat(recreated.isLeader()).as("the former leader, back with a controller").isFalse();
+    }
+
+    /**
+     * The broker leading the controller shuts down while a stream consumer is subscribed: the
+     * consumer subscribes again with the broker that takes the controller lock over, keeps
+     * receiving, and gets the segments of a split made by the new leader.
+     */
+    @Test
+    public void testStreamConsumerFollowsControllerLeaderShutdown() throws Exception {
+        String topic = newScalableTopic(1);
+        TopicName tn = TopicName.get(topic);
+        PulsarService leader = SharedMultiBrokerPulsarCluster.get().startExtraBroker();
+        try {
+            // The extra broker creates its controller first and leads; the others follow it.
+            leader.getBrokerService().getScalableTopicService().getOrCreateController(tn).get();
+            for (int i = 0; i < brokers.size(); i++) {
+                scalableTopicService(i).getOrCreateController(tn).get();
+            }
+            @Cleanup
+            Producer<String> producer = v5Client.newProducer(Schema.string())
+                    .topic(topic)
+                    .create();
+            @Cleanup
+            StreamConsumer<String> consumer = v5Client.newStreamConsumer(Schema.string())
+                    .topic(topic)
+                    .subscriptionName("sub")
+                    .subscriptionInitialPosition(SubscriptionInitialPosition.EARLIEST)
+                    .subscribe();
+            producer.newMessage().key("k").value("before").send();
+            receiveUntil(consumer, Set.of("before"));
+
+            leader.close();
+
+            producer.newMessage().key("k").value("after").send();
+            receiveUntil(consumer, Set.of("after"));
+
+            // The new leader splits the segment: the consumer is assigned the children once it
+            // drained the parent, and receives what is published to them.
+            admin.scalableTopics().splitSegment(topic, 0);
+            Set<String> sent = new HashSet<>();
+            for (int i = 0; i < 20; i++) {
+                String value = "split-" + i;
+                producer.newMessage().key("key-" + i).value(value).send();
+                sent.add(value);
+            }
+            receiveUntil(consumer, sent);
+        } finally {
+            leader.close();
+        }
+    }
+
+    /** Receive, acknowledging, until every one of {@code values} was received. */
+    private static void receiveUntil(StreamConsumer<String> consumer, Set<String> values) {
+        Set<String> received = new HashSet<>();
+        Awaitility.await().atMost(Duration.ofSeconds(60)).until(() -> {
+            Message<String> message = consumer.receive(Duration.ofSeconds(1));
+            if (message != null) {
+                received.add(message.value());
+                consumer.acknowledgeCumulative(message.id());
+            }
+            return received.containsAll(values);
+        });
+    }
+
+    private ScalableTopicService scalableTopicService(int brokerIndex) {
+        return brokers.get(brokerIndex).getBrokerService().getScalableTopicService();
+    }
+}