You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/22 08:31:55 UTC

(pulsar) branch branch-5.0 updated (c914f486271c -> 99ba3dea77aa)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a change to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git


    from c914f486271c [fix][build] Include the java.compiler module in the Pulsar image JVM (#27035)
     new 650885b36a22 [fix][broker] Ignore replicated subscription updates for the compaction subscription (#27032)
     new 1ad5b2884d95 [fix][test] Fix flaky OffloadPrefixTest.terminateTriggersAutomaticOffloadForLastLedger (#27040)
     new da4a1c544678 [improve][broker] Add a random value upto 5% to avoid rollover multiple cursor ledgers  (#26542)
     new 2d370b757ee2 [fix][broker] Shared subscription stalls forever on a chunked message (#26396)
     new d9de47ef195e [fix][broker] Keep the topic-level permissions of a topic migrated to a scalable topic (#26953)
     new d0d92107ee8f [improve][broker] improve readability for the class TopicPolicyListenerWrapper (#26277)
     new 069c0af10f7b [fix][ml] Don't drop a durable cursor reset when an ack is queued behind it (#26305)
     new 96a828bf4785 [fix][fn] Honour consumerCryptoFailureAction in the Python function runtime (#26490)
     new 8fe1446de723 [fix][managed-ledger] Honor batch-index cap in isCursorDataFullyPersistable (#26553)
     new 36d1c98aa2bd [fix][broker] Fail compacted reads when the compacted ledger is missing (#26631)
     new d029efdfa42b [fix][broker] Fix non-persistent subscribe cleanup during migration (#27003)
     new 99ba3dea77aa [fix][broker] Honor secondary tenant admin roles in HTTP admin requests (#26911)

The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../bookkeeper/mledger/impl/ManagedCursorImpl.java |  48 +++-
 .../impl/ManagedCursorBatchAckRecoveryTest.java    |  34 +++
 .../bookkeeper/mledger/impl/ManagedCursorTest.java | 153 ++++++++++++
 .../bookkeeper/mledger/impl/OffloadPrefixTest.java |   5 +-
 pip/pip-475.md                                     |  54 ++++-
 .../authorization/AuthorizationProvider.java       |  19 ++
 .../broker/authorization/AuthorizationService.java |   9 +
 .../authorization/PulsarAuthorizationProvider.java |  67 ++++++
 .../broker/resources/NamespaceResources.java       |   8 +
 .../broker/resources/ScalableTopicResources.java   | 111 +++++++++
 .../PulsarAuthorizationProviderTest.java           | 109 +++++++++
 .../pulsar/broker/admin/v2/ScalableTopics.java     |  55 ++++-
 .../pulsar/broker/service/AbstractTopic.java       |  12 +-
 .../broker/service/SharedConsumerAssignor.java     |  63 ++++-
 .../broker/service/TopicPolicyListenerWrapper.java | 180 ++++++++-------
 .../service/nonpersistent/NonPersistentTopic.java  |  27 +--
 .../PersistentDispatcherMultipleConsumers.java     |   9 +-
 .../service/persistent/PersistentSubscription.java |   5 +-
 .../persistent/PulsarCompactorSubscription.java    |  20 +-
 .../ReplicatedSubscriptionsController.java         |   8 +
 .../service/scalable/ScalableTopicService.java     |   5 +-
 .../pulsar/broker/web/PulsarWebResource.java       |  12 +-
 .../pulsar/compaction/CompactedTopicImpl.java      |  30 +--
 .../compaction/PulsarTopicCompactionService.java   |  11 +-
 .../ScalableTopicMigrationPermissionsTest.java     | 256 +++++++++++++++++++++
 .../broker/admin/v2/ScalableTopicsMigrateTest.java |  12 +-
 .../resources/ScalableTopicCreationLockTest.java   | 198 ++++++++++++++++
 .../broker/service/SharedConsumerAssignorTest.java | 125 ++++++++++
 .../service/TopicPolicyListenerWrapperTest.java    | 216 +++++++++--------
 .../nonpersistent/NonPersistentTopicTest.java      | 131 ++++++++++-
 .../ReplicatedSubscriptionsControllerTest.java     |  32 ++-
 .../MultiRolesTokenAuthorizationProviderTest.java  |  98 ++++++++
 .../api/SubscriptionPauseOnAckStatPersistTest.java |  80 +++++++
 .../apache/pulsar/compaction/CompactionTest.java   |  85 +------
 .../compaction/MissingCompactedLedgerTest.java     | 215 +++++++++++++++++
 .../instance/src/main/python/python_instance.py    |  33 +++
 .../src/test/python/test_python_instance.py        |  88 +++++++
 37 files changed, 2257 insertions(+), 366 deletions(-)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicMigrationPermissionsTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/resources/ScalableTopicCreationLockTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/compaction/MissingCompactedLedgerTest.java


(pulsar) 07/12: [fix][ml] Don't drop a durable cursor reset when an ack is queued behind it (#26305)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 069c0af10f7b48a8d390fa50e7ea19a60f54b14b
Author: water <67...@qq.com>
AuthorDate: Thu Oct 22 09:49:31 2026 +0800

    [fix][ml] Don't drop a durable cursor reset when an ack is queued behind it (#26305)
    
    Co-authored-by: Matteo Merli <mm...@apache.org>
    (cherry picked from commit 33e9b0e8cca9c4447050c37e2d19309ba1d3b3ed)
---
 .../bookkeeper/mledger/impl/ManagedCursorImpl.java | 24 +++++--
 .../bookkeeper/mledger/impl/ManagedCursorTest.java | 82 ++++++++++++++++++++++
 2 files changed, 101 insertions(+), 5 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index a01dfdb1d3cd..6cc16f69ab25 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -249,6 +249,9 @@ public class ManagedCursorImpl implements ManagedCursor {
         final Object ctx;
         final Map<String, Long> properties;
         final Runnable alignAcknowledgeStatusAfterPersisted;
+        // Only a cursor reset supplies its own alignAcknowledgeStatusAfterPersisted, which replaces the whole
+        // acknowledgment state, so it must be persisted and applied even when other entries are queued behind it.
+        final boolean resetsCursor;
 
         // If the callbackGroup is set, it means this mark-delete request was done on behalf of a group of request (just
         // persist the last one in the chain). In this case we need to trigger the callbacks for every request in the
@@ -262,6 +265,7 @@ public class ManagedCursorImpl implements ManagedCursor {
 
         public MarkDeleteEntry(Position newPosition, Map<String, Long> properties,
                 MarkDeleteCallback callback, Object ctx, Runnable alignAcknowledgeStatusAfterPersisted) {
+            this.resetsCursor = alignAcknowledgeStatusAfterPersisted != null;
             if (alignAcknowledgeStatusAfterPersisted == null) {
                 alignAcknowledgeStatusAfterPersisted = () -> {
                     if (batchDeletedIndexes != null) {
@@ -1732,6 +1736,7 @@ public class ManagedCursorImpl implements ManagedCursor {
                         .log("Reset readPosition to, skipping from current readPosition");
             }
             readPosition = newReadPosition;
+            persistentMarkDeletePosition = newMarkDeletePosition;
         };
 
         VoidCallback finalCallback = new VoidCallback() {
@@ -2669,7 +2674,9 @@ public class ManagedCursorImpl implements ManagedCursor {
     }
 
     void internalMarkDelete(final MarkDeleteEntry mdEntry) {
-        if (persistentMarkDeletePosition != null
+        // A cursor reset may move the mark-delete position backward, so it is never skipped in favor of a later
+        // position persisted or being persisted by an acknowledgment
+        if (!mdEntry.resetsCursor && persistentMarkDeletePosition != null
                 && mdEntry.newPosition.compareTo(persistentMarkDeletePosition) < 0) {
             log.info()
                     .attr("position", mdEntry.newPosition)
@@ -2681,7 +2688,7 @@ public class ManagedCursorImpl implements ManagedCursor {
         }
 
         Position inProgressLatest = INPROGRESS_MARKDELETE_PERSIST_POSITION_UPDATER.updateAndGet(this, current -> {
-            if (current != null && current.compareTo(mdEntry.newPosition) > 0) {
+            if (!mdEntry.resetsCursor && current != null && current.compareTo(mdEntry.newPosition) > 0) {
                 return current;
             } else {
                 return mdEntry.newPosition;
@@ -3629,11 +3636,18 @@ public class ManagedCursorImpl implements ManagedCursor {
     }
 
     void internalFlushPendingMarkDeletes() {
-        MarkDeleteEntry lastEntry = pendingMarkDeleteOps.getLast();
-        lastEntry.callbackGroup = Lists.newArrayList(pendingMarkDeleteOps);
+        // Persist the last entry, unless a cursor reset is queued: the entries queued behind it were computed
+        // from the acknowledgment state before the reset, which the reset replaces once it is persisted
+        MarkDeleteEntry entryToPersist = pendingMarkDeleteOps.getLast();
+        for (MarkDeleteEntry entry : pendingMarkDeleteOps) {
+            if (entry.resetsCursor) {
+                entryToPersist = entry;
+            }
+        }
+        entryToPersist.callbackGroup = Lists.newArrayList(pendingMarkDeleteOps);
         pendingMarkDeleteOps.clear();
 
-        internalMarkDelete(lastEntry);
+        internalMarkDelete(entryToPersist);
     }
 
     void createNewMetadataLedger(final VoidCallback callback) {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
index cc9691820904..8a852e8ab3a5 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
@@ -7853,6 +7853,88 @@ public class ManagedCursorTest extends MockedBookKeeperTestCase {
         assertEquals(spyCursor.getProperties(), properties);
     }
 
+    @Test(timeOut = 30000)
+    public void testResetCursorIsNotDroppedByAckQueuedBehindIt() throws Exception {
+        CompletableFuture<Void> releaseReads = new CompletableFuture<>();
+        AtomicBoolean holdReads = new AtomicBoolean(false);
+        bkc.setReadHandleInterceptor((ledgerId, firstEntry, lastEntry, entries) ->
+                holdReads.get() ? releaseReads.thenApply(__ -> entries) : CompletableFuture.completedFuture(entries));
+
+        String ledgerName = "testResetCursorIsNotDroppedByAckQueuedBehindIt";
+        ManagedLedger ledger = factory.open(ledgerName, defaultConfig());
+        ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+        cursor.setInactive(); // disable caching when adding entries, so reads go to the bookie
+        List<Position> positions = new ArrayList<>();
+        for (int i = 0; i < 10; i++) {
+            positions.add(ledger.addEntry(("entry-" + i).getBytes(Encoding)));
+        }
+        cursor.markDelete(positions.get(4));
+
+        // Keep a read in flight, so mark-delete requests are queued
+        holdReads.set(true);
+        CompletableFuture<List<Entry>> readFuture = new CompletableFuture<>();
+        cursor.asyncReadEntries(1, new ReadEntriesCallback() {
+            @Override
+            public void readEntriesComplete(List<Entry> entries, Object ctx) {
+                readFuture.complete(entries);
+            }
+
+            @Override
+            public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
+                readFuture.completeExceptionally(exception);
+            }
+        }, null, PositionFactory.LATEST);
+        Awaitility.await().until(() -> cursor.getPendingReadOpsCount() == 1);
+
+        // Queue a reset, then an ack that advances the mark-delete position behind it
+        CompletableFuture<Void> resetFuture = new CompletableFuture<>();
+        cursor.asyncResetCursor(positions.get(0), false, new AsyncCallbacks.ResetCursorCallback() {
+            @Override
+            public void resetComplete(Object ctx) {
+                resetFuture.complete(null);
+            }
+
+            @Override
+            public void resetFailed(ManagedLedgerException exception, Object ctx) {
+                resetFuture.completeExceptionally(exception);
+            }
+        });
+        Awaitility.await().until(() -> cursor.pendingMarkDeleteOps.size() == 1);
+        cursor.asyncDelete(positions.get(5), new DeleteCallback() {
+            @Override
+            public void deleteComplete(Object ctx) {
+            }
+
+            @Override
+            public void deleteFailed(ManagedLedgerException exception, Object ctx) {
+            }
+        }, null);
+        assertEquals(cursor.pendingMarkDeleteOps.size(), 2);
+        assertEquals(cursor.pendingMarkDeleteOps.getLast().newPosition, positions.get(5));
+
+        // Completing the read flushes the queued mark-delete requests
+        releaseReads.complete(null);
+        readFuture.get(10, TimeUnit.SECONDS).forEach(Entry::release);
+        resetFuture.get(10, TimeUnit.SECONDS);
+
+        Position resetMarkDeletePosition = ((ManagedLedgerImpl) ledger).getPreviousPosition(positions.get(0));
+        assertEquals(cursor.getReadPosition(), positions.get(0));
+        assertEquals(cursor.getMarkDeletedPosition(), resetMarkDeletePosition);
+        assertEquals(cursor.getPersistentMarkDeletedPosition(), resetMarkDeletePosition);
+        assertFalse(cursor.isMessageDeleted(positions.get(5)));
+
+        // Acknowledgments after the reset are persisted
+        holdReads.set(false);
+        cursor.markDelete(positions.get(2));
+        ledger.close();
+
+        ManagedLedger reopenedLedger = factory.open(ledgerName, defaultConfig());
+        ManagedCursorImpl reopenedCursor = (ManagedCursorImpl) reopenedLedger.openCursor("c1");
+        assertEquals(reopenedCursor.getMarkDeletedPosition(), positions.get(2));
+        assertFalse(reopenedCursor.isMessageDeleted(positions.get(5)));
+        reopenedLedger.close();
+    }
+
     @SuppressWarnings("try")
     class TestPulsarMockBookKeeper extends PulsarMockBookKeeper {
         Map<Long, Integer> ledgerErrors = new HashMap<>();


(pulsar) 08/12: [fix][fn] Honour consumerCryptoFailureAction in the Python function runtime (#26490)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 96a828bf478542c9167ee2a4a9bc20df855eb3c2
Author: QIAOZHENYU <14...@qq.com>
AuthorDate: Thu Oct 22 09:50:07 2026 +0800

    [fix][fn] Honour consumerCryptoFailureAction in the Python function runtime (#26490)
    
    Signed-off-by: Matteo Merli <mm...@apache.org>
    Co-authored-by: Matteo Merli <mm...@apache.org>
    (cherry picked from commit 74ba0c68f37fecff22860386c8ac73d73bf2683d)
---
 .../instance/src/main/python/python_instance.py    | 33 ++++++++
 .../src/test/python/test_python_instance.py        | 88 ++++++++++++++++++++++
 2 files changed, 121 insertions(+)

diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py
index 1b3fa49c6a9e..7fd0d1c7ccec 100755
--- a/pulsar-functions/instance/src/main/python/python_instance.py
+++ b/pulsar-functions/instance/src/main/python/python_instance.py
@@ -583,6 +583,7 @@ class PythonInstance(object):
         "dead_letter_policy": dead_letter_policy
       }
       consumer_args.update(nack_args)
+      consumer_args.update(self.get_crypto_failure_action_args(consumer_conf))
       if consumer_conf.HasField("receiverQueueSize"):
         consumer_args["receiver_queue_size"] = consumer_conf.receiverQueueSize.value
 
@@ -654,3 +655,35 @@ class PythonInstance(object):
       except Exception as e:
         Log.error("Failed to load the crypto key reader from spec: %s, error: %s" % (crypto_spec, e))
     return crypto_key_reader
+
+  def get_crypto_failure_action_args(self, consumer_conf):
+    """Build the crypto failure action argument for Client.subscribe().
+
+    Returns a dict to splat into the subscribe() call: either empty, or carrying
+    crypto_failure_action.
+
+    ConsumerSpec.cryptoSpec.consumerCryptoFailureAction tells the client what to do when an
+    input message cannot be decrypted. The Java runtime applies it in PulsarSource
+    (cb.cryptoFailureAction(...)); the Python runtime previously dropped it, so the client
+    default (FAIL) always applied and a configured DISCARD or CONSUME was silently ignored.
+
+    The argument is omitted when no cryptoSpec is configured rather than passed as None,
+    because subscribe() validates it with _check_type(ConsumerCryptoFailureAction, ...) rather
+    than _check_type_or_none, so None would fail for every function without crypto - the same
+    constraint get_negative_ack_args documents for its delay. When a cryptoSpec is present the
+    action is always forwarded: the proto3 enum default (FAIL) equals the client default, so an
+    unset action keeps today's behavior while DISCARD and CONSUME now reach the client. The
+    producer-only value (SEND) is not meaningful for a consumer and falls back to FAIL, like
+    the compression mapping treats values without a consumer equivalent. The Java runtime
+    rejects SEND instead (CryptoUtils.getConsumerCryptoFailureAction throws); the Python
+    runtime keeps the function running on the client default rather than fail at startup.
+    """
+    if not consumer_conf.HasField("cryptoSpec"):
+      return {}
+
+    if consumer_conf.cryptoSpec.consumerCryptoFailureAction == Function_pb2.CryptoSpec.FailureAction.Value("DISCARD"):
+      return {"crypto_failure_action": pulsar.ConsumerCryptoFailureAction.DISCARD}
+    if consumer_conf.cryptoSpec.consumerCryptoFailureAction == Function_pb2.CryptoSpec.FailureAction.Value("CONSUME"):
+      return {"crypto_failure_action": pulsar.ConsumerCryptoFailureAction.CONSUME}
+
+    return {"crypto_failure_action": pulsar.ConsumerCryptoFailureAction.FAIL}
diff --git a/pulsar-functions/instance/src/test/python/test_python_instance.py b/pulsar-functions/instance/src/test/python/test_python_instance.py
index ffd1c1ed4e02..7d02952627c5 100644
--- a/pulsar-functions/instance/src/test/python/test_python_instance.py
+++ b/pulsar-functions/instance/src/test/python/test_python_instance.py
@@ -613,3 +613,91 @@ class TestDeadLetterPolicy(unittest.TestCase):
         self.assertEqual(3, policy.max_redeliver_count)
         self.assertEqual("persistent://public/default/my-dlq", policy.dead_letter_topic)
 
+
+
+class TestConsumerCryptoFailureAction(unittest.TestCase):
+  """Covers ConsumerSpec.cryptoSpec.consumerCryptoFailureAction reaching the consumer.
+
+  The Java runtime applies the configured action in PulsarSource (cb.cryptoFailureAction(...));
+  the Python runtime dropped it, so the client default (FAIL) always applied and a configured
+  DISCARD or CONSUME was silently ignored.
+  """
+
+  def _instance_and_consumer_conf(self, failure_action=None, with_crypto_spec=False):
+    function_details = Function_pb2.FunctionDetails()
+    function_details.sink.topic = "test_sink_topic"
+    consumer_spec = function_details.source.inputSpecs["test_input_topic"]
+    if failure_action is not None:
+      consumer_spec.cryptoSpec.consumerCryptoFailureAction = failure_action
+    elif with_crypto_spec:
+      # a cryptoSpec present with the action left unset reads as the proto3 default (FAIL)
+      consumer_spec.cryptoSpec.cryptoKeyReaderClassName = "test.CryptoKeyReader"
+
+    instance = PythonInstance('test_instance', 'test_func', '1.0', function_details, 100, 30,
+                              'user_code', Mock(), Mock(), 'test_cluster', 'test_url', None)
+    consumer_conf = instance.instance_config.function_details.source.inputSpecs["test_input_topic"]
+    return instance, consumer_conf
+
+  def test_action_is_omitted_without_a_crypto_spec(self):
+    # subscribe() validates crypto_failure_action with _check_type rather than
+    # _check_type_or_none, so it must be omitted rather than passed as None - passing None
+    # would fail for every function that does not configure crypto at all.
+    instance, consumer_conf = self._instance_and_consumer_conf()
+    self.assertEqual({}, instance.get_crypto_failure_action_args(consumer_conf))
+
+  def test_unset_action_in_a_present_crypto_spec_keeps_the_client_default(self):
+    # proto3 enum default (FAIL) equals the client default, so a cryptoSpec that does not set
+    # the action leaves the pre-change behavior in place.
+    instance, consumer_conf = self._instance_and_consumer_conf(with_crypto_spec=True)
+    args = instance.get_crypto_failure_action_args(consumer_conf)
+    self.assertEqual(pulsar.ConsumerCryptoFailureAction.FAIL, args["crypto_failure_action"])
+
+  def test_discard_is_forwarded(self):
+    instance, consumer_conf = self._instance_and_consumer_conf(
+        failure_action=Function_pb2.CryptoSpec.FailureAction.Value("DISCARD"))
+    args = instance.get_crypto_failure_action_args(consumer_conf)
+    self.assertEqual(pulsar.ConsumerCryptoFailureAction.DISCARD, args["crypto_failure_action"])
+
+  def test_consume_is_forwarded(self):
+    instance, consumer_conf = self._instance_and_consumer_conf(
+        failure_action=Function_pb2.CryptoSpec.FailureAction.Value("CONSUME"))
+    args = instance.get_crypto_failure_action_args(consumer_conf)
+    self.assertEqual(pulsar.ConsumerCryptoFailureAction.CONSUME, args["crypto_failure_action"])
+
+  def test_producer_only_send_falls_back_to_fail(self):
+    # SEND only exists for producers; a consumer cannot honor it, so fall back to the client
+    # default rather than crash the instance at startup.
+    instance, consumer_conf = self._instance_and_consumer_conf(
+        failure_action=Function_pb2.CryptoSpec.FailureAction.Value("SEND"))
+    args = instance.get_crypto_failure_action_args(consumer_conf)
+    self.assertEqual(pulsar.ConsumerCryptoFailureAction.FAIL, args["crypto_failure_action"])
+
+  def test_result_is_splattable_into_subscribe_kwargs(self):
+    # The value is consumed via consumer_args.update(...), so it must be a dict with exactly the
+    # keyword subscribe() expects.
+    instance, consumer_conf = self._instance_and_consumer_conf(
+        failure_action=Function_pb2.CryptoSpec.FailureAction.Value("DISCARD"))
+    args = instance.get_crypto_failure_action_args(consumer_conf)
+    self.assertIsInstance(args, dict)
+    self.assertEqual(["crypto_failure_action"], list(args.keys()))
+
+  def test_action_reaches_subscribe_on_the_input_specs_paths(self):
+    # The helper returning the right dict is only half of it: setup_consumers() has to pass it to
+    # Client.subscribe(), on the plain and the regex inputSpecs paths alike.
+    def discard(regex):
+      def configure(details):
+        topic = "persistent://public/default/.*" if regex else "persistent://public/default/in"
+        details.source.inputSpecs[topic].cryptoSpec.consumerCryptoFailureAction = \
+          Function_pb2.CryptoSpec.FailureAction.Value("DISCARD")
+      return configure
+
+    subscribe_args = TestConsumerSubscribeArgs()
+    for regex in (False, True):
+      with self.subTest(regex=regex):
+        kwargs = subscribe_args._subscribe_kwargs(via_input_specs=True, regex=regex,
+                                                  configure=discard(regex))
+        self.assertEqual(pulsar.ConsumerCryptoFailureAction.DISCARD, kwargs["crypto_failure_action"])
+
+  def test_action_is_not_passed_to_subscribe_without_a_crypto_spec(self):
+    kwargs = TestConsumerSubscribeArgs()._subscribe_kwargs(via_input_specs=True)
+    self.assertNotIn("crypto_failure_action", kwargs)


(pulsar) 11/12: [fix][broker] Fix non-persistent subscribe cleanup during migration (#27003)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit d029efdfa42b7de83017a211b9c30b7c79a939c0
Author: sinan liu <li...@gmail.com>
AuthorDate: Thu Oct 22 09:53:17 2026 +0800

    [fix][broker] Fix non-persistent subscribe cleanup during migration (#27003)
    
    (cherry picked from commit 327e41bf4c7d899cb8b66232b2061ad2ce442615)
---
 .../service/nonpersistent/NonPersistentTopic.java  |  27 ++---
 .../nonpersistent/NonPersistentTopicTest.java      | 131 +++++++++++++++++++--
 2 files changed, 132 insertions(+), 26 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
index fc8bd5b5e496..9a03e50d15c7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
@@ -56,6 +56,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedExcepti
 import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException;
 import org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException;
 import org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException;
+import org.apache.pulsar.broker.service.BrokerServiceException.TopicMigratedException;
 import org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException;
 import org.apache.pulsar.broker.service.Consumer;
 import org.apache.pulsar.broker.service.Dispatcher;
@@ -340,24 +341,18 @@ public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPol
             NonPersistentSubscription subscription = subscriptions.computeIfAbsent(subscriptionName,
                     name -> new NonPersistentSubscription(this, subscriptionName, subscriptionProperties));
 
-            Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName,
-                    false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest,
-                    DEFAULT_CONSUMER_EPOCH, schemaType);
-            consumer.checkAndApplyTopicMigrationAsync().thenCompose(migrated -> {
+            CompletableFuture<Boolean> migrationCheck = subscription.isSubscriptionMigrated()
+                    ? isClusterMigrationEnabled(brokerService.getPulsar(), topic)
+                    : CompletableFuture.completedFuture(false);
+            migrationCheck.thenCompose(migrated -> {
                 if (migrated) {
-                    // The topic is migrated: checkAndApplyTopicMigrationAsync() has already sent the
-                    // TopicMigrated redirect and disconnected the consumer (which also released the usage
-                    // count taken by handleConsumerAdded above). Skip addConsumerToSubscription so the consumer
-                    // is never attached to the subscription on the old cluster. Sequencing the migration check
-                    // through thenCompose (instead of the previous fire-and-forget thenAccept that raced with
-                    // addConsumerToSubscription) is what guarantees the consumer is not added once migrated.
-                    log.info()
-                            .attr("subscription", subscriptionName)
-                            .attr("consumerId", consumerId)
-                            .log("Skipped subscription on migrated topic; consumer was redirected");
-                    future.complete(consumer);
-                    return CompletableFuture.<Void>completedFuture(null);
+                    // Roll back the subscribe usage count through the failure path. ServerCnx redirects the
+                    // request without closing a consumer that was never attached to the dispatcher.
+                    return CompletableFuture.<Void>failedFuture(new TopicMigratedException("Topic is migrated"));
                 }
+                Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName,
+                        false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest,
+                        DEFAULT_CONSUMER_EPOCH, schemaType);
                 return addConsumerToSubscription(subscription, consumer).thenRun(() -> {
                     if (!cnx.isActive()) {
                         try {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java
index c7d665de05ea..b3b6b2eed813 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java
@@ -18,7 +18,9 @@
  */
 package org.apache.pulsar.broker.service.nonpersistent;
 
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNotSame;
 import static org.testng.Assert.assertThrows;
@@ -28,23 +30,34 @@ import java.util.Collections;
 import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import lombok.Cleanup;
+import org.apache.pulsar.broker.resources.PulsarResources;
 import org.apache.pulsar.broker.service.AbstractTopic;
+import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionFencedException;
+import org.apache.pulsar.broker.service.BrokerServiceException.TopicMigratedException;
 import org.apache.pulsar.broker.service.BrokerTestBase;
 import org.apache.pulsar.broker.service.PulsarCommandSender;
 import org.apache.pulsar.broker.service.SubscriptionOption;
 import org.apache.pulsar.broker.service.TransportCnx;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.InjectedClientCnxClientBuilder;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionMode;
 import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.ClientBuilderImpl;
+import org.apache.pulsar.client.impl.ClientCnx;
+import org.apache.pulsar.client.impl.metrics.InstrumentProvider;
 import org.apache.pulsar.common.api.proto.CommandSubscribe;
+import org.apache.pulsar.common.api.proto.CommandTopicMigrated;
+import org.apache.pulsar.common.api.proto.CommandTopicMigrated.ResourceType;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl;
 import org.apache.pulsar.common.policies.data.TopicStats;
@@ -52,11 +65,17 @@ import org.awaitility.Awaitility;
 import org.mockito.Mockito;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 @Test(groups = "broker")
 public class NonPersistentTopicTest extends BrokerTestBase {
 
+    @Override
+    protected BrokerService customizeNewBrokerService(BrokerService brokerService) {
+        return Mockito.spy(brokerService);
+    }
+
     @BeforeMethod(alwaysRun = true)
     @Override
     protected void setup() throws Exception {
@@ -140,8 +159,8 @@ public class NonPersistentTopicTest extends BrokerTestBase {
      * {@code addConsumerToSubscription}, and the consumer must NOT be attached to the subscription on the
      * old cluster. The previous code ran {@code getMigratedClusterUrlAsync().thenAccept(consumer::topicMigrated)}
      * concurrently with {@code addConsumerToSubscription}, so the consumer could be added before the redirect
-     * and disconnect ran. The fix sequences the check through {@link org.apache.pulsar.broker.service.Consumer
-     * #checkAndApplyTopicMigrationAsync()} and skips the add when migrated.
+     * and disconnect ran. The migration check must fail the subscribe before attaching a consumer, so that
+     * the subscribe failure path releases the usage count and ServerCnx can redirect the request.
      */
     @Test
     public void testSubscribeOnMigratedTopicSkipsAddingConsumer() throws Exception {
@@ -155,7 +174,7 @@ public class NonPersistentTopicTest extends BrokerTestBase {
                 (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
 
         // Mark the local cluster as migrated so AbstractTopic.getMigratedClusterUrlAsync() (used by
-        // Consumer.checkAndApplyTopicMigrationAsync()) resolves to a present migrated-cluster URL.
+        // the subscribe migration check) resolves to a present migrated-cluster URL.
         ClusterUrl migratedUrl = new ClusterUrl("http://migrated:8080", "https://migrated:8443",
                 "pulsar://migrated:6650", "pulsar+ssl://migrated:6651");
         admin.clusters().updateClusterMigration(conf.getClusterName(), true, migratedUrl);
@@ -200,23 +219,115 @@ public class NonPersistentTopicTest extends BrokerTestBase {
 
         long usageBefore = spyTopic.currentUsageCount();
 
-        org.apache.pulsar.broker.service.Consumer consumer =
-                spyTopic.subscribe(option).get(10, TimeUnit.SECONDS);
-        assertNotNull(consumer);
+        assertThatThrownBy(() -> spyTopic.subscribe(option).get(10, TimeUnit.SECONDS))
+                .hasCauseInstanceOf(TopicMigratedException.class);
 
-        // The migration redirect must have been sent to the client.
-        Mockito.verify(commandSender).sendTopicMigrated(Mockito.any(), Mockito.eq(1L),
-                Mockito.eq(migratedUrl.getBrokerServiceUrl()), Mockito.eq(migratedUrl.getBrokerServiceUrlTls()));
+        // ServerCnx handles the failed subscribe and sends the redirect; the topic must not disconnect a consumer.
+        Mockito.verify(commandSender, Mockito.never()).sendTopicMigrated(Mockito.any(), Mockito.anyLong(),
+                Mockito.any(), Mockito.any());
 
         // The core regression assertion: a migrated topic must never attach the consumer to the
         // subscription. The buggy code added it before the async redirect/disconnect could run.
         Mockito.verify(spySubscription, Mockito.never()).addConsumer(Mockito.any());
         assertTrue(spySubscription.getConsumers().isEmpty());
 
-        // No usage-count leak: handleConsumerAdded's increment is balanced by the disconnect's removeConsumer.
+        // The subscribe failure must balance handleConsumerAdded's increment without dispatcher removal.
         assertEquals(spyTopic.currentUsageCount(), usageBefore);
     }
 
+    @DataProvider
+    public Object[][] migrationSubscriptionTypes() {
+        return new Object[][] {
+                {SubscriptionType.Exclusive}, {SubscriptionType.Failover},
+                {SubscriptionType.Shared}, {SubscriptionType.Key_Shared}
+        };
+    }
+
+    @Test(dataProvider = "migrationSubscriptionTypes")
+    public void testMigratedSubscribeReleasesUsageCount(SubscriptionType subscriptionType) throws Exception {
+        String topicName = "non-persistent://prop/ns-abc/migration-cleanup-" + UUID.randomUUID();
+        String subName = "migration-sub";
+        @Cleanup
+        Consumer<byte[]> existingConsumer = pulsarClient.newConsumer().topic(topicName)
+                .subscriptionName(subName).subscriptionType(subscriptionType).subscribe();
+        NonPersistentTopic topic =
+                (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).orElseThrow();
+        NonPersistentSubscription subscription = topic.getSubscriptions().get(subName);
+        BrokerService brokerService = pulsar.getBrokerService();
+        PulsarResources resources = Mockito.spy(pulsar.getPulsarResources());
+        Mockito.doReturn(resources).when(pulsar).getPulsarResources();
+        assertEquals(topic.currentUsageCount(), 1L);
+
+        // Let the new client finish lookup, but delay its subscribe ownership check until migration starts.
+        CompletableFuture<Void> resumeSubscribe = new CompletableFuture<>();
+        CountDownLatch subscribeStarted = new CountDownLatch(1);
+        Mockito.doAnswer(invocation -> {
+            CompletableFuture<Void> ownership = (CompletableFuture<Void>) invocation.callRealMethod();
+            return ownership.thenCompose(__ -> {
+                subscribeStarted.countDown();
+                return resumeSubscribe;
+            });
+        }).when(brokerService).checkTopicNsOwnership(topicName);
+        // Observe only the pending subscribe's client; the existing consumer also receives a migration notification.
+        CompletableFuture<CommandTopicMigrated> migrationRedirect = new CompletableFuture<>();
+        ClientBuilderImpl clientBuilder = (ClientBuilderImpl) PulsarClient.builder()
+                .serviceUrl(pulsar.getBrokerServiceUrl());
+        @Cleanup
+        PulsarClient reconnectingClient = InjectedClientCnxClientBuilder.create(clientBuilder,
+                (clientConf, eventLoopGroup) -> new ClientCnx(InstrumentProvider.NOOP, clientConf, eventLoopGroup) {
+                    @Override
+                    protected void handleTopicMigrated(CommandTopicMigrated command) {
+                        super.handleTopicMigrated(command);
+                        migrationRedirect.complete(new CommandTopicMigrated().copyFrom(command));
+                    }
+                });
+        reconnectingClient.newConsumer().topic(topicName).subscriptionName(subName)
+                .subscriptionType(subscriptionType).subscribeAsync();
+        assertTrue(subscribeStarted.await(10, TimeUnit.SECONDS));
+
+        ClusterUrl migratedUrl = new ClusterUrl("http://migrated:8080", null, "pulsar://migrated:6650", null);
+        admin.clusters().updateClusterMigration(conf.getClusterName(), true, migratedUrl);
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() ->
+                AbstractTopic.getMigratedClusterUrlAsync(pulsar, topicName).get().isPresent());
+        CompletableFuture<Void> migration;
+        try {
+            // Keep the existing dispatcher in the subscription while the new subscribe observes migration.
+            // The migration flag is set before checkClusterMigration disconnects the existing consumer.
+            synchronized (subscription.getDispatcher()) {
+                migration = CompletableFuture.completedFuture(null)
+                        .thenComposeAsync(__ -> topic.checkClusterMigration(), pulsar.getExecutor());
+                Awaitility.await().atMost(10, TimeUnit.SECONDS).until(topic::isMigrated);
+                CountDownLatch migrationPolicyRead = new CountDownLatch(1);
+                Mockito.doAnswer(invocation -> {
+                    migrationPolicyRead.countDown();
+                    return invocation.callRealMethod();
+                }).when(resources).getClusterResources();
+                pulsar.getExecutor().execute(() -> resumeSubscribe.complete(null));
+                assertTrue(migrationPolicyRead.await(10, TimeUnit.SECONDS),
+                        "The pending subscribe must reach its migration check with the existing subscription");
+                assertTrue(topic.getSubscriptions().get(subName) == subscription);
+                assertEquals(subscription.getConsumers().size(), 1,
+                        "The old consumer must still be attached when the pending subscribe checks migration");
+            }
+            migration.get(10, TimeUnit.SECONDS);
+            Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() ->
+                    assertEquals(topic.currentUsageCount(), 0L,
+                            "Both the existing consumer and the redirected subscribe must release their usage"));
+            assertTrue(subscription.getConsumers().isEmpty());
+
+            CommandTopicMigrated redirect = migrationRedirect.get(10, TimeUnit.SECONDS);
+            assertEquals(redirect.getResourceType(), ResourceType.Consumer);
+            assertEquals(redirect.getBrokerServiceUrl(), migratedUrl.getBrokerServiceUrl());
+            assertFalse(redirect.hasBrokerServiceUrlTls());
+            admin.topics().delete(topicName, true);
+        } finally {
+            resumeSubscribe.complete(null);
+            Mockito.doCallRealMethod().when(brokerService).checkTopicNsOwnership(topicName);
+            Mockito.doCallRealMethod().when(resources).getClusterResources();
+            Mockito.doCallRealMethod().when(pulsar).getPulsarResources();
+        }
+    }
+
     @Test
     public void testSubscriptionsOnNonPersistentTopic() throws Exception {
         final String topicName = "non-persistent://prop/ns-abc/topic_" + UUID.randomUUID();


(pulsar) 12/12: [fix][broker] Honor secondary tenant admin roles in HTTP admin requests (#26911)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 99ba3dea77aa177d899869359ee29505556a4c26
Author: sinan liu <li...@gmail.com>
AuthorDate: Thu Oct 22 09:53:45 2026 +0800

    [fix][broker] Honor secondary tenant admin roles in HTTP admin requests (#26911)
    
    (cherry picked from commit 3ad7e6e91b6add1b7a7518737a5142120cdafcd6)
---
 .../pulsar/broker/web/PulsarWebResource.java       | 12 +--
 .../MultiRolesTokenAuthorizationProviderTest.java  | 98 ++++++++++++++++++++++
 2 files changed, 100 insertions(+), 10 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
index 3eeb50ca036a..42fce8ef5459 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
@@ -345,7 +345,6 @@ public abstract class PulsarWebResource {
                     if (tenantInfoOptional.isEmpty()) {
                         throw new RestException(Status.NOT_FOUND, "Tenant does not exist");
                     }
-                    TenantInfo tenantInfo = tenantInfoOptional.get();
                     if (pulsar.getConfiguration().isAuthenticationEnabled() && pulsar.getConfiguration()
                             .isAuthorizationEnabled()) {
                         if (!isClientAuthenticated(clientAppId)) {
@@ -377,15 +376,8 @@ public abstract class PulsarWebResource {
                         } else {
                             return pulsar.getBrokerService()
                                     .getAuthorizationService()
-                                    .isSuperUser(clientAppId, authenticationData)
-                                    .thenCompose(isSuperUser -> {
-                                        if (!isSuperUser) {
-                                            return pulsar.getBrokerService().getAuthorizationService()
-                                                    .isTenantAdmin(tenant, clientAppId, tenantInfo, authenticationData);
-                                        } else {
-                                            return CompletableFuture.completedFuture(true);
-                                        }
-                                    }).thenAccept(authorized -> {
+                                    .isSuperUserOrTenantAdmin(tenant, clientAppId, authenticationData)
+                                    .thenAccept(authorized -> {
                                         if (!authorized) {
                                             throw new RestException(Status.UNAUTHORIZED,
                                                     "Don't have permission to administrate resources on this tenant");
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java
index 9c8e90d70deb..23ac046f17eb 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java
@@ -18,6 +18,8 @@
  */
 package org.apache.pulsar.client.api;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertThrows;
@@ -56,6 +58,7 @@ import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.awaitility.Awaitility;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 public class MultiRolesTokenAuthorizationProviderTest extends MockedPulsarServiceBaseTest {
@@ -217,6 +220,101 @@ public class MultiRolesTokenAuthorizationProviderTest extends MockedPulsarServic
                         .allowedClusters(Sets.newHashSet(configClusterName)).build()));
     }
 
+    @DataProvider(name = "truncateTopics")
+    public Object[][] truncateTopics() {
+        return new Object[][] {{true}, {false}};
+    }
+
+    @Test(dataProvider = "truncateTopics")
+    public void testTruncateWithSecondaryTenantAdminRole(boolean partitioned) throws Exception {
+        final String tenant = "multi-roles-truncate-tenant-" + partitioned;
+        final String otherTenant = tenant + "-other";
+        final String namespace = tenant + "/ns";
+        final String topic = "persistent://" + namespace + "/test-topic";
+        final String subscription = "sub";
+        final String primaryRole = "truncate-user";
+        final String tenantAdminRole = "truncate-tenant-admin";
+        final String otherTenantAdminRole = "truncate-other-tenant-admin";
+        final int numMessages = 6;
+        Map<String, Object> claims = new HashMap<>();
+        // The authenticated principal is the first role, while the second role grants tenant admin access.
+        claims.put("roles", List.of(primaryRole, tenantAdminRole));
+        final String tenantAdminToken = Jwts.builder().setClaims(claims).signWith(secretKey).compact();
+        claims.put("roles", List.of(primaryRole));
+        final String userToken = Jwts.builder().setClaims(claims).signWith(secretKey).compact();
+        claims.put("roles", List.of(primaryRole, otherTenantAdminRole));
+        final String otherTenantAdminToken = Jwts.builder().setClaims(claims).signWith(secretKey).compact();
+
+        try (PulsarAdmin superUserAdmin = newPulsarAdmin(superUserToken);
+             PulsarAdmin tenantAdmin = newPulsarAdmin(tenantAdminToken);
+             PulsarAdmin userAdmin = newPulsarAdmin(userToken);
+             PulsarAdmin otherTenantAdmin = newPulsarAdmin(otherTenantAdminToken);
+             PulsarClient client = newPulsarClient(superUserToken)) {
+            superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+                    .adminRoles(Set.of(tenantAdminRole))
+                    .allowedClusters(Set.of(configClusterName)).build());
+            superUserAdmin.tenants().createTenant(otherTenant, TenantInfo.builder()
+                    .adminRoles(Set.of(otherTenantAdminRole))
+                    .allowedClusters(Set.of(configClusterName)).build());
+            assertThat(otherTenantAdmin.namespaces().getNamespaces(otherTenant))
+                    .as("the secondary role grants administration of the other tenant")
+                    .isEmpty();
+            superUserAdmin.namespaces().createNamespace(namespace);
+            if (partitioned) {
+                superUserAdmin.topics().createPartitionedTopic(topic, 2);
+            } else {
+                superUserAdmin.topics().createNonPartitionedTopic(topic);
+            }
+            superUserAdmin.topics().createSubscription(topic, subscription, MessageId.earliest);
+            try (Producer<byte[]> producer = client.newProducer().topic(topic).enableBatching(false).create()) {
+                for (int i = 0; i < numMessages; i++) {
+                    producer.send(new byte[] {(byte) i});
+                }
+            }
+            assertThat(getBacklog(superUserAdmin, topic, subscription, partitioned)).isEqualTo(numMessages);
+            assertThat(tenantAdmin.topics().getPartitionedTopicMetadata(topic).partitions)
+                    .as("topic lookup recognizes the secondary tenant admin role")
+                    .isEqualTo(partitioned ? 2 : 0);
+
+            superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, primaryRole,
+                    Set.of(AuthAction.produce, AuthAction.consume));
+            assertThatThrownBy(() -> userAdmin.topics().truncate(topic))
+                    .isInstanceOf(PulsarAdminException.NotAuthorizedException.class);
+            assertThat(getBacklog(superUserAdmin, topic, subscription, partitioned)).isEqualTo(numMessages);
+
+            assertThatThrownBy(() -> otherTenantAdmin.topics().truncate(topic))
+                    .isInstanceOf(PulsarAdminException.NotAuthorizedException.class);
+            assertThat(getBacklog(superUserAdmin, topic, subscription, partitioned)).isEqualTo(numMessages);
+
+            tenantAdmin.topics().truncate(topic);
+            assertThat(getBacklog(superUserAdmin, topic, subscription, partitioned)).isZero();
+
+            if (partitioned) {
+                final String partition = TopicName.get(topic).getPartition(0).toString();
+                try (Producer<byte[]> producer = client.newProducer().topic(partition)
+                        .enableBatching(false).create()) {
+                    producer.send(new byte[] {1});
+                }
+                assertThat(getBacklog(superUserAdmin, topic, subscription, true)).isEqualTo(1);
+                assertThatThrownBy(() -> userAdmin.topics().truncate(partition))
+                        .isInstanceOf(PulsarAdminException.NotAuthorizedException.class);
+                assertThat(getBacklog(superUserAdmin, topic, subscription, true)).isEqualTo(1);
+                assertThatThrownBy(() -> otherTenantAdmin.topics().truncate(partition))
+                        .isInstanceOf(PulsarAdminException.NotAuthorizedException.class);
+                assertThat(getBacklog(superUserAdmin, topic, subscription, true)).isEqualTo(1);
+                tenantAdmin.topics().truncate(partition);
+                assertThat(getBacklog(superUserAdmin, topic, subscription, true)).isZero();
+            }
+        }
+    }
+
+    private long getBacklog(PulsarAdmin admin, String topic, String subscription, boolean partitioned)
+            throws PulsarAdminException {
+        return partitioned ? admin.topics().getPartitionedStats(topic, false)
+                .getSubscriptions().get(subscription).getMsgBacklog()
+                : admin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog();
+    }
+
     @Test
     public void testProduceAndConsumeWithNormalUserToken() throws Exception {
         String tenant = "normaluser-client-tenant";


(pulsar) 05/12: [fix][broker] Keep the topic-level permissions of a topic migrated to a scalable topic (#26953)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit d9de47ef195e0eb144560d18a3159cf8fb587cd8
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Wed Oct 21 18:38:09 2026 -0700

    [fix][broker] Keep the topic-level permissions of a topic migrated to a scalable topic (#26953)
    
    (cherry picked from commit 764040d902d9b18fef4b5993ff4e78331d868b99)
---
 pip/pip-475.md                                     |  54 ++++-
 .../authorization/AuthorizationProvider.java       |  19 ++
 .../broker/authorization/AuthorizationService.java |   9 +
 .../authorization/PulsarAuthorizationProvider.java |  67 ++++++
 .../broker/resources/NamespaceResources.java       |   8 +
 .../broker/resources/ScalableTopicResources.java   | 111 +++++++++
 .../PulsarAuthorizationProviderTest.java           | 109 +++++++++
 .../pulsar/broker/admin/v2/ScalableTopics.java     |  55 ++++-
 .../service/scalable/ScalableTopicService.java     |   5 +-
 .../ScalableTopicMigrationPermissionsTest.java     | 256 +++++++++++++++++++++
 .../broker/admin/v2/ScalableTopicsMigrateTest.java |  12 +-
 .../resources/ScalableTopicCreationLockTest.java   | 198 ++++++++++++++++
 12 files changed, 881 insertions(+), 22 deletions(-)

diff --git a/pip/pip-475.md b/pip/pip-475.md
index 8df12773554d..c429264c3113 100644
--- a/pip/pip-475.md
+++ b/pip/pip-475.md
@@ -125,7 +125,9 @@ Step 2 — Operator runs:
     pulsar-admin scalable-topics migrate persistent://t/n/x [--force]
 
   The broker:
-   2a. Validates that the topic exists and is not already scalable (else
+   2a. Takes the creation lock of topic://t/n/x, which creating that
+       scalable topic also takes, and holds it up to the commit (2e).
+       Validates that the topic exists and is not already scalable (else
        404 / 409). Unless --force is set, validates that no legacy v4
        producer/consumer connection is attached (counting only connections
        without the V5-managed marker); if any are, fails with HTTP 409.
@@ -144,11 +146,14 @@ Step 2 — Operator runs:
        protocol the controller already uses for segment splits / merges.
    2c. Creates the backing segment topics for the new active children
        (the sealed parents reuse the existing persistent:// managed ledgers).
-   2d. Atomically writes the metadata, taking the topic from
+   2d. Grants on topic://t/n/x the permissions granted on
+       persistent://t/n/x, so the roles allowed on the regular topic keep
+       their access through the new segments (see Security Considerations).
+   2e. Atomically writes the metadata, taking the topic from
        "regular" to "scalable" in one CAS on the metadata store. This is
        the commit point; connected lookup sessions are pushed the real DAG
        via the metadata-store watch.
-   2e. Terminates the old persistent:// topic(s) so no further v4 writes
+   2f. Terminates the old persistent:// topic(s) so no further v4 writes
        can land — they become the drainable sealed parent segments.
 
 Step 3 — Connected V5 clients receive the layout-update push on their
@@ -219,7 +224,7 @@ No request body.
 
 Unless `force=true`, the broker enumerates the producers/consumers attached to the source topic (across all partitions, via the topic-stats path) and rejects the migration with HTTP 409 if any *legacy v4* connection is present. A connection is recognised as V5-managed — and therefore excluded — by a reserved metadata marker the V5 SDK sets on every per-segment v4 producer/consumer it creates; everything else counts as a legacy v4 connection. `force=true` skips this check.
 
-Response: `204 No Content` on success. `409 Conflict` if scalable metadata already exists or legacy v4 connections are still attached (and `force` is not set). `404` if the topic doesn't exist.
+Response: `204 No Content` on success. `409 Conflict` if scalable metadata already exists, a creation of the scalable topic is in progress, or legacy v4 connections are still attached (and `force` is not set). `404` if the topic doesn't exist.
 
 #### 2.2 Admin client
 
@@ -243,10 +248,10 @@ pulsar-admin scalable-topics migrate persistent://t/n/x [--force]
 
 #### 2.4 Broker-side migration steps
 
-Executed by the topic's owning broker:
+Executed by the topic's owning broker, with steps 1 to 5 under the creation lock of `topic://t/n/x` (see below):
 
 1. **Precheck**:
-   - No `ScalableTopicMetadata` already exists at the path. If it does, fail 409.
+   - No `ScalableTopicMetadata` already exists at the path, as read from the metadata store rather than the broker's cache. If it does, fail 409.
    - Topic exists (as either partitioned or non-partitioned). If not, fail 404.
    - Unless `force`, no *legacy v4* connection is attached (counting only producers/consumers whose metadata lacks the V5-managed marker). If any are, fail 409.
 2. **Build initial layout** — `ScalableTopicController.createMigratedMetadata`:
@@ -255,14 +260,19 @@ Executed by the topic's owning broker:
    - DAG edges: every child has all N parents as predecessors. The subscription controller's drain-before-assign protocol (already used for splits / merges) preserves per-key ordering.
    - Set `epoch = 0`, `nextSegmentId = 2N`.
 3. **Create child segment topics**: materialize the backing topic for each new active child. Sealed legacy parents reuse the existing `persistent://` managed ledgers, so they are skipped.
-4. **Atomic flip**: write `ScalableTopicMetadata` to `/topics/t/n/x` via metadata-store CAS. This is the commit point — once it succeeds, the topic is scalable. Connected lookup sessions transition from the synthetic layout to the real DAG via the metadata-store watch.
-5. **Terminate the old topic(s)**: terminate `persistent://t/n/x` (or every partition, for a partitioned source) so no further v4 writes can land. Termination is exactly the sealed-parent state — existing data is still drainable by consumers, and the sealed legacy parent segments reference these terminated topics.
+4. **Carry over the topic's permissions**: grant on `topic://t/n/x` the permissions granted on `persistent://t/n/x` itself, adding them to those already granted on `topic://t/n/x`. Done before the commit, so the V5 clients that follow the layout change are authorized on the new segments as soon as they see them. See [Security Considerations](#security-considerations).
+5. **Atomic flip**: write `ScalableTopicMetadata` to `/topics/t/n/x` via metadata-store CAS. This is the commit point — once it succeeds, the topic is scalable. Connected lookup sessions transition from the synthetic layout to the real DAG via the metadata-store watch.
+6. **Terminate the old topic(s)**: terminate `persistent://t/n/x` (or every partition, for a partitioned source) so no further v4 writes can land. Termination is exactly the sealed-parent state — existing data is still drainable by consumers, and the sealed legacy parent segments reference these terminated topics.
 
-Failures before step 4 leave the topic untouched; failures after step 4 leave the topic scalable.
+Failures before step 5 leave the topic untouched (the permissions already granted on `topic://t/n/x` stay, and a retried migration grants them again); failures after step 5 leave the topic scalable.
+
+Creating a scalable topic takes the creation lock of `topic://t/n/x` too, through the admin API or by auto-creation on lookup, while it writes the topic's metadata. A migration and a creation of the same scalable topic therefore exclude each other, on any broker, and no creation can happen between the check of step 1 and the commit of step 5, where it would receive the permissions granted at step 4. A creation or a migration that finds the lock taken retries for about a second, which cov [...]
+
+Each acquisition of the lock creates an ephemeral record of its own in the metadata store, outside the topic's own path, at `/topic-creation-locks/t/n/x:<token>` with a random token, and holds the lock if no other record of the topic exists then: of two acquisitions, the second one to create its record sees the other one. A broker that goes away doesn't keep the lock. Releasing the lock deletes the acquisition's own record only, so a broker whose metadata session expired, and with it its [...]
 
 ### 3. v4 write guard
 
-Step 5 above (terminating the old topics) is the primary guard: an already-connected v4 producer's `send()` fails with `TopicTerminatedError`, and a freshly-connected one hits the same wall. This keeps stale v4 producers from writing to a topic that has been migrated.
+Step 6 above (terminating the old topics) is the primary guard: an already-connected v4 producer's `send()` fails with `TopicTerminatedError`, and a freshly-connected one hits the same wall. This keeps stale v4 producers from writing to a topic that has been migrated.
 
 To make the "once scalable, always scalable" guarantee robust even after the old (terminated, drained) topics are eventually removed by retention, the broker also refuses to **auto-create** a `persistent://t/n/x` topic when `ScalableTopicMetadata` exists for the equivalent `topic://t/n/x`. This prevents a stray v4 client from recreating a regular topic that would shadow the scalable one. The rejection reuses `TopicTerminatedError` (binary) / the equivalent REST status; no new error code  [...]
 
@@ -279,10 +289,12 @@ No `legacyModNRouting` flag or other migration-specific routing knob is needed:
 ### REST API
 
 New endpoint:
-- `POST /admin/v2/scalable-topics/{tenant}/{namespace}/{topic}/migrate` — requires `produce` permission on the topic; query param `force` (default `false`); no request body; 204 on success, 409 if already scalable or legacy v4 connections are still attached (and `force` is not set), 404 if topic missing.
+- `POST /admin/v2/scalable-topics/{tenant}/{namespace}/{topic}/migrate` — requires `produce` permission on the topic; query param `force` (default `false`); no request body; 204 on success, 409 if already scalable, being created as a scalable topic, or legacy v4 connections are still attached (and `force` is not set), 404 if topic missing.
 
 Modified behaviour:
 - Auto-creation of a `persistent://t/n/x` topic is refused when scalable metadata exists for the equivalent `topic://t/n/x` (reusing `TopicTerminatedError`); together with the post-migration termination of the old topics, this keeps v4 clients off a migrated topic.
+- Migrating a topic grants its topic-level permissions on the scalable topic (see [Security Considerations](#security-considerations)).
+- Creating a scalable topic, through the admin API or by auto-creation on lookup, takes the topic's creation lock ([2.4](#24-broker-side-migration-steps)): while a migration to that topic is in progress, the creation fails with 409, or fails the lookup.
 
 ### Binary protocol
 
@@ -293,6 +305,10 @@ Modified behaviour:
 
 - `pulsar-admin scalable-topics migrate <topic> [--force]`
 
+### Authorization provider
+
+- `AuthorizationProvider` gains a default method, `CompletableFuture<Void> copyPermissionsAsync(TopicName sourceTopicName, TopicName targetTopicName)`, that grants on the target topic the permissions granted on the source topic itself (not those of its namespace), adding them to those already granted on the target topic. The default implementation copies nothing; `PulsarAuthorizationProvider` implements it.
+
 ### V5 SDK behavior
 
 - V5 builders accept `persistent://...` and short-form names; the SDK opens a scalable-topic lookup session that the broker resolves to either a real DAG or a synthetic layout.
@@ -301,6 +317,22 @@ Modified behaviour:
 
 ---
 
+## Security Considerations
+
+The migration command is authorized as described in [2.1](#21-admin-rest-endpoint).
+
+A scalable topic and the regular topic it is migrated from are distinct identities, each with permissions of its own: the permissions granted on `topic://t/n/x` authorize the scalable topic and its `segment://` segments, while a legacy segment is authorized as the `persistent://` topic it wraps. A role granted access to `persistent://t/n/x` alone would lose it at the commit: the per-segment producers and consumers of the V5 clients move to the new segments, which the permissions of `pers [...]
+
+So the migration carries the topic's permissions over (step 4 of [2.4](#24-broker-side-migration-steps)): it grants on `topic://t/n/x` the permissions granted on `persistent://t/n/x`, and applications keep working through the migration, whether they address the topic as `persistent://t/n/x` or as `topic://t/n/x`. A role keeps the permissions already granted to it on `topic://t/n/x`, and gains those granted to it on `persistent://t/n/x`. The migration only grants a role what it was grante [...]
+
+- **Namespace permissions** are not copied: they apply to both topics already, and as permissions of the scalable topic they would outlive their revocation on the namespace.
+- **The permissions of a single partition** (`persistent://t/n/x-partition-K`) are not copied: they allow access to that partition's data only, while every new segment receives keys from all the partitions, so they would widen to the whole topic. The legacy segment that wraps the partition keeps honoring them.
+- **After the migration**, each name keeps its own permissions: a change on one doesn't apply to the other. The permissions of the scalable topic are managed on `topic://t/n/x` (`pulsar-admin scalable-topics grant-permission` / `revoke-permission`). Those of `persistent://t/n/x` keep authorizing the legacy segments, which hold the data published before the migration, and the lookups and subscriptions of the V5 clients that address the topic as `persistent://t/n/x`. Taking a role's access [...]
+- **Concurrent creation.** The permissions are granted under the creation lock of `topic://t/n/x` ([2.4](#24-broker-side-migration-steps)), which every creation of that scalable topic also takes, so they can't land on a scalable topic created concurrently under the same name. A migration that finds the scalable topic already created fails with 409 and grants nothing.
+- The broker copies the permissions through its authorization provider (see [Authorization provider](#authorization-provider)). A third-party provider that doesn't implement the copy copies nothing, and the permissions of the scalable topic must then be granted through it.
+
+---
+
 ## Backward & forward compatibility
 
 ### Upgrade
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java
index 7ca7945d7b1d..6f3b96551584 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java
@@ -401,6 +401,25 @@ public interface AuthorizationProvider extends Closeable {
         return CompletableFuture.completedFuture(null);
     }
 
+    /**
+     * Grant on a topic the permissions granted on another topic, for a topic that takes over from another one, such
+     * as the scalable topic a regular topic is migrated to: the roles allowed on the source topic keep their access.
+     *
+     * <p>Both topics are in the same namespace. Only the permissions granted on the source topic itself are copied,
+     * not those of the namespace. A role keeps the permissions already granted to it on the target topic, and gains
+     * those granted to it on the source topic.
+     *
+     * <p>The default implementation copies nothing: a provider that stores the permissions granted on topics should
+     * override it.
+     *
+     * @param sourceTopicName the topic whose permissions are copied
+     * @param targetTopicName the topic to grant them on
+     * @return a future that completes once the permissions are granted on the target topic
+     */
+    default CompletableFuture<Void> copyPermissionsAsync(TopicName sourceTopicName, TopicName targetTopicName) {
+        return CompletableFuture.completedFuture(null);
+    }
+
     /**
      * Get authorization-action permissions on a topic.
      * @param topicName
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
index f63318963611..23c1b99b66b8 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
@@ -1031,6 +1031,15 @@ public class AuthorizationService {
         return provider.removePermissionsAsync(topicName);
     }
 
+    /**
+     * Grant on a topic the permissions granted on another topic itself.
+     *
+     * @see AuthorizationProvider#copyPermissionsAsync(TopicName, TopicName)
+     */
+    public CompletableFuture<Void> copyPermissionsAsync(TopicName sourceTopicName, TopicName targetTopicName) {
+        return provider.copyPermissionsAsync(sourceTopicName, targetTopicName);
+    }
+
     public CompletableFuture<Map<String, Set<AuthAction>>> getPermissionsAsync(TopicName topicName) {
         return provider.getPermissionsAsync(topicName);
     }
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
index ab10e7ce3a7a..89d78fd301fe 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
@@ -26,16 +26,19 @@ import jakarta.ws.rs.core.Response;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Stream;
 import lombok.CustomLog;
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.resources.NamespaceResources;
 import org.apache.pulsar.broker.resources.PulsarResources;
 import org.apache.pulsar.client.admin.GrantTopicPermissionOptions;
 import org.apache.pulsar.client.admin.RevokeTopicPermissionOptions;
@@ -846,6 +849,70 @@ public class PulsarAuthorizationProvider implements AuthorizationProvider {
         });
     }
 
+    @Override
+    public CompletableFuture<Void> copyPermissionsAsync(TopicName sourceTopicName, TopicName targetTopicName) {
+        NamespaceName namespace = sourceTopicName.getNamespaceObject();
+        if (!namespace.equals(targetTopicName.getNamespaceObject())) {
+            return FutureUtil.failedFuture(new IllegalArgumentException("Can't copy the permissions of "
+                    + sourceTopicName + " to a topic of another namespace: " + targetTopicName));
+        }
+        String sourceTopicUri = sourceTopicName.toString();
+        String targetTopicUri = targetTopicName.toString();
+        NamespaceResources namespaceResources = pulsarResources.getNamespaceResources();
+        // Not the cached policies: they may miss a permission granted on another broker just before.
+        return namespaceResources.refreshAndGetPoliciesAsync(namespace).thenCompose(latestPolicies -> {
+            if (!latestPolicies.map(policies -> hasTopicPermissions(policies, sourceTopicUri)).orElse(false)) {
+                // Nothing to copy: don't require the policies to be writable.
+                return CompletableFuture.completedFuture(null);
+            }
+            return getPoliciesReadOnlyAsync().thenCompose(readonly -> {
+                if (readonly) {
+                    log.debug("Policies are read-only. Broker cannot do read-write operations");
+                    throw new IllegalStateException("policies are in readonly mode");
+                }
+                AtomicReference<Set<String>> copiedRoles = new AtomicReference<>(Set.of());
+                return namespaceResources.setPoliciesAsync(namespace, policies -> {
+                    // Read the source permissions from the version of the policies being updated, which is the
+                    // latest one when the update is retried after a concurrent change: a permission revoked
+                    // meanwhile must not be copied.
+                    Map<String, Map<String, Set<AuthAction>>> topicPermissions =
+                            policies.auth_policies.getTopicAuthentication();
+                    Map<String, Set<AuthAction>> sourceRoles = topicPermissions.getOrDefault(sourceTopicUri, Map.of());
+                    copiedRoles.set(Set.copyOf(sourceRoles.keySet()));
+                    if (!sourceRoles.isEmpty()) {
+                        Map<String, Set<AuthAction>> targetRoles =
+                                topicPermissions.computeIfAbsent(targetTopicUri, __ -> new HashMap<>());
+                        sourceRoles.forEach((role, actions) -> {
+                            Set<AuthAction> granted = new HashSet<>(actions);
+                            granted.addAll(targetRoles.getOrDefault(role, Set.of()));
+                            targetRoles.put(role, granted);
+                        });
+                    }
+                    return policies;
+                }).whenComplete((__, ex) -> {
+                    if (ex != null) {
+                        log.error()
+                                .attr("sourceTopic", sourceTopicName)
+                                .attr("targetTopic", targetTopicName)
+                                .exception(ex)
+                                .log("Failed to copy permissions to topic");
+                    } else {
+                        log.info()
+                                .attr("sourceTopic", sourceTopicName)
+                                .attr("targetTopic", targetTopicName)
+                                .attr("roles", copiedRoles.get())
+                                .log("Successfully copied permissions to topic");
+                    }
+                });
+            });
+        });
+    }
+
+    private static boolean hasTopicPermissions(Policies policies, String topicUri) {
+        Map<String, Set<AuthAction>> roles = policies.auth_policies.getTopicAuthentication().get(topicUri);
+        return roles != null && !roles.isEmpty();
+    }
+
     @Override
     public CompletableFuture<Map<String, Set<AuthAction>>> getPermissionsAsync(TopicName topicName) {
         return getPoliciesReadOnlyAsync().thenCompose(readonly -> {
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java
index f80427fb84bf..d52ddb7d185d 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java
@@ -144,6 +144,14 @@ public class NamespaceResources extends BaseResources<Policies> {
         return getCache().get(joinPath(BASE_POLICIES_PATH, ns.toString()));
     }
 
+    /**
+     * Get the namespace policies from the metadata store, rather than from the cache, which may not have caught up
+     * with an update made on another broker yet.
+     */
+    public CompletableFuture<Optional<Policies>> refreshAndGetPoliciesAsync(NamespaceName ns) {
+        return refreshAndGetAsync(joinPath(BASE_POLICIES_PATH, ns.toString()));
+    }
+
     public void setPolicies(NamespaceName ns, Function<Policies, Policies> function) throws MetadataStoreException {
         set(joinPath(BASE_POLICIES_PATH, ns.toString()), function);
     }
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicResources.java
index 452ef739fdca..7da17395f3b2 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicResources.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ScalableTopicResources.java
@@ -26,8 +26,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
@@ -45,6 +50,7 @@ import org.apache.pulsar.metadata.api.MetadataCache;
 import org.apache.pulsar.metadata.api.MetadataStore;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.Notification;
+import org.apache.pulsar.metadata.api.Option;
 import org.apache.pulsar.metadata.api.ScanConsumer;
 
 /**
@@ -60,12 +66,20 @@ import org.apache.pulsar.metadata.api.ScanConsumer;
  *       {@link SubscriptionMetadata}</li>
  *   <li>{@code /topics/{tenant}/{namespace}/{encodedTopicName}/subscriptions/{subscription}
  *       /consumers/{consumerName}} — {@link ConsumerRegistration} (durable session entry)</li>
+ *   <li>{@code /topic-creation-locks/{tenant}/{namespace}/{encodedTopicName}:{token}} — one
+ *       ephemeral record per acquisition of the topic's creation lock, held while the topic is
+ *       created or a regular topic is migrated to it (see {@link #runWithCreationLock}); outside
+ *       the topic's path, so taking it doesn't create the topic's node</li>
  * </ul>
  */
 @CustomLog
 public class ScalableTopicResources extends BaseResources<ScalableTopicMetadata> {
 
     private static final String SCALABLE_TOPIC_PATH = "/topics";
+    private static final String CREATION_LOCK_PATH = "/topic-creation-locks";
+    /** Attempts to take a creation lock, waiting 10-20 ms, 20-40 ms, 40-80 ms... in between: about a second. */
+    private static final int CREATION_LOCK_ATTEMPTS = 7;
+    private static final long CREATION_LOCK_FIRST_RETRY_DELAY_MS = 10;
     private static final String SUBSCRIPTIONS_SEGMENT = "subscriptions";
     private static final String CONSUMERS_SEGMENT = "consumers";
     private static final String SEGMENTS_SEGMENT = "segments";
@@ -287,6 +301,103 @@ public class ScalableTopicResources extends BaseResources<ScalableTopicMetadata>
         return getCache().create(topicPath(tn), metadata, PROPERTY_INDEX_EXTRACTOR);
     }
 
+    /** The creation lock of a scalable topic, while {@link #runWithCreationLock} holds it. */
+    public interface CreationLock {
+        /**
+         * Complete normally if the lock is still held, or fail with {@link MetadataStoreException.LockBusyException}.
+         * An action checks it before each change the lock guards: a broker whose metadata session expired lost the
+         * lock, which another broker may have taken since.
+         */
+        CompletableFuture<Void> checkHeld();
+    }
+
+    /**
+     * Run {@code action} while holding the creation lock of the scalable topic {@code tn}.
+     *
+     * <p>Creating a scalable topic and migrating a regular topic to it take this lock, so they exclude each other,
+     * on every broker: a migration checks that the scalable topic doesn't exist, grants the regular topic's
+     * permissions on it and creates it, with no creation of the same topic in between that would get those
+     * permissions. A lock held by another creation or migration is retried for about a second, which covers a
+     * concurrent creation, as it holds the lock only while it writes the topic's metadata. Then this fails with
+     * {@link MetadataStoreException.LockBusyException}, without running {@code action}.
+     *
+     * <p>Each acquisition creates an ephemeral record of its own, named after the topic and a random token, and
+     * holds the lock if no other record of the topic exists then. So a broker that goes away doesn't keep the lock;
+     * releasing it deletes that record only, never the one of an acquisition made after this broker's metadata
+     * session expired; and a create that the store retried after its response was lost finds its own record.
+     */
+    public <T> CompletableFuture<T> runWithCreationLock(TopicName tn,
+                                                        Function<CreationLock, CompletableFuture<T>> action) {
+        String lockDir = joinPath(CREATION_LOCK_PATH, tn.getNamespace());
+        String prefix = tn.getEncodedLocalName() + ":";
+        String name = prefix + UUID.randomUUID();
+        String path = joinPath(lockDir, name);
+        CreationLock lock = () -> getStore().get(path).thenAccept(record -> {
+            if (record.isEmpty() || !record.get().getStat().isCreatedBySelf()) {
+                throw new CompletionException(new MetadataStoreException.LockBusyException(
+                        "Lost the creation lock of " + tn));
+            }
+        });
+        return acquireCreationLock(lockDir, prefix, name, 1).thenCompose(__ -> {
+            CompletableFuture<T> result = FutureUtil.supplySafely(() -> action.apply(lock));
+            return result.handle((___, ____) -> null)
+                    .thenCompose(___ -> releaseCreationLock(path))
+                    .thenCompose(___ -> result);
+        });
+    }
+
+    private CompletableFuture<Void> acquireCreationLock(String lockDir, String prefix, String name, int attempt) {
+        String path = joinPath(lockDir, name);
+        return getStore().put(path, new byte[0], Optional.of(-1L), Set.of(Option.Ephemeral.INSTANCE))
+                .<Void>thenApply(__ -> null)
+                .exceptionallyCompose(ex -> {
+                    if (FutureUtil.unwrapCompletionException(ex)
+                            instanceof MetadataStoreException.BadVersionException) {
+                        // The record exists: the store retried a create of this acquisition whose response was
+                        // lost. No other acquisition uses this name.
+                        return CompletableFuture.completedFuture(null);
+                    }
+                    return CompletableFuture.failedFuture(ex);
+                })
+                // Read the store, not the cache, which may miss the record of a concurrent acquisition.
+                .thenCompose(__ -> getStore().getChildrenFromStore(lockDir))
+                .exceptionallyCompose(ex ->
+                        // The record may have been created: delete it, or the lock would stay taken.
+                        releaseCreationLock(path).thenCompose(__ -> CompletableFuture.failedFuture(ex)))
+                .thenCompose(records -> {
+                    if (records.stream().noneMatch(record -> record.startsWith(prefix) && !record.equals(name))) {
+                        return CompletableFuture.completedFuture(null);
+                    }
+                    // Another acquisition holds the lock, or is taking it: of two acquisitions, the second one to
+                    // create its record sees the other one, and the first one may. Back off and retry.
+                    return releaseCreationLock(path).thenCompose(__ -> {
+                        if (attempt == CREATION_LOCK_ATTEMPTS) {
+                            return CompletableFuture.failedFuture(new MetadataStoreException.LockBusyException(
+                                    "Another creation or migration of the scalable topic is in progress: " + path));
+                        }
+                        // With jitter, so two acquisitions that see each other don't keep retrying together.
+                        long delayMs = CREATION_LOCK_FIRST_RETRY_DELAY_MS << (attempt - 1);
+                        Executor delayed = CompletableFuture.delayedExecutor(
+                                delayMs + ThreadLocalRandom.current().nextLong(delayMs), TimeUnit.MILLISECONDS);
+                        return CompletableFuture.supplyAsync(() -> null, delayed)
+                                .thenCompose(___ -> acquireCreationLock(lockDir, prefix, name, attempt + 1));
+                    });
+                });
+    }
+
+    /** Delete the record of an acquisition, if it's there. It's named after the acquisition, so it's no other's. */
+    private CompletableFuture<Void> releaseCreationLock(String path) {
+        return getStore().delete(path, Optional.empty())
+                .exceptionally(ex -> {
+                    if (!(FutureUtil.unwrapCompletionException(ex)
+                            instanceof MetadataStoreException.NotFoundException)) {
+                        // It goes away with the broker's metadata session anyway.
+                        log.warn().attr("path", path).exceptionMessage(ex).log("Failed to release a creation lock");
+                    }
+                    return null;
+                });
+    }
+
     public CompletableFuture<Optional<ScalableTopicMetadata>> getScalableTopicMetadataAsync(TopicName tn) {
         return getAsync(topicPath(tn));
     }
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java
index 6396f9e73bf4..b9e47797286c 100644
--- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java
@@ -33,6 +33,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
 import org.apache.logging.log4j.Level;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.resources.NamespaceResources;
@@ -209,4 +210,112 @@ public class PulsarAuthorizationProviderTest {
         assertTrue(provider.canProduceAsync(TopicName.get("persistent://tenant/ns/orders-partition-1"), ROLE, null)
                 .get(), "A partition of a partitioned topic still gets the permissions of the partitioned topic");
     }
+
+    /** A provider whose updates of the namespace policies are applied to {@code policies}. */
+    private static PulsarAuthorizationProvider providerUpdatingPolicies(Policies policies, boolean readOnly)
+            throws Exception {
+        return providerUpdatingPolicies(policies, policies, readOnly);
+    }
+
+    /**
+     * A provider that reads {@code readPolicies} from the metadata store, and whose updates of the namespace policies
+     * are applied to {@code updatedPolicies}, as when another broker changed them in between and the update is
+     * retried on their latest version. Its cache hasn't caught up with the store: it has no permissions at all.
+     */
+    private static PulsarAuthorizationProvider providerUpdatingPolicies(Policies readPolicies,
+                                                                       Policies updatedPolicies, boolean readOnly)
+            throws Exception {
+        NamespaceResources namespaceResources = mock(NamespaceResources.class);
+        when(namespaceResources.getPoliciesAsync(any()))
+                .thenReturn(CompletableFuture.completedFuture(Optional.of(new Policies())));
+        when(namespaceResources.refreshAndGetPoliciesAsync(any()))
+                .thenReturn(CompletableFuture.completedFuture(Optional.of(readPolicies)));
+        when(namespaceResources.getPoliciesReadOnlyAsync()).thenReturn(CompletableFuture.completedFuture(readOnly));
+        when(namespaceResources.setPoliciesAsync(any(), any())).thenAnswer(invocation -> {
+            Function<Policies, Policies> update = invocation.getArgument(1);
+            update.apply(updatedPolicies);
+            return CompletableFuture.completedFuture(null);
+        });
+        PulsarResources pulsarResources = mock(PulsarResources.class);
+        when(pulsarResources.getNamespaceResources()).thenReturn(namespaceResources);
+        return new PulsarAuthorizationProvider(new ServiceConfiguration(), pulsarResources);
+    }
+
+    /**
+     * Copying the permissions of a regular topic to the scalable topic it is migrated to adds, for each role, the
+     * permissions granted on the regular topic itself to those already granted on the scalable topic.
+     */
+    @Test
+    public void testCopyPermissionsAddsThePermissionsOfTheSourceTopic() throws Exception {
+        String source = "persistent://tenant/ns/orders";
+        String target = "topic://tenant/ns/orders";
+        Policies policies = new Policies();
+        Map<String, Map<String, Set<AuthAction>>> topicPermissions = policies.auth_policies.getTopicAuthentication();
+        topicPermissions.put(source, new HashMap<>(Map.of(
+                "producer", Set.of(AuthAction.produce),
+                "consumer", Set.of(AuthAction.consume))));
+        topicPermissions.put(source + "-partition-0",
+                new HashMap<>(Map.of("partition-reader", Set.of(AuthAction.consume))));
+        topicPermissions.put(target, new HashMap<>(Map.of(
+                "consumer", Set.of(AuthAction.produce),
+                "scalable-reader", Set.of(AuthAction.consume))));
+        policies.auth_policies.getNamespaceAuthentication().put("namespace-reader", Set.of(AuthAction.consume));
+        PulsarAuthorizationProvider provider = providerUpdatingPolicies(policies, false);
+
+        provider.copyPermissionsAsync(TopicName.get(source), TopicName.get(target)).get();
+
+        assertEquals(topicPermissions.get(target), Map.of(
+                "producer", Set.of(AuthAction.produce),
+                "consumer", Set.of(AuthAction.produce, AuthAction.consume),
+                "scalable-reader", Set.of(AuthAction.consume)));
+        assertEquals(topicPermissions.get(source), Map.of(
+                "producer", Set.of(AuthAction.produce),
+                "consumer", Set.of(AuthAction.consume)));
+    }
+
+    /**
+     * The permissions are copied from the version of the policies being updated: a permission revoked on the source
+     * topic after the policies were read, by another broker, is not restored on the target topic.
+     */
+    @Test
+    public void testCopyPermissionsDoesNotRestoreAPermissionRevokedMeanwhile() throws Exception {
+        String source = "persistent://tenant/ns/orders";
+        String target = "topic://tenant/ns/orders";
+        Policies readPolicies = new Policies();
+        readPolicies.auth_policies.getTopicAuthentication().put(source, new HashMap<>(Map.of(
+                "revoked", Set.of(AuthAction.produce),
+                "kept", Set.of(AuthAction.consume))));
+        Policies updatedPolicies = new Policies();
+        updatedPolicies.auth_policies.getTopicAuthentication()
+                .put(source, new HashMap<>(Map.of("kept", Set.of(AuthAction.consume))));
+        PulsarAuthorizationProvider provider = providerUpdatingPolicies(readPolicies, updatedPolicies, false);
+
+        provider.copyPermissionsAsync(TopicName.get(source), TopicName.get(target)).get();
+
+        assertEquals(updatedPolicies.auth_policies.getTopicAuthentication().get(target),
+                Map.of("kept", Set.of(AuthAction.consume)));
+    }
+
+    /** A topic without permissions of its own has nothing to copy: the policies are left alone, even read-only. */
+    @Test
+    public void testCopyPermissionsOfTopicWithoutPermissions() throws Exception {
+        Policies policies = policiesGrantingOnTopic("persistent://tenant/ns/payments", ROLE, AuthAction.produce);
+        PulsarAuthorizationProvider provider = providerUpdatingPolicies(policies, true);
+
+        provider.copyPermissionsAsync(TopicName.get("persistent://tenant/ns/orders"),
+                TopicName.get("topic://tenant/ns/orders")).get();
+
+        assertFalse(policies.auth_policies.getTopicAuthentication().containsKey("topic://tenant/ns/orders"));
+    }
+
+    /** Permissions that can't be copied fail the copy, rather than letting their roles lose access. */
+    @Test
+    public void testCopyPermissionsFailsWithReadOnlyPolicies() throws Exception {
+        PulsarAuthorizationProvider provider = providerUpdatingPolicies(
+                policiesGrantingOnTopic("persistent://tenant/ns/orders", ROLE, AuthAction.produce), true);
+
+        ExecutionException e = expectThrows(ExecutionException.class, () -> provider.copyPermissionsAsync(
+                TopicName.get("persistent://tenant/ns/orders"), TopicName.get("topic://tenant/ns/orders")).get());
+        assertTrue(e.getCause() instanceof IllegalStateException, "Unexpected failure: " + e.getCause());
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
index f1a606f3b364..18ed34ef53fe 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ScalableTopics.java
@@ -176,7 +176,8 @@ public class ScalableTopics extends AdminResource {
             @ApiResponse(responseCode = "401",
                     description = "Don't have permission to administrate resources on this tenant"),
             @ApiResponse(responseCode = "403", description = "Don't have admin permission on the namespace"),
-            @ApiResponse(responseCode = "409", description = "Scalable topic already exists"),
+            @ApiResponse(responseCode = "409", description = "Scalable topic already exists, or is being created "
+                    + "or migrated to"),
             @ApiResponse(responseCode = "412", description = "Invalid configuration"),
             @ApiResponse(responseCode = "500", description = "Internal server error")})
     public void createScalableTopic(
@@ -208,7 +209,10 @@ public class ScalableTopics extends AdminResource {
                             pulsar().getConfiguration().getScalableTopicEntryBucketBudget(),
                             pulsar().getConfiguration().getScalableTopicEntryBucketMaxPerSegment(),
                             props);
-                    return resources().createScalableTopicAsync(tn, metadata)
+                    // The creation lock keeps the creation from interleaving with a migration of the
+                    // regular topic of the same name.
+                    return resources().runWithCreationLock(tn,
+                                    lock -> resources().createScalableTopicAsync(tn, metadata))
                             .thenCompose(ignored -> createInitialSegmentTopicsAsync(tn, metadata));
                 })
                 .thenAccept(__ -> {
@@ -222,6 +226,9 @@ public class ScalableTopics extends AdminResource {
                     if (cause instanceof MetadataStoreException.AlreadyExistsException) {
                         asyncResponse.resume(new RestException(Response.Status.CONFLICT,
                                 "Scalable topic already exists: " + tn));
+                    } else if (cause instanceof MetadataStoreException.LockBusyException) {
+                        asyncResponse.resume(new RestException(Response.Status.CONFLICT,
+                                "A migration or another creation of the scalable topic is in progress: " + tn));
                     } else {
                         log.error().attr("clientAppId", clientAppId()).attr("topic", tn)
                                 .exception(ex).log("Failed to create scalable topic");
@@ -272,8 +279,8 @@ public class ScalableTopics extends AdminResource {
                     description = "Don't have permission to administrate resources on this tenant"),
             @ApiResponse(responseCode = "403", description = "Don't have produce permission on the topic"),
             @ApiResponse(responseCode = "404", description = "Topic doesn't exist"),
-            @ApiResponse(responseCode = "409", description = "Already a scalable topic, or legacy v4 clients are "
-                    + "still connected and force was not set"),
+            @ApiResponse(responseCode = "409", description = "Already a scalable topic, being created as one, or "
+                    + "legacy v4 clients are still connected and force was not set"),
             @ApiResponse(responseCode = "500", description = "Internal server error")})
     public void migrateToScalable(
             @Suspended final AsyncResponse asyncResponse,
@@ -305,6 +312,10 @@ public class ScalableTopics extends AdminResource {
                     if (cause instanceof MetadataStoreException.AlreadyExistsException) {
                         asyncResponse.resume(new RestException(Response.Status.CONFLICT,
                                 "Topic is already scalable: " + scalableName));
+                    } else if (cause instanceof MetadataStoreException.LockBusyException) {
+                        asyncResponse.resume(new RestException(Response.Status.CONFLICT,
+                                "A creation of the scalable topic, or another migration, is in progress: "
+                                        + scalableName));
                     } else {
                         log.error().attr("clientAppId", clientAppId()).attr("topic", scalableName)
                                 .exception(ex).log("Failed to migrate topic to scalable");
@@ -317,21 +328,40 @@ public class ScalableTopics extends AdminResource {
     /**
      * Orchestrate a regular-to-scalable migration:
      * <ol>
+     *   <li>take the creation lock of the scalable topic, so no creation of the same topic, on any
+     *       broker, can interleave with the steps up to the commit (see
+     *       {@link ScalableTopicResources#runWithCreationLock});</li>
      *   <li>reject if scalable metadata already exists;</li>
      *   <li>resolve the source topic's existence + partition count;</li>
      *   <li>unless {@code force}, reject if any legacy v4 client is still connected;</li>
      *   <li>build the migrated layout (sealed legacy parents + active children);</li>
      *   <li>create the new child segment topics;</li>
+     *   <li>grant on the scalable topic the permissions granted on the regular topic, so the
+     *       roles allowed on it keep their access through the new segments;</li>
      *   <li>atomically write the scalable metadata (the commit point — connected V5 lookup
      *       sessions transition from the synthetic layout to the real DAG via the metadata
-     *       watch);</li>
+     *       watch), and release the lock;</li>
      *   <li>terminate the old topics so no further v4 writes can land — they become the
      *       drainable sealed parent segments.</li>
      * </ol>
      */
     private CompletableFuture<Void> doMigrateToScalableAsync(TopicName scalableName,
                                                              TopicName persistentBase, boolean force) {
-        return resources().getScalableTopicMetadataAsync(scalableName).thenCompose(existing -> {
+        return resources().runWithCreationLock(scalableName,
+                        lock -> createMigratedScalableTopicAsync(scalableName, persistentBase, force, lock))
+                .thenCompose(partitions -> terminateLegacyTopicsAsync(persistentBase, partitions));
+    }
+
+    /**
+     * The steps of {@link #doMigrateToScalableAsync} that run under the creation lock of the scalable
+     * topic, up to its creation. Completes with the partition count of the regular topic.
+     */
+    private CompletableFuture<Integer> createMigratedScalableTopicAsync(TopicName scalableName,
+                                                                       TopicName persistentBase, boolean force,
+                                                                       ScalableTopicResources.CreationLock lock) {
+        // Read the store, not the cache, which may miss a creation completed on another broker
+        // just before the lock was taken.
+        return resources().getScalableTopicMetadataAsync(scalableName, true).thenCompose(existing -> {
             if (existing.isPresent()) {
                 throw new RestException(Response.Status.CONFLICT,
                         "Topic is already scalable: " + scalableName);
@@ -354,9 +384,20 @@ public class ScalableTopics extends AdminResource {
                     ScalableTopicController.createMigratedMetadata(persistentBase, partitions,
                             pulsar().getConfiguration().getScalableTopicEntryBucketBudget(),
                             pulsar().getConfiguration().getScalableTopicEntryBucketMaxPerSegment());
+            // The new segments are authorized with the permissions of the scalable topic, a distinct
+            // identity, while the legacy segments keep those of the persistent:// topics they wrap. So
+            // grant the regular topic's permissions on the scalable topic before the commit: the V5
+            // clients that follow the layout change are then authorized on the new segments as soon as
+            // they see them. Namespace permissions apply to both topics already. Those of a single
+            // partition are not copied: the new segments mix its data with the other partitions' data.
+            // Both steps check that the creation lock is still held first: a broker that lost it, as when
+            // its metadata session expired during the steps above, must not make them.
             return createMigratedChildTopicsAsync(scalableName, metadata)
+                    .thenCompose(__ -> lock.checkHeld())
+                    .thenCompose(__ -> getAuthorizationService().copyPermissionsAsync(persistentBase, scalableName))
+                    .thenCompose(__ -> lock.checkHeld())
                     .thenCompose(__ -> resources().createScalableTopicAsync(scalableName, metadata))
-                    .thenCompose(__ -> terminateLegacyTopicsAsync(persistentBase, partitions));
+                    .thenApply(__ -> partitions);
         });
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
index e930db40aa3b..34cd77ddc5b6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/scalable/ScalableTopicService.java
@@ -296,8 +296,9 @@ public class ScalableTopicService {
         // (re)materialized on demand the first time a client connects to it — see the
         // active-segment reconciliation in BrokerService.isAllowAutoTopicCreationAsync — so
         // eager materialization here is a happy-path latency optimization, not a correctness
-        // requirement.
-        return resources.createScalableTopicAsync(topic, metadata)
+        // requirement. The creation lock keeps it from interleaving with a migration of the
+        // regular topic of the same name (see ScalableTopicResources#runWithCreationLock).
+        return resources.runWithCreationLock(topic, __ -> resources.createScalableTopicAsync(topic, metadata))
                 .thenCompose(__ -> {
                     List<CompletableFuture<Void>> segmentFutures =
                             metadata.getSegments().values().stream()
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicMigrationPermissionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicMigrationPermissionsTest.java
new file mode 100644
index 000000000000..253a6a299d25
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ScalableTopicMigrationPermissionsTest.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doCallRealMethod;
+import io.jsonwebtoken.Jwts;
+import java.time.Duration;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import lombok.SneakyThrows;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.v5.Message;
+import org.apache.pulsar.client.api.v5.Producer;
+import org.apache.pulsar.client.api.v5.PulsarClient;
+import org.apache.pulsar.client.api.v5.QueueConsumer;
+import org.apache.pulsar.client.api.v5.auth.AuthenticationFactory;
+import org.apache.pulsar.client.api.v5.config.SubscriptionInitialPosition;
+import org.apache.pulsar.client.api.v5.schema.Schema;
+import org.apache.pulsar.client.impl.auth.AuthenticationToken;
+import org.apache.pulsar.common.policies.data.AuthAction;
+import org.mockito.invocation.InvocationOnMock;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Topic-level permissions across the PIP-475 migration of a regular topic to a scalable topic, with the default
+ * authorization provider. After the migration, the V5 clients produce to and consume from new {@code segment://}
+ * segments, which are authorized with the permissions of the scalable topic's {@code topic://} name: the migration
+ * must grant there the permissions of the regular topic, for the roles allowed on it to keep their access.
+ */
+@Test(groups = "broker-admin")
+public class ScalableTopicMigrationPermissionsTest extends AuthZTest {
+
+    private static final String NAMESPACE = "public/default";
+    private static final String CLIENT_ROLE = "migrated-topic-client";
+    private static final String CLIENT_TOKEN = Jwts.builder()
+            .claim("sub", CLIENT_ROLE).signWith(SECRET_KEY).compact();
+    private static final String PARTITION_ROLE = "migrated-partition-client";
+    private static final String NAMESPACE_ROLE = "migrated-namespace-client";
+    private static final int NUM_MESSAGES = 20;
+
+    @SneakyThrows
+    @BeforeClass(alwaysRun = true)
+    public void setup() {
+        configureTokenAuthentication();
+        configureDefaultAuthorization();
+        start();
+        this.superUserAdmin = PulsarAdmin.builder()
+                .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+                .authentication(new AuthenticationToken(SUPER_USER_TOKEN))
+                .build();
+    }
+
+    @SneakyThrows
+    @AfterClass(alwaysRun = true)
+    public void cleanup() {
+        close();
+    }
+
+    private PulsarClient newClient() throws Exception {
+        return PulsarClient.builder()
+                .serviceUrl(getPulsarService().getBrokerServiceUrl())
+                .authentication(AuthenticationFactory.token(CLIENT_TOKEN))
+                .build();
+    }
+
+    /** Send keyed messages, so they're routed to every segment, and add their values to {@code sent}. */
+    private static void send(Producer<String> producer, String prefix, Set<String> sent) throws Exception {
+        for (int i = 0; i < NUM_MESSAGES; i++) {
+            String value = prefix + i;
+            producer.newMessage().key("key-" + i).value(value).send();
+            sent.add(value);
+        }
+    }
+
+    private static Set<String> receiveValues(QueueConsumer<String> consumer, int count) throws Exception {
+        Set<String> values = new HashSet<>();
+        for (int i = 0; i < count; i++) {
+            Message<String> message = consumer.receive(Duration.ofSeconds(10));
+            assertThat(message).as("message %s of %s", i, count).isNotNull();
+            values.add(message.value());
+            consumer.acknowledge(message.id());
+        }
+        return values;
+    }
+
+    /**
+     * An application allowed on the regular topic keeps producing through the migration: once the migration
+     * commits, its producer moves to the new segments of the scalable topic.
+     */
+    @Test
+    public void testProducerKeepsProducingThroughMigration() throws Exception {
+        String topic = "persistent://" + NAMESPACE + "/" + UUID.randomUUID();
+        superUserAdmin.topics().createPartitionedTopic(topic, 2);
+        superUserAdmin.topics().grantPermission(topic, CLIENT_ROLE, Set.of(AuthAction.produce, AuthAction.consume));
+
+        @Cleanup
+        PulsarClient client = newClient();
+        @Cleanup
+        Producer<String> producer = client.newProducer(Schema.string()).topic(topic).create();
+        Set<String> sent = new HashSet<>();
+        send(producer, "pre-", sent);
+
+        superUserAdmin.scalableTopics().migrateToScalable(topic, false);
+
+        send(producer, "post-", sent);
+
+        @Cleanup
+        QueueConsumer<String> consumer = client.newQueueConsumer(Schema.string())
+                .topic(topic)
+                .subscriptionName("sub")
+                .subscriptionInitialPosition(SubscriptionInitialPosition.EARLIEST)
+                .subscribe();
+        // The messages sent before the migration come from the legacy segments, the others from the new ones.
+        assertThat(receiveValues(consumer, sent.size())).isEqualTo(sent);
+    }
+
+    /** After the migration, an application allowed on the regular topic can use its scalable topic:// name. */
+    @Test
+    public void testMigratedTopicIsAccessibleThroughItsScalableName() throws Exception {
+        String name = NAMESPACE + "/" + UUID.randomUUID();
+        superUserAdmin.topics().createNonPartitionedTopic("persistent://" + name);
+        superUserAdmin.topics().grantPermission("persistent://" + name, CLIENT_ROLE,
+                Set.of(AuthAction.produce, AuthAction.consume));
+
+        @Cleanup
+        PulsarClient client = newClient();
+        Set<String> sent = new HashSet<>();
+        try (Producer<String> producer = client.newProducer(Schema.string()).topic("persistent://" + name).create()) {
+            send(producer, "pre-", sent);
+        }
+
+        superUserAdmin.scalableTopics().migrateToScalable(name, false);
+
+        String scalableTopic = "topic://" + name;
+        @Cleanup
+        Producer<String> producer = client.newProducer(Schema.string()).topic(scalableTopic).create();
+        send(producer, "post-", sent);
+        @Cleanup
+        QueueConsumer<String> consumer = client.newQueueConsumer(Schema.string())
+                .topic(scalableTopic)
+                .subscriptionName("sub")
+                .subscriptionInitialPosition(SubscriptionInitialPosition.EARLIEST)
+                .subscribe();
+        assertThat(receiveValues(consumer, sent.size())).isEqualTo(sent);
+    }
+
+    /**
+     * Only the permissions granted on the regular topic itself are copied: not those of its namespace, which apply to
+     * the scalable topic already, nor those granted on a single partition.
+     */
+    @Test
+    public void testMigrationCopiesOnlyThePermissionsOfTheTopic() throws Exception {
+        String name = NAMESPACE + "/" + UUID.randomUUID();
+        String topic = "persistent://" + name;
+        superUserAdmin.topics().createPartitionedTopic(topic, 2);
+        superUserAdmin.topics().grantPermission(topic, CLIENT_ROLE, Set.of(AuthAction.consume));
+        superUserAdmin.topics().grantPermission(topic + "-partition-0", PARTITION_ROLE, Set.of(AuthAction.consume));
+        superUserAdmin.namespaces().grantPermissionOnNamespace(NAMESPACE, NAMESPACE_ROLE, Set.of(AuthAction.consume));
+        try {
+            superUserAdmin.scalableTopics().migrateToScalable(topic, false);
+        } finally {
+            superUserAdmin.namespaces().revokePermissionsOnNamespace(NAMESPACE, NAMESPACE_ROLE);
+        }
+
+        assertThat(superUserAdmin.scalableTopics().getPermissions("topic://" + name))
+                .containsEntry(CLIENT_ROLE, Set.of(AuthAction.consume))
+                .doesNotContainKeys(PARTITION_ROLE, NAMESPACE_ROLE);
+    }
+
+    /**
+     * A migration holds the creation lock of the scalable topic from its check that the topic doesn't exist to its
+     * creation, so a creation of the same topic can't get in between and receive the permissions the migration
+     * grants: it's refused while the migration is about to copy them.
+     */
+    @Test
+    public void testCreationIsRefusedWhileAMigrationCopiesPermissions() throws Exception {
+        String name = NAMESPACE + "/" + UUID.randomUUID();
+        superUserAdmin.topics().createNonPartitionedTopic("persistent://" + name);
+        superUserAdmin.topics().grantPermission("persistent://" + name, CLIENT_ROLE, Set.of(AuthAction.produce));
+
+        CompletableFuture<Void> copyStarted = new CompletableFuture<>();
+        CompletableFuture<Void> resumeCopy = new CompletableFuture<>();
+        doAnswer(invocation -> {
+            copyStarted.complete(null);
+            return resumeCopy.thenCompose(__ -> callRealMethod(invocation));
+        }).when(authorizationService).copyPermissionsAsync(any(), any());
+        try {
+            CompletableFuture<Void> migration = superUserAdmin.scalableTopics().migrateToScalableAsync(name, false);
+            copyStarted.get(30, TimeUnit.SECONDS);
+
+            assertThatThrownBy(() -> superUserAdmin.scalableTopics().createScalableTopic("topic://" + name, 1))
+                    .isInstanceOf(PulsarAdminException.ConflictException.class);
+
+            resumeCopy.complete(null);
+            migration.get(30, TimeUnit.SECONDS);
+        } finally {
+            resumeCopy.complete(null);
+            doCallRealMethod().when(authorizationService).copyPermissionsAsync(any(), any());
+        }
+
+        // The scalable topic is the migrated one, with the permissions of the regular topic.
+        assertThat(superUserAdmin.scalableTopics().getMetadata("topic://" + name).getSegments().values())
+                .anyMatch(segment -> segment.getLegacyTopicName() != null);
+        assertThat(superUserAdmin.scalableTopics().getPermissions("topic://" + name))
+                .containsEntry(CLIENT_ROLE, Set.of(AuthAction.produce));
+    }
+
+    /** A migration that finds the scalable topic already created fails, and grants nothing on it. */
+    @Test
+    public void testMigrationToAnExistingScalableTopicGrantsNothingOnIt() throws Exception {
+        String name = NAMESPACE + "/" + UUID.randomUUID();
+        superUserAdmin.topics().createNonPartitionedTopic("persistent://" + name);
+        superUserAdmin.topics().grantPermission("persistent://" + name, CLIENT_ROLE, Set.of(AuthAction.produce));
+        superUserAdmin.scalableTopics().createScalableTopic("topic://" + name, 1);
+
+        assertThatThrownBy(() -> superUserAdmin.scalableTopics().migrateToScalable(name, false))
+                .isInstanceOf(PulsarAdminException.ConflictException.class);
+        assertThat(superUserAdmin.scalableTopics().getPermissions("topic://" + name)).doesNotContainKey(CLIENT_ROLE);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static CompletableFuture<Void> callRealMethod(InvocationOnMock invocation) {
+        try {
+            return (CompletableFuture<Void>) invocation.callRealMethod();
+        } catch (Throwable t) {
+            return CompletableFuture.failedFuture(t);
+        }
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v2/ScalableTopicsMigrateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v2/ScalableTopicsMigrateTest.java
index bf9bbfbfecee..18a9a0ab02a0 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v2/ScalableTopicsMigrateTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v2/ScalableTopicsMigrateTest.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.admin.v2;
 
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -26,10 +27,12 @@ import static org.testng.Assert.assertEquals;
 import jakarta.ws.rs.container.AsyncResponse;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.function.Function;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.resources.PulsarResources;
 import org.apache.pulsar.broker.resources.ScalableTopicResources;
+import org.apache.pulsar.broker.resources.ScalableTopicResources.CreationLock;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.TopicOperation;
 import org.mockito.ArgumentCaptor;
@@ -96,7 +99,12 @@ public class ScalableTopicsMigrateTest {
         when(pulsar.getPulsarResources()).thenReturn(pulsarResources);
         when(pulsarResources.getScalableTopicResources()).thenReturn(scalableResources);
         when(pulsar.getNamespaceService()).thenReturn(namespaceService);
-        when(scalableResources.getScalableTopicMetadataAsync(any(TopicName.class)))
+        // The creation lock is free: run the migration steps it guards, with the lock held throughout.
+        when(scalableResources.runWithCreationLock(any(TopicName.class), any())).thenAnswer(invocation -> {
+            Function<CreationLock, CompletableFuture<Object>> action = invocation.getArgument(1);
+            return action.apply(() -> CompletableFuture.completedFuture(null));
+        });
+        when(scalableResources.getScalableTopicMetadataAsync(any(TopicName.class), eq(true)))
                 .thenReturn(CompletableFuture.completedFuture(Optional.empty()));
         // Stop the pipeline at the existence check; every identity the endpoint
         // derives has been observed by then.
@@ -118,7 +126,7 @@ public class ScalableTopicsMigrateTest {
         assertEquals(resource.authorizedTopic, expectedPersistent);
 
         ArgumentCaptor<TopicName> metadataLookup = ArgumentCaptor.forClass(TopicName.class);
-        verify(scalableResources).getScalableTopicMetadataAsync(metadataLookup.capture());
+        verify(scalableResources).getScalableTopicMetadataAsync(metadataLookup.capture(), eq(true));
         assertEquals(metadataLookup.getValue(), expectedScalable);
 
         ArgumentCaptor<TopicName> existenceCheck = ArgumentCaptor.forClass(TopicName.class);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resources/ScalableTopicCreationLockTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resources/ScalableTopicCreationLockTest.java
new file mode 100644
index 000000000000..dd0d97992133
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resources/ScalableTopicCreationLockTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resources;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.resources.ScalableTopicResources.CreationLock;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.metadata.TestZKServer;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.Option;
+import org.apache.pulsar.metadata.api.Stat;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.api.extended.SessionEvent;
+import org.apache.pulsar.metadata.impl.DualMetadataStore;
+import org.apache.pulsar.metadata.impl.LocalMemoryMetadataStore;
+import org.apache.pulsar.metadata.impl.ZKMetadataStore;
+import org.awaitility.Awaitility;
+import org.testng.annotations.Test;
+
+/**
+ * The creation lock of a scalable topic ({@link ScalableTopicResources#runWithCreationLock}), which a creation of the
+ * topic and a migration of a regular topic to it take, so they exclude each other on every broker.
+ */
+public class ScalableTopicCreationLockTest {
+
+    private static final TopicName TOPIC = TopicName.get("topic://tenant/ns/orders");
+
+    /** Take the lock, complete {@code held} with it, and hold it until {@code release} completes. */
+    private static CompletableFuture<Void> holdLock(ScalableTopicResources resources,
+                                                    CompletableFuture<CreationLock> held,
+                                                    CompletableFuture<Void> release) {
+        return resources.runWithCreationLock(TOPIC, lock -> {
+            held.complete(lock);
+            return release;
+        });
+    }
+
+    private static void assertLockBusy(CompletableFuture<?> future) {
+        assertThatThrownBy(() -> future.get(10, TimeUnit.SECONDS))
+                .hasCauseInstanceOf(MetadataStoreException.LockBusyException.class);
+    }
+
+    @Test
+    public void testAcquisitionsOfATopicExcludeEachOther() throws Exception {
+        @Cleanup
+        MetadataStoreExtended store = new LocalMemoryMetadataStore("memory:exclusion",
+                MetadataStoreConfig.builder().build());
+        ScalableTopicResources resources = new ScalableTopicResources(store, 30);
+        CompletableFuture<CreationLock> held = new CompletableFuture<>();
+        CompletableFuture<Void> release = new CompletableFuture<>();
+        CompletableFuture<Void> holder = holdLock(resources, held, release);
+        held.get(10, TimeUnit.SECONDS);
+
+        // Refused even on the same broker, with the same metadata session.
+        AtomicBoolean ran = new AtomicBoolean();
+        assertLockBusy(resources.runWithCreationLock(TOPIC, lock -> {
+            ran.set(true);
+            return CompletableFuture.completedFuture(null);
+        }));
+        assertThat(ran).isFalse();
+        // The lock of another topic is another lock.
+        resources.runWithCreationLock(TopicName.get("topic://tenant/ns/payments"), CreationLock::checkHeld)
+                .get(10, TimeUnit.SECONDS);
+
+        release.complete(null);
+        holder.get(10, TimeUnit.SECONDS);
+        resources.runWithCreationLock(TOPIC, CreationLock::checkHeld).get(10, TimeUnit.SECONDS);
+    }
+
+    /**
+     * When the store retried the create of the lock's record after losing its response, and found the record there,
+     * the acquisition holds the lock: the record is its own.
+     */
+    @Test
+    public void testAcquisitionWhoseCreateResponseWasLost() throws Exception {
+        AtomicBoolean responseLost = new AtomicBoolean();
+        @Cleanup
+        MetadataStoreExtended store = new LocalMemoryMetadataStore("memory:lost-response",
+                MetadataStoreConfig.builder().build()) {
+            @Override
+            public CompletableFuture<Stat> storePut(String path, byte[] data, Optional<Long> optExpectedVersion,
+                                                    Set<Option> opts) {
+                CompletableFuture<Stat> put = super.storePut(path, data, optExpectedVersion, opts);
+                if (path.startsWith("/topic-creation-locks/") && responseLost.compareAndSet(false, true)) {
+                    // The record is created, but the response is lost, and the create retried finds the record.
+                    return put.thenCompose(__ -> CompletableFuture.failedFuture(
+                            new MetadataStoreException.BadVersionException("The record exists")));
+                }
+                return put;
+            }
+        };
+        ScalableTopicResources resources = new ScalableTopicResources(store, 30);
+
+        assertThat(resources.runWithCreationLock(TOPIC, lock -> lock.checkHeld().thenApply(__ -> "created"))
+                .get(10, TimeUnit.SECONDS)).isEqualTo("created");
+        assertThat(responseLost).isTrue();
+        // Released: the next acquisition takes the lock.
+        resources.runWithCreationLock(TOPIC, CreationLock::checkHeld).get(10, TimeUnit.SECONDS);
+    }
+
+    /** An acquisition that fails after creating its record, as when it can't list the records, deletes it. */
+    @Test
+    public void testAcquisitionThatFailsToListTheRecords() throws Exception {
+        AtomicBoolean listingFailed = new AtomicBoolean();
+        @Cleanup
+        MetadataStoreExtended store = new LocalMemoryMetadataStore("memory:failed-listing",
+                MetadataStoreConfig.builder().build()) {
+            @Override
+            public CompletableFuture<List<String>> getChildrenFromStore(String path, Set<Option> opts) {
+                if (path.startsWith("/topic-creation-locks/") && listingFailed.compareAndSet(false, true)) {
+                    return CompletableFuture.failedFuture(new MetadataStoreException("Injected listing failure"));
+                }
+                return super.getChildrenFromStore(path, opts);
+            }
+        };
+        ScalableTopicResources resources = new ScalableTopicResources(store, 30);
+
+        AtomicBoolean ran = new AtomicBoolean();
+        assertThatThrownBy(() -> resources.runWithCreationLock(TOPIC, lock -> {
+                    ran.set(true);
+                    return CompletableFuture.completedFuture(null);
+                }).get(10, TimeUnit.SECONDS))
+                .hasRootCauseMessage("Injected listing failure");
+        assertThat(ran).isFalse();
+        // Its record was deleted: the next acquisition takes the lock.
+        resources.runWithCreationLock(TOPIC, CreationLock::checkHeld).get(10, TimeUnit.SECONDS);
+    }
+
+    /**
+     * A broker whose metadata session expired lost the lock, which another broker can take. The first broker no
+     * longer holds it, and releasing it when its action completes leaves the lock of the other broker in place.
+     */
+    @Test
+    public void testLockLostWithTheMetadataSession() throws Exception {
+        @Cleanup
+        TestZKServer zks = new TestZKServer();
+        @Cleanup
+        MetadataStoreExtended storeA = MetadataStoreExtended.create(zks.getConnectionString(),
+                MetadataStoreConfig.builder().build());
+        @Cleanup
+        MetadataStoreExtended storeB = MetadataStoreExtended.create(zks.getConnectionString(),
+                MetadataStoreConfig.builder().build());
+        ScalableTopicResources resourcesA = new ScalableTopicResources(storeA, 30);
+        ScalableTopicResources resourcesB = new ScalableTopicResources(storeB, 30);
+
+        CompletableFuture<CreationLock> heldByA = new CompletableFuture<>();
+        CompletableFuture<Void> releaseA = new CompletableFuture<>();
+        CompletableFuture<Void> holderA = holdLock(resourcesA, heldByA, releaseA);
+        CreationLock lockA = heldByA.get(10, TimeUnit.SECONDS);
+
+        BlockingQueue<SessionEvent> sessionEventsA = new LinkedBlockingQueue<>();
+        storeA.registerSessionListener(sessionEventsA::add);
+        zks.expireSession(((ZKMetadataStore) ((DualMetadataStore) storeA).getSourceStore()).getZkClient()
+                .getSessionId());
+        Awaitility.await().until(() -> sessionEventsA.contains(SessionEvent.SessionReestablished));
+
+        CompletableFuture<CreationLock> heldByB = new CompletableFuture<>();
+        CompletableFuture<Void> releaseB = new CompletableFuture<>();
+        CompletableFuture<Void> holderB = holdLock(resourcesB, heldByB, releaseB);
+        CreationLock lockB = heldByB.get(10, TimeUnit.SECONDS);
+
+        assertLockBusy(lockA.checkHeld());
+        releaseA.complete(null);
+        holderA.get(10, TimeUnit.SECONDS);
+
+        lockB.checkHeld().get(10, TimeUnit.SECONDS);
+        assertLockBusy(resourcesA.runWithCreationLock(TOPIC, CreationLock::checkHeld));
+        releaseB.complete(null);
+        holderB.get(10, TimeUnit.SECONDS);
+    }
+}


(pulsar) 04/12: [fix][broker] Shared subscription stalls forever on a chunked message (#26396)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 2d370b757ee247cbd930355a3fc8fb1e2040b169
Author: Bäm <de...@sandchaschte.ch>
AuthorDate: Thu Oct 22 02:00:34 2026 +0200

    [fix][broker] Shared subscription stalls forever on a chunked message (#26396)
    
    Co-authored-by: Matteo Merli <mm...@apache.org>
    (cherry picked from commit 91aac70e6b9a579ef81df688621c34d8b532bbc5)
---
 .../broker/service/SharedConsumerAssignor.java     |  63 ++++++++++-
 .../PersistentDispatcherMultipleConsumers.java     |   9 +-
 .../broker/service/SharedConsumerAssignorTest.java | 125 +++++++++++++++++++++
 3 files changed, 190 insertions(+), 7 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java
index 4814d0e22150..b2f2fc2c151f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java
@@ -28,7 +28,9 @@ import java.util.function.Supplier;
 import lombok.CustomLog;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import org.apache.bookkeeper.mledger.Position;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.jspecify.annotations.Nullable;
 
 /**
  * The assigner to assign entries to the proper {@link Consumer} in the shared subscription.
@@ -56,6 +58,19 @@ public class SharedConsumerAssignor {
 
     public Map<Consumer, List<EntryAndMetadata>> assign(final List<EntryAndMetadata> entryAndMetadataList,
                                                         final int numConsumers) {
+        return assign(entryAndMetadataList, numConsumers, null);
+    }
+
+    /**
+     * Assign the entries to consumers.
+     *
+     * @param firstReplayPosition the first position waiting in the replay queue when the entries come from a normal
+     *                            read, or {@code null}. An orphan chunk after this position goes back to the replay
+     *                            queue instead of being dispatched, since its first chunk may be waiting there.
+     */
+    public Map<Consumer, List<EntryAndMetadata>> assign(final List<EntryAndMetadata> entryAndMetadataList,
+                                                        final int numConsumers,
+                                                        final @Nullable Position firstReplayPosition) {
         assert numConsumers >= 0;
         consumerToPermits.clear();
         final Map<Consumer, List<EntryAndMetadata>> consumerToEntries = new IdentityHashMap<>();
@@ -92,7 +107,8 @@ public class SharedConsumerAssignor {
             if (metadata == null || !metadata.hasUuid() || !metadata.hasChunkId() || !metadata.hasNumChunksFromMsg()) {
                 consumerToEntries.computeIfAbsent(consumer, __ -> new ArrayList<>()).add(entryAndMetadata);
             } else {
-                final Consumer consumerForUuid = getConsumerForUuid(metadata, consumer);
+                final Consumer consumerForUuid =
+                        getConsumerForUuid(entryAndMetadata, metadata, consumer, firstReplayPosition);
                 if (consumerForUuid == null) {
                     unassignedMessageProcessor.accept(entryAndMetadata);
                     continue;
@@ -123,16 +139,51 @@ public class SharedConsumerAssignor {
         return null;
     }
 
-    private Consumer getConsumerForUuid(final MessageMetadata metadata, final Consumer defaultConsumer) {
+    /**
+     * Remove every uuid mapping that points to the given consumer. Must be called when a consumer is removed from the
+     * dispatcher: a mapping to a disconnected consumer has no permits left, so the remaining chunks of that uuid stay
+     * unassignable and stall the subscription. Also bounds the map when the last chunk is never published.
+     */
+    public void removeConsumer(final Consumer consumer) {
+        uuidToConsumer.values().removeIf(cachedConsumer -> cachedConsumer == consumer);
+    }
+
+    /** Drop all uuid mappings, e.g. after all consumers have been removed from the dispatcher. */
+    public void clear() {
+        uuidToConsumer.clear();
+    }
+
+    private Consumer getConsumerForUuid(final EntryAndMetadata entryAndMetadata, final MessageMetadata metadata,
+                                        final Consumer defaultConsumer, final @Nullable Position firstReplayPosition) {
         final String uuid = metadata.getUuid();
         Consumer consumer = uuidToConsumer.get(uuid);
         if (consumer == null) {
-            if (metadata.getChunkId() != 0) {
-                // Not the first chunk, skip it
+            consumer = defaultConsumer;
+            if (metadata.getChunkId() == 0) {
+                uuidToConsumer.put(uuid, consumer);
+            } else if (firstReplayPosition != null
+                    && firstReplayPosition.compareTo(entryAndMetadata.getPosition()) < 0) {
+                // A normal read that was in flight when the consumer owning this uuid was removed: chunk 0 may have
+                // just been added to the replay queue. Replay this chunk too, replay reads are in position order
+                // so chunk 0 will recreate the mapping before this chunk is assigned again.
                 return null;
+            } else {
+                // Orphan chunk: only chunk 0 creates the mapping, and replay reads come before normal reads and are
+                // in position order, so a missing mapping here means chunk 0 is acknowledged and gone.
+                // Never replay it: the dispatcher drains the replay queue before every normal read, so an entry that
+                // can never be assigned stalls the whole subscription. Dispatch it and let the client discard it.
+                if (subscription != null) {
+                    log.warn()
+                            .attr("topic", subscription.getTopic().getName())
+                            .attr("subscription", subscription.getName())
+                            .attr("uuid", uuid)
+                            .attr("chunkId", metadata.getChunkId())
+                            .attr("numChunks", metadata.getNumChunksFromMsg())
+                            .attr("consumer", defaultConsumer)
+                            .log("Dispatching orphan chunk whose first chunk is gone. The client should discard and"
+                                    + " acknowledge it");
+                }
             }
-            consumer = defaultConsumer;
-            uuidToConsumer.put(uuid, consumer);
         }
         final int permits = consumerToPermits.computeIfAbsent(consumer, Consumer::getAvailablePermits);
         if (permits <= 0) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 6592e169fde1..41b4869161fb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -252,6 +252,8 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
 
     @Override
     public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceException {
+        // Drop this consumer's chunk uuid mappings, otherwise its pending chunks stay unassignable forever.
+        assignor.removeConsumer(consumer);
         if (removeConsumerInstance(consumer)) {
             // decrement unack-message count for removed consumer. Only the removal that actually
             // unregisters the consumer may debit it, otherwise removing an already-removed consumer
@@ -312,6 +314,7 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
 
         redeliveryMessages.clear();
         redeliveryTracker.clear();
+        assignor.clear();
         if (closeFuture != null) {
             log.info("All consumers removed. Subscription is disconnected");
             closeFuture.complete(null);
@@ -1014,8 +1017,12 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
             originalEntryAndMetadataList.add(EntryAndMetadata.create(entries.get(i), metadataArray[i]));
         }
 
+        // On a normal read, an orphan chunk after the first replay position goes back to replay: its first chunk
+        // may be waiting there, see SharedConsumerAssignor#assign
+        final Position firstReplayPosition = readType == ReadType.Normal
+                ? redeliveryMessages.getFirstPositionInReplay().orElse(null) : null;
         final Map<Consumer, List<EntryAndMetadata>> assignResult =
-                assignor.assign(originalEntryAndMetadataList, consumerList.size());
+                assignor.assign(originalEntryAndMetadataList, consumerList.size(), firstReplayPosition);
         long totalMessagesSent = 0;
         long totalBytesSent = 0;
         long totalEntries = 0;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedConsumerAssignorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedConsumerAssignorTest.java
index d9e675fa7e54..f7e67bf786ea 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedConsumerAssignorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedConsumerAssignorTest.java
@@ -153,6 +153,131 @@ public class SharedConsumerAssignorTest {
         assertTrue(assignor.getUuidToConsumer().isEmpty());
     }
 
+    /**
+     * An orphan chunk is a chunk whose uuid is not tracked and whose chunk id is not 0, e.g. entry 5 (A-1-2-3) when
+     * the earlier chunks of "A-1" were acknowledged before the mapping existed. That happens after a topic unload, a
+     * broker restart, or when the consumer that owned the uuid disconnected.
+     * <p>
+     * Such an entry can never become assignable, so handing it to the unassigned message processor (the replay queue)
+     * stalls the subscription forever: {@code PersistentDispatcherMultipleConsumers#readMoreEntries} drains the replay
+     * queue before every normal read, so the dispatcher keeps replaying the same entry, never reads anything new and
+     * the mark-delete position never moves. It must be dispatched instead, so that the client can discard and
+     * acknowledge the incomplete chunked message.
+     */
+    @Test
+    public void testOrphanChunkIsDispatchedInsteadOfReplayedForever() {
+        final Consumer consumer = new Consumer("A", 100);
+        roundRobinConsumerSelector.addConsumers(consumer);
+        assertTrue(assignor.getUuidToConsumer().isEmpty());
+
+        // The last chunk of "A-1" is read while the uuid was never cached
+        Map<Consumer, List<EntryAndMetadata>> result = assignor.assign(entryAndMetadataList.subList(5, 6), 1);
+        assertEquals(toString(result.getOrDefault(consumer, Collections.emptyList())),
+                Collections.singletonList("0:5@A-1-2-3"));
+        // The entry must not go back to the replay queue, otherwise the dispatcher can never make progress
+        assertTrue(replayQueue.isEmpty());
+        // An orphan chunk must not create a mapping either, there is no first chunk to pin to a consumer
+        assertTrue(assignor.getUuidToConsumer().isEmpty());
+
+        // The same holds for a middle chunk, not only for the last one
+        result = assignor.assign(entryAndMetadataList.subList(2, 3), 1);
+        assertEquals(toString(result.getOrDefault(consumer, Collections.emptyList())),
+                Collections.singletonList("0:2@A-1-1-3"));
+        assertTrue(replayQueue.isEmpty());
+        assertTrue(assignor.getUuidToConsumer().isEmpty());
+    }
+
+    /**
+     * When the consumer that owns a uuid is removed from the dispatcher, the mapping must be dropped. Otherwise the
+     * remaining chunks stay pinned to a disconnected consumer that has no available permits, so they are replayed
+     * forever and the subscription stalls.
+     */
+    @Test
+    public void testUuidMappingIsRemovedWhenConsumerIsRemoved() {
+        final Consumer consumerA = new Consumer("A", 3);
+        final Consumer consumerB = new Consumer("B", 100);
+        roundRobinConsumerSelector.addConsumers(consumerA);
+
+        // consumerA receives A-0, A-1-0-3 and A-1-1-3, so the uuid "A-1" gets pinned to consumerA
+        assignor.assign(entryAndMetadataList.subList(0, 3), 1);
+        assertTrue(replayQueue.isEmpty());
+        assertEquals(assignor.getUuidToConsumer().keySet(), Sets.newHashSet("A-1"));
+        assertSame(assignor.getUuidToConsumer().get("A-1"), consumerA);
+
+        // consumerA disconnects while the chunked message "A-1" is still incomplete
+        assignor.removeConsumer(consumerA);
+        assertTrue(assignor.getUuidToConsumer().isEmpty());
+
+        roundRobinConsumerSelector.clear();
+        roundRobinConsumerSelector.addConsumers(consumerB);
+
+        // The last chunk of "A-1" must be dispatched to the remaining consumer instead of getting stuck
+        Map<Consumer, List<EntryAndMetadata>> result = assignor.assign(entryAndMetadataList.subList(5, 6), 1);
+        assertNull(result.get(consumerA));
+        assertEquals(toString(result.getOrDefault(consumerB, Collections.emptyList())),
+                Collections.singletonList("0:5@A-1-2-3"));
+        assertTrue(replayQueue.isEmpty());
+    }
+
+    /**
+     * A normal read can still be in flight when the consumer that owns a uuid is removed. Its pending chunk 0 then
+     * sits in the replay queue while the read returns a later chunk with no mapping. That chunk must go back to the
+     * replay queue rather than be dispatched as an orphan, otherwise it reaches a consumer before its chunk 0 and the
+     * client discards a message that could have been delivered.
+     */
+    @Test
+    public void testChunkAfterPendingReplayIsReplayedInOrder() {
+        final Consumer consumerA = new Consumer("A", 2);
+        final Consumer consumerB = new Consumer("B", 100);
+        roundRobinConsumerSelector.addConsumers(consumerA);
+
+        // consumerA receives A-0 and A-1-0-3, so the uuid "A-1" gets pinned to consumerA
+        assignor.assign(entryAndMetadataList.subList(0, 2), 1);
+        assertSame(assignor.getUuidToConsumer().get("A-1"), consumerA);
+
+        // consumerA disconnects, its pending A-1-0-3 is added to the replay queue
+        assignor.removeConsumer(consumerA);
+        roundRobinConsumerSelector.clear();
+        roundRobinConsumerSelector.addConsumers(consumerB);
+        final EntryAndMetadata firstChunk = entryAndMetadataList.get(1);
+
+        // The in-flight normal read returns A-1-1-3, which must wait behind its chunk 0 in the replay queue
+        Map<Consumer, List<EntryAndMetadata>> result =
+                assignor.assign(entryAndMetadataList.subList(2, 3), 1, firstChunk.getPosition());
+        assertTrue(result.isEmpty());
+        assertEquals(toString(replayQueue), Collections.singletonList("0:2@A-1-1-3"));
+        assertTrue(assignor.getUuidToConsumer().isEmpty());
+
+        // The replay read returns both chunks in position order, and they go to the same consumer
+        replayQueue.clear();
+        result = assignor.assign(entryAndMetadataList.subList(1, 3), 1);
+        assertEquals(toString(result.get(consumerB)), Arrays.asList("0:1@A-1-0-3", "0:2@A-1-1-3"));
+        assertSame(assignor.getUuidToConsumer().get("A-1"), consumerB);
+        assertTrue(replayQueue.isEmpty());
+
+        // An orphan chunk before the first replay position is still dispatched
+        assignor.clear();
+        result = assignor.assign(entryAndMetadataList.subList(5, 6), 1, entryAndMetadataList.get(6).getPosition());
+        assertEquals(toString(result.get(consumerB)), Collections.singletonList("0:5@A-1-2-3"));
+        assertTrue(replayQueue.isEmpty());
+    }
+
+    /**
+     * The uuid mappings must not outlive the consumers of the dispatcher, otherwise they leak for every chunked
+     * message whose last chunk is never published.
+     */
+    @Test
+    public void testClearRemovesAllUuidMappings() {
+        final Consumer consumer = new Consumer("A", 100);
+        roundRobinConsumerSelector.addConsumers(consumer);
+
+        assignor.assign(entryAndMetadataList.subList(0, 5), 1);
+        assertEquals(assignor.getUuidToConsumer().keySet(), Sets.newHashSet("A-1", "B-1"));
+
+        assignor.clear();
+        assertTrue(assignor.getUuidToConsumer().isEmpty());
+    }
+
     @RequiredArgsConstructor
     static class ConsumerSelector implements Supplier<Consumer> {
 


(pulsar) 03/12: [improve][broker] Add a random value upto 5% to avoid rollover multiple cursor ledgers (#26542)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit da4a1c5446787dd37cfacb29f031a2a5e3b5a749
Author: ken <16...@qq.com>
AuthorDate: Thu Oct 22 07:35:55 2026 +0800

    [improve][broker] Add a random value upto 5% to avoid rollover multiple cursor ledgers  (#26542)
    
    Co-authored-by: fanjianye <fa...@bigo.sg>
    Co-authored-by: Matteo Merli <mm...@apache.org>
    (cherry picked from commit 586a0bac5661cc013be04435cda1c1f81e94b1db)
---
 .../bookkeeper/mledger/impl/ManagedCursorImpl.java | 18 +++++-
 .../bookkeeper/mledger/impl/ManagedCursorTest.java | 71 ++++++++++++++++++++++
 2 files changed, 87 insertions(+), 2 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 3ca33fa85575..a01dfdb1d3cd 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -51,6 +51,7 @@ import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -314,6 +315,12 @@ public class ManagedCursorImpl implements ManagedCursor {
     @SuppressWarnings("unused")
     private volatile int pendingMarkDeletedSubmittedCount = 0;
     private volatile long lastLedgerSwitchTimestamp;
+    // Stable per-cursor jitter so cursors created together don't roll over their ledgers together. It is
+    // drawn once, so it doesn't change on each check, and applied to the live config, so config updates
+    // are not missed. The time jitter is in [1.0, 1.05), like the managed ledger's. The entries jitter is
+    // in (0.95, 1.0], so the configured max entries per ledger is never exceeded.
+    private final double rolloverTimeJitter;
+    private final double rolloverEntriesJitter;
     private final Clock clock;
 
     private static final AtomicLongFieldUpdater<ManagedCursorImpl> LAST_ACTIVE_UPDATER =
@@ -394,6 +401,8 @@ public class ManagedCursorImpl implements ManagedCursor {
         this.clock = getConfig().getClock();
         this.lastActive = this.clock.millis();
         this.lastLedgerSwitchTimestamp = this.clock.millis();
+        this.rolloverTimeJitter = 1 + ThreadLocalRandom.current().nextDouble() * 0.05;
+        this.rolloverEntriesJitter = 1 - ThreadLocalRandom.current().nextDouble() * 0.05;
 
         if (getConfig().getThrottleMarkDelete() > 0.0) {
             markDeleteLimiter = RateLimiter.create(getConfig().getThrottleMarkDelete());
@@ -3971,9 +3980,14 @@ public class ManagedCursorImpl implements ManagedCursor {
 
     boolean shouldCloseLedger(LedgerHandle lh) {
         long now = clock.millis();
+        // Rounded up, so small limits (below 20) are not changed by the jitter
+        long maximumLedgerRolloverEntries = (long)
+                Math.ceil(getConfig().getMetadataMaxEntriesPerLedger() * rolloverEntriesJitter);
+        long maximumLedgerRolloverTimeMs = (long)
+                (getConfig().getLedgerRolloverTimeout() * 1000L * rolloverTimeJitter);
         if (ledger.getFactory().isMetadataServiceAvailable()
-                && (lh.getLastAddConfirmed() >= getConfig().getMetadataMaxEntriesPerLedger()
-                || lastLedgerSwitchTimestamp < (now - getConfig().getLedgerRolloverTimeout() * 1000))
+                && (lh.getLastAddConfirmed() >= maximumLedgerRolloverEntries
+                || lastLedgerSwitchTimestamp < (now - maximumLedgerRolloverTimeMs))
                 && !state.isClosed()) {
             // It's safe to modify the timestamp since this method will be only called from a callback, implying that
             // calls will be serialized on one single thread
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
index 86334c9a42bc..cc9691820904 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
@@ -132,6 +132,7 @@ import org.apache.bookkeeper.mledger.proto.ManagedLedgerInfo;
 import org.apache.bookkeeper.mledger.proto.PositionInfo;
 import org.apache.bookkeeper.mledger.util.ManagedLedgerTestUtil;
 import org.apache.bookkeeper.mledger.util.ManagedLedgerUtils;
+import org.apache.bookkeeper.mledger.util.MockClock;
 import org.apache.bookkeeper.stats.NullStatsLogger;
 import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
 import org.apache.commons.collections4.iterators.EmptyIterator;
@@ -6039,6 +6040,76 @@ public class ManagedCursorTest extends MockedBookKeeperTestCase {
         assertNotEquals(cursor.getCursorLedger(), initialLedgerId);
     }
 
+    @Test
+    public void testShouldCloseLedgerObservesUpdatedMetadataMaxEntriesPerLedger() throws Exception {
+        ManagedLedgerConfig config = new ManagedLedgerConfig();
+        config.setMetadataMaxEntriesPerLedger(10_000);
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(
+                "testShouldCloseLedgerObservesUpdatedMetadataMaxEntriesPerLedger", config);
+        ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+
+        LedgerHandle lh = mock(LedgerHandle.class);
+        when(lh.getLastAddConfirmed()).thenReturn(1L);
+
+        // Cursor was created with a large rollover threshold, so a single persisted entry is not enough.
+        assertFalse(cursor.shouldCloseLedger(lh));
+
+        ManagedLedgerConfig updated = new ManagedLedgerConfig();
+        updated.setMetadataMaxEntriesPerLedger(1);
+        ledger.setConfig(updated);
+
+        // shouldCloseLedger must read the live config, not a cached derived threshold.
+        assertTrue(cursor.shouldCloseLedger(lh));
+    }
+
+    @Test
+    public void testShouldCloseLedgerObservesUpdatedLedgerRolloverTimeout() throws Exception {
+        MockClock clock = new MockClock();
+        ManagedLedgerConfig config = new ManagedLedgerConfig();
+        config.setClock(clock);
+        config.setLedgerRolloverTimeout(14400);
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(
+                "testShouldCloseLedgerObservesUpdatedLedgerRolloverTimeout", config);
+        ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+
+        LedgerHandle lh = mock(LedgerHandle.class);
+        // Keep lastAddConfirmed below the entries threshold so only the time check can close.
+        when(lh.getLastAddConfirmed()).thenReturn(0L);
+
+        assertFalse(cursor.shouldCloseLedger(lh));
+
+        // 2s is far below 14400s, but past the 1s timeout even with 5% jitter.
+        clock.advance(2, TimeUnit.SECONDS);
+        assertFalse(cursor.shouldCloseLedger(lh));
+
+        ManagedLedgerConfig updated = new ManagedLedgerConfig();
+        updated.setClock(clock);
+        updated.setLedgerRolloverTimeout(1);
+        ledger.setConfig(updated);
+
+        assertTrue(cursor.shouldCloseLedger(lh));
+    }
+
+    @Test
+    public void testShouldCloseLedgerEntriesJitterStaysWithinConfiguredMax() throws Exception {
+        ManagedLedgerConfig config = new ManagedLedgerConfig();
+        config.setMetadataMaxEntriesPerLedger(1000);
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(
+                "testShouldCloseLedgerEntriesJitterStaysWithinConfiguredMax", config);
+
+        LedgerHandle belowJitterRange = mock(LedgerHandle.class);
+        when(belowJitterRange.getLastAddConfirmed()).thenReturn(949L);
+        LedgerHandle atConfiguredMax = mock(LedgerHandle.class);
+        when(atConfiguredMax.getLastAddConfirmed()).thenReturn(1000L);
+
+        // Every cursor draws its own jitter: none may roll over before 95% of the limit or go past the limit.
+        for (int i = 0; i < 50; i++) {
+            ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c" + i);
+            assertFalse(cursor.shouldCloseLedger(belowJitterRange));
+            assertTrue(cursor.shouldCloseLedger(atConfiguredMax));
+        }
+    }
+
     @Test
     public void testReadEmptyEntryList() throws Exception {
         ManagedLedgerConfig managedLedgerConfig = defaultConfig();


(pulsar) 02/12: [fix][test] Fix flaky OffloadPrefixTest.terminateTriggersAutomaticOffloadForLastLedger (#27040)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 1ad5b2884d95e5fa2659b16279efdeecc4a574d4
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Thu Oct 22 02:19:31 2026 +0300

    [fix][test] Fix flaky OffloadPrefixTest.terminateTriggersAutomaticOffloadForLastLedger (#27040)
    
    (cherry picked from commit 31dd6f5fbcf49541cda56116768baf9bc088d220)
---
 .../java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
index 55a9aa2356b2..e84f09717a13 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
@@ -1579,10 +1579,11 @@ public class OffloadPrefixTest extends MockedBookKeeperTestCase {
         Position lastPosition = ledger.terminate();
 
         assertEquals(lastPosition.getLedgerId(), lastLedgerId);
-        assertEventuallyTrue(() -> offloader.offloadedLedgers().equals(Set.of(lastLedgerId)));
+        Awaitility.await().untilAsserted(() -> assertEquals(offloader.offloadedLedgers(), Set.of(lastLedgerId)));
         assertTrue(ledger.isTerminated());
         assertEquals(ledger.getLedgersInfoAsList().size(), 1);
-        assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().isComplete());
+        // the offloader completes before the ledger info is marked complete with an async metadata update
+        Awaitility.await().until(() -> ledger.getLedgersInfoAsList().get(0).getOffloadContext().isComplete());
     }
 
 


(pulsar) 10/12: [fix][broker] Fail compacted reads when the compacted ledger is missing (#26631)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 36d1c98aa2bd1459472a753340c73b737b9f4802
Author: sinan liu <li...@gmail.com>
AuthorDate: Thu Oct 22 09:50:43 2026 +0800

    [fix][broker] Fail compacted reads when the compacted ledger is missing (#26631)
    
    Co-authored-by: Matteo Merli <mm...@apache.org>
    (cherry picked from commit da0b07d5351147437387a625c147c3970dc5a5d7)
---
 .../service/persistent/PersistentSubscription.java |   5 +-
 .../persistent/PulsarCompactorSubscription.java    |  20 +-
 .../pulsar/compaction/CompactedTopicImpl.java      |  30 +--
 .../compaction/PulsarTopicCompactionService.java   |  11 +-
 .../apache/pulsar/compaction/CompactionTest.java   |  85 ++------
 .../compaction/MissingCompactedLedgerTest.java     | 215 +++++++++++++++++++++
 6 files changed, 268 insertions(+), 98 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index e700f87b5c87..3facfbcd8067 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -1029,7 +1029,10 @@ public class PersistentSubscription extends AbstractSubscription {
                         forceReset.complete(false);
                     }
                 }).exceptionally(ex -> {
-                    forceReset.completeExceptionally(ex);
+                    // Without readable compacted data, let the managed ledger validate the reset
+                    // against the original entries. Ordinary readers must still be able to seek;
+                    // compacted reads continue to surface the compaction failure independently.
+                    forceReset.complete(false);
                     return null;
                 });
             }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java
index 8b4cd321ac1b..3e0d2aa96ce9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java
@@ -24,11 +24,14 @@ import io.github.merlimat.slog.Logger;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
+import org.apache.bookkeeper.client.BKException;
 import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.util.Errors;
 import org.apache.pulsar.common.api.proto.CommandAck.AckType;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.compaction.CompactedTopic;
 import org.apache.pulsar.compaction.CompactedTopicContext;
 import org.apache.pulsar.compaction.CompactedTopicImpl;
@@ -121,13 +124,24 @@ public class PulsarCompactorSubscription extends PersistentSubscription {
     }
 
     CompletableFuture<Void> cleanCompactedLedger() {
-        final CompletableFuture<CompactedTopicContext> compactedTopicContextFuture =
-                ((CompactedTopicImpl) compactedTopic).getCompactedTopicContextFuture();
+        final CompletableFuture<CompactedTopicContext> compactedTopicContextFuture;
+        synchronized (compactedTopic) {
+            compactedTopicContextFuture = ((CompactedTopicImpl) compactedTopic).getCompactedTopicContextFuture();
+            // The subscription was explicitly deleted. Clear its registration even if the ledger
+            // could not be opened, before another compaction can install a replacement context.
+            ((CompactedTopicImpl) compactedTopic).reset();
+        }
         if (compactedTopicContextFuture != null) {
             return compactedTopicContextFuture.thenCompose(context -> {
                 long compactedLedgerId = context.getLedger().getId();
-                ((CompactedTopicImpl) compactedTopic).reset();
                 return compactedTopic.deleteCompactedLedger(compactedLedgerId);
+            }).exceptionallyCompose(ex -> {
+                Throwable cause = FutureUtil.unwrapCompletionException(ex);
+                if (cause instanceof BKException bkException
+                        && Errors.isNoSuchLedgerExistsException(bkException.getCode())) {
+                    return CompletableFuture.completedFuture(null);
+                }
+                return CompletableFuture.failedFuture(cause);
             });
         } else {
             return CompletableFuture.completedFuture(null);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java
index 6be44926b978..e478b1f8866a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java
@@ -76,23 +76,16 @@ public class CompactedTopicImpl implements CompactedTopic {
 
             compactionHorizon = p;
 
-            // The compacted ledger may no longer exist: for example, a cursor recovery rolled the cursor
-            // properties back to a metadata-store snapshot that still referenced a ledger a newer
-            // compaction has already deleted. Reads at or before the compaction horizon would then fail
-            // on the failed open instead of reading the original topic data. Unregister the stale
-            // reference so that readCompacted falls back to the original data.
+            // Keep the failed context and horizon if the referenced ledger is missing. The original
+            // entries may already have been trimmed, so treating this as an uncompacted topic could
+            // return an incomplete key set and let the next compaction persist that incomplete view.
             newCompactedLedger.whenComplete((context, exception) -> {
                 if (exception != null && isNoSuchLedgerExists(exception)) {
-                    synchronized (CompactedTopicImpl.this) {
-                        if (compactedTopicContext == newCompactedLedger) {
-                            log.warn()
-                                    .attr("compactedLedgerId", compactedLedgerId)
-                                    .attr("compactionHorizon", p)
-                                    .log("Compacted ledger no longer exists, falling back to reading"
-                                            + " uncompacted data until the next compaction");
-                            reset();
-                        }
-                    }
+                    log.warn()
+                            .attr("compactedLedgerId", compactedLedgerId)
+                            .attr("compactionHorizon", p)
+                            .log("Compacted ledger no longer exists; compacted reads will fail"
+                                    + " because the original data may have been removed");
                 }
             });
 
@@ -276,9 +269,7 @@ public class CompactedTopicImpl implements CompactedTopic {
 
     @Override
     public CompletableFuture<Entry> readLastEntryOfCompactedLedger() {
-        // Capture the context once: the missing-ledger callback may clear the field between a null
-        // check and the composition below, which would dereference null a second time and throw a
-        // synchronous NullPointerException instead of failing through the returned future.
+        // Capture the context once so an explicit reset cannot race the null check and composition.
         CompletableFuture<CompactedTopicContext> context = compactedTopicContext;
         if (compactionHorizon == null || context == null) {
             return CompletableFuture.completedFuture(null);
@@ -359,7 +350,7 @@ public class CompactedTopicImpl implements CompactedTopic {
         return Optional.ofNullable(this.compactionHorizon);
     }
 
-    public void reset() {
+    public synchronized void reset() {
         this.compactionHorizon = null;
         this.compactedTopicContext = null;
     }
@@ -369,4 +360,3 @@ public class CompactedTopicImpl implements CompactedTopic {
         return compactedTopicContext;
     }
 }
-
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java
index 15712c502cfe..bc32e1039d86 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java
@@ -104,7 +104,16 @@ public class PulsarTopicCompactionService implements TopicCompactionService {
 
     @Override
     public CompletableFuture<Position> getLastCompactedPosition() {
-        return CompletableFuture.completedFuture(compactedTopic.getCompactionHorizon().orElse(null));
+        CompletableFuture<CompactedTopicContext> context;
+        Position horizon;
+        synchronized (compactedTopic) {
+            context = compactedTopic.getCompactedTopicContextFuture();
+            horizon = compactedTopic.getCompactionHorizon().orElse(null);
+        }
+        // A missing compacted ledger is not the same as a topic that has never been compacted.
+        // Propagate open failures even if the reader's position is already beyond the horizon:
+        // retention may have advanced it past data that now exists only in the compacted ledger.
+        return context == null ? CompletableFuture.completedFuture(null) : context.thenApply(ignored -> horizon);
     }
 
     @Override
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java
index 272437d82113..d2f8abd4b3f7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.compaction;
 import static org.apache.pulsar.broker.BrokerTestUtil.newUniqueName;
 import static org.apache.pulsar.broker.BrokerTestUtil.spyWithoutRecordingInvocations;
 import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.Mockito.anyLong;
@@ -44,7 +45,6 @@ import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -69,7 +69,6 @@ import org.apache.bookkeeper.client.BKException;
 import org.apache.bookkeeper.client.BookKeeper;
 import org.apache.bookkeeper.client.api.OpenBuilder;
 import org.apache.bookkeeper.mledger.AsyncCallbacks;
-import org.apache.bookkeeper.mledger.Entry;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.bookkeeper.mledger.ManagedLedgerInfo;
@@ -3019,8 +3018,8 @@ public class CompactionTest extends MockedPulsarServiceBaseTest {
      * snapshot, which may still reference a compacted ledger that a newer compaction has already
      * deleted (the snapshot only advances on rollover or graceful close, while the previous
      * compacted ledger is deleted as soon as the compactor subscription's mark-delete is persisted
-     * in the cursor ledger). readCompacted reads must then fall back to the original topic data
-     * instead of failing at or before the stale compaction horizon.
+     * in the cursor ledger). Keep this as an explicit failure: the original entries are not
+     * guaranteed to exist, so recovery must not silently serve an incomplete uncompacted view.
      */
     @Test
     public void testReadCompactedAfterRecoveryRestoredDeletedCompactedLedger() throws Exception {
@@ -3029,8 +3028,8 @@ public class CompactionTest extends MockedPulsarServiceBaseTest {
 
         // Roll the cursor ledger over on every persist, so the metadata-store snapshot carries the
         // compactor subscription's mark-delete position and properties. The infinite retention
-        // keeps the consumed data ledgers around: their entries are the fallback data for the
-        // readCompacted reads after the recovery below.
+        // keeps the consumed data ledgers around so that the recovered position is not clamped
+        // to a newer ledger. MissingCompactedLedgerTest also covers real retention trimming.
         PersistentTopic setupTopic = (PersistentTopic)
                 pulsar.getBrokerService().getTopic(topic, true).get().orElseThrow();
         setupTopic.getManagedLedger().getConfig()
@@ -3092,72 +3091,12 @@ public class CompactionTest extends MockedPulsarServiceBaseTest {
         assertEquals(compactionCursor.getProperties().get(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY),
                 (Long) compactedLedger1);
 
-        // The stale compacted ledger must not stay registered: no compaction horizon is served
-        // and readCompacted falls back to reading the original entries instead of failing.
-        Awaitility.await().until(() ->
-                persistentTopic.getTopicCompactionService().getLastCompactedPosition().get() == null);
-        try (Reader<byte[]> reader = pulsarClient.newReader().topic(topic).readCompacted(true)
-                .startMessageId(MessageId.earliest).create()) {
-            List<String> received = new ArrayList<>();
-            while (reader.hasMessageAvailable()) {
-                Message<byte[]> m = reader.readNext(2, TimeUnit.SECONDS);
-                received.add(m.getKey() + "=" + new String(m.getData()));
-            }
-            assertEquals(received, List.of("k1=v1a", "k2=v2a", "k1=v1b", "k2=v2b"));
-        }
-
-        // a new compaction restores the compacted view
-        try (Producer<byte[]> producer = pulsarClient.newProducer()
-                .topic(topic).enableBatching(false).create()) {
-            producer.newMessage().key("k3").value("v3".getBytes()).send();
-        }
-        compact(topic);
-        try (Reader<byte[]> reader = pulsarClient.newReader().topic(topic).readCompacted(true)
-                .startMessageId(MessageId.earliest).create()) {
-            Set<String> compacted = new HashSet<>();
-            while (reader.hasMessageAvailable()) {
-                Message<byte[]> m = reader.readNext(2, TimeUnit.SECONDS);
-                compacted.add(m.getKey() + "=" + new String(m.getData()));
-            }
-            assertEquals(compacted, Set.of("k1=v1b", "k2=v2b", "k3=v3"));
-        }
-    }
-
-    /**
-     * A read overlapping the missing-ledger reset must complete through the returned future instead
-     * of throwing a synchronous NullPointerException from a torn null check of the context field:
-     * the last-entry read passes the check while the open is still pending, and the missing-ledger
-     * callback clears the field before the composition dereferences it again.
-     */
-    @Test
-    public void testReadLastEntryOverlappingMissingCompactedLedgerReset() throws Exception {
-        CompactedTopicImpl compactedTopic = new CompactedTopicImpl(bk);
-        long missingLedgerId = 1234567890L;
-        // Hold the failed open so the last-entry read below overlaps the reset callback.
-        pulsarTestContext.getMockBookKeeper().delay(300);
-
-        CompletableFuture<CompactedTopicContext> registration =
-                compactedTopic.newCompactedLedger(PositionFactory.create(1, 1), missingLedgerId);
-
-        // The read must return a future rather than throw while the open is still pending, and it
-        // must complete through that future (exceptionally here) instead of hanging or throwing.
-        CompletableFuture<Entry> lastEntry = compactedTopic.readLastEntryOfCompactedLedger();
-        assertNotNull(lastEntry);
-        try {
-            assertNull(lastEntry.get(5, TimeUnit.SECONDS));
-        } catch (ExecutionException e) {
-            // acceptable: the read observed the failed open
-        }
-
-        // the registration failed with the missing ledger and the state was reset
-        try {
-            registration.get(5, TimeUnit.SECONDS);
-            fail("registration of a missing compacted ledger should have failed");
-        } catch (ExecutionException e) {
-            assertTrue(e.getCause() instanceof BKException);
-        }
-        assertTrue(compactedTopic.getCompactionHorizon().isEmpty());
-        // after the reset, reads answer "no compacted data" again
-        assertNull(compactedTopic.readLastEntryOfCompactedLedger().get(5, TimeUnit.SECONDS));
+        // A persisted reference to a missing compacted ledger must remain distinguishable from
+        // a topic with no compacted data, even when the original entries happen to be retained.
+        TopicCompactionService service = persistentTopic.getTopicCompactionService();
+        assertThatThrownBy(() -> service.getLastCompactedPosition().get(5, TimeUnit.SECONDS))
+                .isInstanceOf(ExecutionException.class).hasCauseInstanceOf(BKException.class);
+        assertThatThrownBy(() -> service.getLastMessagePosition().get(5, TimeUnit.SECONDS))
+                .isInstanceOf(ExecutionException.class).hasCauseInstanceOf(BKException.class);
     }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/MissingCompactedLedgerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/MissingCompactedLedgerTest.java
new file mode 100644
index 000000000000..3cb902edf0f6
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/MissingCompactedLedgerTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.compaction;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.mledger.ManagedLedger;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl;
+import org.apache.pulsar.broker.service.SharedPulsarBaseTest;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClientException.BrokerMetadataException;
+import org.apache.pulsar.client.api.Reader;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.apache.pulsar.common.policies.data.RetentionPolicies;
+import org.awaitility.Awaitility;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker", timeOut = 60000)
+public class MissingCompactedLedgerTest extends SharedPulsarBaseTest {
+
+    @DataProvider
+    public Object[][] missingLedgerCases() {
+        return new Object[][] {{false, false}, {false, true}, {true, false}};
+    }
+
+    @Test(dataProvider = "missingLedgerCases")
+    public void testMissingLedgerDoesNotBecomeAnUncompactedTopic(boolean retainSource, boolean appendTail)
+            throws Exception {
+        String topic = createTopicWithMissingLedger(retainSource, appendTail);
+        try (Reader<String> reader = reader(topic, true)) {
+            assertThatThrownBy(() -> reader.getLastMessageIdsAsync().get(10, TimeUnit.SECONDS))
+                    .as("missing compacted data must not be reported as an empty or partial topic")
+                    .isInstanceOf(ExecutionException.class).hasCauseInstanceOf(BrokerMetadataException.class);
+        }
+        assertThatThrownBy(() -> compactor().compact(topic).get(20, TimeUnit.SECONDS))
+                .as("another compaction must not publish an incomplete replacement")
+                .isInstanceOf(ExecutionException.class).hasCauseInstanceOf(BrokerMetadataException.class);
+
+        // Ordinary readers can still consume the retained source entries. Only compacted reads
+        // promise the old key that may no longer exist in those entries.
+        try (Reader<String> reader = reader(topic, false)) {
+            Set<String> keys = new HashSet<>();
+            while (reader.hasMessageAvailable()) {
+                var message = reader.readNext(10, TimeUnit.SECONDS);
+                assertThat(message).isNotNull();
+                keys.add(message.getKey());
+            }
+            if (retainSource) {
+                assertThat(keys).contains("old-key");
+            } else {
+                assertThat(keys).doesNotContain("old-key");
+            }
+            if (appendTail) {
+                assertThat(keys).contains("tail-key");
+            }
+        }
+    }
+
+    @Test
+    public void testOrdinaryReaderCanSeekWithMissingCompactedLedger() throws Exception {
+        String topic = createTopicWithMissingLedger(true, false);
+        try (Reader<String> reader = reader(topic, false)) {
+            assertThat(reader.readNext(10, TimeUnit.SECONDS)).isNotNull()
+                    .satisfies(message -> assertThat(message.getKey()).isEqualTo("old-key"));
+            reader.seekAsync(MessageId.earliest).get(10, TimeUnit.SECONDS);
+            assertThat(reader.readNext(10, TimeUnit.SECONDS)).isNotNull()
+                    .satisfies(message -> assertThat(message.getKey()).isEqualTo("old-key"));
+        }
+        // Seeking an ordinary reader must not clear the failure for compacted readers.
+        try (Reader<String> reader = reader(topic, true)) {
+            assertThatThrownBy(() -> reader.getLastMessageIdsAsync().get(10, TimeUnit.SECONDS))
+                    .isInstanceOf(ExecutionException.class).hasCauseInstanceOf(BrokerMetadataException.class);
+        }
+    }
+
+    @Test
+    public void testDeleteCompactionSubscriptionClearsMissingLedger() throws Exception {
+        String topic = createTopicWithMissingLedger(true, false);
+        PersistentTopic persistentTopic = topic(topic);
+        admin.topics().deleteSubscription(topic, Compactor.COMPACTION_SUBSCRIPTION);
+        // Unsubscribe completes before the in-memory compaction cleanup callback runs.
+        Awaitility.await().until(() -> persistentTopic.getTopicCompactionService().getLastCompactedPosition()
+                .handle((position, error) -> error == null && position == null).get(5, TimeUnit.SECONDS));
+        assertThat(persistentTopic.getSubscription(Compactor.COMPACTION_SUBSCRIPTION)).isNull();
+        assertThat(topic(topic)).isSameAs(persistentTopic);
+        try (Reader<String> reader = reader(topic, true)) {
+            assertThat(reader.readNext(10, TimeUnit.SECONDS)).isNotNull()
+                    .satisfies(message -> assertThat(message.getKey()).isEqualTo("old-key"));
+        }
+        // Explicitly discarding compaction allows a fresh compaction from the retained originals.
+        assertThat(compactor().compact(topic).get(20, TimeUnit.SECONDS)).isGreaterThanOrEqualTo(0);
+        try (Reader<String> reader = reader(topic, true)) {
+            assertThat(reader.readNext(10, TimeUnit.SECONDS)).isNotNull()
+                    .satisfies(message -> assertThat(message.getKey()).isEqualTo("old-key"));
+            assertThat(reader.readNext(10, TimeUnit.SECONDS)).isNotNull()
+                    .satisfies(message -> assertThat(message.getKey()).isEqualTo("new-key"));
+        }
+    }
+
+    @Test
+    public void testNeverCompactedTopicStillReadsOriginalEntries() throws Exception {
+        String topic = newTopicName();
+        admin.namespaces().setRetention(getNamespace(), new RetentionPolicies(-1, -1));
+        try (Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create()) {
+            producer.newMessage().key("key").value("value").send();
+        }
+        try (Reader<String> reader = reader(topic, true)) {
+            assertThat(reader.hasMessageAvailable()).isTrue();
+            assertThat(reader.readNext(10, TimeUnit.SECONDS)).isNotNull()
+                    .satisfies(message -> assertThat(message.getValue()).isEqualTo("value"));
+        }
+    }
+
+    private String createTopicWithMissingLedger(boolean retainSource, boolean appendTail) throws Exception {
+        String topic = newTopicName();
+        admin.namespaces().setRetention(getNamespace(), new RetentionPolicies(-1, -1));
+        admin.namespaces().setDeduplicationStatus(getNamespace(), false);
+        PersistentTopic persistentTopic = topic(topic);
+        ManagedLedger ledger = persistentTopic.getManagedLedger();
+        ledger.getConfig().setMaxEntriesPerLedger(1);
+        ledger.getConfig().setMinimumRolloverTime(0, TimeUnit.SECONDS);
+        ledger.getConfig().setThrottleMarkDelete(0);
+
+        MessageId oldMessage;
+        try (Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topic).enableBatching(false).create()) {
+            oldMessage = producer.newMessage().key("old-key").value("only-value").send();
+            producer.newMessage().key("new-key").value("new-value").send();
+        }
+
+        BookKeeper bookKeeper = bookKeeper();
+        Compactor compactor = compactor();
+        long compactedLedger = compactor.compact(topic).get(20, TimeUnit.SECONDS);
+        assertThat(compactedLedger).isGreaterThanOrEqualTo(0);
+
+        if (!retainSource) {
+            // Use the normal retention/trim path. The old key is now only available through compaction.
+            ledger.getConfig().setRetentionTime(0, TimeUnit.SECONDS).setRetentionSizeInMB(0);
+            CompletableFuture<Void> trimmed = new CompletableFuture<>();
+            ledger.trimConsumedLedgersInBackground(trimmed);
+            trimmed.get(10, TimeUnit.SECONDS);
+            Awaitility.await().untilAsserted(() -> assertThat(ledger.getLedgersInfo())
+                    .doesNotContainKey(((MessageIdImpl) oldMessage).getLedgerId()));
+        }
+
+        // Prove that real compaction preserved the key even when its source ledger was removed.
+        try (Reader<String> reader = reader(topic, true)) {
+            assertThat(reader.readNext(10, TimeUnit.SECONDS)).isNotNull()
+                    .satisfies(message -> assertThat(message.getKey()).isEqualTo("old-key"));
+        }
+        if (appendTail) {
+            try (Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+                    .topic(topic).enableBatching(false).create()) {
+                producer.newMessage().key("tail-key").value("tail-value").send();
+            }
+        }
+
+        // Persist and reload the real cursor. Delete through BookKeeper, without fabricating cursor
+        // metadata, replacing futures, or stubbing the recovery/read path.
+        admin.topics().unload(topic);
+        bookKeeper.deleteLedger(compactedLedger);
+        PersistentTopic recovered = topic(topic);
+        assertThat(recovered).isNotSameAs(persistentTopic);
+        assertThat(recovered.getSubscription(Compactor.COMPACTION_SUBSCRIPTION).getCursor().getProperties())
+                .containsEntry(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, compactedLedger);
+
+        return topic;
+    }
+
+    private BookKeeper bookKeeper() throws Exception {
+        return ((ManagedLedgerFactoryImpl) getPulsar().getDefaultManagedLedgerFactory())
+                .getBookKeeper().get(10, TimeUnit.SECONDS);
+    }
+
+    private Compactor compactor() throws Exception {
+        return new PublishingOrderCompactor(getConfig(), pulsarClient, bookKeeper(),
+                getPulsar().getCompactorExecutor());
+    }
+
+    private PersistentTopic topic(String topic) throws Exception {
+        return (PersistentTopic) getPulsar().getBrokerService().getTopic(topic, true)
+                .get(10, TimeUnit.SECONDS).orElseThrow();
+    }
+
+    private Reader<String> reader(String topic, boolean readCompacted) throws Exception {
+        return pulsarClient.newReader(Schema.STRING).topic(topic).readCompacted(readCompacted)
+                .startMessageId(MessageId.earliest).create();
+    }
+}


(pulsar) 01/12: [fix][broker] Ignore replicated subscription updates for the compaction subscription (#27032)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 650885b36a22734ac12ba0de318c3b42a1ef1743
Author: kaminski-dev <39...@users.noreply.github.com>
AuthorDate: Thu Oct 22 00:38:22 2026 +0200

    [fix][broker] Ignore replicated subscription updates for the compaction subscription (#27032)
    
    (cherry picked from commit 5511b5a441adfe38e5c64a1726e39a10ba42cf35)
---
 .../ReplicatedSubscriptionsController.java         |  8 ++++++
 .../ReplicatedSubscriptionsControllerTest.java     | 32 +++++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java
index bf3a6513146a..03c43da88c0d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java
@@ -198,6 +198,14 @@ public class ReplicatedSubscriptionsController implements AutoCloseable, Topic.P
     }
 
     private void receiveSubscriptionUpdated(ReplicatedSubscriptionsUpdate update) {
+        if (PersistentTopic.isCompactionSubscription(update.getSubscriptionName())) {
+            // The compactor owns this cursor: it may only move together with the compacted ledger
+            log.debug()
+                    .attr("subscriptionName", update.getSubscriptionName())
+                    .log("Ignoring replicated subscription update for the compaction subscription");
+            return;
+        }
+
         MarkersMessageIdData updatedMessageId = null;
         for (int i = 0, size = update.getClustersCount(); i < size; i++) {
             ClusterMessageId cmid = update.getClusterAt(i);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java
index 09b86418b253..320bf11177fe 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.broker.service.persistent;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
@@ -68,6 +69,7 @@ import org.apache.pulsar.common.policies.data.BacklogQuota;
 import org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.protocol.Markers;
+import org.apache.pulsar.compaction.Compactor;
 import org.apache.pulsar.utils.TestLogAppender;
 import org.mockito.Mockito;
 import org.testng.Assert;
@@ -155,9 +157,37 @@ public class ReplicatedSubscriptionsControllerTest {
         return subscription;
     }
 
+    @Test
+    public void testReplicatedSubscriptionUpdateForCompactionSubscriptionIsIgnored() {
+        PersistentTopic topic = mockTopicForUpdates();
+        PulsarCompactorSubscription compactorSubscription = mock(PulsarCompactorSubscription.class);
+        // PulsarCompactorSubscription rejects acks without the compacted ledger property synchronously
+        when(compactorSubscription.acknowledgeMessageAsync(any(), any(), any()))
+                .thenThrow(new IllegalArgumentException());
+        when(topic.getSubscription(Compactor.COMPACTION_SUBSCRIPTION)).thenReturn(compactorSubscription);
+
+        receiveUpdate(topic, Compactor.COMPACTION_SUBSCRIPTION, 1, 2);
+
+        verify(compactorSubscription, never()).acknowledgeMessageAsync(any(), any(), any());
+    }
+
+    @Test
+    public void testReplicatedSubscriptionUpdateDoesNotCreateCompactionSubscription() {
+        PersistentTopic topic = mockTopicForUpdates();
+        when(topic.getSubscription(Compactor.COMPACTION_SUBSCRIPTION)).thenReturn(null);
+
+        receiveUpdate(topic, Compactor.COMPACTION_SUBSCRIPTION, 1, 2);
+
+        verify(topic, never()).createSubscription(any(), any(), anyBoolean(), any());
+    }
+
     private static void receiveUpdate(PersistentTopic topic, long ledgerId, long entryId) {
+        receiveUpdate(topic, "sub", ledgerId, entryId);
+    }
+
+    private static void receiveUpdate(PersistentTopic topic, String subscriptionName, long ledgerId, long entryId) {
         ReplicatedSubscriptionsController controller = new ReplicatedSubscriptionsController(topic, "local");
-        ByteBuf marker = Markers.newReplicatedSubscriptionsUpdate("sub",
+        ByteBuf marker = Markers.newReplicatedSubscriptionsUpdate(subscriptionName,
                 Map.of("local", new MarkersMessageIdData().setLedgerId(ledgerId).setEntryId(entryId)));
         try {
             Commands.skipMessageMetadata(marker);


(pulsar) 06/12: [improve][broker] improve readability for the class TopicPolicyListenerWrapper (#26277)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit d0d92107ee8feb862697d76f72110ca0109873f2
Author: fengyubiao <yu...@streamnative.io>
AuthorDate: Thu Oct 22 09:49:13 2026 +0800

    [improve][broker] improve readability for the class TopicPolicyListenerWrapper (#26277)
    
    Co-authored-by: Matteo Merli <mm...@apache.org>
    (cherry picked from commit 119e1b93133d21ee52f5d3a2eda5e8b142c3cb89)
---
 .../pulsar/broker/service/AbstractTopic.java       |  12 +-
 .../broker/service/TopicPolicyListenerWrapper.java | 180 ++++++++---------
 .../service/TopicPolicyListenerWrapperTest.java    | 216 ++++++++++++---------
 3 files changed, 216 insertions(+), 192 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
index a39abf155b5c..a029144fbb9a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
@@ -123,7 +123,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener {
 
     // Wraps this topic as a TopicPolicyListener so topic-policy updates received while the initial policy is still
     // loading are buffered and applied in order once initTopicPolicy() completes initialization.
-    protected final TopicPolicyListenerWrapper topicPolicyListener = new TopicPolicyListenerWrapper(this);
+    protected final TopicPolicyListenerWrapper topicPolicyListener;
 
     // Prefix for replication cursors
     protected final String replicatorPrefix;
@@ -211,6 +211,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener {
 
     public AbstractTopic(String topic, BrokerService brokerService) {
         this.topic = topic;
+        this.topicPolicyListener = new TopicPolicyListenerWrapper(this, topic);
         this.log = LOG.with().attr("topic", topic).build();
         this.namespace = TopicName.get(topic).getNamespaceObject();
         // Pin the per-topic policies-notify thread once. BrokerService#getTopicPoliciesNotifyThread centralizes
@@ -618,17 +619,14 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener {
      * topic load, which removes the need to broadcast every topic's policy when a namespace's policy cache finishes
      * loading (see {@code topicPolicyListenerReplayEnabled}).
      *
-     * <p>Each call re-initializes the listener wrapper and, whatever the outcome, always completes its initialization
-     * afterwards, so the wrapper never stays in the buffering phase (dropping updates) even if policy loading fails.
-     * This makes the method safe to run again (e.g. a future retry); runs are expected to be serialized.
+     * <p>This method is invoked once during topic initialization; the listener wrapper is one-shot for a topic
+     * instance. Whatever the outcome, the wrapper's initialization is always completed afterwards, so it never stays
+     * in the buffering phase (dropping updates) even if policy loading fails.
      */
     protected CompletableFuture<Void> initTopicPolicy() {
         final var topicPoliciesService = brokerService.getPulsar().getTopicPoliciesService();
         final var partitionedTopicName = TopicName.getPartitionedTopicName(topic);
 
-        // Begin a fresh initialization phase: updates are buffered until initialization completes below. This resets
-        // any previous phase so the method can be run again.
-        topicPolicyListener.startInitialization();
         CompletableFuture<Void> initTopicPolicyFuture =
                 topicPoliciesService.registerListenerAsync(partitionedTopicName, topicPolicyListener)
                         .thenCompose(registered -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java
index da22b44a632c..21fe62120120 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java
@@ -19,59 +19,64 @@
 
 package org.apache.pulsar.broker.service;
 
-import java.util.Optional;
+import io.github.merlimat.slog.Logger;
 import java.util.concurrent.TimeUnit;
-import lombok.CustomLog;
 import org.apache.pulsar.common.policies.data.TopicPolicies;
+import org.jspecify.annotations.Nullable;
 
 /**
- * This TopicPolicyListener is used as a wrapper for the real TopicPolicyListener.
- * This prevents a race condition in initialization where the topic policy state can change while the topic policy
- * state is being applied to the topic in AbstractTopic#initTopicPolicy(). The impact of the race condition is that the
- * topic policy state would be left inconsistent until another update arrives. This is a rare corner case, but possible.
+ * Wraps the real {@link TopicPolicyListener} of a topic while its initial topic policies are being loaded.
  *
- * <p>Updates received while initializing are buffered (only the latest per scope is kept) and applied by
- * {@link #completeInitialization}; updates received afterwards are forwarded immediately. The wrapper is reusable so
- * that AbstractTopic#initTopicPolicy() can be run again -- for example to retry it, which this enables but does not
- * implement. {@link #startInitialization()} begins a new buffering phase and initialization completes at most once per
- * phase. Concurrent initialization phases are not supported; {@code initTopicPolicy} runs are serialized by the caller.
+ * <ul>
+ *   <li>Loaded policy: the policy read by {@code AbstractTopic#initTopicPolicy()} when the topic is loaded.</li>
+ *   <li>Live policy: a policy update notified by {@link TopicPoliciesService} after the listener is registered.</li>
+ * </ul>
+ *
+ * <p>A live policy has higher priority than the loaded policy of the same scope (local or global), which prevents
+ * the following race:
+ * <ol>
+ *   <li>init thread: creates the topic and registers this listener with {@link TopicPoliciesService}.</li>
+ *   <li>notification thread: receives a live policy.</li>
+ *   <li>init thread: applies the loaded (older) policy, overwriting the live one.</li>
+ * </ol>
+ *
+ * <p>Until initialization completes, live updates are buffered (only the latest per scope is kept) instead of being
+ * forwarded. Initialization completes, at most once, on whichever comes first:
+ * <ul>
+ *   <li>{@link #completeInitialization}, with the loaded policies;</li>
+ *   <li>live updates for both scopes having been received, so the loaded policies are no longer needed;</li>
+ *   <li>a live delete ({@code onUpdate(null)}), which supersedes both scopes;</li>
+ *   <li>{@link #completeInitializationUnlessAlreadyCompleted}, the safety net run after every
+ *       {@code initTopicPolicy()} outcome, so the wrapper never keeps buffering (and dropping) updates.</li>
+ * </ul>
+ * Afterwards every update is forwarded immediately. The wrapper is one-shot for a topic instance.
  */
-@CustomLog
 public class TopicPolicyListenerWrapper implements TopicPolicyListener {
+
+    private static final Logger LOG = Logger.get(TopicPolicyListenerWrapper.class);
+    private static final long INITIALIZATION_WARNING_LOG_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30);
+    private final Logger log;
+
     private final TopicPolicyListener realTopicListener;
-    // The latest value received during initialization, per scope. A null reference means no update was
-    // received during initialization (the loaded value should be used); an Optional that is present holds the
-    // received policies, and an empty Optional records that a delete (onUpdate(null)) was received, so the
-    // loaded value must not be applied. Optional is used because the map-like field cannot itself hold null
-    // while still distinguishing "not received" (null) from "received a delete" (Optional.empty()).
-    private Optional<TopicPolicies> latestGlobalPolicies;
-    private Optional<TopicPolicies> latestLocalPolicies;
+    // The latest live value received during initialization, per scope. A null reference means no update was
+    // received during initialization for that scope (the loaded value should be used).
+    private TopicPolicies latestGlobalPolicies;
+    private TopicPolicies latestLocalPolicies;
+    // Timestamp when the wrapper was created, i.e. when the topic object was constructed. Used only to warn if
+    // initialization takes too long (i.e. completeInitialization was never called).
+    private final long createdNanos;
     private boolean initialized;
-    // Timestamp when the current initialization phase started, set by startInitialization(). Used only to warn if the
-    // phase takes too long (i.e. completeInitialization was never called after policy loading started).
-    private long initializationStartedNanos;
-    private static final long INITIALIZATION_WARNING_LOG_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30);
     private int lastIntervalLogged;
 
-    public TopicPolicyListenerWrapper(TopicPolicyListener realTopicListener) {
+    public TopicPolicyListenerWrapper(TopicPolicyListener realTopicListener, String topic) {
         this.realTopicListener = realTopicListener;
-        startInitialization();
+        this.log = LOG.with().attr("topic", topic).build();
+        this.createdNanos = System.nanoTime();
     }
 
     /**
-     * Starts (or restarts) the initialization phase: {@link #onUpdate} buffers updates (keeping only the latest per
-     * scope) instead of forwarding them, until {@link #completeInitialization} applies them. Called at the start of
-     * every {@code initTopicPolicy} run so the method can be re-run cleanly (which is what would let a future change
-     * retry it; no retry is implemented here). Runs before the listener is registered, so no update can arrive before
-     * the phase (and its warning timer) has started.
+     * Handles live updates. A live update supersedes the loaded policy of its scope.
      */
-    public synchronized void startInitialization() {
-        initialized = false;
-        latestGlobalPolicies = null;
-        latestLocalPolicies = null;
-        initializationStartedNanos = System.nanoTime();
-    }
-
     @Override
     public synchronized void onUpdate(TopicPolicies data) {
         if (initialized) {
@@ -81,84 +86,83 @@ public class TopicPolicyListenerWrapper implements TopicPolicyListener {
 
         maybeLogWarning();
 
-        // Record the latest value received during initialization so it can be applied (preferring it over the
-        // loaded value) in completeInitialization. A received value is stored as Optional.of(data) and a delete
-        // as Optional.empty(), so the delete is propagated downstream instead of being lost.
         if (data == null) {
-            // A delete (onUpdate(null)) does not carry the global/local scope through the listener interface,
-            // so record it for both scopes; a later scoped update received during initialization still
-            // overrides its own scope.
-            latestGlobalPolicies = Optional.empty();
-            latestLocalPolicies = Optional.empty();
-        } else if (data.isGlobalPolicies()) {
-            latestGlobalPolicies = Optional.of(data);
+            // A delete does not carry the scope through the listener interface and removes the topic policies of
+            // both scopes, so it supersedes every buffered and loaded value: forward it and finish initialization.
+            initialized = true;
+            latestLocalPolicies = null;
+            latestGlobalPolicies = null;
+            realTopicListener.onUpdate(null);
+            return;
+        }
+        if (data.isGlobalPolicies()) {
+            latestGlobalPolicies = data;
         } else {
-            latestLocalPolicies = Optional.of(data);
+            latestLocalPolicies = data;
+        }
+        // With the latest value of both scopes at hand, the loaded policies are not needed anymore.
+        if (latestGlobalPolicies != null && latestLocalPolicies != null) {
+            doInitPolicies(latestLocalPolicies, latestGlobalPolicies);
         }
     }
 
     /**
-     * Complete initialization of the TopicPolicyListenerWrapper and emit the latest policies to the real listener.
+     * Completes initialization with the loaded policies, unless it has already completed. For each scope, the latest
+     * live value received during initialization is preferred over the loaded one.
      *
-     * @param loadedGlobalPolicies the loaded global policies
-     * @param loadedLocalPolicies  the loaded local policies
+     * @param globalLoaded the loaded global policies, or null if there are none
+     * @param localLoaded  the loaded local policies, or null if there are none
      */
-    public synchronized void completeInitialization(TopicPolicies loadedGlobalPolicies,
-                                                    TopicPolicies loadedLocalPolicies) {
-        // Idempotent: an initialization phase completes at most once. initTopicPolicy runs a terminal
-        // completeInitializationUnlessAlreadyCompleted() after applying the loaded policies, so a later call must be a
-        // no-op and must not re-emit policies. A new phase is started explicitly via startInitialization().
+    public synchronized void completeInitialization(@Nullable TopicPolicies globalLoaded,
+                                                    @Nullable TopicPolicies localLoaded) {
         if (initialized) {
             return;
         }
-
-        // The listener might have received a newer value (or a delete) than the loaded one while the loading
-        // was happening; prefer the latest value received during initialization, falling back to the loaded
-        // value only when nothing was received for that scope.
-        //
-        // Emit the local policy before the global policy. A local topic policy takes precedence over a global one,
-        // so applying the local value first means that by the time the global value is applied the local override is
-        // already in place and the merged (local-wins) result is what takes effect. Emitting the global value first
-        // would briefly apply it on its own and let a global-only setting act before the local policy overrides it --
-        // e.g. a compaction subscription being created for a global compaction policy even though the local policy
-        // disables compaction. This does not fully solve such ordering hazards, but it removes them whenever a local
-        // policy exists. When no local policy exists nothing is emitted for the local scope (see emitInitialPolicies),
-        // so this ordering does not change behavior for topics that only have a global policy.
-        emitInitialPolicies(latestLocalPolicies, loadedLocalPolicies);
-        emitInitialPolicies(latestGlobalPolicies, loadedGlobalPolicies);
-
-        latestGlobalPolicies = null;
-        latestLocalPolicies = null;
-        initialized = true;
+        TopicPolicies local = latestLocalPolicies != null ? latestLocalPolicies : localLoaded;
+        TopicPolicies global = latestGlobalPolicies != null ? latestGlobalPolicies : globalLoaded;
+        doInitPolicies(local, global);
     }
 
     /**
-     * Completes initialization with no loaded policies, unless it has already completed. Used as a safety net at the
-     * end of {@code initTopicPolicy} so the wrapper always leaves the buffering phase -- even when the listener was not
-     * registered or policy loading failed -- and therefore stops dropping updates: it emits any buffered value and
-     * forwards all future live updates. A no-op once initialization has completed (e.g. with loaded policies).
+     * Completes initialization with no loaded policies, unless it has already completed. Run at the end of
+     * {@code AbstractTopic#initTopicPolicy()} whatever its outcome -- the listener not being registered, policy
+     * loading failing or being skipped -- so the wrapper always leaves the buffering phase: it emits any buffered
+     * value and forwards all future live updates.
      */
     public synchronized void completeInitializationUnlessAlreadyCompleted() {
         completeInitialization(null, null);
     }
 
-    private void emitInitialPolicies(Optional<TopicPolicies> latestReceived, TopicPolicies loaded) {
-        if (latestReceived != null) {
-            // A value (or a delete) was received during initialization; it supersedes the loaded value.
-            realTopicListener.onUpdate(latestReceived.orElse(null));
-        } else if (loaded != null) {
-            realTopicListener.onUpdate(loaded);
+    private void doInitPolicies(@Nullable TopicPolicies local, @Nullable TopicPolicies global) {
+        initialized = true;
+        latestLocalPolicies = null;
+        latestGlobalPolicies = null;
+        // Emit the local policy before the global policy. A local topic policy takes precedence over a global one,
+        // so applying the local value first means that by the time the global value is applied the local override is
+        // already in place and the merged (local-wins) result is what takes effect. Emitting the global value first
+        // would briefly apply it on its own and let a global-only setting act before the local policy overrides it
+        // -- e.g. a compaction subscription being created for a global compaction policy even though the local
+        // policy disables compaction.
+        //
+        // Nothing is emitted for a scope without a value: null means "policies deleted" to the listener.
+        try {
+            if (local != null) {
+                realTopicListener.onUpdate(local);
+            }
+        } finally {
+            if (global != null) {
+                realTopicListener.onUpdate(global);
+            }
         }
     }
 
     // warn if the initialization takes too long and updates have been received
     // this helps detect issues where completeInitialization didn't get called after loading policies
     private void maybeLogWarning() {
-        long durationNanos = System.nanoTime() - initializationStartedNanos;
+        long durationNanos = System.nanoTime() - createdNanos;
         int warningLogIntervalCount = (int) (durationNanos / INITIALIZATION_WARNING_LOG_INTERVAL_NANOS);
         if (warningLogIntervalCount > lastIntervalLogged) {
-            log.warn().attr("topicPolicyListener", realTopicListener)
-                    .attr("sinceInitializationStartedMs", TimeUnit.NANOSECONDS.toMillis(durationNanos))
+            log.warn().attr("sinceCreatedMs", TimeUnit.NANOSECONDS.toMillis(durationNanos))
                     .log("TopicPolicyUpdate buffered. TopicPolicyListenerWrapper initialization phase took too long. "
                             + "completeInitialization should have been called to complete the phase.");
             lastIntervalLogged = warningLogIntervalCount;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java
index 9d32342aab5d..44d766dcda77 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java
@@ -19,7 +19,7 @@
 package org.apache.pulsar.broker.service;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.pulsar.common.policies.data.TopicPolicies;
@@ -28,6 +28,8 @@ import org.testng.annotations.Test;
 @Test(groups = "broker")
 public class TopicPolicyListenerWrapperTest {
 
+    private static final String TOPIC = "public/default/test-topic";
+
     private static TopicPolicies globalPolicies() {
         return TopicPolicies.builder().isGlobal(true).build();
     }
@@ -46,167 +48,187 @@ public class TopicPolicyListenerWrapperTest {
     }
 
     @Test
-    public void shouldBufferUpdatesUntilInitializedThenForwardLive() {
+    public void shouldApplyLoadedLocalBeforeGlobalWhenNoLiveUpdates() {
         RecordingListener real = new RecordingListener();
-        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real);
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
 
-        // Updates received before initialization are buffered, not forwarded.
-        TopicPolicies bufferedLocal = localPolicies();
-        wrapper.onUpdate(bufferedLocal);
-        assertThat(real.updates).isEmpty();
+        TopicPolicies loadedGlobal = globalPolicies();
+        TopicPolicies loadedLocal = localPolicies();
+        wrapper.completeInitialization(loadedGlobal, loadedLocal);
+        assertThat(real.updates).containsExactly(loadedLocal, loadedGlobal);
+    }
+
+    @Test
+    public void shouldSkipLoadedLocalWhenLocalAlreadyUpdated() {
+        RecordingListener real = new RecordingListener();
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
+
+        TopicPolicies liveLocal = localPolicies();
+        wrapper.onUpdate(liveLocal);
 
-        // On completion, the buffered local value wins over the loaded local value; the loaded global value
-        // is applied since none was buffered. The local policy is emitted before the global one.
         TopicPolicies loadedGlobal = globalPolicies();
         wrapper.completeInitialization(loadedGlobal, localPolicies());
-        assertThat(real.updates).containsExactly(bufferedLocal, loadedGlobal);
-
-        // After initialization, updates are forwarded immediately.
-        TopicPolicies liveUpdate = localPolicies();
-        wrapper.onUpdate(liveUpdate);
-        assertThat(real.updates).containsExactly(bufferedLocal, loadedGlobal, liveUpdate);
+        assertThat(real.updates).containsExactly(liveLocal, loadedGlobal);
     }
 
     @Test
-    public void shouldPreferBufferedOverLoadedForBothScopes() {
+    public void shouldSkipLoadedGlobalWhenGlobalAlreadyUpdated() {
         RecordingListener real = new RecordingListener();
-        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real);
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
 
-        TopicPolicies bufferedGlobal = globalPolicies();
-        TopicPolicies bufferedLocal = localPolicies();
-        wrapper.onUpdate(bufferedGlobal);
-        wrapper.onUpdate(bufferedLocal);
+        TopicPolicies liveGlobal = globalPolicies();
+        wrapper.onUpdate(liveGlobal);
 
-        wrapper.completeInitialization(globalPolicies(), localPolicies());
-        assertThat(real.updates).containsExactly(bufferedLocal, bufferedGlobal);
+        TopicPolicies loadedLocal = localPolicies();
+        wrapper.completeInitialization(globalPolicies(), loadedLocal);
+        assertThat(real.updates).containsExactly(loadedLocal, liveGlobal);
     }
 
     @Test
-    public void shouldApplyLoadedWhenNothingBuffered() {
+    public void shouldSkipBothLoadedWhenBothAlreadyUpdated() {
         RecordingListener real = new RecordingListener();
-        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real);
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
 
-        // The local policy is emitted before the global policy, so a local topic policy takes precedence over a
-        // global one once both have been applied.
-        TopicPolicies loadedGlobal = globalPolicies();
-        TopicPolicies loadedLocal = localPolicies();
-        wrapper.completeInitialization(loadedGlobal, loadedLocal);
-        assertThat(real.updates).containsExactly(loadedLocal, loadedGlobal);
+        TopicPolicies liveGlobal = globalPolicies();
+        TopicPolicies liveLocal = localPolicies();
+        wrapper.onUpdate(liveGlobal);
+        wrapper.onUpdate(liveLocal);
+        assertThat(real.updates).containsExactly(liveLocal, liveGlobal);
+
+        wrapper.completeInitialization(globalPolicies(), localPolicies());
+        assertThat(real.updates).containsExactly(liveLocal, liveGlobal);
     }
 
+
     @Test
-    public void shouldSuppressLoadedValuesWhenDeletedBeforeInitialization() {
+    public void shouldForwardDeleteAndSkipLoadedPoliciesWhenDeletedBeforeInitialization() {
         RecordingListener real = new RecordingListener();
-        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real);
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
 
-        // A delete (null) arriving before initialization must not NPE and must not be forwarded yet (#26037).
-        assertThatCode(() -> wrapper.onUpdate(null)).doesNotThrowAnyException();
-        assertThat(real.updates).isEmpty();
+        // A delete supersedes both scopes, so it is forwarded right away and completes initialization.
+        wrapper.onUpdate(localPolicies());
+        wrapper.onUpdate(null);
+        assertThat(real.updates).containsExactly((TopicPolicies) null);
 
-        // The delete supersedes the (now-stale) loaded values: they are not applied, and the delete (null) is
-        // propagated downstream instead.
         wrapper.completeInitialization(globalPolicies(), localPolicies());
-        assertThat(real.updates).containsExactly(null, null);
+        assertThat(real.updates).containsExactly((TopicPolicies) null);
     }
 
     @Test
-    public void shouldApplyLatestScopedUpdateOverEarlierDeleteDuringInitialization() {
+    public void shouldForwardPerScopeUpdatesAfterDeleteBeforeInitialization() {
         RecordingListener real = new RecordingListener();
-        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real);
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
 
-        // A delete records empty for both scopes, then a newer global update overrides only the global scope.
         wrapper.onUpdate(null);
         TopicPolicies newerGlobal = globalPolicies();
         wrapper.onUpdate(newerGlobal);
 
         wrapper.completeInitialization(globalPolicies(), localPolicies());
-        // Local (emitted first): the delete (null) wins over the loaded local value; Global: the newer update wins.
         assertThat(real.updates).containsExactly(null, newerGlobal);
     }
 
     @Test
-    public void shouldNotEmitLocalScopeWhenNoLocalPolicyExists() {
+    public void shouldForwardLiveUpdatesAfterInitialization() {
+        RecordingListener real = new RecordingListener();
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
+
+        wrapper.completeInitialization(globalPolicies(), localPolicies());
+        real.updates.clear();
+
+        TopicPolicies liveLocal = localPolicies();
+        wrapper.onUpdate(liveLocal);
+        assertThat(real.updates).containsExactly(liveLocal);
+
+        TopicPolicies liveGlobal = globalPolicies();
+        wrapper.onUpdate(liveGlobal);
+        assertThat(real.updates).containsExactly(liveLocal, liveGlobal);
+    }
+
+    @Test
+    public void shouldEmitNothingForAScopeWithoutAValue() {
         RecordingListener real = new RecordingListener();
-        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real);
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
 
-        // With no local policy, only the global policy is emitted; no local onUpdate happens, so the
-        // local-before-global ordering leaves behavior unchanged for topics that only have a global policy.
+        // null means "policies deleted" to the listener, so an absent scope must not be emitted as null.
         TopicPolicies loadedGlobal = globalPolicies();
         wrapper.completeInitialization(loadedGlobal, null);
         assertThat(real.updates).containsExactly(loadedGlobal);
+
+        RecordingListener real2 = new RecordingListener();
+        TopicPolicyListenerWrapper wrapper2 = new TopicPolicyListenerWrapper(real2, TOPIC);
+        TopicPolicies loadedLocal = localPolicies();
+        wrapper2.completeInitialization(null, loadedLocal);
+        assertThat(real2.updates).containsExactly(loadedLocal);
+
+        RecordingListener real3 = new RecordingListener();
+        TopicPolicyListenerWrapper wrapper3 = new TopicPolicyListenerWrapper(real3, TOPIC);
+        wrapper3.completeInitialization(null, null);
+        assertThat(real3.updates).isEmpty();
+    }
+
+    @Test
+    public void shouldSkipLoadedLocalEvenWhenNullLoadedGlobalAfterLiveUpdate() {
+        RecordingListener real = new RecordingListener();
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
+
+        TopicPolicies liveLocal = localPolicies();
+        wrapper.onUpdate(liveLocal);
+
+        wrapper.completeInitialization(null, localPolicies());
+        assertThat(real.updates).containsExactly(liveLocal);
     }
 
     @Test
-    public void shouldIgnoreCompleteInitializationAfterAlreadyCompleted() {
+    public void shouldBeNoopWhenInitializationCalledAgain() {
         RecordingListener real = new RecordingListener();
-        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real);
-        wrapper.startInitialization();
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
 
         TopicPolicies loadedLocal = localPolicies();
         wrapper.completeInitialization(null, loadedLocal);
         assertThat(real.updates).containsExactly(loadedLocal);
 
-        // Completing again (e.g. from initTopicPolicy's terminal handler) must be a no-op and must not re-emit.
         wrapper.completeInitialization(globalPolicies(), localPolicies());
         wrapper.completeInitializationUnlessAlreadyCompleted();
         assertThat(real.updates).containsExactly(loadedLocal);
     }
 
     @Test
-    public void shouldEmitBufferedValueAndForwardLiveUpdatesWhenCompletedWithoutLoadedPolicies() {
+    public void shouldForwardBufferedAndLiveUpdatesWhenLoadingNeverCompletes() {
         RecordingListener real = new RecordingListener();
-        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real);
-        wrapper.startInitialization();
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real, TOPIC);
 
-        // A policy update arrives while initializing and is buffered.
-        TopicPolicies buffered = localPolicies();
-        wrapper.onUpdate(buffered);
+        // Policy loading failed, so completeInitialization is never called: only the safety net runs.
+        TopicPolicies bufferedLocal = localPolicies();
+        wrapper.onUpdate(bufferedLocal);
         assertThat(real.updates).isEmpty();
 
-        // initTopicPolicy's terminal handler completes initialization with no loaded policies -- the path taken after
-        // a policy-load error or when the listener was not registered. The buffered value is emitted and the wrapper
-        // leaves the buffering phase.
-        wrapper.completeInitializationUnlessAlreadyCompleted();
-        assertThat(real.updates).containsExactly(buffered);
-
-        // Subsequent live updates now flow through instead of being dropped.
-        TopicPolicies live = globalPolicies();
-        wrapper.onUpdate(live);
-        assertThat(real.updates).containsExactly(buffered, live);
-    }
-
-    @Test
-    public void shouldForwardLiveUpdatesAfterCompletingWithNothingBuffered() {
-        RecordingListener real = new RecordingListener();
-        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real);
-        wrapper.startInitialization();
-
-        // Completed with nothing buffered and no loaded policies (e.g. after a failed load): nothing is emitted, but
-        // the wrapper still leaves the buffering phase so later live updates are forwarded rather than dropped.
         wrapper.completeInitializationUnlessAlreadyCompleted();
-        assertThat(real.updates).isEmpty();
+        assertThat(real.updates).containsExactly(bufferedLocal);
 
-        TopicPolicies live = localPolicies();
-        wrapper.onUpdate(live);
-        assertThat(real.updates).containsExactly(live);
+        TopicPolicies liveLocal = localPolicies();
+        wrapper.onUpdate(liveLocal);
+        assertThat(real.updates).containsExactly(bufferedLocal, liveLocal);
     }
 
     @Test
-    public void shouldRebufferAndReapplyAfterStartInitializationIsCalledAgain() {
-        RecordingListener real = new RecordingListener();
-        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real);
-        wrapper.startInitialization();
-        TopicPolicies firstLocal = localPolicies();
-        wrapper.completeInitialization(null, firstLocal);
-        assertThat(real.updates).containsExactly(firstLocal);
-
-        // A new initialization phase (e.g. re-running initTopicPolicy): updates are buffered again until it completes,
-        // and a value buffered during the phase is applied on completion.
-        wrapper.startInitialization();
-        TopicPolicies bufferedGlobal = globalPolicies();
-        wrapper.onUpdate(bufferedGlobal);
-        assertThat(real.updates).containsExactly(firstLocal);
-        wrapper.completeInitialization(null, null);
-        assertThat(real.updates).containsExactly(firstLocal, bufferedGlobal);
+    public void shouldEmitGlobalEvenWhenTheListenerThrowsOnLocal() {
+        TopicPolicies loadedLocal = localPolicies();
+        TopicPolicies loadedGlobal = globalPolicies();
+        List<TopicPolicies> updates = new ArrayList<>();
+        TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(data -> {
+            updates.add(data);
+            if (data == loadedLocal) {
+                throw new IllegalStateException("listener failure");
+            }
+        }, TOPIC);
+
+        assertThatThrownBy(() -> wrapper.completeInitialization(loadedGlobal, loadedLocal))
+                .isInstanceOf(IllegalStateException.class);
+        assertThat(updates).containsExactly(loadedLocal, loadedGlobal);
+
+        // Initialization still completed, so later updates are forwarded.
+        TopicPolicies liveGlobal = globalPolicies();
+        wrapper.onUpdate(liveGlobal);
+        assertThat(updates).containsExactly(loadedLocal, loadedGlobal, liveGlobal);
     }
 }


(pulsar) 09/12: [fix][managed-ledger] Honor batch-index cap in isCursorDataFullyPersistable (#26553)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-5.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 8fe1446de7239a44a073b0508b4e61f7a130ac3e
Author: dev_Hakaze <af...@gmail.com>
AuthorDate: Thu Oct 22 08:50:22 2026 +0700

    [fix][managed-ledger] Honor batch-index cap in isCursorDataFullyPersistable (#26553)
    
    Signed-off-by: arimu1 <19...@users.noreply.github.com>
    Co-authored-by: arimu1 <19...@users.noreply.github.com>
    Co-authored-by: Matteo Merli <mm...@apache.org>
    (cherry picked from commit 9761be7e2b879804c685fc780e7706f6fc6da8b7)
---
 .../bookkeeper/mledger/impl/ManagedCursorImpl.java |  6 +-
 .../impl/ManagedCursorBatchAckRecoveryTest.java    | 34 +++++++++
 .../api/SubscriptionPauseOnAckStatPersistTest.java | 80 ++++++++++++++++++++++
 3 files changed, 119 insertions(+), 1 deletion(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 6cc16f69ab25..110569e259af 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -432,7 +432,11 @@ public class ManagedCursorImpl implements ManagedCursor {
     public boolean isCursorDataFullyPersistable() {
         lock.readLock().lock();
         try {
-            return individualDeletedMessages.size() <= getConfig().getMaxUnackedRangesToPersist();
+            if (individualDeletedMessages.size() > getConfig().getMaxUnackedRangesToPersist()) {
+                return false;
+            }
+            return batchDeletedIndexes == null
+                    || batchDeletedIndexes.size() <= getConfig().getMaxBatchDeletedIndexToPersist();
         } finally {
             lock.readLock().unlock();
         }
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckRecoveryTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckRecoveryTest.java
index c2f7631ca886..03ea76cee6d7 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckRecoveryTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckRecoveryTest.java
@@ -60,6 +60,40 @@ public class ManagedCursorBatchAckRecoveryTest extends MockedBookKeeperTestCase
         return new Object[][] {{false}, {true}};
     }
 
+    @Test
+    public void testIsCursorDataFullyPersistableReflectsBatchDeletedIndexLimit() throws Exception {
+        ManagedLedgerConfig config = defaultConfig();
+        config.setDeletionAtBatchIndexLevelEnabled(true);
+        config.setMaxBatchDeletedIndexToPersist(2);
+        config.setThrottleMarkDelete(0);
+        String name = "tenant/ns/persistent/cursor-data-fully-persistable-batch-index";
+        @Cleanup
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(name, config);
+        ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("sub");
+        List<Position> positions = new ArrayList<>();
+        for (int i = 0; i <= 3; i++) {
+            positions.add(ledger.addEntry(new byte[] {(byte) i}));
+        }
+        cursor.delete(positions.get(0));
+        Awaitility.await().until(() -> cursor.getStats().getPersistLedgerSucceed() > 0);
+
+        long[][] ackSets = {{2L}, {Long.MIN_VALUE, 1L}, {5L, 0L, 3L}};
+        for (int i = 1; i < positions.size(); i++) {
+            Position position = positions.get(i);
+            cursor.delete(AckSetStateUtil.createPositionWithAckSet(
+                    position.getLedgerId(), position.getEntryId(), ackSets[i - 1]));
+            if (i < 3) {
+                assertThat(cursor.isCursorDataFullyPersistable())
+                        .as("batch deleted index count %s is within limit 2", i)
+                        .isTrue();
+            } else {
+                assertThat(cursor.isCursorDataFullyPersistable())
+                        .as("third batch deleted index record exceeds limit 2")
+                        .isFalse();
+            }
+        }
+    }
+
     @Test(dataProvider = "booleans")
     public void testCumulativeAckPreservesIndividualBatchAcks(boolean metadataStore) throws Exception {
         ManagedLedgerConfig config = defaultConfig()
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionPauseOnAckStatPersistTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionPauseOnAckStatPersistTest.java
index a47e5c7209d2..f90b68455c4a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionPauseOnAckStatPersistTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionPauseOnAckStatPersistTest.java
@@ -21,11 +21,16 @@ package org.apache.pulsar.client.api;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import lombok.CustomLog;
 import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.apache.bookkeeper.mledger.impl.AckSetStateUtil;
 import org.apache.bookkeeper.mledger.impl.ActiveManagedCursorContainer;
 import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer;
 import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
@@ -36,6 +41,7 @@ import org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService;
 import org.apache.pulsar.broker.service.persistent.AbstractPersistentDispatcherMultipleConsumers;
 import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers;
 import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer;
+import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.client.admin.GetStatsOptions;
 import org.apache.pulsar.client.impl.MessageIdImpl;
@@ -56,6 +62,7 @@ import org.testng.annotations.Test;
 public class SubscriptionPauseOnAckStatPersistTest extends ProducerConsumerBase {
 
     private static final int MAX_UNACKED_RANGES_TO_PERSIST = 50;
+    private static final int MAX_BATCH_DELETED_INDEX_TO_PERSIST = 2;
 
     @BeforeClass(alwaysRun = true)
     @Override
@@ -72,6 +79,8 @@ public class SubscriptionPauseOnAckStatPersistTest extends ProducerConsumerBase
 
     protected void doInitConf() throws Exception {
         conf.setManagedLedgerMaxUnackedRangesToPersist(MAX_UNACKED_RANGES_TO_PERSIST);
+        conf.setManagedLedgerMaxBatchDeletedIndexToPersist(MAX_BATCH_DELETED_INDEX_TO_PERSIST);
+        conf.setAcknowledgmentAtBatchIndexLevelEnabled(true);
     }
 
     private void enablePolicyDispatcherPauseOnAckStatePersistent(String tpName) {
@@ -276,6 +285,77 @@ public class SubscriptionPauseOnAckStatPersistTest extends ProducerConsumerBase
         });
     }
 
+    @Test
+    public void testPauseOnBatchDeletedIndexLimitExceeded() throws Exception {
+        final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp");
+        final String subscription = "s1";
+
+        enablePolicyDispatcherPauseOnAckStatePersistent(tpName);
+        admin.topics().createNonPartitionedTopic(tpName);
+        admin.topics().createSubscription(tpName, subscription, MessageId.earliest);
+
+        List<MessageIdImpl> messageIds = new ArrayList<>();
+        Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(tpName)
+                .enableBatching(false).create();
+        for (int i = 0; i < 4; i++) {
+            messageIds.add((MessageIdImpl) producer.send("m-" + i));
+        }
+
+        PersistentTopic persistentTopic =
+                (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get();
+        PersistentSubscription persistentSubscription =
+                (PersistentSubscription) persistentTopic.getSubscription(subscription);
+        ManagedCursorImpl cursor = (ManagedCursorImpl) persistentSubscription.getCursor();
+        cursor.delete(PositionFactory.create(messageIds.get(0).getLedgerId(), messageIds.get(0).getEntryId()));
+        long[][] ackSets = {{2L}, {Long.MIN_VALUE, 1L}, {5L, 0L, 3L}};
+        for (int i = 0; i < ackSets.length; i++) {
+            MessageIdImpl messageId = messageIds.get(i + 1);
+            cursor.delete(AckSetStateUtil.createPositionWithAckSet(
+                    messageId.getLedgerId(), messageId.getEntryId(), ackSets[i]));
+        }
+        Assert.assertFalse(cursor.isCursorDataFullyPersistable());
+
+        // Zero queue + no receive yet: avoid ConsumerImpl's post-subscribe Flow race with
+        // afterAckMessages() that can deliver backlog before the dispatcher is paused.
+        Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(tpName)
+                .subscriptionName(subscription).subscriptionType(SubscriptionType.Shared)
+                .receiverQueueSize(0).enableBatchIndexAcknowledgment(true).isAckReceiptEnabled(true)
+                .subscribe();
+        persistentSubscription.getDispatcher().afterAckMessages(null, null);
+
+        cancelPendingRead(tpName, subscription);
+        triggerNewReadMoreEntries(tpName, subscription);
+
+        final String specifiedMessage = "9876543210";
+        producer.send(specifiedMessage);
+        CompletableFuture<Message<String>> pausedReceive = consumer.receiveAsync();
+        try {
+            pausedReceive.get(2, TimeUnit.SECONDS);
+            Assert.fail("expected timeout while dispatcher is paused");
+        } catch (TimeoutException expected) {
+            // Dispatcher should stay paused while batch-deleted index exceeds the persist cap.
+        }
+        Assert.assertFalse(pausedReceive.isDone());
+
+        for (int i = 0; i < ackSets.length; i++) {
+            MessageIdImpl messageId = messageIds.get(i + 1);
+            cursor.delete(PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()));
+        }
+        Assert.assertTrue(cursor.isCursorDataFullyPersistable());
+        persistentSubscription.getDispatcher().afterAckMessages(null, null);
+
+        cancelPendingRead(tpName, subscription);
+        triggerNewReadMoreEntries(tpName, subscription);
+
+        Message<String> resumedReceive = pausedReceive.get(5, TimeUnit.SECONDS);
+        Assert.assertNotNull(resumedReceive);
+        Assert.assertEquals(resumedReceive.getValue(), specifiedMessage);
+
+        producer.close();
+        consumer.close();
+        admin.topics().delete(tpName, false);
+    }
+
     @Test(dataProvider = "multiConsumerSubscriptionTypes")
     @SuppressWarnings("unchecked")
     public void testPauseOnAckStatPersist(SubscriptionType subscriptionType) throws Exception {