You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by zi...@apache.org on 2026/08/28 07:16:56 UTC

(pulsar) branch master updated: [fix][broker] Bound the topic deletion retries triggered by a replication cluster removal (#26432)

This is an automated email from the ASF dual-hosted git repository.

nodece pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 4ce0cd509573 [fix][broker] Bound the topic deletion retries triggered by a replication cluster removal (#26432)
4ce0cd509573 is described below

commit 4ce0cd509573d4c8bd72cb74fa2b06a510e14b45
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Aug 28 10:16:49 2026 +0300

    [fix][broker] Bound the topic deletion retries triggered by a replication cluster removal (#26432)
---
 .../broker/service/persistent/PersistentTopic.java |  92 +++++++++++++++++--
 .../PersistentTopicProtectedMethodsTest.java       | 102 +++++++++++++++++++++
 2 files changed, 186 insertions(+), 8 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 1b835682859a..e8c3bd7ca89e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -348,6 +348,15 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
     private final CompletableFuture<Void> initialReplicationCheck = new CompletableFuture<>();
     private final AtomicBoolean initialReplicationCheckInitialized = new AtomicBoolean(false);
 
+    /**
+     * Set while a deletion triggered by {@link #removeTopicIfLocalClusterNotAllowed()} is in progress, and while
+     * a failed one is waiting to be retried. Every failure path of {@link #delete} un-fences the topic, and
+     * un-fencing runs {@link #checkReplication()} again, which would start the same deletion again straight away.
+     * Without this guard, a deletion that keeps failing is retried in a tight loop for as long as the local
+     * cluster stays out of the replication clusters.
+     */
+    private final AtomicBoolean clusterRemovalDeletionInProgress = new AtomicBoolean(false);
+
     /***
      * We use 3 futures to prevent a new closing if there is an in-progress deletion or closing.  We make Pulsar return
      * the in-progress one when it is called the second time.
@@ -1637,6 +1646,9 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
                     new CloseFutures(new CompletableFuture<>(), new CompletableFuture<>(), new CompletableFuture<>());
 
             AtomicBoolean alreadyUnFenced = new AtomicBoolean();
+            // The outer whenComplete below sees the same failure the inner handlers already reported, so it only
+            // logs the ones they did not.
+            AtomicBoolean deleteFailureLogged = new AtomicBoolean();
             CompletableFuture<Void> res = getBrokerService().getPulsar().getPulsarResources().getNamespaceResources()
                         .getPartitionedTopicResources().runWithMarkDeleteAsync(TopicName.get(topic), () -> {
                 CompletableFuture<Void> deleteFuture = new CompletableFuture<>();
@@ -1675,6 +1687,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
                                 .thenCompose(ignore -> transactionBufferCleanupAndClose())
                                 .whenComplete((v, ex) -> {
                                     if (ex != null) {
+                                        deleteFailureLogged.set(true);
                                         log.error().exception(ex).log("Error deleting topic");
                                         alreadyUnFenced.set(true);
                                         unfenceTopicToResume();
@@ -1685,6 +1698,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
 
                                     FutureUtil.waitForAll(subsDeleteFutures).whenComplete((f, e) -> {
                                         if (e != null) {
+                                            deleteFailureLogged.set(true);
                                             log.error().exception(e).log("Error deleting topic");
                                             alreadyUnFenced.set(true);
                                             unfenceTopicToResume();
@@ -1716,6 +1730,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
                                                                 .log("Topic is already deleted");
                                                         deleteLedgerComplete(ctx);
                                                     } else {
+                                                        deleteFailureLogged.set(true);
                                                         log.error()
                                                                 .exception(exception)
                                                                 .log("Error deleting topic");
@@ -1743,7 +1758,9 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
                 return deleteFuture;
                 }).whenComplete((value, ex) -> {
                     if (ex != null) {
-                        log.error().exception(ex).log("Error deleting topic");
+                        if (!deleteFailureLogged.get()) {
+                            log.error().exception(ex).log("Error deleting topic");
+                        }
                         if (!alreadyUnFenced.get()) {
                             unfenceTopicToResume();
                         }
@@ -2250,17 +2267,76 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
     protected CompletableFuture<Boolean> removeTopicIfLocalClusterNotAllowed() {
         final String localCluster = brokerService.pulsar().getConfiguration().getClusterName();
         return checkAllowedCluster(localCluster).thenCompose(isAllowed -> {
-            if (!isAllowed) {
-                // if local cluster is removed from global namespace cluster-list : then delete topic forcefully
-                // because pulsar doesn't serve global topic without local repl-cluster configured.
-                return deleteForcefully().thenCompose(ignore -> {
-                    return deleteSchemaAndPoliciesIfClusterRemoved().thenApply(__ -> true);
-                });
+            if (isAllowed) {
+                return CompletableFuture.completedFuture(false);
             }
-            return CompletableFuture.completedFuture(false);
+            // if local cluster is removed from global namespace cluster-list : then delete topic forcefully
+            // because pulsar doesn't serve global topic without local repl-cluster configured.
+            if (!clusterRemovalDeletionInProgress.compareAndSet(false, true)) {
+                // A deletion is already in progress, or a failed one is waiting to be retried. Report the topic
+                // as removed so the caller does not start replicators for a topic that is being deleted.
+                return CompletableFuture.completedFuture(true);
+            }
+            return deleteForcefully()
+                    .thenCompose(ignore -> deleteSchemaAndPoliciesIfClusterRemoved())
+                    .thenApply(__ -> true)
+                    .whenComplete((__, ex) -> {
+                        if (ex != null) {
+                            scheduleClusterRemovalDeletionRetry();
+                        }
+                    });
         });
     }
 
+    /**
+     * Schedules another attempt at the deletion triggered by the local cluster removal, so one that failed for a
+     * transient reason is still retried, at a bounded rate instead of in a tight loop. Must only be called while
+     * {@link #clusterRemovalDeletionInProgress} is set, which is what keeps a re-entrant check from starting a
+     * deletion of its own before the delay elapses.
+     */
+    private void scheduleClusterRemovalDeletionRetry() {
+        try {
+            brokerService.executor().schedule(this::retryClusterRemovalDeletion,
+                    getClusterRemovalDeletionRetryDelaySeconds(), SECONDS);
+        } catch (RejectedExecutionException e) {
+            // The broker is shutting down, so there is nothing left to retry.
+            clusterRemovalDeletionInProgress.set(false);
+        }
+    }
+
+    private void retryClusterRemovalDeletion() {
+        clusterRemovalDeletionInProgress.set(false);
+        if (isClosed()) {
+            return;
+        }
+        try {
+            checkReplication().exceptionally(ex -> {
+                rescheduleClusterRemovalDeletionAfterFailedCheck();
+                return null;
+            });
+        } catch (Throwable t) {
+            // checkReplication is overridable, so an implementation can fail before returning its future.
+            rescheduleClusterRemovalDeletionAfterFailedCheck();
+        }
+    }
+
+    /**
+     * Handles a replication check that failed before it could start a deletion, for instance because reading the
+     * namespace policies failed: nothing downstream scheduled the next attempt, so it has to be scheduled here. A
+     * failure of the deletion itself has already scheduled one and left {@link #clusterRemovalDeletionInProgress}
+     * set, which is what the compareAndSet detects, so the two paths never schedule competing retries.
+     */
+    private void rescheduleClusterRemovalDeletionAfterFailedCheck() {
+        if (clusterRemovalDeletionInProgress.compareAndSet(false, true)) {
+            scheduleClusterRemovalDeletionRetry();
+        }
+    }
+
+    @VisibleForTesting
+    long getClusterRemovalDeletionRetryDelaySeconds() {
+        return POLICY_UPDATE_FAILURE_RETRY_TIME_SECONDS;
+    }
+
     protected CompletableFuture<Boolean> checkAllowedCluster(String localCluster) {
         List<String> topicRepls = topicPolicies.getReplicationClusters().get();
         return brokerService.pulsar().getPulsarResources().getNamespaceResources()
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicProtectedMethodsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicProtectedMethodsTest.java
index f7ee1e36c172..a479b8546566 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicProtectedMethodsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicProtectedMethodsTest.java
@@ -18,9 +18,17 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
+import java.time.Duration;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import lombok.CustomLog;
 import org.apache.bookkeeper.mledger.Position;
 import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
@@ -29,6 +37,7 @@ import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.client.api.PulsarClientException;
 import org.awaitility.Awaitility;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -155,4 +164,97 @@ public class PersistentTopicProtectedMethodsTest extends ProducerConsumerBase {
         c1.close();
         admin.topics().delete(tp, false);
     }
+
+    /***
+     * When the local cluster is removed from the replication clusters, checkReplication deletes the topic through
+     * removeTopicIfLocalClusterNotAllowed. Every failure path of PersistentTopic.delete un-fences the topic, and
+     * un-fencing runs checkReplication again, so a deletion that keeps failing must not be restarted every time
+     * the previous attempt reports its failure: that is an unbounded hot loop for as long as the local cluster
+     * stays out of the replication clusters. Retrying is only allowed again after a delay.
+     */
+    @Test
+    public void testClusterRemovalDeletionIsNotRetriedInALoop() throws Exception {
+        final String tp = BrokerTestUtil.newUniqueName("public/default/tp");
+        admin.topics().createNonPartitionedTopic(tp);
+        PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopic(tp, false).join().get();
+
+        PersistentTopic spyTopic = spy(persistentTopic);
+        CompletableFuture<Void> firstDeletion = new CompletableFuture<>();
+        AtomicInteger deletions = new AtomicInteger();
+        doReturn(CompletableFuture.completedFuture(false)).when(spyTopic).checkAllowedCluster(anyString());
+        doAnswer(invocation -> deletions.incrementAndGet() == 1
+                ? firstDeletion : CompletableFuture.completedFuture(null))
+                .when(spyTopic).deleteForcefully();
+        // The scheduled retry is covered by testClusterRemovalDeletionIsRetriedWhenTheReplicationCheckFails.
+        // Push it beyond the lifetime of this test so none of the assertions below can race the timer.
+        doReturn(TimeUnit.HOURS.toSeconds(1)).when(spyTopic).getClusterRemovalDeletionRetryDelaySeconds();
+
+        // The first check starts the deletion.
+        CompletableFuture<Boolean> firstCheck = spyTopic.removeTopicIfLocalClusterNotAllowed();
+        assertFalse(firstCheck.isDone(), "The deletion is still in progress");
+
+        // A concurrent check must not start a second deletion, and must report the topic as being removed so the
+        // caller does not start replicators for it. checkAllowedCluster is stubbed with an already completed
+        // future, so deleteForcefully has been called by the time the check returns.
+        CompletableFuture<Boolean> concurrentCheck = spyTopic.removeTopicIfLocalClusterNotAllowed();
+        assertEquals(deletions.get(), 1);
+        assertTrue(concurrentCheck.get(5, TimeUnit.SECONDS));
+
+        // The deletion fails. This is what un-fences the topic and runs checkReplication again in production, so
+        // the check that immediately follows the failure must not start yet another deletion.
+        firstDeletion.completeExceptionally(
+                new PulsarClientException.AlreadyClosedException("Producer already closed"));
+        assertTrue(firstCheck.isCompletedExceptionally(), "The failure is still reported to the caller");
+        CompletableFuture<Boolean> checkAfterFailure = spyTopic.removeTopicIfLocalClusterNotAllowed();
+        assertEquals(deletions.get(), 1, "The failed deletion must not be restarted immediately");
+        assertTrue(checkAfterFailure.get(5, TimeUnit.SECONDS));
+
+        admin.topics().delete(tp, false);
+    }
+
+    /***
+     * A scheduled retry of the deletion goes through checkReplication, which can fail before it reaches the
+     * deletion at all, for example when reading the namespace policies fails. Nothing downstream schedules the
+     * next attempt in that case, so the retry has to be rescheduled by the check itself; otherwise the topic is
+     * left undeleted until an unrelated event happens to run another replication check. checkReplication is
+     * overridable through TopicFactory, so it can also fail synchronously, before returning its future.
+     */
+    @Test
+    public void testClusterRemovalDeletionIsRetriedWhenTheReplicationCheckFails() throws Exception {
+        final String tp = BrokerTestUtil.newUniqueName("public/default/tp");
+        admin.topics().createNonPartitionedTopic(tp);
+        PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopic(tp, false).join().get();
+
+        PersistentTopic spyTopic = spy(persistentTopic);
+        AtomicInteger clusterChecks = new AtomicInteger();
+        AtomicInteger deletions = new AtomicInteger();
+        // Checks 2 and 3 are the ones the scheduled retries run. Fail both before they can start a deletion, the
+        // first synchronously and the second by returning a failed future.
+        doAnswer(invocation -> {
+            switch (clusterChecks.incrementAndGet()) {
+                case 2: throw new RuntimeException("mocked synchronous policy read failure");
+                case 3: return CompletableFuture.failedFuture(new RuntimeException("mocked policy read failure"));
+                default: return CompletableFuture.completedFuture(false);
+            }
+        }).when(spyTopic).checkAllowedCluster(anyString());
+        doAnswer(invocation -> deletions.incrementAndGet() == 1
+                ? CompletableFuture.failedFuture(
+                        new PulsarClientException.AlreadyClosedException("Producer already closed"))
+                : CompletableFuture.completedFuture(null))
+                .when(spyTopic).deleteForcefully();
+        doReturn(1L).when(spyTopic).getClusterRemovalDeletionRetryDelaySeconds();
+
+        // Check 1: the deletion is started and fails, so a retry is scheduled.
+        CompletableFuture<Boolean> firstCheck = spyTopic.removeTopicIfLocalClusterNotAllowed();
+        assertTrue(firstCheck.isCompletedExceptionally());
+        assertEquals(deletions.get(), 1);
+
+        // Checks 2 and 3 run from the scheduled retries and fail before starting a deletion. Check 4 must
+        // therefore still happen, and its deletion succeeds.
+        Awaitility.await().atMost(Duration.ofSeconds(60))
+                .untilAsserted(() -> assertEquals(deletions.get(), 2));
+        assertEquals(clusterChecks.get(), 4);
+
+        admin.topics().delete(tp, false);
+    }
 }