You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by zi...@apache.org on 2026/08/28 07:16:56 UTC
(pulsar) branch master updated: [fix][broker] Bound the topic deletion retries triggered by a replication cluster removal (#26432)
This is an automated email from the ASF dual-hosted git repository.
nodece pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 4ce0cd509573 [fix][broker] Bound the topic deletion retries triggered by a replication cluster removal (#26432)
4ce0cd509573 is described below
commit 4ce0cd509573d4c8bd72cb74fa2b06a510e14b45
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Aug 28 10:16:49 2026 +0300
[fix][broker] Bound the topic deletion retries triggered by a replication cluster removal (#26432)
---
.../broker/service/persistent/PersistentTopic.java | 92 +++++++++++++++++--
.../PersistentTopicProtectedMethodsTest.java | 102 +++++++++++++++++++++
2 files changed, 186 insertions(+), 8 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 1b835682859a..e8c3bd7ca89e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -348,6 +348,15 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
private final CompletableFuture<Void> initialReplicationCheck = new CompletableFuture<>();
private final AtomicBoolean initialReplicationCheckInitialized = new AtomicBoolean(false);
+ /**
+ * Set while a deletion triggered by {@link #removeTopicIfLocalClusterNotAllowed()} is in progress, and while
+ * a failed one is waiting to be retried. Every failure path of {@link #delete} un-fences the topic, and
+ * un-fencing runs {@link #checkReplication()} again, which would start the same deletion again straight away.
+ * Without this guard, a deletion that keeps failing is retried in a tight loop for as long as the local
+ * cluster stays out of the replication clusters.
+ */
+ private final AtomicBoolean clusterRemovalDeletionInProgress = new AtomicBoolean(false);
+
/***
* We use 3 futures to prevent a new closing if there is an in-progress deletion or closing. We make Pulsar return
* the in-progress one when it is called the second time.
@@ -1637,6 +1646,9 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
new CloseFutures(new CompletableFuture<>(), new CompletableFuture<>(), new CompletableFuture<>());
AtomicBoolean alreadyUnFenced = new AtomicBoolean();
+ // The outer whenComplete below sees the same failure the inner handlers already reported, so it only
+ // logs the ones they did not.
+ AtomicBoolean deleteFailureLogged = new AtomicBoolean();
CompletableFuture<Void> res = getBrokerService().getPulsar().getPulsarResources().getNamespaceResources()
.getPartitionedTopicResources().runWithMarkDeleteAsync(TopicName.get(topic), () -> {
CompletableFuture<Void> deleteFuture = new CompletableFuture<>();
@@ -1675,6 +1687,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
.thenCompose(ignore -> transactionBufferCleanupAndClose())
.whenComplete((v, ex) -> {
if (ex != null) {
+ deleteFailureLogged.set(true);
log.error().exception(ex).log("Error deleting topic");
alreadyUnFenced.set(true);
unfenceTopicToResume();
@@ -1685,6 +1698,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
FutureUtil.waitForAll(subsDeleteFutures).whenComplete((f, e) -> {
if (e != null) {
+ deleteFailureLogged.set(true);
log.error().exception(e).log("Error deleting topic");
alreadyUnFenced.set(true);
unfenceTopicToResume();
@@ -1716,6 +1730,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
.log("Topic is already deleted");
deleteLedgerComplete(ctx);
} else {
+ deleteFailureLogged.set(true);
log.error()
.exception(exception)
.log("Error deleting topic");
@@ -1743,7 +1758,9 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
return deleteFuture;
}).whenComplete((value, ex) -> {
if (ex != null) {
- log.error().exception(ex).log("Error deleting topic");
+ if (!deleteFailureLogged.get()) {
+ log.error().exception(ex).log("Error deleting topic");
+ }
if (!alreadyUnFenced.get()) {
unfenceTopicToResume();
}
@@ -2250,17 +2267,76 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
protected CompletableFuture<Boolean> removeTopicIfLocalClusterNotAllowed() {
final String localCluster = brokerService.pulsar().getConfiguration().getClusterName();
return checkAllowedCluster(localCluster).thenCompose(isAllowed -> {
- if (!isAllowed) {
- // if local cluster is removed from global namespace cluster-list : then delete topic forcefully
- // because pulsar doesn't serve global topic without local repl-cluster configured.
- return deleteForcefully().thenCompose(ignore -> {
- return deleteSchemaAndPoliciesIfClusterRemoved().thenApply(__ -> true);
- });
+ if (isAllowed) {
+ return CompletableFuture.completedFuture(false);
}
- return CompletableFuture.completedFuture(false);
+ // if local cluster is removed from global namespace cluster-list : then delete topic forcefully
+ // because pulsar doesn't serve global topic without local repl-cluster configured.
+ if (!clusterRemovalDeletionInProgress.compareAndSet(false, true)) {
+ // A deletion is already in progress, or a failed one is waiting to be retried. Report the topic
+ // as removed so the caller does not start replicators for a topic that is being deleted.
+ return CompletableFuture.completedFuture(true);
+ }
+ return deleteForcefully()
+ .thenCompose(ignore -> deleteSchemaAndPoliciesIfClusterRemoved())
+ .thenApply(__ -> true)
+ .whenComplete((__, ex) -> {
+ if (ex != null) {
+ scheduleClusterRemovalDeletionRetry();
+ }
+ });
});
}
+ /**
+ * Schedules another attempt at the deletion triggered by the local cluster removal, so one that failed for a
+ * transient reason is still retried, at a bounded rate instead of in a tight loop. Must only be called while
+ * {@link #clusterRemovalDeletionInProgress} is set, which is what keeps a re-entrant check from starting a
+ * deletion of its own before the delay elapses.
+ */
+ private void scheduleClusterRemovalDeletionRetry() {
+ try {
+ brokerService.executor().schedule(this::retryClusterRemovalDeletion,
+ getClusterRemovalDeletionRetryDelaySeconds(), SECONDS);
+ } catch (RejectedExecutionException e) {
+ // The broker is shutting down, so there is nothing left to retry.
+ clusterRemovalDeletionInProgress.set(false);
+ }
+ }
+
+ private void retryClusterRemovalDeletion() {
+ clusterRemovalDeletionInProgress.set(false);
+ if (isClosed()) {
+ return;
+ }
+ try {
+ checkReplication().exceptionally(ex -> {
+ rescheduleClusterRemovalDeletionAfterFailedCheck();
+ return null;
+ });
+ } catch (Throwable t) {
+ // checkReplication is overridable, so an implementation can fail before returning its future.
+ rescheduleClusterRemovalDeletionAfterFailedCheck();
+ }
+ }
+
+ /**
+ * Handles a replication check that failed before it could start a deletion, for instance because reading the
+ * namespace policies failed: nothing downstream scheduled the next attempt, so it has to be scheduled here. A
+ * failure of the deletion itself has already scheduled one and left {@link #clusterRemovalDeletionInProgress}
+ * set, which is what the compareAndSet detects, so the two paths never schedule competing retries.
+ */
+ private void rescheduleClusterRemovalDeletionAfterFailedCheck() {
+ if (clusterRemovalDeletionInProgress.compareAndSet(false, true)) {
+ scheduleClusterRemovalDeletionRetry();
+ }
+ }
+
+ @VisibleForTesting
+ long getClusterRemovalDeletionRetryDelaySeconds() {
+ return POLICY_UPDATE_FAILURE_RETRY_TIME_SECONDS;
+ }
+
protected CompletableFuture<Boolean> checkAllowedCluster(String localCluster) {
List<String> topicRepls = topicPolicies.getReplicationClusters().get();
return brokerService.pulsar().getPulsarResources().getNamespaceResources()
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicProtectedMethodsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicProtectedMethodsTest.java
index f7ee1e36c172..a479b8546566 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicProtectedMethodsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicProtectedMethodsTest.java
@@ -18,9 +18,17 @@
*/
package org.apache.pulsar.broker.service.persistent;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
+import java.time.Duration;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import lombok.CustomLog;
import org.apache.bookkeeper.mledger.Position;
import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
@@ -29,6 +37,7 @@ import org.apache.pulsar.broker.BrokerTestUtil;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.client.api.PulsarClientException;
import org.awaitility.Awaitility;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -155,4 +164,97 @@ public class PersistentTopicProtectedMethodsTest extends ProducerConsumerBase {
c1.close();
admin.topics().delete(tp, false);
}
+
+ /***
+ * When the local cluster is removed from the replication clusters, checkReplication deletes the topic through
+ * removeTopicIfLocalClusterNotAllowed. Every failure path of PersistentTopic.delete un-fences the topic, and
+ * un-fencing runs checkReplication again, so a deletion that keeps failing must not be restarted every time
+ * the previous attempt reports its failure: that is an unbounded hot loop for as long as the local cluster
+ * stays out of the replication clusters. Retrying is only allowed again after a delay.
+ */
+ @Test
+ public void testClusterRemovalDeletionIsNotRetriedInALoop() throws Exception {
+ final String tp = BrokerTestUtil.newUniqueName("public/default/tp");
+ admin.topics().createNonPartitionedTopic(tp);
+ PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopic(tp, false).join().get();
+
+ PersistentTopic spyTopic = spy(persistentTopic);
+ CompletableFuture<Void> firstDeletion = new CompletableFuture<>();
+ AtomicInteger deletions = new AtomicInteger();
+ doReturn(CompletableFuture.completedFuture(false)).when(spyTopic).checkAllowedCluster(anyString());
+ doAnswer(invocation -> deletions.incrementAndGet() == 1
+ ? firstDeletion : CompletableFuture.completedFuture(null))
+ .when(spyTopic).deleteForcefully();
+ // The scheduled retry is covered by testClusterRemovalDeletionIsRetriedWhenTheReplicationCheckFails.
+ // Push it beyond the lifetime of this test so none of the assertions below can race the timer.
+ doReturn(TimeUnit.HOURS.toSeconds(1)).when(spyTopic).getClusterRemovalDeletionRetryDelaySeconds();
+
+ // The first check starts the deletion.
+ CompletableFuture<Boolean> firstCheck = spyTopic.removeTopicIfLocalClusterNotAllowed();
+ assertFalse(firstCheck.isDone(), "The deletion is still in progress");
+
+ // A concurrent check must not start a second deletion, and must report the topic as being removed so the
+ // caller does not start replicators for it. checkAllowedCluster is stubbed with an already completed
+ // future, so deleteForcefully has been called by the time the check returns.
+ CompletableFuture<Boolean> concurrentCheck = spyTopic.removeTopicIfLocalClusterNotAllowed();
+ assertEquals(deletions.get(), 1);
+ assertTrue(concurrentCheck.get(5, TimeUnit.SECONDS));
+
+ // The deletion fails. This is what un-fences the topic and runs checkReplication again in production, so
+ // the check that immediately follows the failure must not start yet another deletion.
+ firstDeletion.completeExceptionally(
+ new PulsarClientException.AlreadyClosedException("Producer already closed"));
+ assertTrue(firstCheck.isCompletedExceptionally(), "The failure is still reported to the caller");
+ CompletableFuture<Boolean> checkAfterFailure = spyTopic.removeTopicIfLocalClusterNotAllowed();
+ assertEquals(deletions.get(), 1, "The failed deletion must not be restarted immediately");
+ assertTrue(checkAfterFailure.get(5, TimeUnit.SECONDS));
+
+ admin.topics().delete(tp, false);
+ }
+
+ /***
+ * A scheduled retry of the deletion goes through checkReplication, which can fail before it reaches the
+ * deletion at all, for example when reading the namespace policies fails. Nothing downstream schedules the
+ * next attempt in that case, so the retry has to be rescheduled by the check itself; otherwise the topic is
+ * left undeleted until an unrelated event happens to run another replication check. checkReplication is
+ * overridable through TopicFactory, so it can also fail synchronously, before returning its future.
+ */
+ @Test
+ public void testClusterRemovalDeletionIsRetriedWhenTheReplicationCheckFails() throws Exception {
+ final String tp = BrokerTestUtil.newUniqueName("public/default/tp");
+ admin.topics().createNonPartitionedTopic(tp);
+ PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopic(tp, false).join().get();
+
+ PersistentTopic spyTopic = spy(persistentTopic);
+ AtomicInteger clusterChecks = new AtomicInteger();
+ AtomicInteger deletions = new AtomicInteger();
+ // Checks 2 and 3 are the ones the scheduled retries run. Fail both before they can start a deletion, the
+ // first synchronously and the second by returning a failed future.
+ doAnswer(invocation -> {
+ switch (clusterChecks.incrementAndGet()) {
+ case 2: throw new RuntimeException("mocked synchronous policy read failure");
+ case 3: return CompletableFuture.failedFuture(new RuntimeException("mocked policy read failure"));
+ default: return CompletableFuture.completedFuture(false);
+ }
+ }).when(spyTopic).checkAllowedCluster(anyString());
+ doAnswer(invocation -> deletions.incrementAndGet() == 1
+ ? CompletableFuture.failedFuture(
+ new PulsarClientException.AlreadyClosedException("Producer already closed"))
+ : CompletableFuture.completedFuture(null))
+ .when(spyTopic).deleteForcefully();
+ doReturn(1L).when(spyTopic).getClusterRemovalDeletionRetryDelaySeconds();
+
+ // Check 1: the deletion is started and fails, so a retry is scheduled.
+ CompletableFuture<Boolean> firstCheck = spyTopic.removeTopicIfLocalClusterNotAllowed();
+ assertTrue(firstCheck.isCompletedExceptionally());
+ assertEquals(deletions.get(), 1);
+
+ // Checks 2 and 3 run from the scheduled retries and fail before starting a deletion. Check 4 must
+ // therefore still happen, and its deletion succeeds.
+ Awaitility.await().atMost(Duration.ofSeconds(60))
+ .untilAsserted(() -> assertEquals(deletions.get(), 2));
+ assertEquals(clusterChecks.get(), 4);
+
+ admin.topics().delete(tp, false);
+ }
}