You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/20 22:21:31 UTC
(pulsar) 10/21: [fix][broker] Producer still publishes messages to Broker after Topic was unloaded (#24793)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 0d457f42186df8f0cc92d212029b458b3f47b422
Author: zhouyifan279 <88...@users.noreply.github.com>
AuthorDate: Sat Oct 17 22:42:55 2026 +0800
[fix][broker] Producer still publishes messages to Broker after Topic was unloaded (#24793)
Co-authored-by: Matteo Merli <mm...@apache.org>
(cherry picked from commit 4c49b154431fdf74988f5be4b2759c35f96f09fd)
---
.../pulsar/broker/service/AbstractTopic.java | 69 +++++++++++-----------
.../pulsar/broker/service/AbstractTopicTest.java | 62 +++++++++++++++++++
2 files changed, 98 insertions(+), 33 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
index 821849605127..9c12fab97bf9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
@@ -890,31 +890,15 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener {
return brokerService.checkTopicNsOwnership(getName())
.thenCompose(__ ->
incrementTopicEpochIfNeeded(producer, producerQueuedFuture))
- .thenCompose(producerEpoch -> {
- lock.writeLock().lock();
- try {
- checkTopicFenced();
- if (isMigrated()) {
- log.warn("[{}] Attempting to add producer to a migrated topic", topic);
- throw new TopicMigratedException("Topic was already migrated");
- } else if (isTerminated()) {
- log.warn("[{}] Attempting to add producer to a terminated topic", topic);
- throw new TopicTerminatedException("Topic was already terminated");
- }
- return internalAddProducer(producer).thenApply(ignore -> {
+ .thenCompose(producerEpoch ->
+ internalAddProducer(producer).thenApply(ignore -> {
USAGE_COUNT_UPDATER.incrementAndGet(this);
if (log.isDebugEnabled()) {
log.debug("[{}] [{}] Added producer -- count: {}", topic, producer.getProducerName(),
USAGE_COUNT_UPDATER.get(this));
}
return producerEpoch;
- });
- } catch (BrokerServiceException e) {
- return FutureUtil.failedFuture(e);
- } finally {
- lock.writeLock().unlock();
- }
- });
+ }));
}
protected CompletableFuture<Optional<Long>> incrementTopicEpochIfNeeded(Producer producer,
@@ -1094,23 +1078,42 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener {
}
protected CompletableFuture<Void> internalAddProducer(Producer producer) {
- if (isSameAddressProducersExceeded(producer)) {
- log.warn("[{}] Attempting to add producer to topic which reached max same address producers limit", topic);
- return CompletableFuture.failedFuture(new BrokerServiceException.ProducerBusyException(
- "Topic '" + topic + "' reached max same address producers limit"));
- }
+ // Check the topic state under the lock on every attempt, including the retry after the old producer's
+ // connection liveness check, since the topic may have been closed while that check was in progress.
+ lock.writeLock().lock();
+ try {
+ checkTopicFenced();
+ if (isMigrated()) {
+ log.warn("[{}] Attempting to add producer to a migrated topic", topic);
+ throw new TopicMigratedException("Topic was already migrated");
+ } else if (isTerminated()) {
+ log.warn("[{}] Attempting to add producer to a terminated topic", topic);
+ throw new TopicTerminatedException("Topic was already terminated");
+ }
- if (log.isDebugEnabled()) {
- log.debug("[{}] {} Got request to create producer ", topic, producer.getProducerName());
- }
+ if (isSameAddressProducersExceeded(producer)) {
+ log.warn("[{}] Attempting to add producer to topic which reached max same address producers limit",
+ topic);
+ return CompletableFuture.failedFuture(new BrokerServiceException.ProducerBusyException(
+ "Topic '" + topic + "' reached max same address producers limit"));
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] {} Got request to create producer ", topic, producer.getProducerName());
+ }
- Producer existProducer = producers.putIfAbsent(producer.getProducerName(), producer);
- if (existProducer != null) {
- return tryOverwriteOldProducer(existProducer, producer);
- } else if (!producer.isRemote()) {
- USER_CREATED_PRODUCER_COUNTER_UPDATER.incrementAndGet(this);
+ Producer existProducer = producers.putIfAbsent(producer.getProducerName(), producer);
+ if (existProducer != null) {
+ return tryOverwriteOldProducer(existProducer, producer);
+ } else if (!producer.isRemote()) {
+ USER_CREATED_PRODUCER_COUNTER_UPDATER.incrementAndGet(this);
+ }
+ return CompletableFuture.completedFuture(null);
+ } catch (BrokerServiceException e) {
+ return FutureUtil.failedFuture(e);
+ } finally {
+ lock.writeLock().unlock();
}
- return CompletableFuture.completedFuture(null);
}
private CompletableFuture<Void> tryOverwriteOldProducer(Producer oldProducer, Producer newProducer) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java
index 1e15bcf12c08..00c6457dd8e3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java
@@ -18,16 +18,26 @@
*/
package org.apache.pulsar.broker.service;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.withSettings;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertSame;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.expectThrows;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.broker.qos.AsyncTokenBucket;
+import org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException;
+import org.apache.pulsar.common.api.proto.ProducerAccessMode;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -49,6 +59,7 @@ public class AbstractTopicTest {
when(pulsarService.getConfiguration()).thenReturn(serviceConfiguration);
when(brokerService.getBacklogQuotaManager()).thenReturn(backlogQuotaManager);
doReturn(AsyncTokenBucket.DEFAULT_SNAPSHOT_CLOCK).when(pulsarService).getMonotonicClock();
+ when(brokerService.checkTopicNsOwnership(anyString())).thenReturn(CompletableFuture.completedFuture(null));
topic = mock(AbstractTopic.class, withSettings()
.useConstructor("topic", brokerService)
@@ -72,4 +83,55 @@ public class AbstractTopicTest {
when(subscription.getBytesOutCounter()).thenReturn(2L);
assertEquals(topic.getBytesOutCounter(), 3L);
}
+
+ @Test
+ public void testOverwriteOldProducerAfterTopicFenced() {
+ TransportCnx oldCnx = mock(TransportCnx.class);
+ Producer oldProducer = mockProducer(oldCnx);
+ topic.addProducer(oldProducer, new CompletableFuture<>()).join();
+
+ // The old producer's connection is gone, but the liveness check completes only after the topic is fenced
+ CompletableFuture<Optional<Boolean>> liveness = new CompletableFuture<>();
+ when(oldCnx.checkConnectionLiveness()).thenReturn(liveness);
+
+ Producer newProducer = mockProducer(mock(TransportCnx.class));
+ CompletableFuture<Optional<Long>> addFuture = topic.addProducer(newProducer, new CompletableFuture<>());
+ assertFalse(addFuture.isDone());
+
+ // Topic close fences the topic and disconnects the producers it knows about
+ topic.isFenced = true;
+ topic.getProducers().remove(oldProducer.getProducerName(), oldProducer);
+ liveness.complete(Optional.of(false));
+
+ CompletionException e = expectThrows(CompletionException.class, addFuture::join);
+ assertTrue(e.getCause() instanceof TopicFencedException, e.getCause().toString());
+ assertTrue(topic.getProducers().isEmpty());
+ }
+
+ @Test
+ public void testOverwriteOldProducerWithDeadConnection() {
+ TransportCnx oldCnx = mock(TransportCnx.class);
+ Producer oldProducer = mockProducer(oldCnx);
+ topic.addProducer(oldProducer, new CompletableFuture<>()).join();
+
+ CompletableFuture<Optional<Boolean>> liveness = new CompletableFuture<>();
+ when(oldCnx.checkConnectionLiveness()).thenReturn(liveness);
+
+ Producer newProducer = mockProducer(mock(TransportCnx.class));
+ CompletableFuture<Optional<Long>> addFuture = topic.addProducer(newProducer, new CompletableFuture<>());
+ topic.getProducers().remove(oldProducer.getProducerName(), oldProducer);
+ liveness.complete(Optional.of(false));
+
+ addFuture.join();
+ assertSame(topic.getProducers().get(newProducer.getProducerName()), newProducer);
+ }
+
+ private Producer mockProducer(TransportCnx cnx) {
+ Producer producer = mock(Producer.class);
+ when(producer.getTopic()).thenReturn(topic);
+ when(producer.getProducerName()).thenReturn("producer");
+ when(producer.getAccessMode()).thenReturn(ProducerAccessMode.Shared);
+ when(producer.getCnx()).thenReturn(cnx);
+ return producer;
+ }
}