You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by xy...@apache.org on 2025/07/28 03:30:36 UTC
(pulsar) 01/02: [fix][client] Partitioned topics are unexpectedly created by client after deletion (#24554)
This is an automated email from the ASF dual-hosted git repository.
xyz pushed a commit to branch bewaremypower/4.0-pr-24454
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 0c5eb23173651a2b1a81f99333585c9c0a427120
Author: Yunze Xu <xy...@163.com>
AuthorDate: Fri Jul 25 20:59:01 2025 +0800
[fix][client] Partitioned topics are unexpectedly created by client after deletion (#24554)
(cherry picked from commit 969920794840d5be884d039d9626acf64cb362a8)
---
.../bookkeeper/mledger/impl/ManagedLedgerImpl.java | 15 +++++-
.../pulsar/broker/admin/TopicAutoCreationTest.java | 54 +++++++++++++++++++---
.../client/impl/MultiTopicsConsumerImpl.java | 2 +-
.../client/impl/PartitionedProducerImpl.java | 2 +-
4 files changed, 63 insertions(+), 10 deletions(-)
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 11be72e2055..a76beb37938 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -3063,7 +3063,7 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
// Truncate to ensure the offloaded data is not orphaned.
// Also ensures the BK ledgers are deleted and not just scheduled for deletion
- CompletableFuture<Void> truncateFuture = asyncTruncate();
+ CompletableFuture<Void> truncateFuture = asyncTruncate(true);
truncateFuture.whenComplete((ignore, exc) -> {
if (exc != null) {
log.error("[{}] Error truncating ledger for deletion", name, exc);
@@ -4460,6 +4460,12 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
@Override
public CompletableFuture<Void> asyncTruncate() {
+ return asyncTruncate(false);
+ }
+
+ // When asyncTruncate is called by asyncDelete, the argument should be true because cursors will not be accessed
+ // after the managed ledger is deleted.
+ private CompletableFuture<Void> asyncTruncate(boolean ignoreCursorFailure) {
final List<CompletableFuture<Void>> futures = new ArrayList();
for (ManagedCursor cursor : cursors) {
@@ -4472,7 +4478,12 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
@Override
public void clearBacklogFailed(ManagedLedgerException exception, Object ctx) {
- future.completeExceptionally(exception);
+ if (ignoreCursorFailure) {
+ log.warn("Failed to clear backlog for cursor {}", cursor.getName(), exception);
+ future.complete(null);
+ } else {
+ future.completeExceptionally(exception);
+ }
}
}, null);
futures.add(future);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java
index ca2091816cc..791b9bca6a6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java
@@ -24,7 +24,9 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
+import java.io.Closeable;
import java.net.InetSocketAddress;
+import java.time.Duration;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@@ -36,18 +38,21 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.impl.LookupService;
import org.apache.pulsar.client.impl.LookupTopicResult;
import org.apache.pulsar.client.impl.PulsarClientImpl;
import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride;
import org.apache.pulsar.common.policies.data.TopicType;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@Test(groups = "broker-admin")
@@ -55,7 +60,7 @@ import org.testng.annotations.Test;
public class TopicAutoCreationTest extends ProducerConsumerBase {
@Override
- @BeforeMethod
+ @BeforeClass
protected void setup() throws Exception {
conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED);
conf.setAllowAutoTopicCreation(true);
@@ -71,7 +76,7 @@ public class TopicAutoCreationTest extends ProducerConsumerBase {
}
@Override
- @AfterMethod(alwaysRun = true)
+ @AfterClass(alwaysRun = true)
protected void cleanup() throws Exception {
super.internalCleanup();
}
@@ -87,9 +92,11 @@ public class TopicAutoCreationTest extends ProducerConsumerBase {
.create();
List<String> partitionedTopics = admin.topics().getPartitionedTopicList(namespaceName);
+ assertTrue(partitionedTopics.contains(topic));
List<String> topics = admin.topics().getList(namespaceName);
- assertEquals(partitionedTopics.size(), 1);
- assertEquals(topics.size(), 3);
+ for (int i = 0; i < conf.getDefaultNumPartitions(); i++) {
+ assertTrue(topics.contains(topic + TopicName.PARTITIONED_TOPIC_SUFFIX + i));
+ }
producer.close();
for (String t : topics) {
@@ -248,4 +255,39 @@ public class TopicAutoCreationTest extends ProducerConsumerBase {
admin.namespaces().deleteNamespace(namespace, true);
}
+ @Test
+ public void testPartitionsNotCreatedAfterDeletion() throws Exception {
+ @Cleanup final var client = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).build();
+ final var topicName = TopicName.get("my-property/my-ns/testPartitionsNotCreatedAfterDeletion");
+ final var topic = topicName.toString();
+ final var interval = Duration.ofSeconds(1);
+ final ThrowableConsumer<ThrowableSupplier<Closeable>> verifier = creator -> {
+ admin.topics().createPartitionedTopic(topic, 1);
+ try (final var ignored = creator.get()) {
+ admin.topics().terminatePartitionedTopic(topic);
+ admin.topics().deletePartitionedTopic(topic, true);
+ Thread.sleep(interval.toMillis() + 500); // wait until the auto update partitions task has run
+
+ final var topics = admin.topics().getList(topicName.getNamespace()).stream()
+ .filter(__ -> __.contains(topicName.getLocalName())).toList();
+ assertTrue(topics.isEmpty(), "topics are " + topics);
+ }
+ };
+ verifier.accept(() -> client.newProducer().topic(topic)
+ .autoUpdatePartitionsInterval(interval.toSecondsPart(), TimeUnit.SECONDS).create());
+ verifier.accept(() -> client.newConsumer().topic(topic).subscriptionName("sub")
+ .autoUpdatePartitionsInterval(interval.toSecondsPart(), TimeUnit.SECONDS).subscribe());
+ verifier.accept(() -> client.newReader().topic(topic).startMessageId(MessageId.earliest)
+ .autoUpdatePartitionsInterval(interval.toSecondsPart(), TimeUnit.SECONDS).create());
+ }
+
+ private interface ThrowableConsumer<T> {
+
+ void accept(T value) throws Exception;
+ }
+
+ public interface ThrowableSupplier<T> {
+
+ T get() throws Exception;
+ }
}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
index 727a71a9b8d..a31b72c348c 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
@@ -1401,7 +1401,7 @@ public class MultiTopicsConsumerImpl<T> extends ConsumerBase<T> {
private CompletableFuture<Void> subscribeIncreasedTopicPartitions(String topicName) {
int oldPartitionNumber = partitionedTopics.get(topicName);
- return client.getPartitionsForTopic(topicName).thenCompose(list -> {
+ return client.getPartitionsForTopic(topicName, false).thenCompose(list -> {
int currentPartitionNumber = Long.valueOf(list.stream()
.filter(t -> TopicName.get(t).isPartitioned()).count()).intValue();
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java
index 903a1beaaee..cca18998515 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java
@@ -390,7 +390,7 @@ public class PartitionedProducerImpl<T> extends ProducerBase<T> {
return future;
}
- client.getPartitionsForTopic(topic).thenCompose(list -> {
+ client.getPartitionsForTopic(topic, false).thenCompose(list -> {
int oldPartitionNumber = topicMetadata.numPartitions();
int currentPartitionNumber = list.size();