You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2026/10/12 06:21:16 UTC
(pulsar) branch master updated: [fix][client] Fix ZeroQueueConsumer stuck on pending receiveAsync after reconnection (#26814)
This is an automated email from the ASF dual-hosted git repository.
codelipenghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new dcd94d90c3da [fix][client] Fix ZeroQueueConsumer stuck on pending receiveAsync after reconnection (#26814)
dcd94d90c3da is described below
commit dcd94d90c3da41c3adcb111522ab4406ac69b480
Author: subodhwandile <49...@users.noreply.github.com>
AuthorDate: Mon Oct 12 11:51:10 2026 +0530
[fix][client] Fix ZeroQueueConsumer stuck on pending receiveAsync after reconnection (#26814)
---
.../pulsar/client/impl/ZeroQueueSizeTest.java | 193 +++++++++++++++++++++
.../pulsar/client/impl/ZeroQueueConsumerImpl.java | 48 +++--
2 files changed, 228 insertions(+), 13 deletions(-)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java
index 300b1c8bb325..f671cc01ce17 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java
@@ -30,10 +30,14 @@ import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import lombok.Cleanup;
import lombok.CustomLog;
import org.apache.pulsar.broker.service.SharedPulsarBaseTest;
import org.apache.pulsar.client.admin.PulsarAdminException;
@@ -47,9 +51,12 @@ import org.apache.pulsar.client.api.MessagePayloadProcessor;
import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.ProducerBuilder;
+import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.policies.data.ConsumerStats;
+import org.awaitility.Awaitility;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -508,6 +515,192 @@ public class ZeroQueueSizeTest extends SharedPulsarBaseTest {
producer.close();
}
+ @Test(timeOut = 30000)
+ public void testReceiveAsyncNotStuckAfterTopicUnload() throws Exception {
+ String topicName = newTopicName();
+ Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
+ .topic(topicName)
+ .subscriptionName("sub")
+ .subscriptionType(SubscriptionType.Shared)
+ .receiverQueueSize(0)
+ .subscribe();
+ Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+ .topic(topicName)
+ .enableBatching(false)
+ .create();
+
+ try {
+ ConsumerImpl<?> consumerImpl = (ConsumerImpl<?>) consumer;
+ long lastDisconnectedTimestamp = consumerImpl.getLastDisconnectedTimestamp();
+ CompletableFuture<Message<String>> receiveFuture = consumer.receiveAsync();
+
+ admin.topics().unload(topicName);
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)
+ .until(() -> consumerImpl.getLastDisconnectedTimestamp() > lastDisconnectedTimestamp);
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).until(consumerImpl::isConnected);
+
+ producer.send("message");
+ assertEquals(receiveFuture.get(10, TimeUnit.SECONDS).getValue(), "message");
+ } finally {
+ consumer.close();
+ producer.close();
+ }
+ }
+
+ @Test(timeOut = 30000)
+ public void testMultiplePendingReceiveAsyncNotStuckAfterTopicUnload() throws Exception {
+ String topicName = newTopicName();
+ Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
+ .topic(topicName)
+ .subscriptionName("sub")
+ .subscriptionType(SubscriptionType.Shared)
+ .receiverQueueSize(0)
+ .subscribe();
+ Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+ .topic(topicName)
+ .enableBatching(false)
+ .create();
+
+ try {
+ ConsumerImpl<?> consumerImpl = (ConsumerImpl<?>) consumer;
+ long lastDisconnectedTimestamp = consumerImpl.getLastDisconnectedTimestamp();
+ List<CompletableFuture<Message<String>>> receiveFutures = List.of(
+ consumer.receiveAsync(), consumer.receiveAsync(), consumer.receiveAsync());
+
+ admin.topics().unload(topicName);
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)
+ .until(() -> consumerImpl.getLastDisconnectedTimestamp() > lastDisconnectedTimestamp);
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).until(consumerImpl::isConnected);
+
+ producer.send("message-0");
+ producer.send("message-1");
+ producer.send("message-2");
+
+ CompletableFuture.allOf(receiveFutures.toArray(CompletableFuture[]::new)).get(10, TimeUnit.SECONDS);
+ Set<String> receivedMessages = new HashSet<>();
+ for (CompletableFuture<Message<String>> receiveFuture : receiveFutures) {
+ receivedMessages.add(receiveFuture.join().getValue());
+ }
+ assertEquals(receivedMessages, Set.of("message-0", "message-1", "message-2"));
+ } finally {
+ consumer.close();
+ producer.close();
+ }
+ }
+
+ private int getBrokerAvailablePermits(String topicName) throws Exception {
+ List<? extends ConsumerStats> consumers = admin.topics().getStats(topicName)
+ .getSubscriptions().get("sub").getConsumers();
+ assertEquals(consumers.size(), 1);
+ return consumers.get(0).getAvailablePermits();
+ }
+
+ @Test(timeOut = 30000)
+ public void testReceiveAsyncRegisteredAfterReconnectionNotStuck() throws Exception {
+ String topicName = newTopicName();
+ @Cleanup PulsarClient client = newPulsarClient();
+ Consumer<String> consumer = client.newConsumer(Schema.STRING)
+ .topic(topicName)
+ .subscriptionName("sub")
+ .subscriptionType(SubscriptionType.Shared)
+ .receiverQueueSize(0)
+ .subscribe();
+ Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+ .topic(topicName)
+ .enableBatching(false)
+ .create();
+
+ CountDownLatch blocked = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ try {
+ ConsumerImpl<?> consumerImpl = (ConsumerImpl<?>) consumer;
+ // Block the consumer's internal executor so the receiveAsync registration task
+ // cannot run until the reconnection has completed.
+ consumerImpl.internalPinnedExecutor.execute(() -> {
+ blocked.countDown();
+ try {
+ release.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+ assertTrue(blocked.await(10, TimeUnit.SECONDS));
+
+ long lastDisconnectedTimestamp = consumerImpl.getLastDisconnectedTimestamp();
+ CompletableFuture<Message<String>> receiveFuture = consumer.receiveAsync();
+
+ admin.topics().unload(topicName);
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)
+ .until(() -> consumerImpl.getLastDisconnectedTimestamp() > lastDisconnectedTimestamp);
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).until(consumerImpl::isConnected);
+
+ release.countDown();
+
+ // The pending receiveAsync must be counted by the reconnection path: exactly one permit.
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)
+ .untilAsserted(() -> assertEquals(getBrokerAvailablePermits(topicName), 1));
+
+ producer.send("message");
+ assertEquals(receiveFuture.get(10, TimeUnit.SECONDS).getValue(), "message");
+ } finally {
+ release.countDown();
+ consumer.close();
+ producer.close();
+ }
+ }
+
+ @Test(timeOut = 30000)
+ public void testMixedReceiveAndReceiveAsyncNotStuckAfterTopicUnload() throws Exception {
+ String topicName = newTopicName();
+ Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
+ .topic(topicName)
+ .subscriptionName("sub")
+ .subscriptionType(SubscriptionType.Shared)
+ .receiverQueueSize(0)
+ .subscribe();
+ Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+ .topic(topicName)
+ .enableBatching(false)
+ .create();
+ ExecutorService receiveExecutor = Executors.newSingleThreadExecutor();
+
+ try {
+ ConsumerImpl<?> consumerImpl = (ConsumerImpl<?>) consumer;
+ Future<Message<String>> blockingReceive =
+ receiveExecutor.submit(() -> consumer.receive());
+ List<CompletableFuture<Message<String>>> receiveFutures = List.of(
+ consumer.receiveAsync(), consumer.receiveAsync(), consumer.receiveAsync());
+
+ // 1 permit for the blocking receive() plus 1 for each pending receiveAsync().
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)
+ .untilAsserted(() -> assertEquals(getBrokerAvailablePermits(topicName), 4));
+
+ long lastDisconnectedTimestamp = consumerImpl.getLastDisconnectedTimestamp();
+ admin.topics().unload(topicName);
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)
+ .until(() -> consumerImpl.getLastDisconnectedTimestamp() > lastDisconnectedTimestamp);
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).until(consumerImpl::isConnected);
+
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)
+ .untilAsserted(() -> assertEquals(getBrokerAvailablePermits(topicName), 4));
+
+ for (int i = 0; i < 4; i++) {
+ producer.send("message-" + i);
+ }
+
+ Set<String> receivedMessages = new HashSet<>();
+ receivedMessages.add(blockingReceive.get(10, TimeUnit.SECONDS).getValue());
+ for (CompletableFuture<Message<String>> receiveFuture : receiveFutures) {
+ receivedMessages.add(receiveFuture.get(10, TimeUnit.SECONDS).getValue());
+ }
+ assertEquals(receivedMessages, Set.of("message-0", "message-1", "message-2", "message-3"));
+ } finally {
+ receiveExecutor.shutdownNow();
+ consumer.close();
+ producer.close();
+ }
+ }
+
@Test(timeOut = 30000)
public void testPauseAndResumeNoReconnection() throws Exception {
final String topicName = newTopicName();
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
index 8cfb3853c407..bcf28a4bebe1 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
@@ -36,6 +36,7 @@ import org.apache.pulsar.client.util.ExecutorProvider;
import org.apache.pulsar.common.api.proto.BrokerEntryMetadata;
import org.apache.pulsar.common.api.proto.MessageIdData;
import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.util.CompletableFutureCancellationHandler;
public class ZeroQueueConsumerImpl<T> extends ConsumerImpl<T> {
@@ -82,13 +83,26 @@ public class ZeroQueueConsumerImpl<T> extends ConsumerImpl<T> {
@Override
protected CompletableFuture<Message<T>> internalReceiveAsync() {
- CompletableFuture<Message<T>> future = super.internalReceiveAsync();
- if (!future.isDone()) {
- // We expect the message to be not in the queue yet
- increaseAvailablePermits(cnx());
- }
-
- return future;
+ CompletableFutureCancellationHandler cancellationHandler = new CompletableFutureCancellationHandler();
+ CompletableFuture<Message<T>> result = cancellationHandler.createFuture();
+ internalPinnedExecutor.execute(() -> {
+ Message<T> message = incomingMessages.poll();
+ if (message != null) {
+ messageProcessed(message);
+ result.complete(beforeConsume(message));
+ return;
+ }
+ // Hold the same lock as consumerIsReconnectedToBroker() so exactly one permit is sent:
+ // either reconnect counts this future, or this task sends it after reconnection.
+ synchronized (ZeroQueueConsumerImpl.this) {
+ pendingReceives.add(result);
+ cancellationHandler.setCancelAction(() -> pendingReceives.remove(result));
+ if (isConnected()) {
+ increaseAvailablePermits(cnx());
+ }
+ }
+ });
+ return result;
}
private Message<T> fetchSingleMessageFromBroker() throws PulsarClientException {
@@ -102,8 +116,8 @@ public class ZeroQueueConsumerImpl<T> extends ConsumerImpl<T> {
Message<T> message;
try {
// if cnx is null or if the connection breaks the connectionOpened function will send the flow again
- waitingOnReceiveForZeroQueueSize = true;
synchronized (this) {
+ waitingOnReceiveForZeroQueueSize = true;
if (isConnected()) {
increaseAvailablePermits(cnx());
}
@@ -140,12 +154,20 @@ public class ZeroQueueConsumerImpl<T> extends ConsumerImpl<T> {
protected void consumerIsReconnectedToBroker(ClientCnx cnx, int currentQueueSize) {
super.consumerIsReconnectedToBroker(cnx, currentQueueSize);
- // For zerosize queue : If the connection is reset and someone is waiting for the messages
- // or queue was not empty: send a flow command
- if (waitingOnReceiveForZeroQueueSize
+ // Send one permit per pending receiveAsync(), plus one for a blocking receive(),
+ // idle listener, or non-empty queue.
+ int pendingAsyncReceives = 0;
+ for (CompletableFuture<Message<T>> pendingReceive : pendingReceives) {
+ if (!pendingReceive.isDone()) {
+ pendingAsyncReceives++;
+ }
+ }
+ boolean shouldSendPermit = waitingOnReceiveForZeroQueueSize
|| currentQueueSize > 0
- || (listener != null && !waitingOnListenerForZeroQueueSize)) {
- increaseAvailablePermits(cnx);
+ || (listener != null && !waitingOnListenerForZeroQueueSize);
+ int permits = pendingAsyncReceives + (shouldSendPermit ? 1 : 0);
+ if (permits > 0) {
+ increaseAvailablePermits(cnx, permits);
}
}