You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2026/10/12 06:21:16 UTC

(pulsar) branch master updated: [fix][client] Fix ZeroQueueConsumer stuck on pending receiveAsync after reconnection (#26814)

This is an automated email from the ASF dual-hosted git repository.

codelipenghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new dcd94d90c3da [fix][client] Fix ZeroQueueConsumer stuck on pending receiveAsync after reconnection (#26814)
dcd94d90c3da is described below

commit dcd94d90c3da41c3adcb111522ab4406ac69b480
Author: subodhwandile <49...@users.noreply.github.com>
AuthorDate: Mon Oct 12 11:51:10 2026 +0530

    [fix][client] Fix ZeroQueueConsumer stuck on pending receiveAsync after reconnection (#26814)
---
 .../pulsar/client/impl/ZeroQueueSizeTest.java      | 193 +++++++++++++++++++++
 .../pulsar/client/impl/ZeroQueueConsumerImpl.java  |  48 +++--
 2 files changed, 228 insertions(+), 13 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java
index 300b1c8bb325..f671cc01ce17 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java
@@ -30,10 +30,14 @@ import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import lombok.Cleanup;
 import lombok.CustomLog;
 import org.apache.pulsar.broker.service.SharedPulsarBaseTest;
 import org.apache.pulsar.client.admin.PulsarAdminException;
@@ -47,9 +51,12 @@ import org.apache.pulsar.client.api.MessagePayloadProcessor;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.ProducerBuilder;
+import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.policies.data.ConsumerStats;
+import org.awaitility.Awaitility;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
@@ -508,6 +515,192 @@ public class ZeroQueueSizeTest extends SharedPulsarBaseTest {
         producer.close();
     }
 
+    @Test(timeOut = 30000)
+    public void testReceiveAsyncNotStuckAfterTopicUnload() throws Exception {
+        String topicName = newTopicName();
+        Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topicName)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .receiverQueueSize(0)
+                .subscribe();
+        Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topicName)
+                .enableBatching(false)
+                .create();
+
+        try {
+            ConsumerImpl<?> consumerImpl = (ConsumerImpl<?>) consumer;
+            long lastDisconnectedTimestamp = consumerImpl.getLastDisconnectedTimestamp();
+            CompletableFuture<Message<String>> receiveFuture = consumer.receiveAsync();
+
+            admin.topics().unload(topicName);
+            Awaitility.await().atMost(10, TimeUnit.SECONDS)
+                    .until(() -> consumerImpl.getLastDisconnectedTimestamp() > lastDisconnectedTimestamp);
+            Awaitility.await().atMost(10, TimeUnit.SECONDS).until(consumerImpl::isConnected);
+
+            producer.send("message");
+            assertEquals(receiveFuture.get(10, TimeUnit.SECONDS).getValue(), "message");
+        } finally {
+            consumer.close();
+            producer.close();
+        }
+    }
+
+    @Test(timeOut = 30000)
+    public void testMultiplePendingReceiveAsyncNotStuckAfterTopicUnload() throws Exception {
+        String topicName = newTopicName();
+        Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topicName)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .receiverQueueSize(0)
+                .subscribe();
+        Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topicName)
+                .enableBatching(false)
+                .create();
+
+        try {
+            ConsumerImpl<?> consumerImpl = (ConsumerImpl<?>) consumer;
+            long lastDisconnectedTimestamp = consumerImpl.getLastDisconnectedTimestamp();
+            List<CompletableFuture<Message<String>>> receiveFutures = List.of(
+                    consumer.receiveAsync(), consumer.receiveAsync(), consumer.receiveAsync());
+
+            admin.topics().unload(topicName);
+            Awaitility.await().atMost(10, TimeUnit.SECONDS)
+                    .until(() -> consumerImpl.getLastDisconnectedTimestamp() > lastDisconnectedTimestamp);
+            Awaitility.await().atMost(10, TimeUnit.SECONDS).until(consumerImpl::isConnected);
+
+            producer.send("message-0");
+            producer.send("message-1");
+            producer.send("message-2");
+
+            CompletableFuture.allOf(receiveFutures.toArray(CompletableFuture[]::new)).get(10, TimeUnit.SECONDS);
+            Set<String> receivedMessages = new HashSet<>();
+            for (CompletableFuture<Message<String>> receiveFuture : receiveFutures) {
+                receivedMessages.add(receiveFuture.join().getValue());
+            }
+            assertEquals(receivedMessages, Set.of("message-0", "message-1", "message-2"));
+        } finally {
+            consumer.close();
+            producer.close();
+        }
+    }
+
+    private int getBrokerAvailablePermits(String topicName) throws Exception {
+        List<? extends ConsumerStats> consumers = admin.topics().getStats(topicName)
+                .getSubscriptions().get("sub").getConsumers();
+        assertEquals(consumers.size(), 1);
+        return consumers.get(0).getAvailablePermits();
+    }
+
+    @Test(timeOut = 30000)
+    public void testReceiveAsyncRegisteredAfterReconnectionNotStuck() throws Exception {
+        String topicName = newTopicName();
+        @Cleanup PulsarClient client = newPulsarClient();
+        Consumer<String> consumer = client.newConsumer(Schema.STRING)
+                .topic(topicName)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .receiverQueueSize(0)
+                .subscribe();
+        Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topicName)
+                .enableBatching(false)
+                .create();
+
+        CountDownLatch blocked = new CountDownLatch(1);
+        CountDownLatch release = new CountDownLatch(1);
+        try {
+            ConsumerImpl<?> consumerImpl = (ConsumerImpl<?>) consumer;
+            // Block the consumer's internal executor so the receiveAsync registration task
+            // cannot run until the reconnection has completed.
+            consumerImpl.internalPinnedExecutor.execute(() -> {
+                blocked.countDown();
+                try {
+                    release.await();
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                }
+            });
+            assertTrue(blocked.await(10, TimeUnit.SECONDS));
+
+            long lastDisconnectedTimestamp = consumerImpl.getLastDisconnectedTimestamp();
+            CompletableFuture<Message<String>> receiveFuture = consumer.receiveAsync();
+
+            admin.topics().unload(topicName);
+            Awaitility.await().atMost(10, TimeUnit.SECONDS)
+                    .until(() -> consumerImpl.getLastDisconnectedTimestamp() > lastDisconnectedTimestamp);
+            Awaitility.await().atMost(10, TimeUnit.SECONDS).until(consumerImpl::isConnected);
+
+            release.countDown();
+
+            // The pending receiveAsync must be counted by the reconnection path: exactly one permit.
+            Awaitility.await().atMost(10, TimeUnit.SECONDS)
+                    .untilAsserted(() -> assertEquals(getBrokerAvailablePermits(topicName), 1));
+
+            producer.send("message");
+            assertEquals(receiveFuture.get(10, TimeUnit.SECONDS).getValue(), "message");
+        } finally {
+            release.countDown();
+            consumer.close();
+            producer.close();
+        }
+    }
+
+    @Test(timeOut = 30000)
+    public void testMixedReceiveAndReceiveAsyncNotStuckAfterTopicUnload() throws Exception {
+        String topicName = newTopicName();
+        Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topicName)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .receiverQueueSize(0)
+                .subscribe();
+        Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topicName)
+                .enableBatching(false)
+                .create();
+        ExecutorService receiveExecutor = Executors.newSingleThreadExecutor();
+
+        try {
+            ConsumerImpl<?> consumerImpl = (ConsumerImpl<?>) consumer;
+            Future<Message<String>> blockingReceive =
+                    receiveExecutor.submit(() -> consumer.receive());
+            List<CompletableFuture<Message<String>>> receiveFutures = List.of(
+                    consumer.receiveAsync(), consumer.receiveAsync(), consumer.receiveAsync());
+
+            // 1 permit for the blocking receive() plus 1 for each pending receiveAsync().
+            Awaitility.await().atMost(10, TimeUnit.SECONDS)
+                    .untilAsserted(() -> assertEquals(getBrokerAvailablePermits(topicName), 4));
+
+            long lastDisconnectedTimestamp = consumerImpl.getLastDisconnectedTimestamp();
+            admin.topics().unload(topicName);
+            Awaitility.await().atMost(10, TimeUnit.SECONDS)
+                    .until(() -> consumerImpl.getLastDisconnectedTimestamp() > lastDisconnectedTimestamp);
+            Awaitility.await().atMost(10, TimeUnit.SECONDS).until(consumerImpl::isConnected);
+
+            Awaitility.await().atMost(10, TimeUnit.SECONDS)
+                    .untilAsserted(() -> assertEquals(getBrokerAvailablePermits(topicName), 4));
+
+            for (int i = 0; i < 4; i++) {
+                producer.send("message-" + i);
+            }
+
+            Set<String> receivedMessages = new HashSet<>();
+            receivedMessages.add(blockingReceive.get(10, TimeUnit.SECONDS).getValue());
+            for (CompletableFuture<Message<String>> receiveFuture : receiveFutures) {
+                receivedMessages.add(receiveFuture.get(10, TimeUnit.SECONDS).getValue());
+            }
+            assertEquals(receivedMessages, Set.of("message-0", "message-1", "message-2", "message-3"));
+        } finally {
+            receiveExecutor.shutdownNow();
+            consumer.close();
+            producer.close();
+        }
+    }
+
     @Test(timeOut = 30000)
     public void testPauseAndResumeNoReconnection() throws Exception {
         final String topicName = newTopicName();
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
index 8cfb3853c407..bcf28a4bebe1 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
@@ -36,6 +36,7 @@ import org.apache.pulsar.client.util.ExecutorProvider;
 import org.apache.pulsar.common.api.proto.BrokerEntryMetadata;
 import org.apache.pulsar.common.api.proto.MessageIdData;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.util.CompletableFutureCancellationHandler;
 
 public class ZeroQueueConsumerImpl<T> extends ConsumerImpl<T> {
 
@@ -82,13 +83,26 @@ public class ZeroQueueConsumerImpl<T> extends ConsumerImpl<T> {
 
     @Override
     protected CompletableFuture<Message<T>> internalReceiveAsync() {
-        CompletableFuture<Message<T>> future = super.internalReceiveAsync();
-        if (!future.isDone()) {
-            // We expect the message to be not in the queue yet
-            increaseAvailablePermits(cnx());
-        }
-
-        return future;
+        CompletableFutureCancellationHandler cancellationHandler = new CompletableFutureCancellationHandler();
+        CompletableFuture<Message<T>> result = cancellationHandler.createFuture();
+        internalPinnedExecutor.execute(() -> {
+            Message<T> message = incomingMessages.poll();
+            if (message != null) {
+                messageProcessed(message);
+                result.complete(beforeConsume(message));
+                return;
+            }
+            // Hold the same lock as consumerIsReconnectedToBroker() so exactly one permit is sent:
+            // either reconnect counts this future, or this task sends it after reconnection.
+            synchronized (ZeroQueueConsumerImpl.this) {
+                pendingReceives.add(result);
+                cancellationHandler.setCancelAction(() -> pendingReceives.remove(result));
+                if (isConnected()) {
+                    increaseAvailablePermits(cnx());
+                }
+            }
+        });
+        return result;
     }
 
     private Message<T> fetchSingleMessageFromBroker() throws PulsarClientException {
@@ -102,8 +116,8 @@ public class ZeroQueueConsumerImpl<T> extends ConsumerImpl<T> {
         Message<T> message;
         try {
             // if cnx is null or if the connection breaks the connectionOpened function will send the flow again
-            waitingOnReceiveForZeroQueueSize = true;
             synchronized (this) {
+                waitingOnReceiveForZeroQueueSize = true;
                 if (isConnected()) {
                     increaseAvailablePermits(cnx());
                 }
@@ -140,12 +154,20 @@ public class ZeroQueueConsumerImpl<T> extends ConsumerImpl<T> {
     protected void consumerIsReconnectedToBroker(ClientCnx cnx, int currentQueueSize) {
         super.consumerIsReconnectedToBroker(cnx, currentQueueSize);
 
-        // For zerosize queue : If the connection is reset and someone is waiting for the messages
-        // or queue was not empty: send a flow command
-        if (waitingOnReceiveForZeroQueueSize
+        // Send one permit per pending receiveAsync(), plus one for a blocking receive(),
+        // idle listener, or non-empty queue.
+        int pendingAsyncReceives = 0;
+        for (CompletableFuture<Message<T>> pendingReceive : pendingReceives) {
+            if (!pendingReceive.isDone()) {
+                pendingAsyncReceives++;
+            }
+        }
+        boolean shouldSendPermit = waitingOnReceiveForZeroQueueSize
                 || currentQueueSize > 0
-                || (listener != null && !waitingOnListenerForZeroQueueSize)) {
-            increaseAvailablePermits(cnx);
+                || (listener != null && !waitingOnListenerForZeroQueueSize);
+        int permits = pendingAsyncReceives + (shouldSendPermit ? 1 : 0);
+        if (permits > 0) {
+            increaseAvailablePermits(cnx, permits);
         }
     }