You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/09/17 14:55:13 UTC

(pulsar) 01/01: [fix][broker] Prevent slow Key_Shared sockets from stalling other consumers

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch lh-fix-key-shared-write-progress
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit afd0f52d3ac340fb3605a003084953ad2c413949
Author: Lari Hotari <lh...@apache.org>
AuthorDate: Thu Sep 17 13:31:15 2026 +0300

    [fix][broker] Prevent slow Key_Shared sockets from stalling other consumers
    
    Assisted-by: Codex
---
 ...istentStickyKeyDispatcherMultipleConsumers.java |   8 +-
 .../persistent/KeySharedSlowConsumerTest.java      | 129 +++++++++++++++++++++
 ...ntStickyKeyDispatcherMultipleConsumersTest.java |  44 +++++++
 3 files changed, 176 insertions(+), 5 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
index ec34fcd92fca..b8acced90c9a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
@@ -30,7 +30,6 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
 import lombok.Getter;
 import org.apache.bookkeeper.mledger.Entry;
@@ -314,7 +313,6 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi
         final Map<Consumer, List<Entry>> entriesByConsumerForDispatching =
                 filterAndGroupEntriesForDispatching(entries, readType, triggerLookAhead);
 
-        AtomicInteger remainingConsumersToFinishSending = new AtomicInteger(entriesByConsumerForDispatching.size());
         for (Map.Entry<Consumer, List<Entry>> current : entriesByConsumerForDispatching.entrySet()) {
             Consumer consumer = current.getKey();
             List<Entry> entriesForConsumer = current.getValue();
@@ -340,9 +338,9 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi
                     sendMessageInfo.getTotalMessages(),
                     sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(),
                     getRedeliveryTracker()).addListener(future -> {
-                if (future.isDone() && remainingConsumersToFinishSending.decrementAndGet() == 0) {
-                    readMoreEntriesAsync();
-                }
+                // One blocked socket must not hold up consumers whose writes have completed.
+                // The conflated read loop rechecks writability and permits before selecting a consumer.
+                readMoreEntriesAsync();
             });
 
             TOTAL_AVAILABLE_PERMITS_UPDATER.getAndAdd(this,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/KeySharedSlowConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/KeySharedSlowConsumerTest.java
new file mode 100644
index 000000000000..3ff64d7f6b07
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/KeySharedSlowConsumerTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.persistent;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
+import io.netty.channel.Channel;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import org.apache.pulsar.broker.service.ServerCnx;
+import org.apache.pulsar.broker.service.SharedPulsarBaseTest;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.ConsumerImpl;
+import org.awaitility.Awaitility;
+import org.testng.annotations.Test;
+
+/** Verifies that a paused Key_Shared socket does not prevent other consumers from making progress. */
+@Test(groups = "broker-api")
+public class KeySharedSlowConsumerTest extends SharedPulsarBaseTest {
+    @Test(timeOut = 120000)
+    public void testSlowKeySharedSocketDoesNotBlockFastConsumer() throws Exception {
+        String topicName = newTopicName();
+        int fastMessageCount = 1024;
+        // Exceed the socket receive window without reaching the dispatcher's replay look-ahead limit.
+        int slowMessagesPerFastMessage = 4;
+        int slowMessageCount = fastMessageCount * slowMessagesPerFastMessage;
+        try (PulsarClient slowClient = newPulsarClient();
+             PulsarClient fastClient = newPulsarClient();
+             Consumer<byte[]> slow = slowClient.newConsumer(Schema.BYTES).topic(topicName)
+                     .subscriptionName("sub").consumerName("slow").subscriptionType(SubscriptionType.Key_Shared)
+                     .receiverQueueSize(32768).subscribe();
+             Consumer<byte[]> fast = fastClient.newConsumer(Schema.BYTES).topic(topicName)
+                     .subscriptionName("sub").consumerName("fast").subscriptionType(SubscriptionType.Key_Shared)
+                     .receiverQueueSize(32768).subscribe();
+             Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES).topic(topicName)
+                     .enableBatching(false).maxPendingMessages(256).blockIfQueueFull(true).create()) {
+            PersistentTopic topic = (PersistentTopic) getTopic(topicName, false).join().orElseThrow();
+            PersistentStickyKeyDispatcherMultipleConsumers dispatcher =
+                    (PersistentStickyKeyDispatcherMultipleConsumers) topic.getSubscription("sub").getDispatcher();
+            var slowBrokerConsumer = dispatcher.getConsumers().stream()
+                    .filter(c -> c.consumerName().equals("slow")).findFirst().orElseThrow();
+            var fastBrokerConsumer = dispatcher.getConsumers().stream()
+                    .filter(c -> c.consumerName().equals("fast")).findFirst().orElseThrow();
+            Awaitility.await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> {
+                assertThat(slowBrokerConsumer.getAvailablePermits()).isEqualTo(32768);
+                assertThat(fastBrokerConsumer.getAvailablePermits()).isEqualTo(32768);
+            });
+            String slowKey = keyForConsumer(dispatcher, "slow");
+            String fastKey = keyForConsumer(dispatcher, "fast");
+            Channel brokerChannel = ((ServerCnx) slowBrokerConsumer.cnx()).ctx().channel();
+            Channel clientChannel = ((ConsumerImpl<byte[]>) slow).getClientCnx().ctx().channel();
+            clientChannel.eventLoop().submit(() -> clientChannel.config().setAutoRead(false)).sync();
+            try {
+                List<CompletableFuture<MessageId>> sends = new ArrayList<>();
+                for (int i = 0; i < slowMessageCount; i++) {
+                    byte[] payload = new byte[16384];
+                    ByteBuffer.wrap(payload).putInt(i);
+                    sends.add(producer.newMessage().key(slowKey).value(payload).sendAsync());
+                    // Spread healthy messages across the slow stream so some remain when its socket fills.
+                    // Keep healthy batches below the watermark: writable events must not hide the barrier.
+                    if (i % slowMessagesPerFastMessage == 0) {
+                        byte[] fastPayload = new byte[128];
+                        ByteBuffer.wrap(fastPayload).putInt(i / slowMessagesPerFastMessage);
+                        sends.add(producer.newMessage().key(fastKey).value(fastPayload).sendAsync());
+                    }
+                }
+                CompletableFuture.allOf(sends.toArray(CompletableFuture[]::new)).get(15, TimeUnit.SECONDS);
+                Awaitility.await().atMost(Duration.ofSeconds(10)).until(() -> !brokerChannel.isWritable());
+                List<Message<byte[]>> fastMessages = new ArrayList<>();
+                // No FLOW or ACK from either consumer: a slow socket must not stop the other hash range.
+                for (int i = 0; i < fastMessageCount; i++) {
+                    Message<byte[]> message = fast.receive(3, TimeUnit.SECONDS);
+                    assertThat(message).as("fast message %s while the slow socket is paused", i).isNotNull();
+                    assertThat(ByteBuffer.wrap(message.getData()).getInt()).isEqualTo(i);
+                    fastMessages.add(message);
+                }
+                clientChannel.eventLoop().submit(() -> clientChannel.config().setAutoRead(true)).sync();
+                for (int i = 0; i < slowMessageCount; i++) {
+                    Message<byte[]> message = slow.receive(5, TimeUnit.SECONDS);
+                    assertThat(message).as("slow message %s after resume", i).isNotNull();
+                    assertThat(ByteBuffer.wrap(message.getData()).getInt()).isEqualTo(i);
+                    slow.acknowledge(message);
+                }
+                for (Message<byte[]> message : fastMessages) {
+                    fast.acknowledge(message);
+                }
+            } finally {
+                clientChannel.eventLoop().submit(() -> clientChannel.config().setAutoRead(true)).sync();
+            }
+        }
+    }
+
+    private static String keyForConsumer(PersistentStickyKeyDispatcherMultipleConsumers dispatcher, String name) {
+        for (int i = 0; i < 10000; i++) {
+            String key = "key-" + i;
+            if (dispatcher.getSelector().select(key.getBytes(UTF_8)).consumerName().equals(name)) {
+                return key;
+            }
+        }
+        throw new AssertionError("No key found for " + name);
+    }
+
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
index 9cfb5271dbea..7d65c17e1b4e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
@@ -45,6 +45,8 @@ import io.netty.buffer.Unpooled;
 import io.netty.channel.EventLoopGroup;
 import io.netty.util.concurrent.EventExecutor;
 import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.ImmediateEventExecutor;
+import io.netty.util.concurrent.Promise;
 import io.netty.util.concurrent.SucceededFuture;
 import java.lang.reflect.Field;
 import java.util.ArrayDeque;
@@ -648,6 +650,48 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest {
                 .containsExactly(PositionFactory.create(1, 1));
     }
 
+    @Test(timeOut = 10000)
+    public void testSlowWriteDoesNotBlockOtherConsumers() throws Exception {
+        succeededFuture = ImmediateEventExecutor.INSTANCE.newSucceededFuture(null);
+        EventLoopGroup brokerExecutor = brokerMock.executor();
+        ArrayDeque<Runnable> readyTasks = new ArrayDeque<>();
+        doAnswer(inv -> {
+            readyTasks.add(inv.getArgument(0, Runnable.class));
+            return null;
+        }).when(brokerExecutor).execute(any(Runnable.class));
+        Consumer slowConsumer = createMockConsumer();
+        doReturn("slow-consumer").when(slowConsumer).consumerName();
+        doReturn(1000).when(slowConsumer).getAvailablePermits();
+        doReturn(true).when(slowConsumer).isWritable();
+        Promise<Void> slowWrite = ImmediateEventExecutor.INSTANCE.newPromise();
+        doAnswer(inv -> {
+            List<Entry> entries = inv.getArgument(0);
+            entries.stream().filter(Objects::nonNull).forEach(Entry::release);
+            // The selected batch fills this socket, but its write promise remains incomplete.
+            doReturn(false).when(slowConsumer).isWritable();
+            return slowWrite;
+        }).when(slowConsumer).sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any());
+        persistentDispatcher.addConsumer(consumerMock).join();
+        persistentDispatcher.addConsumer(slowConsumer).join();
+        persistentDispatcher.totalAvailablePermits = 2000;
+        String fastKey = generateKeyForConsumer(persistentDispatcher.getSelector(), consumerMock);
+        String slowKey = generateKeyForConsumer(persistentDispatcher.getSelector(), slowConsumer);
+        try {
+            persistentDispatcher.readEntriesComplete(new ArrayList<>(List.of(
+                    createEntry(1, 1, "fast", 1, fastKey), createEntry(1, 2, "slow", 2, slowKey))),
+                    PersistentDispatcherMultipleConsumers.ReadType.Normal);
+            verify(consumerMock).sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any());
+            verify(slowConsumer).sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any());
+            assertThat(slowWrite.isDone()).isFalse();
+            // Another bounded read can serve the writable consumer without waiting for the slow socket.
+            assertThat(readyTasks).hasSize(1);
+            readyTasks.remove().run();
+            verify(cursorMock).asyncReadEntriesWithSkipOrWait(anyInt(), anyLong(), any(), any(), any(), any());
+        } finally {
+            slowWrite.trySuccess(null);
+        }
+    }
+
     @Test(timeOut = 10000)
     public void testOutOfOrderReplayFilterIncludesPositionWithoutStickyKeyHash() {
         persistentDispatcher.close();