You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/09/17 14:55:13 UTC
(pulsar) 01/01: [fix][broker] Prevent slow Key_Shared sockets from stalling other consumers
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch lh-fix-key-shared-write-progress
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit afd0f52d3ac340fb3605a003084953ad2c413949
Author: Lari Hotari <lh...@apache.org>
AuthorDate: Thu Sep 17 13:31:15 2026 +0300
[fix][broker] Prevent slow Key_Shared sockets from stalling other consumers
Assisted-by: Codex
---
...istentStickyKeyDispatcherMultipleConsumers.java | 8 +-
.../persistent/KeySharedSlowConsumerTest.java | 129 +++++++++++++++++++++
...ntStickyKeyDispatcherMultipleConsumersTest.java | 44 +++++++
3 files changed, 176 insertions(+), 5 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
index ec34fcd92fca..b8acced90c9a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
@@ -30,7 +30,6 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
import lombok.Getter;
import org.apache.bookkeeper.mledger.Entry;
@@ -314,7 +313,6 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi
final Map<Consumer, List<Entry>> entriesByConsumerForDispatching =
filterAndGroupEntriesForDispatching(entries, readType, triggerLookAhead);
- AtomicInteger remainingConsumersToFinishSending = new AtomicInteger(entriesByConsumerForDispatching.size());
for (Map.Entry<Consumer, List<Entry>> current : entriesByConsumerForDispatching.entrySet()) {
Consumer consumer = current.getKey();
List<Entry> entriesForConsumer = current.getValue();
@@ -340,9 +338,9 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi
sendMessageInfo.getTotalMessages(),
sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(),
getRedeliveryTracker()).addListener(future -> {
- if (future.isDone() && remainingConsumersToFinishSending.decrementAndGet() == 0) {
- readMoreEntriesAsync();
- }
+ // One blocked socket must not hold up consumers whose writes have completed.
+ // The conflated read loop rechecks writability and permits before selecting a consumer.
+ readMoreEntriesAsync();
});
TOTAL_AVAILABLE_PERMITS_UPDATER.getAndAdd(this,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/KeySharedSlowConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/KeySharedSlowConsumerTest.java
new file mode 100644
index 000000000000..3ff64d7f6b07
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/KeySharedSlowConsumerTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.persistent;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
+import io.netty.channel.Channel;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import org.apache.pulsar.broker.service.ServerCnx;
+import org.apache.pulsar.broker.service.SharedPulsarBaseTest;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.ConsumerImpl;
+import org.awaitility.Awaitility;
+import org.testng.annotations.Test;
+
+/** Verifies that a paused Key_Shared socket does not prevent other consumers from making progress. */
+@Test(groups = "broker-api")
+public class KeySharedSlowConsumerTest extends SharedPulsarBaseTest {
+ @Test(timeOut = 120000)
+ public void testSlowKeySharedSocketDoesNotBlockFastConsumer() throws Exception {
+ String topicName = newTopicName();
+ int fastMessageCount = 1024;
+ // Exceed the socket receive window without reaching the dispatcher's replay look-ahead limit.
+ int slowMessagesPerFastMessage = 4;
+ int slowMessageCount = fastMessageCount * slowMessagesPerFastMessage;
+ try (PulsarClient slowClient = newPulsarClient();
+ PulsarClient fastClient = newPulsarClient();
+ Consumer<byte[]> slow = slowClient.newConsumer(Schema.BYTES).topic(topicName)
+ .subscriptionName("sub").consumerName("slow").subscriptionType(SubscriptionType.Key_Shared)
+ .receiverQueueSize(32768).subscribe();
+ Consumer<byte[]> fast = fastClient.newConsumer(Schema.BYTES).topic(topicName)
+ .subscriptionName("sub").consumerName("fast").subscriptionType(SubscriptionType.Key_Shared)
+ .receiverQueueSize(32768).subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES).topic(topicName)
+ .enableBatching(false).maxPendingMessages(256).blockIfQueueFull(true).create()) {
+ PersistentTopic topic = (PersistentTopic) getTopic(topicName, false).join().orElseThrow();
+ PersistentStickyKeyDispatcherMultipleConsumers dispatcher =
+ (PersistentStickyKeyDispatcherMultipleConsumers) topic.getSubscription("sub").getDispatcher();
+ var slowBrokerConsumer = dispatcher.getConsumers().stream()
+ .filter(c -> c.consumerName().equals("slow")).findFirst().orElseThrow();
+ var fastBrokerConsumer = dispatcher.getConsumers().stream()
+ .filter(c -> c.consumerName().equals("fast")).findFirst().orElseThrow();
+ Awaitility.await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> {
+ assertThat(slowBrokerConsumer.getAvailablePermits()).isEqualTo(32768);
+ assertThat(fastBrokerConsumer.getAvailablePermits()).isEqualTo(32768);
+ });
+ String slowKey = keyForConsumer(dispatcher, "slow");
+ String fastKey = keyForConsumer(dispatcher, "fast");
+ Channel brokerChannel = ((ServerCnx) slowBrokerConsumer.cnx()).ctx().channel();
+ Channel clientChannel = ((ConsumerImpl<byte[]>) slow).getClientCnx().ctx().channel();
+ clientChannel.eventLoop().submit(() -> clientChannel.config().setAutoRead(false)).sync();
+ try {
+ List<CompletableFuture<MessageId>> sends = new ArrayList<>();
+ for (int i = 0; i < slowMessageCount; i++) {
+ byte[] payload = new byte[16384];
+ ByteBuffer.wrap(payload).putInt(i);
+ sends.add(producer.newMessage().key(slowKey).value(payload).sendAsync());
+ // Spread healthy messages across the slow stream so some remain when its socket fills.
+ // Keep healthy batches below the watermark: writable events must not hide the barrier.
+ if (i % slowMessagesPerFastMessage == 0) {
+ byte[] fastPayload = new byte[128];
+ ByteBuffer.wrap(fastPayload).putInt(i / slowMessagesPerFastMessage);
+ sends.add(producer.newMessage().key(fastKey).value(fastPayload).sendAsync());
+ }
+ }
+ CompletableFuture.allOf(sends.toArray(CompletableFuture[]::new)).get(15, TimeUnit.SECONDS);
+ Awaitility.await().atMost(Duration.ofSeconds(10)).until(() -> !brokerChannel.isWritable());
+ List<Message<byte[]>> fastMessages = new ArrayList<>();
+ // No FLOW or ACK from either consumer: a slow socket must not stop the other hash range.
+ for (int i = 0; i < fastMessageCount; i++) {
+ Message<byte[]> message = fast.receive(3, TimeUnit.SECONDS);
+ assertThat(message).as("fast message %s while the slow socket is paused", i).isNotNull();
+ assertThat(ByteBuffer.wrap(message.getData()).getInt()).isEqualTo(i);
+ fastMessages.add(message);
+ }
+ clientChannel.eventLoop().submit(() -> clientChannel.config().setAutoRead(true)).sync();
+ for (int i = 0; i < slowMessageCount; i++) {
+ Message<byte[]> message = slow.receive(5, TimeUnit.SECONDS);
+ assertThat(message).as("slow message %s after resume", i).isNotNull();
+ assertThat(ByteBuffer.wrap(message.getData()).getInt()).isEqualTo(i);
+ slow.acknowledge(message);
+ }
+ for (Message<byte[]> message : fastMessages) {
+ fast.acknowledge(message);
+ }
+ } finally {
+ clientChannel.eventLoop().submit(() -> clientChannel.config().setAutoRead(true)).sync();
+ }
+ }
+ }
+
+ private static String keyForConsumer(PersistentStickyKeyDispatcherMultipleConsumers dispatcher, String name) {
+ for (int i = 0; i < 10000; i++) {
+ String key = "key-" + i;
+ if (dispatcher.getSelector().select(key.getBytes(UTF_8)).consumerName().equals(name)) {
+ return key;
+ }
+ }
+ throw new AssertionError("No key found for " + name);
+ }
+
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
index 9cfb5271dbea..7d65c17e1b4e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
@@ -45,6 +45,8 @@ import io.netty.buffer.Unpooled;
import io.netty.channel.EventLoopGroup;
import io.netty.util.concurrent.EventExecutor;
import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.ImmediateEventExecutor;
+import io.netty.util.concurrent.Promise;
import io.netty.util.concurrent.SucceededFuture;
import java.lang.reflect.Field;
import java.util.ArrayDeque;
@@ -648,6 +650,48 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest {
.containsExactly(PositionFactory.create(1, 1));
}
+ @Test(timeOut = 10000)
+ public void testSlowWriteDoesNotBlockOtherConsumers() throws Exception {
+ succeededFuture = ImmediateEventExecutor.INSTANCE.newSucceededFuture(null);
+ EventLoopGroup brokerExecutor = brokerMock.executor();
+ ArrayDeque<Runnable> readyTasks = new ArrayDeque<>();
+ doAnswer(inv -> {
+ readyTasks.add(inv.getArgument(0, Runnable.class));
+ return null;
+ }).when(brokerExecutor).execute(any(Runnable.class));
+ Consumer slowConsumer = createMockConsumer();
+ doReturn("slow-consumer").when(slowConsumer).consumerName();
+ doReturn(1000).when(slowConsumer).getAvailablePermits();
+ doReturn(true).when(slowConsumer).isWritable();
+ Promise<Void> slowWrite = ImmediateEventExecutor.INSTANCE.newPromise();
+ doAnswer(inv -> {
+ List<Entry> entries = inv.getArgument(0);
+ entries.stream().filter(Objects::nonNull).forEach(Entry::release);
+ // The selected batch fills this socket, but its write promise remains incomplete.
+ doReturn(false).when(slowConsumer).isWritable();
+ return slowWrite;
+ }).when(slowConsumer).sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any());
+ persistentDispatcher.addConsumer(consumerMock).join();
+ persistentDispatcher.addConsumer(slowConsumer).join();
+ persistentDispatcher.totalAvailablePermits = 2000;
+ String fastKey = generateKeyForConsumer(persistentDispatcher.getSelector(), consumerMock);
+ String slowKey = generateKeyForConsumer(persistentDispatcher.getSelector(), slowConsumer);
+ try {
+ persistentDispatcher.readEntriesComplete(new ArrayList<>(List.of(
+ createEntry(1, 1, "fast", 1, fastKey), createEntry(1, 2, "slow", 2, slowKey))),
+ PersistentDispatcherMultipleConsumers.ReadType.Normal);
+ verify(consumerMock).sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any());
+ verify(slowConsumer).sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any());
+ assertThat(slowWrite.isDone()).isFalse();
+ // Another bounded read can serve the writable consumer without waiting for the slow socket.
+ assertThat(readyTasks).hasSize(1);
+ readyTasks.remove().run();
+ verify(cursorMock).asyncReadEntriesWithSkipOrWait(anyInt(), anyLong(), any(), any(), any(), any());
+ } finally {
+ slowWrite.trySuccess(null);
+ }
+ }
+
@Test(timeOut = 10000)
public void testOutOfOrderReplayFilterIncludesPositionWithoutStickyKeyHash() {
persistentDispatcher.close();