You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/02 14:34:15 UTC
(pulsar) branch branch-4.2 updated (6d479841d9ed -> aef9f3aa70f3)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a change to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git
from 6d479841d9ed [improve][build][branch-4.2] Bump version to 4.2.6-SNAPSHOT
new bde53f874431 [fix][broker] Fix Exclusive/Failover dispatch rate limiting using counts from another dispatch (#26810)
new aef9f3aa70f3 [fix][broker] Fix dispatcher permit accounting race with recycled batch index acks (#26809)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../PersistentDispatcherMultipleConsumers.java | 11 +-
...rsistentDispatcherMultipleConsumersClassic.java | 11 +-
.../PersistentDispatcherSingleActiveConsumer.java | 12 ++-
...istentStickyKeyDispatcherMultipleConsumers.java | 4 +-
...tickyKeyDispatcherMultipleConsumersClassic.java | 4 +-
...rsistentDispatcherSingleActiveConsumerTest.java | 54 ++++++++++
.../SharedDispatcherPermitAccountingTest.java | 116 +++++++++++++++++++--
7 files changed, 191 insertions(+), 21 deletions(-)
(pulsar) 02/02: [fix][broker] Fix dispatcher permit accounting race with recycled batch index acks (#26809)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit aef9f3aa70f3a394eb431ab54548ced1c8507d95
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Thu Oct 1 22:34:14 2026 -0700
[fix][broker] Fix dispatcher permit accounting race with recycled batch index acks (#26809)
(cherry picked from commit 1046481c970067cbca9c80a25a98257b9179a3da)
[branch-4.x] Also applied to the classic Shared and Key_Shared dispatchers
(PersistentDispatcherMultipleConsumersClassic, PersistentStickyKeyDispatcherMultipleConsumersClassic),
which were removed on master by #26687. The test covers both implementations.
---
.../PersistentDispatcherMultipleConsumers.java | 11 +-
...rsistentDispatcherMultipleConsumersClassic.java | 11 +-
...istentStickyKeyDispatcherMultipleConsumers.java | 4 +-
...tickyKeyDispatcherMultipleConsumersClassic.java | 4 +-
.../SharedDispatcherPermitAccountingTest.java | 116 +++++++++++++++++++--
5 files changed, 130 insertions(+), 16 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 009af8299543..e79f7d1bf29e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -888,6 +888,8 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
entriesForThisConsumer, batchSizes, sendMessageInfo, batchIndexesAcks, cursor,
readType == ReadType.Replay, c);
totalEntriesProcessed += entriesForThisConsumer.size();
+ // read before sendMessages: it hands batchIndexesAcks to the consumer's event loop, which recycles it
+ int totalAckedIndexCount = batchIndexesAcks.getTotalAckedIndexCount();
c.sendMessages(entriesForThisConsumer, batchSizes, batchIndexesAcks, sendMessageInfo.getTotalMessages(),
sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(), redeliveryTracker);
@@ -896,12 +898,11 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
remainingMessages -= msgSent;
start += maxEntriesInThisBatch;
entriesToDispatch -= maxEntriesInThisBatch;
- TOTAL_AVAILABLE_PERMITS_UPDATER.addAndGet(this,
- -(msgSent - batchIndexesAcks.getTotalAckedIndexCount()));
+ TOTAL_AVAILABLE_PERMITS_UPDATER.addAndGet(this, -(msgSent - totalAckedIndexCount));
if (log.isDebugEnabled()) {
log.debug("[{}] Added -({} minus {}) permits to TOTAL_AVAILABLE_PERMITS_UPDATER in "
+ "PersistentDispatcherMultipleConsumers",
- name, msgSent, batchIndexesAcks.getTotalAckedIndexCount());
+ name, msgSent, totalAckedIndexCount);
}
totalMessagesSent += sendMessageInfo.getTotalMessages();
totalBytesSent += sendMessageInfo.getTotalBytes();
@@ -997,6 +998,8 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
totalEntries += filterEntriesForConsumer(entryAndMetadataList, batchSizes, sendMessageInfo,
batchIndexesAcks, cursor, readType == ReadType.Replay, consumer);
totalEntriesProcessed += entryAndMetadataList.size();
+ // read before sendMessages: it hands batchIndexesAcks to the consumer's event loop, which recycles it
+ final int totalAckedIndexCount = batchIndexesAcks.getTotalAckedIndexCount();
consumer.sendMessages(entryAndMetadataList, batchSizes, batchIndexesAcks,
sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes(),
sendMessageInfo.getTotalChunkedMessages(), getRedeliveryTracker()
@@ -1007,7 +1010,7 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
});
TOTAL_AVAILABLE_PERMITS_UPDATER.getAndAdd(this,
- -(sendMessageInfo.getTotalMessages() - batchIndexesAcks.getTotalAckedIndexCount()));
+ -(sendMessageInfo.getTotalMessages() - totalAckedIndexCount));
totalMessagesSent += sendMessageInfo.getTotalMessages();
totalBytesSent += sendMessageInfo.getTotalBytes();
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
index a0bb11bdbb08..9740df2262a1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
@@ -777,6 +777,8 @@ public class PersistentDispatcherMultipleConsumersClassic extends AbstractPersis
totalEntries += filterEntriesForConsumer(metadataArray, start,
entriesForThisConsumer, batchSizes, sendMessageInfo, batchIndexesAcks, cursor,
readType == ReadType.Replay, c);
+ // read before sendMessages: it hands batchIndexesAcks to the consumer's event loop, which recycles it
+ int totalAckedIndexCount = batchIndexesAcks.getTotalAckedIndexCount();
c.sendMessages(entriesForThisConsumer, batchSizes, batchIndexesAcks, sendMessageInfo.getTotalMessages(),
sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(), redeliveryTracker);
@@ -785,12 +787,11 @@ public class PersistentDispatcherMultipleConsumersClassic extends AbstractPersis
remainingMessages -= msgSent;
start += messagesForC;
entriesToDispatch -= messagesForC;
- TOTAL_AVAILABLE_PERMITS_UPDATER.addAndGet(this,
- -(msgSent - batchIndexesAcks.getTotalAckedIndexCount()));
+ TOTAL_AVAILABLE_PERMITS_UPDATER.addAndGet(this, -(msgSent - totalAckedIndexCount));
if (log.isDebugEnabled()) {
log.debug("[{}] Added -({} minus {}) permits to TOTAL_AVAILABLE_PERMITS_UPDATER in "
+ "PersistentDispatcherMultipleConsumers",
- name, msgSent, batchIndexesAcks.getTotalAckedIndexCount());
+ name, msgSent, totalAckedIndexCount);
}
totalMessagesSent += sendMessageInfo.getTotalMessages();
totalBytesSent += sendMessageInfo.getTotalBytes();
@@ -855,6 +856,8 @@ public class PersistentDispatcherMultipleConsumersClassic extends AbstractPersis
totalEntries += filterEntriesForConsumer(entryAndMetadataList, batchSizes, sendMessageInfo,
batchIndexesAcks, cursor, readType == ReadType.Replay, consumer);
+ // read before sendMessages: it hands batchIndexesAcks to the consumer's event loop, which recycles it
+ final int totalAckedIndexCount = batchIndexesAcks.getTotalAckedIndexCount();
consumer.sendMessages(entryAndMetadataList, batchSizes, batchIndexesAcks,
sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes(),
sendMessageInfo.getTotalChunkedMessages(), getRedeliveryTracker()
@@ -865,7 +868,7 @@ public class PersistentDispatcherMultipleConsumersClassic extends AbstractPersis
});
TOTAL_AVAILABLE_PERMITS_UPDATER.getAndAdd(this,
- -(sendMessageInfo.getTotalMessages() - batchIndexesAcks.getTotalAckedIndexCount()));
+ -(sendMessageInfo.getTotalMessages() - totalAckedIndexCount));
totalMessagesSent += sendMessageInfo.getTotalMessages();
totalBytesSent += sendMessageInfo.getTotalBytes();
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
index 1bcdd39736c1..486bbbbf31a4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java
@@ -293,6 +293,8 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi
totalEntries += filterEntriesForConsumer(entriesForConsumer, batchSizes, sendMessageInfo,
batchIndexesAcks, cursor, readType == ReadType.Replay, consumer);
totalEntriesProcessed += entriesForConsumer.size();
+ // read before sendMessages: it hands batchIndexesAcks to the consumer's event loop, which recycles it
+ int totalAckedIndexCount = batchIndexesAcks.getTotalAckedIndexCount();
consumer.sendMessages(entriesForConsumer, batchSizes, batchIndexesAcks,
sendMessageInfo.getTotalMessages(),
sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(),
@@ -303,7 +305,7 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi
});
TOTAL_AVAILABLE_PERMITS_UPDATER.getAndAdd(this,
- -(sendMessageInfo.getTotalMessages() - batchIndexesAcks.getTotalAckedIndexCount()));
+ -(sendMessageInfo.getTotalMessages() - totalAckedIndexCount));
totalMessagesSent += sendMessageInfo.getTotalMessages();
totalBytesSent += sendMessageInfo.getTotalBytes();
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java
index e957ce07c5ef..88ef86f2f6d5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java
@@ -381,6 +381,8 @@ public class PersistentStickyKeyDispatcherMultipleConsumersClassic
EntryBatchIndexesAcks batchIndexesAcks = EntryBatchIndexesAcks.get(messagesForC);
totalEntries += filterEntriesForConsumer(entriesWithSameKey, batchSizes, sendMessageInfo,
batchIndexesAcks, cursor, readType == ReadType.Replay, consumer);
+ // read before sendMessages: it hands batchIndexesAcks to the consumer's event loop, which recycles it
+ int totalAckedIndexCount = batchIndexesAcks.getTotalAckedIndexCount();
consumer.sendMessages(entriesWithSameKey, batchSizes, batchIndexesAcks,
sendMessageInfo.getTotalMessages(),
sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(),
@@ -391,7 +393,7 @@ public class PersistentStickyKeyDispatcherMultipleConsumersClassic
});
TOTAL_AVAILABLE_PERMITS_UPDATER.getAndAdd(this,
- -(sendMessageInfo.getTotalMessages() - batchIndexesAcks.getTotalAckedIndexCount()));
+ -(sendMessageInfo.getTotalMessages() - totalAckedIndexCount));
totalMessagesSent += sendMessageInfo.getTotalMessages();
totalBytesSent += sendMessageInfo.getTotalBytes();
} else {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SharedDispatcherPermitAccountingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SharedDispatcherPermitAccountingTest.java
index 36f7bf900112..82586ef327b3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SharedDispatcherPermitAccountingTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/SharedDispatcherPermitAccountingTest.java
@@ -38,16 +38,25 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.Unpooled;
import io.netty.channel.EventLoopGroup;
import io.netty.util.concurrent.EventExecutor;
import io.netty.util.concurrent.ImmediateEventExecutor;
import java.time.Duration;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.List;
+import java.util.Objects;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.bookkeeper.mledger.Entry;
import org.apache.bookkeeper.mledger.ManagedCursor;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.apache.bookkeeper.mledger.impl.AckSetStateUtil;
+import org.apache.bookkeeper.mledger.impl.EntryImpl;
import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.broker.service.Consumer;
@@ -69,7 +78,9 @@ import org.apache.pulsar.client.api.Range;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
import org.apache.pulsar.common.api.proto.KeySharedMeta;
+import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl;
+import org.apache.pulsar.common.protocol.Commands;
import org.awaitility.Awaitility;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -102,6 +113,19 @@ public class SharedDispatcherPermitAccountingTest extends SharedPulsarBaseTest {
};
}
+ @DataProvider(name = "partiallyAckedBatchDispatchPaths")
+ public Object[][] partiallyAckedBatchDispatchPaths() {
+ return new Object[][] {
+ {false, Shared, false},
+ {true, Shared, false},
+ // A chunk in the read routes the whole read through the Shared chunked-message dispatch path
+ {false, Shared, true},
+ {true, Shared, true},
+ {false, Key_Shared, false},
+ {true, Key_Shared, false}
+ };
+ }
+
@Test(dataProvider = "untrackedSubscriptionVariants", timeOut = 30_000)
public void testUnrelatedSubscriptionDoesNotAccumulatePendingDispatcherPermits(
boolean persistent, SubType subType) throws Exception {
@@ -409,19 +433,78 @@ public class SharedDispatcherPermitAccountingTest extends SharedPulsarBaseTest {
}
}
+ @Test(dataProvider = "partiallyAckedBatchDispatchPaths", timeOut = 30_000)
+ public void testRedeliveredPartiallyAckedBatchDoesNotLosePermits(boolean classic, SubType subType,
+ boolean readHasChunk) throws Exception {
+ int batchSize = 10;
+ int ackedIndexes = 4;
+ int permitsPerConsumer = 100;
+ String topicName = newTopicName();
+ admin.topics().createNonPartitionedTopic(topicName);
+ PersistentTopic topic = (PersistentTopic) getTopic(topicName, false).join().orElseThrow();
+ byte[] batch = serializeEntry(new MessageMetadata().setNumMessagesInBatch(batchSize));
+ byte[] chunk = serializeEntry(new MessageMetadata()
+ .setUuid("chunked-message")
+ .setChunkId(0)
+ .setNumChunksFromMsg(2));
+ Position batchPosition = topic.getManagedLedger().addEntry(batch);
+ Position chunkPosition = topic.getManagedLedger().addEntry(chunk);
+
+ // Ack the first batch indexes, as a consumer did before the batch was redelivered.
+ BitSet unackedIndexes = new BitSet(batchSize);
+ unackedIndexes.set(ackedIndexes, batchSize);
+ ManagedCursor cursor = topic.getManagedLedger().newNonDurableCursor(PositionFactory.EARLIEST);
+ cursor.delete(AckSetStateUtil.createPositionWithAckSet(batchPosition.getLedgerId(),
+ batchPosition.getEntryId(), unackedIndexes.toLongArray()));
+ cursor.close();
+ Dispatcher dispatcher = createTestContext(classic, subType, topic, cursor).dispatcher();
+ int initialPermits = permitsPerConsumer * dispatcher.getConsumers().size();
+ for (Consumer consumer : dispatcher.getConsumers()) {
+ consumer.flowPermits(permitsPerConsumer);
+ }
+ drainBrokerWorkerGroup(topic);
+ assertThat(totalAvailablePermits(dispatcher)).isEqualTo(initialPermits);
+
+ List<Entry> entries = new ArrayList<>();
+ entries.add(EntryImpl.create(batchPosition.getLedgerId(), batchPosition.getEntryId(), batch));
+ if (readHasChunk) {
+ entries.add(EntryImpl.create(chunkPosition.getLedgerId(), chunkPosition.getEntryId(), chunk));
+ }
+ // The command sender from createConsumer recycles batchIndexesAcks before sendMessages returns, like a
+ // consumer event loop that runs the send task before the dispatcher thread continues.
+ if (dispatcher instanceof PersistentDispatcherMultipleConsumers pip379Dispatcher) {
+ pip379Dispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Replay,
+ entries, true);
+ } else {
+ ((PersistentDispatcherMultipleConsumersClassic) dispatcher).sendMessagesToConsumers(
+ PersistentDispatcherMultipleConsumersClassic.ReadType.Replay, entries, true);
+ }
+
+ int deliveredMessages = batchSize - ackedIndexes + (readHasChunk ? 1 : 0);
+ assertThat(totalAvailablePermits(dispatcher)).isEqualTo(initialPermits - deliveredMessages);
+ assertThat(totalAvailablePermits(dispatcher)).isEqualTo(
+ dispatcher.getConsumers().stream().mapToInt(Consumer::getAvailablePermits).sum());
+ }
+
private TestContext createTestContext(boolean classic) throws Exception {
return createTestContext(classic, Shared);
}
private TestContext createTestContext(boolean classic, SubType subType) throws Exception {
String topicName = newTopicName();
- String subscriptionName = "shared-sub";
admin.topics().createNonPartitionedTopic(topicName);
PersistentTopic topic = (PersistentTopic) getTopic(topicName, false).join().get();
ManagedCursor cursor = mock(ManagedCursorImpl.class);
- when(cursor.getName()).thenReturn(subscriptionName);
+ when(cursor.getName()).thenReturn("shared-sub");
when(cursor.isClosed()).thenReturn(true);
+ return createTestContext(classic, subType, topic, cursor);
+ }
+
+ private TestContext createTestContext(boolean classic, SubType subType, PersistentTopic topic,
+ ManagedCursor cursor) {
+ String topicName = topic.getName();
+ String subscriptionName = cursor.getName();
Subscription subscription = mock(PersistentSubscription.class);
when(subscription.getName()).thenReturn(subscriptionName);
when(subscription.getTopic()).thenReturn(topic);
@@ -457,9 +540,20 @@ public class SharedDispatcherPermitAccountingTest extends SharedPulsarBaseTest {
when(cnx.isActive()).thenReturn(true);
when(cnx.isWritable()).thenReturn(true);
when(cnx.getCommandSender()).thenReturn(commandSender);
+ // Release and recycle what the consumer event loop would, before sendMessages returns.
when(commandSender.sendMessagesToConsumer(anyLong(), anyString(), any(), anyInt(), anyList(),
- any(EntryBatchSizes.class), any(EntryBatchIndexesAcks.class), any(RedeliveryTracker.class), anyLong()))
- .thenReturn(ImmediateEventExecutor.INSTANCE.newSucceededFuture(null));
+ any(EntryBatchSizes.class), any(), any(RedeliveryTracker.class), anyLong()))
+ .thenAnswer(invocation -> {
+ List<? extends Entry> entries = invocation.getArgument(4);
+ entries.stream().filter(Objects::nonNull).forEach(Entry::release);
+ EntryBatchSizes batchSizes = invocation.getArgument(5);
+ batchSizes.recyle();
+ EntryBatchIndexesAcks batchIndexesAcks = invocation.getArgument(6);
+ if (batchIndexesAcks != null) {
+ batchIndexesAcks.recycle();
+ }
+ return ImmediateEventExecutor.INSTANCE.newSucceededFuture(null);
+ });
return new Consumer(subscription, subType, topicName, consumerId, 0, "consumer-" + consumerId,
true, cnx, "role", emptyMap(), false, new KeySharedMeta().setKeySharedMode(AUTO_SPLIT),
MessageId.latest, DEFAULT_CONSUMER_EPOCH);
@@ -483,9 +577,19 @@ public class SharedDispatcherPermitAccountingTest extends SharedPulsarBaseTest {
consumer.sendMessages(new ArrayList<>(List.of(entry)), stickyKeyHashes, batchSizes, batchIndexesAcks,
permits, 0, 0, redeliveryTracker, DEFAULT_CONSUMER_EPOCH).syncUninterruptibly();
decrementTotalAvailablePermits(dispatcher, permits);
+ }
- batchSizes.recyle();
- batchIndexesAcks.recycle();
+ private static byte[] serializeEntry(MessageMetadata metadata) {
+ metadata.setProducerName("producer")
+ .setSequenceId(0)
+ .setPublishTime(System.currentTimeMillis());
+ ByteBuf entry = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, metadata,
+ Unpooled.EMPTY_BUFFER);
+ try {
+ return ByteBufUtil.getBytes(entry);
+ } finally {
+ entry.release();
+ }
}
private static void decrementTotalAvailablePermits(Dispatcher dispatcher, int permits) {
(pulsar) 01/02: [fix][broker] Fix Exclusive/Failover dispatch rate limiting using counts from another dispatch (#26810)
Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit bde53f874431a40fe703845b3c16dbd89c45005d
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Thu Oct 1 22:33:57 2026 -0700
[fix][broker] Fix Exclusive/Failover dispatch rate limiting using counts from another dispatch (#26810)
(cherry picked from commit 33dca0ed994718091659b254f4e2b4b0a652fd32)
---
.../PersistentDispatcherSingleActiveConsumer.java | 12 +++--
...rsistentDispatcherSingleActiveConsumerTest.java | 54 ++++++++++++++++++++++
2 files changed, 61 insertions(+), 5 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
index 5cc36df3f4b8..2eaa86109ba3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
@@ -241,14 +241,16 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher
protected void dispatchEntriesToConsumer(Consumer currentConsumer, List<Entry> entries,
EntryBatchSizes batchSizes, EntryBatchIndexesAcks batchIndexesAcks,
SendMessageInfo sendMessageInfo, long epoch) {
+ // sendMessageInfo is a thread-local instance that the next dispatch on this thread can overwrite before
+ // the write listener below runs on the connection's event loop, so capture the counts here.
+ final int totalMessages = sendMessageInfo.getTotalMessages();
+ final long totalBytes = sendMessageInfo.getTotalBytes();
currentConsumer
- .sendMessages(entries, batchSizes, batchIndexesAcks, sendMessageInfo.getTotalMessages(),
- sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(),
- redeliveryTracker, epoch)
+ .sendMessages(entries, batchSizes, batchIndexesAcks, totalMessages, totalBytes,
+ sendMessageInfo.getTotalChunkedMessages(), redeliveryTracker, epoch)
.addListener(future -> {
if (future.isSuccess()) {
- acquirePermitsForDeliveredMessages(topic, cursor, entries.size(),
- sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes());
+ acquirePermitsForDeliveredMessages(topic, cursor, entries.size(), totalMessages, totalBytes);
// Schedule a new read batch operation only after the previous batch has been written to the socket.
executor.execute(() -> readMoreEntries(getActiveConsumer()));
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java
index f24c1bb4d86c..7a205c3ae039 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java
@@ -18,6 +18,16 @@
*/
package org.apache.pulsar.broker.service.persistent;
+import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import io.netty.util.concurrent.ImmediateEventExecutor;
+import io.netty.util.concurrent.Promise;
+import java.util.List;
import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
@@ -28,16 +38,21 @@ import java.util.concurrent.atomic.AtomicReference;
import lombok.Cleanup;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
+import org.apache.bookkeeper.mledger.Entry;
import org.apache.bookkeeper.mledger.ManagedCursor;
import org.apache.bookkeeper.mledger.ManagedLedgerException;
import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
import org.apache.pulsar.broker.intercept.MockBrokerInterceptor;
import org.apache.pulsar.broker.service.BrokerServiceException;
import org.apache.pulsar.broker.service.Consumer;
+import org.apache.pulsar.broker.service.EntryBatchIndexesAcks;
+import org.apache.pulsar.broker.service.EntryBatchSizes;
+import org.apache.pulsar.broker.service.SendMessageInfo;
import org.apache.pulsar.broker.service.ServerCnx;
import org.apache.pulsar.broker.service.SharedPulsarBaseTest;
import org.apache.pulsar.broker.service.SharedPulsarCluster;
import org.apache.pulsar.broker.service.Subscription;
+import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
@@ -126,6 +141,45 @@ public class PersistentDispatcherSingleActiveConsumerTest extends SharedPulsarBa
admin.topics().delete(topicName, false);
}
+ @Test
+ public void testDispatchQuotaWhenSendMessageInfoReusedBeforeWriteCompletes() throws Exception {
+ String topicName = newTopicName();
+ String subscription = "s1";
+ admin.topics().createNonPartitionedTopic(topicName);
+ admin.topics().createSubscription(topicName, subscription, MessageId.earliest);
+ PersistentTopic topic = (PersistentTopic) getTopic(topicName, false).join().orElseThrow();
+ PersistentSubscription sub = topic.getSubscription(subscription);
+ DispatchRateLimiter rateLimiter = mock(DispatchRateLimiter.class);
+ PersistentDispatcherSingleActiveConsumer dispatcher =
+ new PersistentDispatcherSingleActiveConsumer(sub.getCursor(),
+ CommandSubscribe.SubType.Exclusive, 0, topic, sub) {
+ @Override
+ public Optional<DispatchRateLimiter> getRateLimiter() {
+ return Optional.of(rateLimiter);
+ }
+ };
+ // The write to the consumer connection stays pending until the test completes it.
+ Promise<Void> writePromise = ImmediateEventExecutor.INSTANCE.newPromise();
+ Consumer consumer = mock(Consumer.class);
+ when(consumer.sendMessages(any(), any(), any(), anyInt(), anyLong(), anyLong(), any(), anyLong()))
+ .thenReturn(writePromise);
+
+ SendMessageInfo sendMessageInfo = SendMessageInfo.getThreadLocal();
+ sendMessageInfo.setTotalMessages(3);
+ sendMessageInfo.setTotalBytes(300);
+ dispatcher.dispatchEntriesToConsumer(consumer, List.of(mock(Entry.class)), EntryBatchSizes.get(1),
+ EntryBatchIndexesAcks.get(1), sendMessageInfo, DEFAULT_CONSUMER_EPOCH);
+
+ // Another dispatch on this thread reuses the thread-local SendMessageInfo before the write completes.
+ SendMessageInfo nextSendMessageInfo = SendMessageInfo.getThreadLocal();
+ nextSendMessageInfo.setTotalMessages(7);
+ nextSendMessageInfo.setTotalBytes(700);
+ writePromise.setSuccess(null);
+
+ // The dispatch quota must be charged with the counts of the messages that were written.
+ verify(rateLimiter).consumeDispatchQuota(3, 300);
+ }
+
@DataProvider
public static Object[][] closeDelayMs() {
return new Object[][] { { 500 }, { 2000 } };