You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/09/23 12:36:54 UTC
(pulsar) 04/21: [improve][broker] Conflate concurrent readMoreEntries calls (#26539)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 6ab1fafdbd880291f71d85653881041065e5b489
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Sep 11 19:18:35 2026 +0300
[improve][broker] Conflate concurrent readMoreEntries calls (#26539)
(cherry picked from commit d5dec18cba9908318f9dfb0d097fef6e1832087b)
---
...tractPersistentDispatcherMultipleConsumers.java | 57 ++++++++
.../PersistentDispatcherMultipleConsumers.java | 5 +-
...rsistentDispatcherMultipleConsumersClassic.java | 3 +-
.../PersistentDispatcherMultipleConsumersTest.java | 162 +++++++++++++++++++++
...ntStickyKeyDispatcherMultipleConsumersTest.java | 4 +-
5 files changed, 226 insertions(+), 5 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java
index 3a63dc659470..a0c0c0358646 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.broker.service.persistent;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
import org.apache.bookkeeper.mledger.Entry;
import org.apache.bookkeeper.mledger.ManagedCursor;
@@ -32,6 +33,11 @@ import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
public abstract class AbstractPersistentDispatcherMultipleConsumers extends AbstractDispatcherMultipleConsumers
implements Dispatcher, AsyncCallbacks.ReadEntriesCallback {
+ private static final int IDLE = 0;
+ private static final int RUNNING = 1;
+ private static final int RUNNING_REQUESTED = 2;
+ private final AtomicInteger readMoreEntriesState = new AtomicInteger(IDLE);
+
public AbstractPersistentDispatcherMultipleConsumers(Subscription subscription,
ServiceConfiguration serviceConfig) {
super(subscription, serviceConfig);
@@ -41,6 +47,57 @@ public abstract class AbstractPersistentDispatcherMultipleConsumers extends Abst
public abstract void readMoreEntriesAsync();
+ /**
+ * Conflate concurrent and reentrant requests into follow-up passes driven by the owning caller.
+ * If a pass fails, drain registered follow-ups before propagating the first failure to the owner.
+ */
+ public final void readMoreEntries() {
+ for (;;) {
+ int state = readMoreEntriesState.get();
+ if (state == IDLE) {
+ if (readMoreEntriesState.compareAndSet(IDLE, RUNNING)) {
+ break;
+ }
+ } else if (state == RUNNING) {
+ if (readMoreEntriesState.compareAndSet(RUNNING, RUNNING_REQUESTED)) {
+ return;
+ }
+ } else {
+ // A follow-up pass is already requested.
+ return;
+ }
+ }
+ Throwable failure = null;
+ for (;;) {
+ // Requests received before this pass are covered by this pass.
+ readMoreEntriesState.set(RUNNING);
+ try {
+ internalReadMoreEntries();
+ } catch (RuntimeException | Error t) {
+ // Keep ownership so a failure cannot discard an already-registered follow-up.
+ if (failure == null) {
+ failure = t;
+ } else if (failure != t) {
+ failure.addSuppressed(t);
+ }
+ }
+ if (readMoreEntriesState.compareAndSet(RUNNING, IDLE)) {
+ break;
+ }
+ // A request arrived during the pass. Run another pass without recursion, even after a failure.
+ }
+ if (failure instanceof RuntimeException runtimeException) {
+ throw runtimeException;
+ } else if (failure instanceof Error error) {
+ throw error;
+ }
+ }
+
+ /**
+ * Execute one read pass. Implementations must synchronize on this dispatcher to protect read state.
+ */
+ protected abstract void internalReadMoreEntries();
+
public abstract String getName();
public abstract boolean isBlockedDispatcherOnUnackedMsgs();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index fe9d59d0e8b3..86f477ac7ce3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -345,7 +345,8 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
}
}
- public synchronized void readMoreEntries() {
+ @Override
+ protected synchronized void internalReadMoreEntries() {
if (cursor.isClosed()) {
if (log.isDebugEnabled()) {
log.debug("[{}] Cursor is already closed, skipping read more entries.", cursor.getName());
@@ -373,7 +374,7 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
return;
}
- // increment the counter for readMoreEntries calls, to track the number of times readMoreEntries is called
+ // Count executed passes, not conflated requests, for pending-read rescheduling.
readMoreEntriesCallCount++;
// totalAvailablePermits may be updated by other threads
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
index 5745568af833..35407214aefa 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
@@ -314,7 +314,8 @@ public class PersistentDispatcherMultipleConsumersClassic extends AbstractPersis
topic.getBrokerService().executor().execute(this::readMoreEntries);
}
- public synchronized void readMoreEntries() {
+ @Override
+ protected synchronized void internalReadMoreEntries() {
if (cursor.isClosed()) {
if (log.isDebugEnabled()) {
log.debug("[{}] Cursor is already closed, skipping read more entries.", cursor.getName());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java
index fdb9fa9109a3..1da9cc10987c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java
@@ -18,6 +18,8 @@
*/
package org.apache.pulsar.broker.service.persistent;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import com.carrotsearch.hppc.ObjectSet;
import java.util.List;
import java.util.Optional;
@@ -25,6 +27,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -49,6 +52,7 @@ import org.mockito.Mockito;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@Slf4j
@@ -68,6 +72,164 @@ public class PersistentDispatcherMultipleConsumersTest extends ProducerConsumerB
super.internalCleanup();
}
+ @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+ public void testReadMoreEntriesConflatesConcurrentRequests(boolean classic) throws Exception {
+ ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+ AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+ CountDownLatch[] passStarted = {new CountDownLatch(1), new CountDownLatch(1)};
+ CountDownLatch[] releasePass = {new CountDownLatch(1), new CountDownLatch(1)};
+ AtomicInteger passes = new AtomicInteger();
+ Mockito.doAnswer(inv -> {
+ assertThat(Thread.holdsLock(dispatcher)).isTrue();
+ int pass = passes.getAndIncrement();
+ if (pass < passStarted.length) {
+ passStarted[pass].countDown();
+ assertThat(releasePass[pass].await(10, TimeUnit.SECONDS)).isTrue();
+ }
+ return true;
+ }).when(cursor).isClosed();
+
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ Future<?> owner = executor.submit(dispatcher::readMoreEntries);
+ for (int pass = 0; pass < passStarted.length; pass++) {
+ assertThat(passStarted[pass].await(10, TimeUnit.SECONDS)).isTrue();
+ // All callers must return while the owner still holds the dispatcher monitor.
+ executor.submit(() -> {
+ for (int request = 0; request < 100; request++) {
+ dispatcher.readMoreEntries();
+ }
+ }).get(10, TimeUnit.SECONDS);
+ assertThat(passes.get()).isEqualTo(pass + 1);
+ releasePass[pass].countDown();
+ }
+ owner.get(10, TimeUnit.SECONDS);
+ assertThat(passes.get()).isEqualTo(3);
+ dispatcher.readMoreEntries();
+ assertThat(passes.get()).isEqualTo(4);
+ } finally {
+ for (CountDownLatch latch : releasePass) {
+ latch.countDown();
+ }
+ executor.shutdownNow();
+ assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue();
+ }
+ }
+
+ @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+ public void testReadMoreEntriesConflatesReentrantRequests(boolean classic) throws Exception {
+ ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+ AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+ AtomicInteger passes = new AtomicInteger();
+ AtomicInteger depth = new AtomicInteger();
+ Mockito.doAnswer(inv -> {
+ assertThat(depth.incrementAndGet()).isEqualTo(1);
+ try {
+ if (passes.incrementAndGet() < 1000) {
+ dispatcher.readMoreEntries();
+ dispatcher.readMoreEntries();
+ }
+ return true;
+ } finally {
+ depth.decrementAndGet();
+ }
+ }).when(cursor).isClosed();
+
+ dispatcher.readMoreEntries();
+ assertThat(passes.get()).isEqualTo(1000);
+ }
+
+ @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+ public void testReadMoreEntriesRecoversAfterFailure(boolean classic) throws Exception {
+ ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+ AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+ IllegalStateException failure = new IllegalStateException("read failed");
+ Mockito.doThrow(failure).when(cursor).isClosed();
+ assertThatThrownBy(dispatcher::readMoreEntries).isSameAs(failure);
+
+ Mockito.doReturn(true).when(cursor).isClosed();
+ Mockito.clearInvocations(cursor);
+ dispatcher.readMoreEntries();
+ Mockito.verify(cursor).isClosed();
+ }
+
+ @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+ public void testReadMoreEntriesRunsReentrantRequestsAfterFailure(boolean classic) throws Exception {
+ ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+ AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+ IllegalStateException failure = new IllegalStateException("read failed");
+ IllegalStateException followUpFailure = new IllegalStateException("follow-up read failed");
+ AtomicInteger passes = new AtomicInteger();
+ Mockito.doAnswer(inv -> {
+ int pass = passes.incrementAndGet();
+ if (pass <= 3) {
+ dispatcher.readMoreEntries();
+ // Reusing the first exception must not cause self-suppression.
+ throw pass == 2 ? followUpFailure : failure;
+ }
+ return true;
+ }).when(cursor).isClosed();
+
+ assertThatThrownBy(dispatcher::readMoreEntries).isSameAs(failure);
+ assertThat(passes.get()).as("Registered follow-ups run before propagating the failure").isEqualTo(4);
+ assertThat(failure.getSuppressed()).containsExactly(followUpFailure);
+ dispatcher.readMoreEntries();
+ assertThat(passes.get()).isEqualTo(5);
+ }
+
+ @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+ public void testReadMoreEntriesRunsConcurrentRequestAfterFailure(boolean classic) throws Exception {
+ ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+ AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+ IllegalStateException failure = new IllegalStateException("read failed");
+ CountDownLatch passStarted = new CountDownLatch(1);
+ CountDownLatch releasePass = new CountDownLatch(1);
+ AtomicInteger passes = new AtomicInteger();
+ Mockito.doAnswer(inv -> {
+ if (passes.incrementAndGet() == 1) {
+ passStarted.countDown();
+ assertThat(releasePass.await(10, TimeUnit.SECONDS)).isTrue();
+ throw failure;
+ }
+ return true;
+ }).when(cursor).isClosed();
+
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ Future<?> owner = executor.submit(dispatcher::readMoreEntries);
+ assertThat(passStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ executor.submit(dispatcher::readMoreEntries).get(10, TimeUnit.SECONDS);
+ releasePass.countDown();
+ assertThatThrownBy(() -> owner.get(10, TimeUnit.SECONDS)).hasCause(failure);
+ assertThat(passes.get()).as("The registered follow-up runs without a new caller").isEqualTo(2);
+ dispatcher.readMoreEntries();
+ assertThat(passes.get()).isEqualTo(3);
+ } finally {
+ releasePass.countDown();
+ executor.shutdownNow();
+ assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue();
+ }
+ }
+
+ @DataProvider
+ public Object[][] readConflationDispatcherTypes() {
+ return new Object[][] {{false}, {true}};
+ }
+
+ private AbstractPersistentDispatcherMultipleConsumers createReadConflationDispatcher(ManagedCursor cursor,
+ boolean classic)
+ throws Exception {
+ String topicName = newTopicName();
+ admin.topics().createNonPartitionedTopic(topicName);
+ admin.topics().createSubscription(topicName, "s1", MessageId.earliest);
+ PersistentTopic topic = (PersistentTopic) getTopic(topicName, false).join().orElseThrow();
+ Mockito.doReturn("s1").when(cursor).getName();
+ Subscription subscription = Mockito.mock(PersistentSubscription.class);
+ Mockito.doReturn(topic).when(subscription).getTopic();
+ return classic ? new PersistentDispatcherMultipleConsumersClassic(topic, cursor, subscription)
+ : new PersistentDispatcherMultipleConsumers(topic, cursor, subscription);
+ }
+
@Test(timeOut = 30 * 1000)
public void testTopicDeleteIfConsumerSetMismatchConsumerList() throws Exception {
final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp");
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
index 3a5c73315a01..498c000044ba 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
@@ -939,7 +939,7 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest {
}
@Override
- public synchronized void readMoreEntries() {
+ protected synchronized void internalReadMoreEntries() {
readMoreEntriesCalled.incrementAndGet();
}
@@ -960,7 +960,7 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest {
}
@Override
- public synchronized void readMoreEntries() {
+ protected synchronized void internalReadMoreEntries() {
readMoreEntriesCalled.incrementAndGet();
}