You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/09/23 12:36:54 UTC

(pulsar) 04/21: [improve][broker] Conflate concurrent readMoreEntries calls (#26539)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 6ab1fafdbd880291f71d85653881041065e5b489
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Sep 11 19:18:35 2026 +0300

    [improve][broker] Conflate concurrent readMoreEntries calls (#26539)
    
    (cherry picked from commit d5dec18cba9908318f9dfb0d097fef6e1832087b)
---
 ...tractPersistentDispatcherMultipleConsumers.java |  57 ++++++++
 .../PersistentDispatcherMultipleConsumers.java     |   5 +-
 ...rsistentDispatcherMultipleConsumersClassic.java |   3 +-
 .../PersistentDispatcherMultipleConsumersTest.java | 162 +++++++++++++++++++++
 ...ntStickyKeyDispatcherMultipleConsumersTest.java |   4 +-
 5 files changed, 226 insertions(+), 5 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java
index 3a63dc659470..a0c0c0358646 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.broker.service.persistent;
 
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.bookkeeper.mledger.AsyncCallbacks;
 import org.apache.bookkeeper.mledger.Entry;
 import org.apache.bookkeeper.mledger.ManagedCursor;
@@ -32,6 +33,11 @@ import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 
 public abstract class AbstractPersistentDispatcherMultipleConsumers extends AbstractDispatcherMultipleConsumers
         implements Dispatcher, AsyncCallbacks.ReadEntriesCallback {
+    private static final int IDLE = 0;
+    private static final int RUNNING = 1;
+    private static final int RUNNING_REQUESTED = 2;
+    private final AtomicInteger readMoreEntriesState = new AtomicInteger(IDLE);
+
     public AbstractPersistentDispatcherMultipleConsumers(Subscription subscription,
                                                          ServiceConfiguration serviceConfig) {
         super(subscription, serviceConfig);
@@ -41,6 +47,57 @@ public abstract class AbstractPersistentDispatcherMultipleConsumers extends Abst
 
     public abstract void readMoreEntriesAsync();
 
+    /**
+     * Conflate concurrent and reentrant requests into follow-up passes driven by the owning caller.
+     * If a pass fails, drain registered follow-ups before propagating the first failure to the owner.
+     */
+    public final void readMoreEntries() {
+        for (;;) {
+            int state = readMoreEntriesState.get();
+            if (state == IDLE) {
+                if (readMoreEntriesState.compareAndSet(IDLE, RUNNING)) {
+                    break;
+                }
+            } else if (state == RUNNING) {
+                if (readMoreEntriesState.compareAndSet(RUNNING, RUNNING_REQUESTED)) {
+                    return;
+                }
+            } else {
+                // A follow-up pass is already requested.
+                return;
+            }
+        }
+        Throwable failure = null;
+        for (;;) {
+            // Requests received before this pass are covered by this pass.
+            readMoreEntriesState.set(RUNNING);
+            try {
+                internalReadMoreEntries();
+            } catch (RuntimeException | Error t) {
+                // Keep ownership so a failure cannot discard an already-registered follow-up.
+                if (failure == null) {
+                    failure = t;
+                } else if (failure != t) {
+                    failure.addSuppressed(t);
+                }
+            }
+            if (readMoreEntriesState.compareAndSet(RUNNING, IDLE)) {
+                break;
+            }
+            // A request arrived during the pass. Run another pass without recursion, even after a failure.
+        }
+        if (failure instanceof RuntimeException runtimeException) {
+            throw runtimeException;
+        } else if (failure instanceof Error error) {
+            throw error;
+        }
+    }
+
+    /**
+     * Execute one read pass. Implementations must synchronize on this dispatcher to protect read state.
+     */
+    protected abstract void internalReadMoreEntries();
+
     public abstract String getName();
 
     public abstract boolean isBlockedDispatcherOnUnackedMsgs();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index fe9d59d0e8b3..86f477ac7ce3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -345,7 +345,8 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
         }
     }
 
-    public synchronized void readMoreEntries() {
+    @Override
+    protected synchronized void internalReadMoreEntries() {
         if (cursor.isClosed()) {
             if (log.isDebugEnabled()) {
                 log.debug("[{}] Cursor is already closed, skipping read more entries.", cursor.getName());
@@ -373,7 +374,7 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
             return;
         }
 
-        // increment the counter for readMoreEntries calls, to track the number of times readMoreEntries is called
+        // Count executed passes, not conflated requests, for pending-read rescheduling.
         readMoreEntriesCallCount++;
 
         // totalAvailablePermits may be updated by other threads
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
index 5745568af833..35407214aefa 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
@@ -314,7 +314,8 @@ public class PersistentDispatcherMultipleConsumersClassic extends AbstractPersis
         topic.getBrokerService().executor().execute(this::readMoreEntries);
     }
 
-    public synchronized void readMoreEntries() {
+    @Override
+    protected synchronized void internalReadMoreEntries() {
         if (cursor.isClosed()) {
             if (log.isDebugEnabled()) {
                 log.debug("[{}] Cursor is already closed, skipping read more entries.", cursor.getName());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java
index fdb9fa9109a3..1da9cc10987c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java
@@ -18,6 +18,8 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import com.carrotsearch.hppc.ObjectSet;
 import java.util.List;
 import java.util.Optional;
@@ -25,6 +27,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -49,6 +52,7 @@ import org.mockito.Mockito;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 @Slf4j
@@ -68,6 +72,164 @@ public class PersistentDispatcherMultipleConsumersTest extends ProducerConsumerB
         super.internalCleanup();
     }
 
+    @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+    public void testReadMoreEntriesConflatesConcurrentRequests(boolean classic) throws Exception {
+        ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+        AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+        CountDownLatch[] passStarted = {new CountDownLatch(1), new CountDownLatch(1)};
+        CountDownLatch[] releasePass = {new CountDownLatch(1), new CountDownLatch(1)};
+        AtomicInteger passes = new AtomicInteger();
+        Mockito.doAnswer(inv -> {
+            assertThat(Thread.holdsLock(dispatcher)).isTrue();
+            int pass = passes.getAndIncrement();
+            if (pass < passStarted.length) {
+                passStarted[pass].countDown();
+                assertThat(releasePass[pass].await(10, TimeUnit.SECONDS)).isTrue();
+            }
+            return true;
+        }).when(cursor).isClosed();
+
+        ExecutorService executor = Executors.newFixedThreadPool(2);
+        try {
+            Future<?> owner = executor.submit(dispatcher::readMoreEntries);
+            for (int pass = 0; pass < passStarted.length; pass++) {
+                assertThat(passStarted[pass].await(10, TimeUnit.SECONDS)).isTrue();
+                // All callers must return while the owner still holds the dispatcher monitor.
+                executor.submit(() -> {
+                    for (int request = 0; request < 100; request++) {
+                        dispatcher.readMoreEntries();
+                    }
+                }).get(10, TimeUnit.SECONDS);
+                assertThat(passes.get()).isEqualTo(pass + 1);
+                releasePass[pass].countDown();
+            }
+            owner.get(10, TimeUnit.SECONDS);
+            assertThat(passes.get()).isEqualTo(3);
+            dispatcher.readMoreEntries();
+            assertThat(passes.get()).isEqualTo(4);
+        } finally {
+            for (CountDownLatch latch : releasePass) {
+                latch.countDown();
+            }
+            executor.shutdownNow();
+            assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue();
+        }
+    }
+
+    @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+    public void testReadMoreEntriesConflatesReentrantRequests(boolean classic) throws Exception {
+        ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+        AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+        AtomicInteger passes = new AtomicInteger();
+        AtomicInteger depth = new AtomicInteger();
+        Mockito.doAnswer(inv -> {
+            assertThat(depth.incrementAndGet()).isEqualTo(1);
+            try {
+                if (passes.incrementAndGet() < 1000) {
+                    dispatcher.readMoreEntries();
+                    dispatcher.readMoreEntries();
+                }
+                return true;
+            } finally {
+                depth.decrementAndGet();
+            }
+        }).when(cursor).isClosed();
+
+        dispatcher.readMoreEntries();
+        assertThat(passes.get()).isEqualTo(1000);
+    }
+
+    @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+    public void testReadMoreEntriesRecoversAfterFailure(boolean classic) throws Exception {
+        ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+        AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+        IllegalStateException failure = new IllegalStateException("read failed");
+        Mockito.doThrow(failure).when(cursor).isClosed();
+        assertThatThrownBy(dispatcher::readMoreEntries).isSameAs(failure);
+
+        Mockito.doReturn(true).when(cursor).isClosed();
+        Mockito.clearInvocations(cursor);
+        dispatcher.readMoreEntries();
+        Mockito.verify(cursor).isClosed();
+    }
+
+    @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+    public void testReadMoreEntriesRunsReentrantRequestsAfterFailure(boolean classic) throws Exception {
+        ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+        AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+        IllegalStateException failure = new IllegalStateException("read failed");
+        IllegalStateException followUpFailure = new IllegalStateException("follow-up read failed");
+        AtomicInteger passes = new AtomicInteger();
+        Mockito.doAnswer(inv -> {
+            int pass = passes.incrementAndGet();
+            if (pass <= 3) {
+                dispatcher.readMoreEntries();
+                // Reusing the first exception must not cause self-suppression.
+                throw pass == 2 ? followUpFailure : failure;
+            }
+            return true;
+        }).when(cursor).isClosed();
+
+        assertThatThrownBy(dispatcher::readMoreEntries).isSameAs(failure);
+        assertThat(passes.get()).as("Registered follow-ups run before propagating the failure").isEqualTo(4);
+        assertThat(failure.getSuppressed()).containsExactly(followUpFailure);
+        dispatcher.readMoreEntries();
+        assertThat(passes.get()).isEqualTo(5);
+    }
+
+    @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+    public void testReadMoreEntriesRunsConcurrentRequestAfterFailure(boolean classic) throws Exception {
+        ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+        AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+        IllegalStateException failure = new IllegalStateException("read failed");
+        CountDownLatch passStarted = new CountDownLatch(1);
+        CountDownLatch releasePass = new CountDownLatch(1);
+        AtomicInteger passes = new AtomicInteger();
+        Mockito.doAnswer(inv -> {
+            if (passes.incrementAndGet() == 1) {
+                passStarted.countDown();
+                assertThat(releasePass.await(10, TimeUnit.SECONDS)).isTrue();
+                throw failure;
+            }
+            return true;
+        }).when(cursor).isClosed();
+
+        ExecutorService executor = Executors.newFixedThreadPool(2);
+        try {
+            Future<?> owner = executor.submit(dispatcher::readMoreEntries);
+            assertThat(passStarted.await(10, TimeUnit.SECONDS)).isTrue();
+            executor.submit(dispatcher::readMoreEntries).get(10, TimeUnit.SECONDS);
+            releasePass.countDown();
+            assertThatThrownBy(() -> owner.get(10, TimeUnit.SECONDS)).hasCause(failure);
+            assertThat(passes.get()).as("The registered follow-up runs without a new caller").isEqualTo(2);
+            dispatcher.readMoreEntries();
+            assertThat(passes.get()).isEqualTo(3);
+        } finally {
+            releasePass.countDown();
+            executor.shutdownNow();
+            assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue();
+        }
+    }
+
+    @DataProvider
+    public Object[][] readConflationDispatcherTypes() {
+        return new Object[][] {{false}, {true}};
+    }
+
+    private AbstractPersistentDispatcherMultipleConsumers createReadConflationDispatcher(ManagedCursor cursor,
+                                                                                       boolean classic)
+            throws Exception {
+        String topicName = newTopicName();
+        admin.topics().createNonPartitionedTopic(topicName);
+        admin.topics().createSubscription(topicName, "s1", MessageId.earliest);
+        PersistentTopic topic = (PersistentTopic) getTopic(topicName, false).join().orElseThrow();
+        Mockito.doReturn("s1").when(cursor).getName();
+        Subscription subscription = Mockito.mock(PersistentSubscription.class);
+        Mockito.doReturn(topic).when(subscription).getTopic();
+        return classic ? new PersistentDispatcherMultipleConsumersClassic(topic, cursor, subscription)
+                : new PersistentDispatcherMultipleConsumers(topic, cursor, subscription);
+    }
+
     @Test(timeOut = 30 * 1000)
     public void testTopicDeleteIfConsumerSetMismatchConsumerList() throws Exception {
         final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp");
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
index 3a5c73315a01..498c000044ba 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
@@ -939,7 +939,7 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest {
                 }
 
                 @Override
-                public synchronized void readMoreEntries() {
+                protected synchronized void internalReadMoreEntries() {
                     readMoreEntriesCalled.incrementAndGet();
                 }
 
@@ -960,7 +960,7 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest {
                 }
 
                 @Override
-                public synchronized void readMoreEntries() {
+                protected synchronized void internalReadMoreEntries() {
                     readMoreEntriesCalled.incrementAndGet();
                 }