You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/09/17 02:27:16 UTC
(pulsar) 01/01: [improve][broker] Avoid redundant read-completion executor handoffs
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch lh-improve-read-completion
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit bf5615609d1e8ea5d4853e7d98720d2052514a45
Author: Lari Hotari <lh...@apache.org>
AuthorDate: Thu Sep 17 05:14:27 2026 +0300
[improve][broker] Avoid redundant read-completion executor handoffs
Allow self-dispatching read callbacks to complete on the current thread while retaining the Exclusive/Failover dispatcher executor boundary. Add callback affinity and failure coverage, a handoff benchmark, and reproducible YAML profiling scenarios.
Assisted-by: Codex
---
.../apache/bookkeeper/mledger/AsyncCallbacks.java | 9 +
.../bookkeeper/mledger/impl/OpReadEntry.java | 4 +
.../mledger/util/ManagedLedgerUtils.java | 15 ++
.../mledger/impl/ReadCompletionAffinityTest.java | 200 ++++++++++++++++++
.../impl/ReadCompletionHandoffBenchmark.java | 80 ++++++++
.../PersistentDispatcherSingleActiveConsumer.java | 4 +-
.../profiling/AbstractPulsarProfilingTest.java | 37 +++-
.../profiling/PulsarProfilingConfig.java | 99 +++++++--
.../profiling/PulsarProfilingConfigTest.java | 226 +++++++++++++++++++++
.../profiling/PulsarProfilingV4Test.java | 12 ++
tests/performance/README.md | 68 ++++++-
tests/performance/read-completion-isolation.md | 123 +++++++++++
.../{ => scenarios}/pulsar-profiling.yaml | 12 ++
.../read-completion-isolation-256k-128k.yaml | 25 +++
.../read-completion-isolation-64k-32k.yaml | 25 +++
.../read-completion-isolation-failover.yaml | 23 +++
.../read-completion-isolation-shared.yaml | 23 +++
.../read-completion-isolation.yaml} | 36 ++--
18 files changed, 979 insertions(+), 42 deletions(-)
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java
index 70db427afce4..59e276aafdd8 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java
@@ -83,6 +83,15 @@ public interface AsyncCallbacks {
}
interface ReadEntriesCallback {
+ /**
+ * Whether a successful read may invoke this callback on the completing thread, avoiding a handoff to the
+ * managed-ledger executor. The callback must not block and must bound any recursive reads it initiates.
+ * Callbacks that dispatch their own continuation to an executor can opt in without changing its affinity.
+ */
+ default boolean canExecuteOnAnyThread() {
+ return false;
+ }
+
void readEntriesComplete(List<Entry> entries, Object ctx);
void readEntriesFailed(ManagedLedgerException exception, Object ctx);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
index 153db59ab7f4..c82781d93fc5 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
@@ -286,6 +286,10 @@ class OpReadEntry implements ReadEntriesCallback {
}
private void complete(Object ctx) {
+ if (callback.canExecuteOnAnyThread()) {
+ completeNow(ctx);
+ return;
+ }
ThreadBoundExecutor executor = cursor.ledger.getExecutor();
// Run inline on the ledger thread to skip the queue hop. A fully cached read completes synchronously and
// callers such as OpScan and the replicator issue their next read from this callback, so the nesting is
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/ManagedLedgerUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/ManagedLedgerUtils.java
index 8ce8a2eef69d..14869ac08125 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/ManagedLedgerUtils.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/ManagedLedgerUtils.java
@@ -84,8 +84,23 @@ public class ManagedLedgerUtils {
public static CompletableFuture<List<Entry>> readEntriesWithSkipOrWait(
ManagedCursor cursor, int maxEntries, long maxSizeBytes, Position maxPosition,
@Nullable Predicate<Position> skipCondition) {
+ return readEntriesWithSkipOrWait(cursor, maxEntries, maxSizeBytes, maxPosition, skipCondition, false);
+ }
+
+ /**
+ * Reads entries with optional completion on the reading thread. When enabled, callers must dispatch continuations
+ * to their own executor if they require thread affinity or an asynchronous boundary between reads.
+ */
+ public static CompletableFuture<List<Entry>> readEntriesWithSkipOrWait(
+ ManagedCursor cursor, int maxEntries, long maxSizeBytes, Position maxPosition,
+ @Nullable Predicate<Position> skipCondition, boolean callbackCanExecuteOnAnyThread) {
final var future = new CompletableFuture<List<Entry>>();
cursor.asyncReadEntriesWithSkipOrWait(maxEntries, maxSizeBytes, new AsyncCallbacks.ReadEntriesCallback() {
+ @Override
+ public boolean canExecuteOnAnyThread() {
+ return callbackCanExecuteOnAnyThread;
+ }
+
@Override
public void readEntriesComplete(List<Entry> entries, Object ctx) {
future.complete(entries);
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadCompletionAffinityTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadCompletionAffinityTest.java
new file mode 100644
index 000000000000..cc015f462935
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadCompletionAffinityTest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.apache.bookkeeper.mledger.util.ManagedLedgerTestUtil.rawEntryConfig;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotSame;
+import static org.testng.Assert.assertSame;
+import static org.testng.Assert.assertTrue;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.bookkeeper.mledger.ManagedCursor;
+import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.apache.bookkeeper.mledger.util.ManagedLedgerUtils;
+import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class ReadCompletionAffinityTest extends MockedBookKeeperTestCase {
+ @DataProvider
+ public Object[][] inlineCompletion() {
+ return new Object[][] {{false}, {true}};
+ }
+
+ @Test(dataProvider = "inlineCompletion")
+ public void testCacheHitCompletionAffinity(boolean inline) throws Exception {
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("completion-affinity-" + inline,
+ rawEntryConfig());
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ try {
+ ManagedCursor cursor = ledger.openCursor("cursor");
+ ledger.addEntry(new byte[] {1});
+ assertTrue(ledger.entryCache.getSize() > 0, "The test requires a cache hit");
+ Thread worker = blockWorker(ledger, releaseWorker);
+ Thread caller = Thread.currentThread();
+ CompletableFuture<Thread> completed = new CompletableFuture<>();
+ cursor.asyncReadEntries(1, new ReadEntriesCallback() {
+ @Override
+ public boolean canExecuteOnAnyThread() {
+ return inline;
+ }
+
+ @Override
+ public void readEntriesComplete(List<Entry> entries, Object ctx) {
+ entries.forEach(Entry::release);
+ completed.complete(Thread.currentThread());
+ }
+
+ @Override
+ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
+ completed.completeExceptionally(exception);
+ }
+ }, null, PositionFactory.LATEST);
+ // A blocked worker makes the queue boundary deterministic, rather than depending on task timing.
+ assertEquals(completed.isDone(), inline);
+ releaseWorker.countDown();
+ assertSame(completed.get(10, TimeUnit.SECONDS), inline ? caller : worker);
+ } finally {
+ releaseWorker.countDown();
+ ledger.close();
+ }
+ }
+
+ @Test
+ public void testInlineFutureKeepsDispatcherBoundary() throws Exception {
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("completion-dispatcher-boundary",
+ rawEntryConfig());
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ ExecutorService dispatcher = Executors.newSingleThreadExecutor();
+ try {
+ ManagedCursor cursor = ledger.openCursor("cursor");
+ ledger.addEntry(new byte[] {1});
+ assertTrue(ledger.entryCache.getSize() > 0, "The test requires a cache hit");
+ Thread worker = blockWorker(ledger, releaseWorker);
+ Thread caller = Thread.currentThread();
+ CompletableFuture<List<Entry>> read = ManagedLedgerUtils.readEntriesWithSkipOrWait(
+ cursor, 1, Long.MAX_VALUE, PositionFactory.LATEST, null, true);
+ CompletableFuture<Thread> continued = read.thenApplyAsync(entries -> {
+ entries.forEach(Entry::release);
+ return Thread.currentThread();
+ }, dispatcher);
+ assertTrue(read.isDone(), "A cache hit must not wait behind unrelated ledger work");
+ Thread continuationThread = continued.get(10, TimeUnit.SECONDS);
+ assertNotSame(continuationThread, caller);
+ assertNotSame(continuationThread, worker);
+ } finally {
+ releaseWorker.countDown();
+ ledger.close();
+ dispatcher.shutdownNow();
+ assertTrue(dispatcher.awaitTermination(10, TimeUnit.SECONDS));
+ }
+ }
+
+ private static Thread blockWorker(ManagedLedgerImpl ledger, CountDownLatch releaseWorker) throws Exception {
+ CompletableFuture<Thread> started = new CompletableFuture<>();
+ ledger.getExecutor().execute(() -> {
+ started.complete(Thread.currentThread());
+ try {
+ releaseWorker.await(20, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+ return started.get(10, TimeUnit.SECONDS);
+ }
+
+ @Test(dataProvider = "inlineCompletion")
+ public void testCacheMissAcrossLedgers(boolean inline) throws Exception {
+ ManagedLedgerConfig config = rawEntryConfig().setMaxEntriesPerLedger(2);
+ config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("completion-rollover-" + inline, config);
+ AtomicInteger storageReads = new AtomicInteger();
+ try {
+ ManagedCursor cursor = ledger.openCursor("cursor");
+ for (int i = 0; i < 6; i++) {
+ ledger.addEntry(new byte[] {(byte) i});
+ }
+ ledger.entryCache.clear();
+ bkc.setReadHandleInterceptor((ledgerId, first, last, entries) -> {
+ storageReads.incrementAndGet();
+ return CompletableFuture.completedFuture(entries);
+ });
+ List<Entry> entries = ManagedLedgerUtils.readEntriesWithSkipOrWait(
+ cursor, 6, Long.MAX_VALUE, PositionFactory.LATEST, null, inline).get(10, TimeUnit.SECONDS);
+ try {
+ assertThat(entries).hasSize(6);
+ for (int i = 0; i < entries.size(); i++) {
+ assertThat(entries.get(i).getData()).containsExactly((byte) i);
+ }
+ assertThat(entries.stream().map(entry -> entry.getPosition().getLedgerId()).distinct().count())
+ .isEqualTo(3);
+ assertThat(storageReads.get()).isGreaterThanOrEqualTo(3);
+ } finally {
+ entries.forEach(Entry::release);
+ }
+ } finally {
+ bkc.setReadHandleInterceptor(null);
+ ledger.close();
+ }
+ }
+
+ @Test(dataProvider = "inlineCompletion")
+ public void testStorageFailureAndRetry(boolean inline) throws Exception {
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("completion-failure-" + inline,
+ rawEntryConfig());
+ try {
+ ManagedCursor cursor = ledger.openCursor("cursor");
+ ledger.addEntry(new byte[] {1});
+ ledger.entryCache.clear();
+ bkc.setReadHandleInterceptor((ledgerId, first, last, entries) -> {
+ entries.close();
+ return CompletableFuture.failedFuture(BKException.create(BKException.Code.ReadException));
+ });
+ var failed = ManagedLedgerUtils.readEntriesWithSkipOrWait(
+ cursor, 1, Long.MAX_VALUE, PositionFactory.LATEST, null, inline);
+ assertThatThrownBy(() -> failed.get(10, TimeUnit.SECONDS))
+ .hasCauseInstanceOf(ManagedLedgerException.class);
+ bkc.setReadHandleInterceptor(null);
+ List<Entry> entries = ManagedLedgerUtils.readEntriesWithSkipOrWait(
+ cursor, 1, Long.MAX_VALUE, PositionFactory.LATEST, null, inline).get(10, TimeUnit.SECONDS);
+ try {
+ assertThat(entries).hasSize(1);
+ assertThat(entries.get(0).getData()).containsExactly((byte) 1);
+ } finally {
+ entries.forEach(Entry::release);
+ }
+ } finally {
+ bkc.setReadHandleInterceptor(null);
+ ledger.close();
+ }
+ }
+}
diff --git a/microbench/src/main/java/org/apache/bookkeeper/mledger/impl/ReadCompletionHandoffBenchmark.java b/microbench/src/main/java/org/apache/bookkeeper/mledger/impl/ReadCompletionHandoffBenchmark.java
new file mode 100644
index 000000000000..2789ac63e2d0
--- /dev/null
+++ b/microbench/src/main/java/org/apache/bookkeeper/mledger/impl/ReadCompletionHandoffBenchmark.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import io.netty.util.concurrent.DefaultThreadFactory;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.common.util.SingleThreadExecutor;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/** Measures the future-adapter handoff using real executors, without ledger/cache work or producer queue pressure. */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+public class ReadCompletionHandoffBenchmark {
+ @Param({"false", "true"})
+ public boolean inline;
+
+ private SingleThreadExecutor ledger;
+ private SingleThreadExecutor dispatcher;
+
+ @Setup
+ public void setup() {
+ ledger = new SingleThreadExecutor(new DefaultThreadFactory("ledger-benchmark"));
+ dispatcher = new SingleThreadExecutor(new DefaultThreadFactory("dispatcher-benchmark"));
+ }
+
+ @Benchmark
+ public int completeRead() {
+ CompletableFuture<Integer> completed = new CompletableFuture<>();
+ dispatcher.execute(() -> {
+ CompletableFuture<Integer> read = new CompletableFuture<>();
+ if (inline) {
+ read.complete(1);
+ } else {
+ ledger.executeOrRun(() -> read.complete(1));
+ }
+ read.thenAcceptAsync(completed::complete, dispatcher);
+ });
+ return completed.join();
+ }
+
+ @TearDown
+ public void tearDown() throws InterruptedException {
+ ledger.shutdown();
+ dispatcher.shutdown();
+ ledger.awaitTermination(10, TimeUnit.SECONDS);
+ dispatcher.awaitTermination(10, TimeUnit.SECONDS);
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
index 8c2b1d820520..80f7e39492ee 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
@@ -397,8 +397,10 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher
entriesFuture = CompactedTopicUtils.asyncReadCompactedEntries(topic.getTopicCompactionService(),
cursor, messagesToRead, bytesToRead, topic.getMaxReadPosition(), readFromEarliest, true);
} else {
+ // The continuation below already dispatches to our executor. A cache hit must not wait
+ // behind unrelated publishing work on the managed-ledger executor first.
entriesFuture = readEntriesWithSkipOrWait(cursor, messagesToRead, bytesToRead,
- topic.getMaxReadPosition(), null);
+ topic.getMaxReadPosition(), null, true);
}
entriesFuture.whenCompleteAsync((entries, e) -> {
if (e == null) {
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/AbstractPulsarProfilingTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/AbstractPulsarProfilingTest.java
index 675dee4ba329..c2982c821e93 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/AbstractPulsarProfilingTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/AbstractPulsarProfilingTest.java
@@ -19,6 +19,7 @@
package org.apache.pulsar.tests.integration.profiling;
import static org.assertj.core.api.SoftAssertions.assertSoftly;
+import com.github.dockerjava.api.model.Capability;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -33,6 +34,7 @@ import lombok.CustomLog;
import org.apache.pulsar.common.naming.TopicDomain;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.tests.ManualTestUtil;
import org.apache.pulsar.tests.integration.containers.PulsarContainer;
import org.apache.pulsar.tests.integration.suites.PulsarTestSuite;
@@ -162,8 +164,10 @@ public abstract class AbstractPulsarProfilingTest extends PulsarTestSuite {
"bash", "-c", "set -o pipefail; echo $$ > /tmp/command.pid; "
+ "/pulsar/bin/pulsar-perf consume" + commandSuffix + " " + topicName + " "
+ "-u pulsar://" + brokerHostname + ":6650 "
- + "-st Shared "
- + "-q 50000 "
+ + "-st " + load.subscriptionType() + " "
+ + "-q " + load.receiverQueueSize() + " "
+ + "--num-consumers " + load.consumerCount() + " "
+ + "--num-io-threads " + load.consumerIoThreads() + " "
+ isolatedClientsOption(load.isolatedConsumers())
+ "-m " + load.numberOfMessages() + " -ml " + load.consumeMemoryLimit() + " "
+ "--histogram-file=/testoutput/consume" + commandSuffix
@@ -178,7 +182,11 @@ public abstract class AbstractPulsarProfilingTest extends PulsarTestSuite {
+ "-u pulsar://" + brokerHostname + ":6650 "
+ "-au http://" + brokerHostname + ":8080 "
+ "-r " + load.produceRate() + " "
- + "-s " + load.messageSize() + " -db "
+ + "-s " + load.messageSize() + " "
+ + (load.batchingEnabled() ? "" : "-db ")
+ + "--num-producers " + load.producerCount() + " "
+ + "--num-io-threads " + load.producerIoThreads() + " "
+ + "--max-connections 1 "
+ isolatedClientsOption(load.isolatedProducers())
// maxOutstanding only applies to the v4 client; the v5 client accepts
// the flag for back-compat but ignores it
@@ -193,6 +201,19 @@ public abstract class AbstractPulsarProfilingTest extends PulsarTestSuite {
return commandSuffix.equals("-v4") && count > 0 ? "--isolated-clients " + count + " " : "";
}
+ void enableProfiling(String options, String role) {
+ if (options == null || options.isBlank()) {
+ return;
+ }
+ // Permit native CPU sampling, as for the profiled broker containers.
+ withCreateContainerCmdModifier(cmd -> cmd.getHostConfig()
+ .withCapAdd(Capability.PERFMON)
+ .withSecurityOpts(List.of("seccomp=unconfined")));
+ withEnv("PULSAR_EXTRA_OPTS", "-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints "
+ + "-agentpath:/opt/async-profiler/lib/libasyncProfiler.so=start," + options
+ + ",file=/testoutput/client-" + role + "-%t-%p.jfr");
+ }
+
/**
* Polls the given topic admin endpoints and the broker metrics every 10 seconds. The metrics
* are always collected; the topic endpoints may be empty when the domain has no admin stats.
@@ -270,6 +291,8 @@ public abstract class AbstractPulsarProfilingTest extends PulsarTestSuite {
// This matters only on Linux
try {
Files.setPosixFilePermissions(testOutputDir.toPath(), PosixFilePermissions.fromString("rwxrwxrwx"));
+ ObjectMapperFactory.getYamlMapper().getObjectMapper().writeValue(
+ new File(testOutputDir, "resolved-config.yaml"), profilingConfig);
} catch (IOException e) {
throw new UncheckedIOException("Cannot change access to test output directory", e);
}
@@ -352,6 +375,8 @@ public abstract class AbstractPulsarProfilingTest extends PulsarTestSuite {
commandSuffix, load);
perfConsume = new PulsarPerfContainer(testOutputDir, clusterName, brokerHostname, "perf-consume", "-Xmx1g",
commandSuffix, load);
+ perfProduce.enableProfiling(profilingConfig.profiling().producerOptions(), "producer");
+ perfConsume.enableProfiling(profilingConfig.profiling().consumerOptions(), "consumer");
printStats = new PulsarPerfContainer(testOutputDir, clusterName, brokerHostname, "print-stats", "-Xmx1g",
commandSuffix, load);
specBuilder.externalServices(Map.of(
@@ -374,6 +399,7 @@ public abstract class AbstractPulsarProfilingTest extends PulsarTestSuite {
*/
protected void runPulsarPerfBenchmark() throws Exception {
String topicName = generateTopicName("profiletest", getTopicDomain());
+ prepareTopic(topicName);
CompletableFuture<Long> consumeFuture = perfConsume.consume(topicName);
Thread.sleep(1000);
CompletableFuture<Long> produceFuture = perfProduce.produce(topicName);
@@ -381,7 +407,7 @@ public abstract class AbstractPulsarProfilingTest extends PulsarTestSuite {
printStats.stats(getTopicStatsEndpoints(topicName));
// pulsar-perf is sized to finish inside this window, so running out of it is a failure.
FutureUtil.waitForAll(List.of(consumeFuture, produceFuture))
- .orTimeout(3, TimeUnit.MINUTES)
+ .orTimeout(profilingConfig.load().timeoutSeconds(), TimeUnit.SECONDS)
.exceptionally(t -> {
log.error().exception(t).log("Failed to run pulsar-perf");
throw FutureUtil.wrapToCompletionException(t);
@@ -392,4 +418,7 @@ public abstract class AbstractPulsarProfilingTest extends PulsarTestSuite {
softly.assertThat(produceFuture).as("produce should have completed successfully").isCompletedWithValue(0L);
});
}
+
+ protected void prepareTopic(String topicName) throws Exception {
+ }
}
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingConfig.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingConfig.java
index 6df6e7c1d9d8..4249b0a26360 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingConfig.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingConfig.java
@@ -23,7 +23,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.LinkedHashSet;
import java.util.Map;
+import java.util.Set;
+import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.common.util.ObjectMapperFactory;
/** Configuration for the profiling scenario harness. */
@@ -31,7 +34,7 @@ final class PulsarProfilingConfig {
static final String CONFIG_ENV = "PULSAR_PROFILING_CONFIG";
static final String ENV_PREFIX = "PULSAR_PROFILING_";
- record Config(Cluster cluster, Load load, Output output) {
+ record Config(Cluster cluster, Load load, Profiling profiling, Output output) {
static Config read() {
String configFile = System.getenv(CONFIG_ENV);
return read(configFile == null || configFile.isBlank() ? null : Path.of(configFile),
@@ -42,11 +45,7 @@ final class PulsarProfilingConfig {
ObjectMapper mapper = ObjectMapperFactory.getYamlMapper().getObjectMapper();
ObjectNode root = mapper.valueToTree(defaults());
if (configFile != null) {
- try {
- merge(root, mapper.readTree(configFile.toFile()));
- } catch (IOException e) {
- throw new IllegalArgumentException("Cannot read profiling config " + configFile, e);
- }
+ mergeFile(root, configFile, mapper, new LinkedHashSet<>());
}
applyEnvironmentOverrides(root, environment);
try {
@@ -84,7 +83,9 @@ final class PulsarProfilingConfig {
Map.entry("isForceGCAllowWhenNoSpace", "true"),
Map.entry("diskUsageLwmThreshold", "0.75"),
Map.entry("diskCheckInterval", "60"))),
- new Load(20_000_000, "200M", "200M", Integer.MAX_VALUE, 128, 20_000, 10, 0, 0),
+ new Load(20_000_000, "200M", "200M", Integer.MAX_VALUE, 128, 20_000, 10, 0, 0,
+ 1, 1, 1, 1, SubscriptionType.Shared, 50_000, 180, false),
+ new Profiling("", ""),
new Output("build/pulsar-profiling"));
}
}
@@ -96,22 +97,90 @@ final class PulsarProfilingConfig {
record Load(long numberOfMessages, String produceMemoryLimit, String consumeMemoryLimit,
int produceRate, int messageSize, int maxOutstanding, int statsIntervalSeconds,
- int isolatedProducers, int isolatedConsumers) {
+ int isolatedProducers, int isolatedConsumers, int producerCount, int consumerCount,
+ int producerIoThreads, int consumerIoThreads, SubscriptionType subscriptionType,
+ int receiverQueueSize, int timeoutSeconds, boolean batchingEnabled) {
+ Load {
+ if (producerCount < 1 || consumerCount < 1 || producerIoThreads < 1 || consumerIoThreads < 1
+ || isolatedProducers < 0 || isolatedConsumers < 0 || receiverQueueSize < 1 || timeoutSeconds < 1
+ || numberOfMessages < 1 || produceRate < 1 || subscriptionType == null) {
+ throw new IllegalArgumentException("Profiling counts, rate, queue size and timeout must be positive");
+ }
+ // pulsar-perf divides the message count and rate between workers using integer division.
+ int workers = Math.max(1, isolatedProducers);
+ if (isolatedProducers > producerCount || isolatedConsumers > consumerCount
+ || numberOfMessages % workers != 0 || produceRate < workers) {
+ throw new IllegalArgumentException("Isolated clients require at least one producer/consumer "
+ + "per client, a message count divisible by producer clients, and a rate >= producer clients");
+ }
+ if (subscriptionType == SubscriptionType.Exclusive && consumerCount != 1) {
+ throw new IllegalArgumentException("Exclusive subscriptions require exactly one consumer");
+ }
+ }
}
record Output(String directory) {
}
- private static void merge(ObjectNode target, JsonNode source) {
- if (source == null || !source.isObject()) {
- return;
+ /** Empty options disable profiling for that client process. */
+ record Profiling(String producerOptions, String consumerOptions) {
+ }
+
+ private static void mergeFile(ObjectNode target, Path file, ObjectMapper mapper, Set<Path> activeFiles) {
+ try {
+ Path path = file.toRealPath();
+ if (!activeFiles.add(path)) {
+ throw new IllegalArgumentException("Profiling config inheritance cycle: "
+ + activeFiles + " -> " + path);
+ }
+ try {
+ JsonNode source = mapper.readTree(path.toFile());
+ if (!(source instanceof ObjectNode object)) {
+ throw new IllegalArgumentException("Profiling config must be a YAML mapping: " + path);
+ }
+ JsonNode parents = object.remove("extends");
+ if (parents != null) {
+ if (parents.isTextual()) {
+ mergeParent(target, parents, path, mapper, activeFiles);
+ } else if (parents.isArray()) {
+ for (JsonNode parent : parents) {
+ mergeParent(target, parent, path, mapper, activeFiles);
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "Profiling config 'extends' must be a path or list of paths: " + path);
+ }
+ }
+ // Apply directly to the accumulated config so inherited nulls also remove earlier values/defaults.
+ merge(target, object);
+ } finally {
+ activeFiles.remove(path);
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Cannot read profiling config " + file, e);
+ }
+ }
+
+ private static void mergeParent(ObjectNode target, JsonNode parent, Path file,
+ ObjectMapper mapper, Set<Path> activeFiles) {
+ if (!parent.isTextual() || parent.textValue().isBlank()) {
+ throw new IllegalArgumentException("Profiling config 'extends' entries must be non-empty paths: " + file);
}
+ mergeFile(target, file.getParent().resolve(parent.textValue()), mapper, activeFiles);
+ }
+
+ private static void merge(ObjectNode target, ObjectNode source) {
source.properties().forEach(entry -> {
- JsonNode current = target.get(entry.getKey());
- if (current != null && current.isObject() && entry.getValue().isObject()) {
- merge((ObjectNode) current, entry.getValue());
+ String key = entry.getKey();
+ JsonNode value = entry.getValue();
+ if (value.isNull()) {
+ target.remove(key);
+ } else if (value instanceof ObjectNode object) {
+ JsonNode current = target.get(key);
+ ObjectNode child = current instanceof ObjectNode ? (ObjectNode) current : target.putObject(key);
+ merge(child, object);
} else {
- target.set(entry.getKey(), entry.getValue());
+ target.set(key, value);
}
});
}
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingConfigTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingConfigTest.java
index 472d8f6a4894..5515b0cfa449 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingConfigTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingConfigTest.java
@@ -19,9 +19,14 @@
package org.apache.pulsar.tests.integration.profiling;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Comparator;
import java.util.Map;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
public class PulsarProfilingConfigTest {
@@ -42,4 +47,225 @@ public class PulsarProfilingConfigTest {
Files.deleteIfExists(file);
}
}
+
+ @Test
+ public void configuresIsolatedProducersAndExclusiveConsumer() {
+ var config = PulsarProfilingConfig.Config.read(null, Map.of(
+ "PULSAR_PROFILING_LOAD_PRODUCER_COUNT", "500",
+ "PULSAR_PROFILING_LOAD_ISOLATED_PRODUCERS", "500",
+ "PULSAR_PROFILING_LOAD_PRODUCER_IO_THREADS", "8",
+ "PULSAR_PROFILING_LOAD_SUBSCRIPTION_TYPE", "Exclusive",
+ "PULSAR_PROFILING_LOAD_TIMEOUT_SECONDS", "480"));
+ assertThat(config.load().producerCount()).isEqualTo(500);
+ assertThat(config.load().isolatedProducers()).isEqualTo(500);
+ assertThat(config.load().producerIoThreads()).isEqualTo(8);
+ assertThat(config.load().subscriptionType()).isEqualTo(SubscriptionType.Exclusive);
+ assertThat(config.load().consumerCount()).isEqualTo(1);
+ assertThat(config.load().timeoutSeconds()).isEqualTo(480);
+ }
+
+ @Test
+ public void inheritsIndependentClientProfilingAndBatchingOptions() throws Exception {
+ try (ScenarioFiles files = new ScenarioFiles()) {
+ files.write("clients.yaml", """
+ profiling:
+ producerOptions: event=cpu,interval=10ms
+ consumerOptions: event=cpu,lock=0
+ load:
+ batchingEnabled: true
+ """);
+ Path child = files.write("scenario.yaml", """
+ extends: clients.yaml
+ profiling:
+ consumerOptions: ~
+ """);
+ var config = PulsarProfilingConfig.Config.read(child, Map.of(
+ "PULSAR_PROFILING_PROFILING_PRODUCER_OPTIONS", "event=cpu,interval=20ms",
+ "PULSAR_PROFILING_LOAD_BATCHING_ENABLED", "false"));
+ assertThat(config.profiling().producerOptions()).isEqualTo("event=cpu,interval=20ms");
+ assertThat(config.profiling().consumerOptions()).isNull();
+ assertThat(config.load().batchingEnabled()).isFalse();
+ assertThat(PulsarProfilingConfig.Config.defaults().profiling().producerOptions()).isEmpty();
+ assertThat(PulsarProfilingConfig.Config.defaults().profiling().consumerOptions()).isEmpty();
+ }
+ }
+
+ @Test
+ public void rejectsWorkloadsThatCannotFinish() {
+ assertThatThrownBy(() -> PulsarProfilingConfig.Config.read(null,
+ Map.of("PULSAR_PROFILING_LOAD_ISOLATED_PRODUCERS", "500")))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> PulsarProfilingConfig.Config.read(null, Map.of(
+ "PULSAR_PROFILING_LOAD_PRODUCER_COUNT", "500",
+ "PULSAR_PROFILING_LOAD_ISOLATED_PRODUCERS", "500",
+ "PULSAR_PROFILING_LOAD_NUMBER_OF_MESSAGES", "501")))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> PulsarProfilingConfig.Config.read(null, Map.of(
+ "PULSAR_PROFILING_LOAD_SUBSCRIPTION_TYPE", "Exclusive",
+ "PULSAR_PROFILING_LOAD_CONSUMER_COUNT", "2")))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ public void mergesRecursiveParentsInOrderBeforeChildAndEnvironment() throws Exception {
+ try (ScenarioFiles files = new ScenarioFiles()) {
+ files.write("common.yaml", """
+ load:
+ messageSize: 256
+ produceRate: 1000
+ cluster:
+ numBookies: 5
+ """);
+ files.write("parents/first.yaml", """
+ extends: ../common.yaml
+ load:
+ messageSize: 512
+ producerCount: 3
+ """);
+ files.write("parents/second.yaml", """
+ load:
+ messageSize: 1024
+ produceRate: 2000
+ """);
+ Path child = files.write("scenario.yaml", """
+ extends: [parents/first.yaml, parents/second.yaml]
+ load:
+ produceRate: 3000
+ """);
+ var config = PulsarProfilingConfig.Config.read(child, Map.of(
+ "PULSAR_PROFILING_CLUSTER_NUM_BOOKIES", "7"));
+ assertThat(config.load().messageSize()).isEqualTo(1024);
+ assertThat(config.load().produceRate()).isEqualTo(3000);
+ assertThat(config.load().producerCount()).isEqualTo(3);
+ assertThat(config.cluster().numBookies()).isEqualTo(7);
+ assertThat(config.load().consumerCount()).isEqualTo(1);
+ }
+ }
+
+ @Test
+ public void removesInheritedAndDefaultEntriesIncludingThroughRecursiveParents() throws Exception {
+ try (ScenarioFiles files = new ScenarioFiles()) {
+ files.write("base.yaml", """
+ cluster:
+ brokerEnvs:
+ customSetting: enabled
+ keptSetting: kept
+ restoredSetting: old
+ """);
+ files.write("removals.yaml", """
+ cluster:
+ brokerEnvs:
+ customSetting: ~
+ preciseDispatcherFlowControl: null
+ restoredSetting: ~
+ absentSetting: ~
+ bookkeeperEnvs: ~
+ """);
+ files.write("parent.yaml", "extends: removals.yaml\n");
+ Path child = files.write("scenario.yaml", """
+ extends: [base.yaml, parent.yaml]
+ cluster:
+ brokerEnvs:
+ restoredSetting: new
+ bookkeeperEnvs:
+ journalSyncData: "true"
+ absentSetting: ~
+ """);
+ var config = PulsarProfilingConfig.Config.read(child, Map.of());
+ assertThat(config.cluster().brokerEnvs())
+ .doesNotContainKeys("customSetting", "preciseDispatcherFlowControl", "absentSetting")
+ .containsEntry("keptSetting", "kept")
+ .containsEntry("restoredSetting", "new")
+ .containsEntry("managedLedgerDefaultEnsembleSize", "1");
+ assertThat(config.cluster().bookkeeperEnvs()).containsExactlyEntriesOf(Map.of("journalSyncData", "true"));
+ }
+ }
+
+ @Test
+ public void allowsSharedAncestorsWithoutTreatingThemAsCycles() throws Exception {
+ try (ScenarioFiles files = new ScenarioFiles()) {
+ files.write("common.yaml", "load:\n messageSize: 256\n");
+ files.write("first.yaml", "extends: common.yaml\nload:\n messageSize: 512\n");
+ files.write("second.yaml", "extends: common.yaml\n");
+ Path child = files.write("scenario.yaml", "extends: [first.yaml, second.yaml]\n");
+ var config = PulsarProfilingConfig.Config.read(child, Map.of());
+ // The second parent's ancestry is applied too, in declaration order.
+ assertThat(config.load().messageSize()).isEqualTo(256);
+ }
+ }
+
+ @Test
+ public void rejectsInheritanceCycles() throws Exception {
+ try (ScenarioFiles files = new ScenarioFiles()) {
+ Path first = files.write("first.yaml", "extends: second.yaml\n");
+ files.write("second.yaml", "extends: ./first.yaml\n");
+ assertThatThrownBy(() -> PulsarProfilingConfig.Config.read(first, Map.of()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("inheritance cycle")
+ .hasMessageContaining("first.yaml")
+ .hasMessageContaining("second.yaml");
+ }
+ }
+
+ @DataProvider
+ public Object[][] invalidInheritance() {
+ return new Object[][] {
+ {"extends: 42\n"},
+ {"extends: ~\n"},
+ {"extends: {}\n"},
+ {"extends: ''\n"},
+ {"extends: [42]\n"},
+ {"extends: [~]\n"},
+ {"extends: missing.yaml\n"},
+ {"- not-a-mapping\n"},
+ {""}
+ };
+ }
+
+ @Test(dataProvider = "invalidInheritance")
+ public void rejectsInvalidInheritance(String yaml) throws Exception {
+ try (ScenarioFiles files = new ScenarioFiles()) {
+ Path child = files.write("scenario.yaml", yaml);
+ assertThatThrownBy(() -> PulsarProfilingConfig.Config.read(child, Map.of()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("config");
+ }
+ }
+
+ @Test
+ public void sharedScenarioOnlyChangesSubscriptionTypeAndOutputDirectory() {
+ Path scenarios = Path.of("../performance/scenarios");
+ var exclusive = PulsarProfilingConfig.Config.read(
+ scenarios.resolve("read-completion-isolation.yaml"), Map.of());
+ var shared = PulsarProfilingConfig.Config.read(scenarios.resolve("read-completion-isolation-shared.yaml"),
+ Map.of());
+ assertThat(shared.cluster()).isEqualTo(exclusive.cluster());
+ assertThat(shared.load()).usingRecursiveComparison().ignoringFields("subscriptionType")
+ .isEqualTo(exclusive.load());
+ assertThat(shared.load().subscriptionType()).isEqualTo(SubscriptionType.Shared);
+ assertThat(shared.output().directory()).isEqualTo("build/pulsar-profiling/read-completion-isolation-shared");
+ }
+
+ private static final class ScenarioFiles implements AutoCloseable {
+ private final Path directory;
+
+ private ScenarioFiles() throws IOException {
+ directory = Files.createTempDirectory("pulsar-profiling-config");
+ }
+
+ private Path write(String name, String yaml) throws IOException {
+ Path path = directory.resolve(name);
+ Files.createDirectories(path.getParent());
+ return Files.writeString(path, yaml);
+ }
+
+ @Override
+ public void close() throws IOException {
+ try (var paths = Files.walk(directory)) {
+ for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) {
+ Files.delete(path);
+ }
+ }
+ }
+ }
}
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingV4Test.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingV4Test.java
index 37762a866262..a2e6d92c2cfb 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingV4Test.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingV4Test.java
@@ -19,6 +19,8 @@
package org.apache.pulsar.tests.integration.profiling;
import java.util.List;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.common.naming.TopicDomain;
import org.testng.annotations.Test;
@@ -42,6 +44,16 @@ public class PulsarProfilingV4Test extends AbstractPulsarProfilingTest {
return "-v4";
}
+ @Override
+ protected void prepareTopic(String topicName) throws Exception {
+ // Establish the subscription before publishing, even if pulsar-perf takes longer to start.
+ try (PulsarAdmin admin = PulsarAdmin.builder()
+ .serviceHttpUrl(pulsarCluster.getAnyBroker().getHttpServiceUrl()).build()) {
+ admin.topics().createNonPartitionedTopic(topicName);
+ admin.topics().createSubscription(topicName, "sub", MessageId.earliest);
+ }
+ }
+
@Override
protected List<TopicStatsEndpoint> getTopicStatsEndpoints(String topicName) {
String basePath = adminV2Path("persistent", topicName);
diff --git a/tests/performance/README.md b/tests/performance/README.md
index 67289353ad8e..b24955dea202 100644
--- a/tests/performance/README.md
+++ b/tests/performance/README.md
@@ -46,13 +46,13 @@ classic `persistent://` topic. Both variants profile a single broker and write r
output under `tests/integration/build/pulsar-profiling`.
The harness accepts a YAML scenario file through `PULSAR_PROFILING_CONFIG`. Start with
-[`pulsar-profiling.yaml`](pulsar-profiling.yaml); omitted values retain the existing defaults. The
-sections correspond to the main components of a run: `cluster`, `load` and `output`. Individual scalar
+[`pulsar-profiling.yaml`](scenarios/pulsar-profiling.yaml); omitted values retain the existing defaults. The
+sections correspond to the main components of a run: `cluster`, `load`, `profiling` and `output`. Individual scalar
values can still be overridden for a one-off run with the `PULSAR_PROFILING_` prefix and an upper-case
path, for example:
```bash
-PULSAR_PROFILING_CONFIG=tests/performance/pulsar-profiling.yaml \
+PULSAR_PROFILING_CONFIG="$PWD/tests/performance/scenarios/pulsar-profiling.yaml" \
PULSAR_PROFILING_LOAD_NUMBER_OF_MESSAGES=1000000 \
./gradlew :tests:integration:profilingIntegrationTest --tests "*PulsarProfilingV4Test"
```
@@ -60,10 +60,70 @@ PULSAR_PROFILING_LOAD_NUMBER_OF_MESSAGES=1000000 \
For the v4 scenario, set `load.isolatedProducers` or `load.isolatedConsumers` to create that many
independent v4 client instances. The corresponding `pulsar-perf` command receives
`--isolated-clients`; v5 ignores these fields. The option is mutually exclusive with the regular
-producer test-thread option and with consumer listener-thread expansion.
+producer test-thread option and with consumer listener-thread expansion. Set `load.producerCount`
+and `load.consumerCount` separately: creating clients does not create producers or consumers.
+`load.subscriptionType` selects the subscription type; `producerIoThreads` and `consumerIoThreads`
+size the shared client IO pools. `maxOutstanding` is per producer, not a global limit.
+Set `load.batchingEnabled: true` to use pulsar-perf's default producer batching; the default is
+`false`, preserving unbatched entry-by-entry measurements.
+
+Client profiling is optional and independent of broker profiling. Set `profiling.producerOptions`
+and/or `profiling.consumerOptions` to async-profiler options, for example
+`event=cpu,interval=10ms,lock=0,alloc=2m,jfrsync=profile`. An empty or null option disables that
+client's profiler. Client recordings are named `client-producer-*.jfr` and `client-consumer-*.jfr`
+in the same output directory. The harness grants native CPU profiling access only to enabled clients.
+The broker continues to use `-Pinttest.asyncprofiler.opts`. To inspect lock contention without
+wall-clock sampling overhead, omit `wall` and use `lock=0` to record all supported lock events.
+Use the same options for baseline and candidate; extra profiling has a measurement cost.
+
+Broker-side variations from the contention investigations need no dedicated environment switches:
+use `cluster.brokerEnvs` for write-buffer watermarks, dispatcher batch size, broker/BookKeeper IO
+thread counts and `PULSAR_GC`; use `cluster.brokerMemory` for heap/direct-memory limits and JVM
+properties such as allocator or transport selection. These maps let YAML scenarios retain the
+complete configuration instead of relying on shell history.
+
+The harness saves `resolved-config.yaml` with inheritance and environment overrides applied in the output directory.
For v4 production, `--num-producers` remains the producer count per topic and is distributed across
the isolated clients; when the counts differ, producers are assigned round-robin as evenly as possible.
+### Inheriting scenario configurations
+
+Use a top-level `extends` to inherit one file or a list of files:
+
+```yaml
+extends: [cluster.yaml, workloads/many-producers.yaml]
+load:
+ subscriptionType: Shared
+cluster:
+ brokerEnvs:
+ preciseDispatcherFlowControl: ~
+output:
+ directory: build/pulsar-profiling/shared
+```
+
+Each path is relative to the file declaring it; absolute paths also work. Parents can themselves
+inherit other files. Starting with the harness defaults, the loader visits each parent recursively
+in the listed order, then applies the current file. Later values win, mappings merge recursively,
+and scalar values and lists replace earlier values. Shared ancestors are applied on each visit;
+inheritance cycles, missing files and invalid `extends` entries are rejected. Environment overrides
+are applied last, and `resolved-config.yaml` contains the resulting values without `extends`.
+
+An explicit YAML `null` or `~` removes an entry, including a harness default. For example, the
+`preciseDispatcherFlowControl` removal above leaves that broker setting to the broker's own default.
+Deleting a mapping removes all its entries; a later mapping starts fresh. Required workload fields
+must still have valid values in the final configuration.
+
+## Reproducible scenarios
+
+- [Read-completion queue isolation](read-completion-isolation.md): 500 producers on separate
+ connections to one persistent topic, with one Exclusive consumer. Includes the
+ [scenario YAML](scenarios/read-completion-isolation.yaml), an inherited
+ [Shared-subscription variant](scenarios/read-completion-isolation-shared.yaml), a
+ [Failover variant](scenarios/read-completion-isolation-failover.yaml), explicit
+ [64/32 KiB](scenarios/read-completion-isolation-64k-32k.yaml) and
+ [256/128 KiB](scenarios/read-completion-isolation-256k-128k.yaml) channel-watermark variations,
+ and baseline/comparison instructions.
+
## Inspecting recordings
Render the CPU, wall-clock, allocation and lock views with:
diff --git a/tests/performance/read-completion-isolation.md b/tests/performance/read-completion-isolation.md
new file mode 100644
index 000000000000..7575799466bf
--- /dev/null
+++ b/tests/performance/read-completion-isolation.md
@@ -0,0 +1,123 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# Read-completion profiling scenario
+
+Use this scenario to measure whether consumer dispatch keeps up when many producers publish to
+one persistent topic. Run the same workload and profiler settings on the base revision and the
+candidate revision. See [README.md](README.md) for the profiling harness and analysis tools.
+
+## Workload
+
+[read-completion-isolation.yaml](scenarios/read-completion-isolation.yaml) runs the v4 `pulsar-perf` commands
+with 500 producers, 500 isolated clients sharing PIP-234 resources, one connection per client,
+and one Exclusive consumer on a non-partitioned `persistent://` topic. It sends 12 million
+unbatched 128-byte messages at unrestricted rate. Each producer permits 40 outstanding sends,
+for a total limit of 20,000. The harness creates the subscription before publishing.
+
+Inherited variations change one aspect of this workload:
+
+- [Shared subscription](scenarios/read-completion-isolation-shared.yaml)
+- [Failover subscription](scenarios/read-completion-isolation-failover.yaml)
+- [64/32 KiB channel high/low watermarks](scenarios/read-completion-isolation-64k-32k.yaml)
+- [256/128 KiB channel high/low watermarks](scenarios/read-completion-isolation-256k-128k.yaml)
+
+Keep inherited YAML files together: parent paths resolve relative to the file declaring them.
+Always use the same variation for the baseline and candidate. A result for one subscription type
+does not establish an improvement for another.
+
+## Run a comparison
+
+Use separate checkouts for the base and candidate revisions. Both must contain the same profiling
+harness and scenario files. If the candidate changes the harness, apply just its harness changes
+to the baseline before measuring. For example, from a candidate checkout based on `origin/master`:
+
+```bash
+git worktree add --detach ../pulsar-read-baseline origin/master
+git diff origin/master...HEAD -- tests/integration tests/performance > /tmp/read-completion-harness.patch
+git -C ../pulsar-read-baseline apply /tmp/read-completion-harness.patch
+```
+
+Run this command from each checkout's root. Replace `baseline` with `candidate` for the candidate
+run and use a fresh output directory for every repetition. Use an absolute YAML path because
+Gradle runs the test from the integration module directory.
+
+```bash
+PULSAR_PROFILING_CONFIG="$PWD/tests/performance/scenarios/read-completion-isolation.yaml" \
+PULSAR_PROFILING_OUTPUT_DIRECTORY="$PWD/tests/integration/build/pulsar-profiling/baseline" \
+./gradlew :tests:integration:profilingIntegrationTest --tests '*PulsarProfilingV4Test' \
+ '-Pinttest.asyncprofiler.opts=event=cpu,interval=10ms,lock=0,alloc=2m,jfrsync=profile'
+```
+
+Build and run the checkouts sequentially because they share Docker image tags. Keep host power
+settings, memory limits and profiler options identical. Avoid other builds or benchmarks during
+measurement; record thermal throttling and available memory. Save the source revision and any local
+diff alongside the output. The harness writes the effective configuration to `resolved-config.yaml`.
+
+To select a variation, change `PULSAR_PROFILING_CONFIG` to its YAML file. For a rate-limited run,
+add `PULSAR_PROFILING_LOAD_PRODUCE_RATE=100000`, which gives 200 messages/s per producer client.
+The message count must be divisible by the number of isolated producer clients.
+
+## Compare results
+
+**Steady dispatch** measures broker delivery while all 500 producers are active. Omit the first
+20 seconds after all producers connect, then use only complete snapshot intervals with 500
+publishers at both endpoints. The window ends before producers start disconnecting. Divide the
+sum of `msgOutCounter` increases by the total interval duration. Calculate **steady ingress**
+the same way from `msgInCounter`. These measure broker activity, not client acknowledgement.
+The cached `msgRateIn` and `msgRateOut` values can lag workload transitions.
+
+**Whole-run throughput** is each `pulsar-perf` client's reported aggregate rate over its own timer.
+Consumer throughput includes ramp-up and backlog drain; the producer timer is separate. A high
+whole-run consumer rate can hide slow dispatch during publishing if the backlog drains quickly
+after producers stop. Compare steady rates and backlog as well as aggregate completion speed.
+
+**Sampled maximum backlog** is the largest `subscriptions.sub.msgBacklog` across all collected
+`stats-v4.*.txt` snapshots. It is a sampled maximum, not an exact peak between snapshots. Report
+the sampling interval alongside it.
+
+Before comparing rates:
+
+- Verify 500 publishers with distinct connection addresses and one consumer of the intended
+ subscription type in the topic stats.
+- Check final client counts and failed ACKs. The consumer must receive 12 million messages;
+ completed-send accounting can let the producer exceed its target by sends already in flight.
+- Reject runs with a timeout, broker restart, OOM or failed persistence, even if clients reconnect.
+- Report percentage changes alongside absolute values and distinguish lower-is-better metrics
+ such as backlog and CPU from higher-is-better throughput.
+
+Analyze the broker recording with Jafar MCP `jfr_diagnose` and `jfr_stackprofile`, saving output
+beside the recording as `<filename>.jfr.analysis.md`. Compare CPU, monitor contention and thread-park
+views within the steady window. Executor queue delay need not appear as monitor contention.
+The command above omits wall-clock sampling to avoid its overhead during lock profiling.
+Compare allocations and actual GC pauses as well; concurrent GC duration is not pause time.
+
+For a focused comparison of completion handoff overhead, run the JMH benchmark separately from the
+integration test:
+
+```bash
+./gradlew :microbench:shadowJar
+java -jar microbench/build/libs/microbench-*-benchmarks.jar \
+ '.*ReadCompletionHandoffBenchmark.*' -prof gc -rf json -rff read-handoff.json
+```
+
+The microbenchmark uses real executors without storage IO or publishing pressure. Its latency and
+allocation results do not predict end-to-end throughput or establish concurrency correctness.
diff --git a/tests/performance/pulsar-profiling.yaml b/tests/performance/scenarios/pulsar-profiling.yaml
similarity index 83%
copy from tests/performance/pulsar-profiling.yaml
copy to tests/performance/scenarios/pulsar-profiling.yaml
index db1316d03a6c..cce6edfff751 100644
--- a/tests/performance/pulsar-profiling.yaml
+++ b/tests/performance/scenarios/pulsar-profiling.yaml
@@ -36,10 +36,22 @@ load:
consumeMemoryLimit: 200M
produceRate: 2147483647
messageSize: 128
+ batchingEnabled: false
maxOutstanding: 20000
statsIntervalSeconds: 10
# Used only by produce-v4/consume-v4. A value of 500 creates up to 500 clients.
+ producerCount: 1
+ consumerCount: 1
+ producerIoThreads: 1
+ consumerIoThreads: 1
+ subscriptionType: Shared
+ receiverQueueSize: 50000
+ timeoutSeconds: 180
isolatedProducers: 0
isolatedConsumers: 0
+profiling:
+ # Empty options leave the client unprofiled. Broker options use -Pinttest.asyncprofiler.opts.
+ producerOptions: ""
+ consumerOptions: ""
output:
directory: build/pulsar-profiling
diff --git a/tests/performance/scenarios/read-completion-isolation-256k-128k.yaml b/tests/performance/scenarios/read-completion-isolation-256k-128k.yaml
new file mode 100644
index 000000000000..674330c8e8b2
--- /dev/null
+++ b/tests/performance/scenarios/read-completion-isolation-256k-128k.yaml
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+extends: read-completion-isolation.yaml
+cluster:
+ brokerEnvs:
+ pulsarChannelWriteBufferHighWaterMark: "262144"
+ pulsarChannelWriteBufferLowWaterMark: "131072"
+output:
+ directory: build/pulsar-profiling/read-completion-isolation-256k-128k
diff --git a/tests/performance/scenarios/read-completion-isolation-64k-32k.yaml b/tests/performance/scenarios/read-completion-isolation-64k-32k.yaml
new file mode 100644
index 000000000000..a675f60ca463
--- /dev/null
+++ b/tests/performance/scenarios/read-completion-isolation-64k-32k.yaml
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+extends: read-completion-isolation.yaml
+cluster:
+ brokerEnvs:
+ pulsarChannelWriteBufferHighWaterMark: "65536"
+ pulsarChannelWriteBufferLowWaterMark: "32768"
+output:
+ directory: build/pulsar-profiling/read-completion-isolation-64k-32k
diff --git a/tests/performance/scenarios/read-completion-isolation-failover.yaml b/tests/performance/scenarios/read-completion-isolation-failover.yaml
new file mode 100644
index 000000000000..7a31b5667695
--- /dev/null
+++ b/tests/performance/scenarios/read-completion-isolation-failover.yaml
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+extends: read-completion-isolation.yaml
+load:
+ subscriptionType: Failover
+output:
+ directory: build/pulsar-profiling/read-completion-isolation-failover
diff --git a/tests/performance/scenarios/read-completion-isolation-shared.yaml b/tests/performance/scenarios/read-completion-isolation-shared.yaml
new file mode 100644
index 000000000000..856fb6e360eb
--- /dev/null
+++ b/tests/performance/scenarios/read-completion-isolation-shared.yaml
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Same 500-producer workload with one Shared consumer instead of Exclusive.
+extends: read-completion-isolation.yaml
+load:
+ subscriptionType: Shared
+output:
+ directory: build/pulsar-profiling/read-completion-isolation-shared
diff --git a/tests/performance/pulsar-profiling.yaml b/tests/performance/scenarios/read-completion-isolation.yaml
similarity index 56%
rename from tests/performance/pulsar-profiling.yaml
rename to tests/performance/scenarios/read-completion-isolation.yaml
index db1316d03a6c..2090b72e6aea 100644
--- a/tests/performance/pulsar-profiling.yaml
+++ b/tests/performance/scenarios/read-completion-isolation.yaml
@@ -15,31 +15,31 @@
# specific language governing permissions and limitations
# under the License.
-# Pass this file with PULSAR_PROFILING_CONFIG=/path/to/pulsar-profiling.yaml.
-# Values not present here keep the harness defaults.
+# 500 producers / 500 TCP connections / one Exclusive consumer on a persistent topic.
+# Run with PulsarProfilingV4Test; see read-completion-isolation.md for a baseline comparison.
cluster:
numBrokers: 1
numBookies: 3
numProxies: 0
- brokerMemory: "-Xms2g -Xmx2g -XX:+UseTransparentHugePages -XX:+AlwaysPreTouch"
- bookkeeperMemory: "-Xmx1g -XX:+UseTransparentHugePages -XX:+AlwaysPreTouch"
- brokerEnvs:
- managedLedgerDefaultEnsembleSize: "1"
- managedLedgerDefaultWriteQuorum: "1"
- managedLedgerDefaultAckQuorum: "1"
- bookkeeperEnvs:
- dbStorage_writeCacheMaxSizeMb: "64"
- dbStorage_readAheadCacheMaxSizeMb: "96"
+ brokerMemory: "-Xms2g -Xmx2g -XX:MaxDirectMemorySize=4g"
+ bookkeeperMemory: "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=512m"
load:
- numberOfMessages: 20000000
+ numberOfMessages: 12000000
+ producerCount: 500
+ isolatedProducers: 500
+ producerIoThreads: 8
+ consumerCount: 1
+ isolatedConsumers: 0
+ consumerIoThreads: 1
+ subscriptionType: Exclusive
+ receiverQueueSize: 50000
+ # This is a per-producer queue limit: 40 * 500 = 20000 outstanding sends.
+ maxOutstanding: 40
produceMemoryLimit: 200M
consumeMemoryLimit: 200M
produceRate: 2147483647
messageSize: 128
- maxOutstanding: 20000
- statsIntervalSeconds: 10
- # Used only by produce-v4/consume-v4. A value of 500 creates up to 500 clients.
- isolatedProducers: 0
- isolatedConsumers: 0
+ statsIntervalSeconds: 5
+ timeoutSeconds: 480
output:
- directory: build/pulsar-profiling
+ directory: build/pulsar-profiling/read-completion-isolation