You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/08/19 09:25:26 UTC

(pulsar) 02/02: [fix][cli][branch-4.0] Don't pass unset pulsar-perf pending-message options to the producer (#26371)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit f17c81162f5fe0bd9b1a28ebbda57c153ac59915
Author: Lari Hotari <lh...@apache.org>
AuthorDate: Wed Aug 19 12:05:43 2026 +0300

    [fix][cli][branch-4.0] Don't pass unset pulsar-perf pending-message options to the producer (#26371)
    
    Backport of the branch-4.2 fix, limited to pulsar-testclient: #26371's hunks in
    ProducerConfigurationData and ProducerBuilderImplTest already arrived on this branch with the
    #26342 backport (4c9e045854e), which carries the same
    "maxPendingMessagesAcrossPartitions needs to be >= 0" relaxation.
    
    The branch-4.2 follow-up d705ea3, which adapted the pulsar-perf test once #26342 landed there,
    is folded in here because on this branch #26342 landed first.
    
    (cherry picked from commit e0d4fd3d880b7b7f6fc9eb8be5b71d131644d19e)
    (cherry picked from commit d705ea36dd7ae86161b271108322635c503f3b0f)
---
 .../pulsar/testclient/PerformanceProducer.java     | 25 ++++++----
 .../pulsar/testclient/PerformanceProducerTest.java | 55 ++++++++++++++++++++++
 2 files changed, 71 insertions(+), 9 deletions(-)

diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
index 8860696321a3..5394838f8a51 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
@@ -22,8 +22,6 @@ import static java.util.Objects.requireNonNull;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_BATCHING_MAX_MESSAGES;
-import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES;
-import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
 import com.google.common.collect.Range;
@@ -64,6 +62,7 @@ import org.apache.pulsar.client.api.ProducerAccessMode;
 import org.apache.pulsar.client.api.ProducerBuilder;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.TypedMessageBuilder;
 import org.apache.pulsar.client.api.transaction.Transaction;
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
@@ -134,12 +133,13 @@ public class PerformanceProducer extends PerformanceTopicListArguments{
             + "larger than allowed max size")
     private boolean chunkingAllowed = false;
 
-    @Option(names = { "-o", "--max-outstanding" }, description = "Max number of outstanding messages")
-    public int maxOutstanding = DEFAULT_MAX_PENDING_MESSAGES;
+    @Option(names = { "-o", "--max-outstanding" }, description = "Max number of outstanding messages. "
+            + "Left to the client default when unset")
+    public Integer maxOutstanding;
 
     @Option(names = { "-p", "--max-outstanding-across-partitions" }, description = "Max number of outstanding "
-            + "messages across partitions")
-    public int maxPendingMessagesAcrossPartitions = DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS;
+            + "messages across partitions. Left to the client default when unset")
+    public Integer maxPendingMessagesAcrossPartitions;
 
     @Option(names = { "-np", "--partitions" }, description = "Create partitioned topics with the given number "
             + "of partitions, set 0 to not try to create the topic")
@@ -447,14 +447,21 @@ public class PerformanceProducer extends PerformanceTopicListArguments{
     }
 
     ProducerBuilder<byte[]> createProducerBuilder(PulsarClient client, int producerId) {
-        ProducerBuilder<byte[]> producerBuilder = client.newProducer() //
+        // pulsar-perf runs with the client memory limit disabled unless --memory-limit is given, so the
+        // client's pending-message defaults are the producer's only backpressure. They are applied to
+        // any producer whose limits are left unset, whichever newProducer() overload is used.
+        ProducerBuilder<byte[]> producerBuilder = client.newProducer(Schema.BYTES) //
                 .sendTimeout(this.sendTimeout, TimeUnit.SECONDS) //
                 .compressionType(this.compression) //
-                .maxPendingMessages(this.maxOutstanding) //
                 .accessMode(this.producerAccessMode)
                 // enable round robin message routing if it is a partitioned topic
                 .messageRoutingMode(MessageRoutingMode.RoundRobinPartition);
-        if (this.maxPendingMessagesAcrossPartitions > 0) {
+        // Only pass a limit the user actually asked for. Their unset value is 0, which the client reads
+        // as "no message-count limit", so passing it through would leave the producer unbounded.
+        if (this.maxOutstanding != null) {
+            producerBuilder.maxPendingMessages(this.maxOutstanding);
+        }
+        if (this.maxPendingMessagesAcrossPartitions != null) {
             producerBuilder.maxPendingMessagesAcrossPartitions(this.maxPendingMessagesAcrossPartitions);
         }
 
diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java
index 47815ddddf8b..277f72bcafc0 100644
--- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java
+++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java
@@ -32,9 +32,12 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
 import org.apache.pulsar.client.api.ClientBuilder;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.ProducerBase;
 import org.apache.pulsar.client.impl.ProducerBuilderImpl;
+import org.apache.pulsar.client.impl.conf.ProducerConfigurationData;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.awaitility.Awaitility;
@@ -236,6 +239,58 @@ public class PerformanceProducerTest extends MockedPulsarServiceBaseTest {
         consumer.close();
     }
 
+    /**
+     * pulsar-perf runs with the client memory limit disabled unless {@code --memory-limit} is given, so
+     * the producer's only backpressure is its pending-message queue. Leaving the options unset has to
+     * leave the client's own defaults in place; passing their unset value of 0 through would read as an
+     * explicit "no message-count limit" and leave the producer unbounded, which is what exhausts direct
+     * memory against a slow broker on a non-partitioned topic.
+     */
+    @Test(timeOut = 20000)
+    public void testPendingMessageLimitsAreLeftToTheClientWhenUnset() throws Exception {
+        PerformanceProducer producer = new PerformanceProducer();
+        producer.topics = List.of(testTopic + UUID.randomUUID());
+        producer.serviceURL = pulsar.getBrokerServiceUrl();
+
+        @Cleanup
+        PulsarClient client = PerfClientUtils.createClientBuilderFromArguments(producer).build();
+        ProducerBuilderImpl<byte[]> builder =
+                (ProducerBuilderImpl<byte[]>) producer.createProducerBuilder(client, 0);
+
+        Assert.assertNull(producer.maxOutstanding);
+        Assert.assertNull(producer.maxPendingMessagesAcrossPartitions);
+        // pulsar-perf must not configure either limit, so that they stay the client's to decide.
+        Assert.assertFalse(builder.isMaxPendingMessagesConfigured());
+        Assert.assertFalse(builder.isMaxPendingMessagesAcrossPartitionsConfigured());
+
+        // The client resolves its no-memory-limit defaults when the producer is created rather than on
+        // the builder, so the effective configuration is where they have to show up.
+        @Cleanup
+        Producer<byte[]> createdProducer = builder.topic(producer.topics.get(0)).create();
+        ProducerConfigurationData conf = ((ProducerBase<byte[]>) createdProducer).getConfiguration();
+        Assert.assertTrue(conf.getMaxPendingMessages() > 0);
+        Assert.assertTrue(conf.getMaxPendingMessagesAcrossPartitions() > 0);
+    }
+
+    /**
+     * An option that is given still wins over the client default, and either one can be given on its
+     * own — the across-partitions budget is allowed to be below the per-producer limit.
+     */
+    @Test(timeOut = 20000)
+    public void testGivenPendingMessageLimitsAreApplied() throws Exception {
+        PerformanceProducer producer = new PerformanceProducer();
+        producer.topics = List.of(testTopic + UUID.randomUUID());
+        producer.serviceURL = pulsar.getBrokerServiceUrl();
+        producer.maxPendingMessagesAcrossPartitions = 500;
+
+        @Cleanup
+        PulsarClient client = PerfClientUtils.createClientBuilderFromArguments(producer).build();
+        ProducerBuilderImpl<byte[]> builder =
+                (ProducerBuilderImpl<byte[]>) producer.createProducerBuilder(client, 0);
+
+        Assert.assertEquals(builder.getConf().getMaxPendingMessagesAcrossPartitions(), 500);
+    }
+
     @Test
     public void testRangeConvert() {
         PerformanceProducer.RangeConvert rangeConvert = new PerformanceProducer.RangeConvert();