You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/08/19 09:25:26 UTC
(pulsar) 02/02: [fix][cli][branch-4.0] Don't pass unset pulsar-perf pending-message options to the producer (#26371)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit f17c81162f5fe0bd9b1a28ebbda57c153ac59915
Author: Lari Hotari <lh...@apache.org>
AuthorDate: Wed Aug 19 12:05:43 2026 +0300
[fix][cli][branch-4.0] Don't pass unset pulsar-perf pending-message options to the producer (#26371)
Backport of the branch-4.2 fix, limited to pulsar-testclient: #26371's hunks in
ProducerConfigurationData and ProducerBuilderImplTest already arrived on this branch with the
#26342 backport (4c9e045854e), which carries the same
"maxPendingMessagesAcrossPartitions needs to be >= 0" relaxation.
The branch-4.2 follow-up d705ea3, which adapted the pulsar-perf test once #26342 landed there,
is folded in here because on this branch #26342 landed first.
(cherry picked from commit e0d4fd3d880b7b7f6fc9eb8be5b71d131644d19e)
(cherry picked from commit d705ea36dd7ae86161b271108322635c503f3b0f)
---
.../pulsar/testclient/PerformanceProducer.java | 25 ++++++----
.../pulsar/testclient/PerformanceProducerTest.java | 55 ++++++++++++++++++++++
2 files changed, 71 insertions(+), 9 deletions(-)
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
index 8860696321a3..5394838f8a51 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
@@ -22,8 +22,6 @@ import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_BATCHING_MAX_MESSAGES;
-import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES;
-import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.collect.Range;
@@ -64,6 +62,7 @@ import org.apache.pulsar.client.api.ProducerAccessMode;
import org.apache.pulsar.client.api.ProducerBuilder;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.TypedMessageBuilder;
import org.apache.pulsar.client.api.transaction.Transaction;
import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
@@ -134,12 +133,13 @@ public class PerformanceProducer extends PerformanceTopicListArguments{
+ "larger than allowed max size")
private boolean chunkingAllowed = false;
- @Option(names = { "-o", "--max-outstanding" }, description = "Max number of outstanding messages")
- public int maxOutstanding = DEFAULT_MAX_PENDING_MESSAGES;
+ @Option(names = { "-o", "--max-outstanding" }, description = "Max number of outstanding messages. "
+ + "Left to the client default when unset")
+ public Integer maxOutstanding;
@Option(names = { "-p", "--max-outstanding-across-partitions" }, description = "Max number of outstanding "
- + "messages across partitions")
- public int maxPendingMessagesAcrossPartitions = DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS;
+ + "messages across partitions. Left to the client default when unset")
+ public Integer maxPendingMessagesAcrossPartitions;
@Option(names = { "-np", "--partitions" }, description = "Create partitioned topics with the given number "
+ "of partitions, set 0 to not try to create the topic")
@@ -447,14 +447,21 @@ public class PerformanceProducer extends PerformanceTopicListArguments{
}
ProducerBuilder<byte[]> createProducerBuilder(PulsarClient client, int producerId) {
- ProducerBuilder<byte[]> producerBuilder = client.newProducer() //
+ // pulsar-perf runs with the client memory limit disabled unless --memory-limit is given, so the
+ // client's pending-message defaults are the producer's only backpressure. They are applied to
+ // any producer whose limits are left unset, whichever newProducer() overload is used.
+ ProducerBuilder<byte[]> producerBuilder = client.newProducer(Schema.BYTES) //
.sendTimeout(this.sendTimeout, TimeUnit.SECONDS) //
.compressionType(this.compression) //
- .maxPendingMessages(this.maxOutstanding) //
.accessMode(this.producerAccessMode)
// enable round robin message routing if it is a partitioned topic
.messageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- if (this.maxPendingMessagesAcrossPartitions > 0) {
+ // Only pass a limit the user actually asked for. Their unset value is 0, which the client reads
+ // as "no message-count limit", so passing it through would leave the producer unbounded.
+ if (this.maxOutstanding != null) {
+ producerBuilder.maxPendingMessages(this.maxOutstanding);
+ }
+ if (this.maxPendingMessagesAcrossPartitions != null) {
producerBuilder.maxPendingMessagesAcrossPartitions(this.maxPendingMessagesAcrossPartitions);
}
diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java
index 47815ddddf8b..277f72bcafc0 100644
--- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java
+++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java
@@ -32,9 +32,12 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.ProducerBase;
import org.apache.pulsar.client.impl.ProducerBuilderImpl;
+import org.apache.pulsar.client.impl.conf.ProducerConfigurationData;
import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.policies.data.TenantInfoImpl;
import org.awaitility.Awaitility;
@@ -236,6 +239,58 @@ public class PerformanceProducerTest extends MockedPulsarServiceBaseTest {
consumer.close();
}
+ /**
+ * pulsar-perf runs with the client memory limit disabled unless {@code --memory-limit} is given, so
+ * the producer's only backpressure is its pending-message queue. Leaving the options unset has to
+ * leave the client's own defaults in place; passing their unset value of 0 through would read as an
+ * explicit "no message-count limit" and leave the producer unbounded, which is what exhausts direct
+ * memory against a slow broker on a non-partitioned topic.
+ */
+ @Test(timeOut = 20000)
+ public void testPendingMessageLimitsAreLeftToTheClientWhenUnset() throws Exception {
+ PerformanceProducer producer = new PerformanceProducer();
+ producer.topics = List.of(testTopic + UUID.randomUUID());
+ producer.serviceURL = pulsar.getBrokerServiceUrl();
+
+ @Cleanup
+ PulsarClient client = PerfClientUtils.createClientBuilderFromArguments(producer).build();
+ ProducerBuilderImpl<byte[]> builder =
+ (ProducerBuilderImpl<byte[]>) producer.createProducerBuilder(client, 0);
+
+ Assert.assertNull(producer.maxOutstanding);
+ Assert.assertNull(producer.maxPendingMessagesAcrossPartitions);
+ // pulsar-perf must not configure either limit, so that they stay the client's to decide.
+ Assert.assertFalse(builder.isMaxPendingMessagesConfigured());
+ Assert.assertFalse(builder.isMaxPendingMessagesAcrossPartitionsConfigured());
+
+ // The client resolves its no-memory-limit defaults when the producer is created rather than on
+ // the builder, so the effective configuration is where they have to show up.
+ @Cleanup
+ Producer<byte[]> createdProducer = builder.topic(producer.topics.get(0)).create();
+ ProducerConfigurationData conf = ((ProducerBase<byte[]>) createdProducer).getConfiguration();
+ Assert.assertTrue(conf.getMaxPendingMessages() > 0);
+ Assert.assertTrue(conf.getMaxPendingMessagesAcrossPartitions() > 0);
+ }
+
+ /**
+ * An option that is given still wins over the client default, and either one can be given on its
+ * own — the across-partitions budget is allowed to be below the per-producer limit.
+ */
+ @Test(timeOut = 20000)
+ public void testGivenPendingMessageLimitsAreApplied() throws Exception {
+ PerformanceProducer producer = new PerformanceProducer();
+ producer.topics = List.of(testTopic + UUID.randomUUID());
+ producer.serviceURL = pulsar.getBrokerServiceUrl();
+ producer.maxPendingMessagesAcrossPartitions = 500;
+
+ @Cleanup
+ PulsarClient client = PerfClientUtils.createClientBuilderFromArguments(producer).build();
+ ProducerBuilderImpl<byte[]> builder =
+ (ProducerBuilderImpl<byte[]>) producer.createProducerBuilder(client, 0);
+
+ Assert.assertEquals(builder.getConf().getMaxPendingMessagesAcrossPartitions(), 500);
+ }
+
@Test
public void testRangeConvert() {
PerformanceProducer.RangeConvert rangeConvert = new PerformanceProducer.RangeConvert();