You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2026/09/10 14:21:29 UTC
(pulsar) branch master updated: [improve][broker] Pre-size individual acknowledgment lists (#26517)
This is an automated email from the ASF dual-hosted git repository.
merlimat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 64db84e142d3 [improve][broker] Pre-size individual acknowledgment lists (#26517)
64db84e142d3 is described below
commit 64db84e142d39b531a9e30933234ef0fdc9b1bac
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Thu Sep 10 17:21:23 2026 +0300
[improve][broker] Pre-size individual acknowledgment lists (#26517)
---
.../service/IndividualAckAllocationBenchmark.java | 90 ++++++++++++++++++++++
.../org/apache/pulsar/broker/service/Consumer.java | 9 ++-
.../apache/pulsar/broker/service/ConsumerTest.java | 49 +++++++++++-
3 files changed, 144 insertions(+), 4 deletions(-)
diff --git a/microbench/src/main/java/org/apache/pulsar/broker/service/IndividualAckAllocationBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/service/IndividualAckAllocationBenchmark.java
new file mode 100644
index 000000000000..3a6c52fdcf51
--- /dev/null
+++ b/microbench/src/main/java/org/apache/pulsar/broker/service/IndividualAckAllocationBenchmark.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.apache.pulsar.common.api.proto.CommandAck;
+import org.apache.pulsar.common.api.proto.MessageIdData;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * Isolates list assembly for non-transactional grouped acknowledgements in Consumer.individualAck.
+ * Both lists escape to model their use by asynchronous persistence and completion callbacks.
+ * This excludes pending-ack lookups and persistence, and does not estimate full broker throughput.
+ */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+public class IndividualAckAllocationBenchmark {
+ @Param({"1", "100", "1000", "10000"})
+ public int messageCount;
+
+ private CommandAck ack;
+
+ @Setup
+ public void setup() {
+ ack = new CommandAck().setConsumerId(1).setAckType(CommandAck.AckType.Individual);
+ for (int i = 0; i < messageCount; i++) {
+ ack.addMessageId().setLedgerId(1).setEntryId(i);
+ }
+ }
+
+ @Benchmark
+ public void growingLists(Blackhole blackhole) {
+ assemble(new ArrayList<>(), new ArrayList<>(), blackhole);
+ }
+
+ @Benchmark
+ public void sizedLists(Blackhole blackhole) {
+ assemble(new ArrayList<>(ack.getMessageIdsCount()),
+ new ArrayList<>(ack.getMessageIdsCount()), blackhole);
+ }
+
+ private void assemble(List<Position> positions, List<Completion> completions, Blackhole blackhole) {
+ for (int i = 0; i < ack.getMessageIdsCount(); i++) {
+ MessageIdData id = ack.getMessageIdAt(i);
+ Position position = PositionFactory.create(id.getLedgerId(), id.getEntryId());
+ positions.add(position);
+ completions.add(new Completion(null, position, false, 1));
+ }
+ blackhole.consume(positions);
+ blackhole.consume(completions);
+ }
+
+ private record Completion(Consumer consumer, Position position, boolean hasAckSet, long ackedCount) {
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index b45947b76e83..97606d5e9d78 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -609,9 +609,10 @@ public class Consumer {
List<Pair<Consumer, MutablePair<Position, Integer>>> txnPositions =
hasTxn ? new ArrayList<>() : null;
// Non-txn path needs plain positions for acknowledgeMessageAsync.
- List<Position> nonTxnPositions = hasTxn ? null : new ArrayList<>();
+ List<Position> nonTxnPositions = hasTxn ? null : new ArrayList<>(ack.getMessageIdsCount());
// Deferred completions for non-txn (applied after persistence).
- List<PendingAckCompletion> pendingAckCompletions = new ArrayList<>();
+ List<PendingAckCompletion> pendingAckCompletions =
+ hasTxn ? null : new ArrayList<>(ack.getMessageIdsCount());
long totalAckCount = 0;
for (int i = 0; i < ack.getMessageIdsCount(); i++) {
@@ -682,7 +683,9 @@ public class Consumer {
totalAckCount += ackedCount;
}
- checkAckValidationError(ack, getAckPosition(hasTxn, msgId));
+ if (ack.hasValidationError()) {
+ checkAckValidationError(ack, getAckPosition(hasTxn, msgId));
+ }
}
final long finalTotalAckCount = totalAckCount;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
index a974655b06c2..39d36e9820c6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
@@ -23,25 +23,37 @@ import static org.apache.pulsar.client.api.MessageId.latest;
import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Exclusive;
import static org.apache.pulsar.common.api.proto.KeySharedMode.AUTO_SPLIT;
import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import java.net.SocketAddress;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import org.apache.bookkeeper.mledger.Position;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.common.api.proto.CommandAck;
import org.apache.pulsar.common.api.proto.KeySharedMeta;
+import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies;
import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl;
+import org.mockito.ArgumentCaptor;
import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@Test(groups = "broker")
public class ConsumerTest {
private Consumer consumer;
+ private Subscription subscription;
private final ConsumerStatsImpl stats = new ConsumerStatsImpl();
@BeforeMethod
public void beforeMethod() {
- Subscription subscription = mock(Subscription.class);
+ subscription = mock(Subscription.class);
ServerCnx cnx = mock(ServerCnx.class);
SocketAddress address = mock(SocketAddress.class);
Topic topic = mock(Topic.class);
@@ -51,6 +63,9 @@ public class ConsumerTest {
when(cnx.clientAddress()).thenReturn(address);
when(subscription.getTopic()).thenReturn(topic);
+ HierarchyTopicPolicies policies = new HierarchyTopicPolicies();
+ policies.getMaxUnackedMessagesOnConsumer().updateBrokerValue(0);
+ when(topic.getHierarchyTopicPolicies()).thenReturn(policies);
when(topic.getBrokerService()).thenReturn(brokerService);
when(brokerService.getPulsar()).thenReturn(pulsarService);
when(pulsarService.getConfiguration()).thenReturn(serviceConfiguration);
@@ -73,4 +88,36 @@ public class ConsumerTest {
consumer.updateStats(stats);
assertEquals(consumer.getBytesOutCounter(), 1L);
}
+
+ @DataProvider
+ public Object[][] groupedAcknowledgements() {
+ return new Object[][] {{0, false}, {1, false}, {1000, false}, {1, true}};
+ }
+
+ @Test(dataProvider = "groupedAcknowledgements")
+ public void testGroupedAcknowledgementsPreservePositionsAndCompletion(int count, boolean validationError) {
+ CompletableFuture<Void> persistence = new CompletableFuture<>();
+ when(subscription.acknowledgeMessageAsync(any(), eq(CommandAck.AckType.Individual), any()))
+ .thenReturn(persistence);
+ CommandAck ack = new CommandAck().setConsumerId(1).setAckType(CommandAck.AckType.Individual);
+ if (validationError) {
+ ack.setValidationError(CommandAck.ValidationError.ChecksumMismatch);
+ }
+ for (int i = 0; i < count; i++) {
+ ack.addMessageId().setLedgerId(7).setEntryId(i);
+ }
+
+ CompletableFuture<Void> result = consumer.messageAcked(ack, true);
+ ArgumentCaptor<List<Position>> positions = ArgumentCaptor.captor();
+ verify(subscription).acknowledgeMessageAsync(positions.capture(),
+ eq(CommandAck.AckType.Individual), eq(emptyMap()));
+ assertThat(positions.getValue()).hasSize(count);
+ for (int i = 0; i < count; i++) {
+ assertThat(positions.getValue().get(i).getLedgerId()).isEqualTo(7);
+ assertThat(positions.getValue().get(i).getEntryId()).isEqualTo(i);
+ }
+ assertThat(result).isNotDone();
+ persistence.complete(null);
+ assertThat(result).isCompletedWithValue(null);
+ }
}