You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2026/09/10 14:21:29 UTC

(pulsar) branch master updated: [improve][broker] Pre-size individual acknowledgment lists (#26517)

This is an automated email from the ASF dual-hosted git repository.

merlimat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 64db84e142d3 [improve][broker] Pre-size individual acknowledgment lists (#26517)
64db84e142d3 is described below

commit 64db84e142d39b531a9e30933234ef0fdc9b1bac
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Thu Sep 10 17:21:23 2026 +0300

    [improve][broker] Pre-size individual acknowledgment lists (#26517)
---
 .../service/IndividualAckAllocationBenchmark.java  | 90 ++++++++++++++++++++++
 .../org/apache/pulsar/broker/service/Consumer.java |  9 ++-
 .../apache/pulsar/broker/service/ConsumerTest.java | 49 +++++++++++-
 3 files changed, 144 insertions(+), 4 deletions(-)

diff --git a/microbench/src/main/java/org/apache/pulsar/broker/service/IndividualAckAllocationBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/service/IndividualAckAllocationBenchmark.java
new file mode 100644
index 000000000000..3a6c52fdcf51
--- /dev/null
+++ b/microbench/src/main/java/org/apache/pulsar/broker/service/IndividualAckAllocationBenchmark.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.apache.pulsar.common.api.proto.CommandAck;
+import org.apache.pulsar.common.api.proto.MessageIdData;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * Isolates list assembly for non-transactional grouped acknowledgements in Consumer.individualAck.
+ * Both lists escape to model their use by asynchronous persistence and completion callbacks.
+ * This excludes pending-ack lookups and persistence, and does not estimate full broker throughput.
+ */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+public class IndividualAckAllocationBenchmark {
+    @Param({"1", "100", "1000", "10000"})
+    public int messageCount;
+
+    private CommandAck ack;
+
+    @Setup
+    public void setup() {
+        ack = new CommandAck().setConsumerId(1).setAckType(CommandAck.AckType.Individual);
+        for (int i = 0; i < messageCount; i++) {
+            ack.addMessageId().setLedgerId(1).setEntryId(i);
+        }
+    }
+
+    @Benchmark
+    public void growingLists(Blackhole blackhole) {
+        assemble(new ArrayList<>(), new ArrayList<>(), blackhole);
+    }
+
+    @Benchmark
+    public void sizedLists(Blackhole blackhole) {
+        assemble(new ArrayList<>(ack.getMessageIdsCount()),
+                new ArrayList<>(ack.getMessageIdsCount()), blackhole);
+    }
+
+    private void assemble(List<Position> positions, List<Completion> completions, Blackhole blackhole) {
+        for (int i = 0; i < ack.getMessageIdsCount(); i++) {
+            MessageIdData id = ack.getMessageIdAt(i);
+            Position position = PositionFactory.create(id.getLedgerId(), id.getEntryId());
+            positions.add(position);
+            completions.add(new Completion(null, position, false, 1));
+        }
+        blackhole.consume(positions);
+        blackhole.consume(completions);
+    }
+
+    private record Completion(Consumer consumer, Position position, boolean hasAckSet, long ackedCount) {
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index b45947b76e83..97606d5e9d78 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -609,9 +609,10 @@ public class Consumer {
         List<Pair<Consumer, MutablePair<Position, Integer>>> txnPositions =
                 hasTxn ? new ArrayList<>() : null;
         // Non-txn path needs plain positions for acknowledgeMessageAsync.
-        List<Position> nonTxnPositions = hasTxn ? null : new ArrayList<>();
+        List<Position> nonTxnPositions = hasTxn ? null : new ArrayList<>(ack.getMessageIdsCount());
         // Deferred completions for non-txn (applied after persistence).
-        List<PendingAckCompletion> pendingAckCompletions = new ArrayList<>();
+        List<PendingAckCompletion> pendingAckCompletions =
+                hasTxn ? null : new ArrayList<>(ack.getMessageIdsCount());
         long totalAckCount = 0;
 
         for (int i = 0; i < ack.getMessageIdsCount(); i++) {
@@ -682,7 +683,9 @@ public class Consumer {
                 totalAckCount += ackedCount;
             }
 
-            checkAckValidationError(ack, getAckPosition(hasTxn, msgId));
+            if (ack.hasValidationError()) {
+                checkAckValidationError(ack, getAckPosition(hasTxn, msgId));
+            }
         }
 
         final long finalTotalAckCount = totalAckCount;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
index a974655b06c2..39d36e9820c6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
@@ -23,25 +23,37 @@ import static org.apache.pulsar.client.api.MessageId.latest;
 import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Exclusive;
 import static org.apache.pulsar.common.api.proto.KeySharedMode.AUTO_SPLIT;
 import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
 import java.net.SocketAddress;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import org.apache.bookkeeper.mledger.Position;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.common.api.proto.CommandAck;
 import org.apache.pulsar.common.api.proto.KeySharedMeta;
+import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies;
 import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl;
+import org.mockito.ArgumentCaptor;
 import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 @Test(groups = "broker")
 public class ConsumerTest {
     private Consumer consumer;
+    private Subscription subscription;
     private final ConsumerStatsImpl stats = new ConsumerStatsImpl();
 
     @BeforeMethod
     public void beforeMethod() {
-        Subscription subscription = mock(Subscription.class);
+        subscription = mock(Subscription.class);
         ServerCnx cnx = mock(ServerCnx.class);
         SocketAddress address = mock(SocketAddress.class);
         Topic topic = mock(Topic.class);
@@ -51,6 +63,9 @@ public class ConsumerTest {
 
         when(cnx.clientAddress()).thenReturn(address);
         when(subscription.getTopic()).thenReturn(topic);
+        HierarchyTopicPolicies policies = new HierarchyTopicPolicies();
+        policies.getMaxUnackedMessagesOnConsumer().updateBrokerValue(0);
+        when(topic.getHierarchyTopicPolicies()).thenReturn(policies);
         when(topic.getBrokerService()).thenReturn(brokerService);
         when(brokerService.getPulsar()).thenReturn(pulsarService);
         when(pulsarService.getConfiguration()).thenReturn(serviceConfiguration);
@@ -73,4 +88,36 @@ public class ConsumerTest {
         consumer.updateStats(stats);
         assertEquals(consumer.getBytesOutCounter(), 1L);
     }
+
+    @DataProvider
+    public Object[][] groupedAcknowledgements() {
+        return new Object[][] {{0, false}, {1, false}, {1000, false}, {1, true}};
+    }
+
+    @Test(dataProvider = "groupedAcknowledgements")
+    public void testGroupedAcknowledgementsPreservePositionsAndCompletion(int count, boolean validationError) {
+        CompletableFuture<Void> persistence = new CompletableFuture<>();
+        when(subscription.acknowledgeMessageAsync(any(), eq(CommandAck.AckType.Individual), any()))
+                .thenReturn(persistence);
+        CommandAck ack = new CommandAck().setConsumerId(1).setAckType(CommandAck.AckType.Individual);
+        if (validationError) {
+            ack.setValidationError(CommandAck.ValidationError.ChecksumMismatch);
+        }
+        for (int i = 0; i < count; i++) {
+            ack.addMessageId().setLedgerId(7).setEntryId(i);
+        }
+
+        CompletableFuture<Void> result = consumer.messageAcked(ack, true);
+        ArgumentCaptor<List<Position>> positions = ArgumentCaptor.captor();
+        verify(subscription).acknowledgeMessageAsync(positions.capture(),
+                eq(CommandAck.AckType.Individual), eq(emptyMap()));
+        assertThat(positions.getValue()).hasSize(count);
+        for (int i = 0; i < count; i++) {
+            assertThat(positions.getValue().get(i).getLedgerId()).isEqualTo(7);
+            assertThat(positions.getValue().get(i).getEntryId()).isEqualTo(i);
+        }
+        assertThat(result).isNotDone();
+        persistence.complete(null);
+        assertThat(result).isCompletedWithValue(null);
+    }
 }