You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2026/09/15 16:40:22 UTC

(pulsar) branch master updated: [improve][broker] Store deferred acknowledgment completion state in arrays (#26592)

This is an automated email from the ASF dual-hosted git repository.

merlimat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 36651b1e5126 [improve][broker] Store deferred acknowledgment completion state in arrays (#26592)
36651b1e5126 is described below

commit 36651b1e512660e27d625c9c021c4897426fc43c
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Sep 15 19:40:16 2026 +0300

    [improve][broker] Store deferred acknowledgment completion state in arrays (#26592)
---
 .../service/PendingAckCompletionBenchmark.java     | 105 +++++++++++++++++++++
 .../org/apache/pulsar/broker/service/Consumer.java |  73 +++++++++++---
 .../apache/pulsar/broker/service/ConsumerTest.java | 105 ++++++++++++++++++++-
 3 files changed, 269 insertions(+), 14 deletions(-)

diff --git a/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAckCompletionBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAckCompletionBenchmark.java
new file mode 100644
index 000000000000..b2b3001ab2cb
--- /dev/null
+++ b/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAckCompletionBenchmark.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/** Compares the previous record-list layout with the actual command-local completion snapshot. */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+public class PendingAckCompletionBenchmark {
+    @Param({"1", "100", "1000"})
+    public int entries;
+
+    @Param({"false", "true"})
+    public boolean batched;
+
+    @Param({"records", "arrays"})
+    public String representation;
+
+    private Position[] positions;
+    private Consumer owner;
+    private volatile Object retained;
+
+    @Setup
+    public void setup() {
+        owner = new Consumer("benchmark", 1000);
+        positions = new Position[entries];
+        for (int i = 0; i < entries; i++) {
+            positions[i] = PositionFactory.create(123, i);
+        }
+    }
+
+    @Benchmark
+    public long buildAndRead(Blackhole blackhole) {
+        long result = 0;
+        if ("records".equals(representation)) {
+            List<LegacyCompletion> completions = new ArrayList<>(entries);
+            for (int i = 0; i < entries; i++) {
+                completions.add(new LegacyCompletion(owner, positions[i], batched, i));
+            }
+            retained = completions;
+            for (LegacyCompletion completion : completions) {
+                blackhole.consume(completion.consumer());
+                result += completion.position().getEntryId();
+                if (completion.hasAckSet()) {
+                    result += Math.max(0, completion.ackedCount());
+                }
+            }
+        } else {
+            Consumer.PendingAckCompletions completions = new Consumer.PendingAckCompletions(entries);
+            for (int i = 0; i < entries; i++) {
+                completions.add(owner, positions[i], batched, i);
+            }
+            retained = completions;
+            for (int i = 0; i < completions.size(); i++) {
+                blackhole.consume(completions.consumerAt(i));
+                result += completions.positionAt(i).getEntryId();
+                if (completions.hasAckSetAt(i)) {
+                    result += Math.max(0, completions.ackedCountAt(i));
+                }
+            }
+        }
+        return result;
+    }
+
+    // Previous production record layout. Positions are prebuilt and owner calls are excluded.
+    private record LegacyCompletion(Consumer consumer, Position position, boolean hasAckSet, long ackedCount) {
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index abc6779e194a..db00accd401c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -610,8 +610,8 @@ public class Consumer {
         // Non-txn path needs plain positions for acknowledgeMessageAsync.
         List<Position> nonTxnPositions = hasTxn ? null : new ArrayList<>(ack.getMessageIdsCount());
         // Deferred completions for non-txn (applied after persistence).
-        List<PendingAckCompletion> pendingAckCompletions =
-                hasTxn ? null : new ArrayList<>(ack.getMessageIdsCount());
+        PendingAckCompletions pendingAckCompletions =
+                hasTxn ? null : new PendingAckCompletions(ack.getMessageIdsCount());
         long totalAckCount = 0;
 
         for (int i = 0; i < ack.getMessageIdsCount(); i++) {
@@ -677,8 +677,7 @@ public class Consumer {
                 long ackedCount = computeAckedCount(msgId, position, ackOwnerConsumer, batchSize);
 
                 nonTxnPositions.add(position);
-                pendingAckCompletions.add(new PendingAckCompletion(ackOwnerConsumer, position,
-                        hasAckSet, ackedCount));
+                pendingAckCompletions.add(ackOwnerConsumer, position, hasAckSet, ackedCount);
                 totalAckCount += ackedCount;
             }
 
@@ -741,20 +740,68 @@ public class Consumer {
     }
 
 
-    private record PendingAckCompletion(Consumer consumer, Position position, boolean hasAckSet, long ackedCount) {
+    /**
+     * A command-local snapshot, independent of the position list passed to the subscription.
+     * Arrays avoid allocating a completion record per message ID and reduce traversal overhead for grouped ACKs.
+     * Built before starting persistence and never mutated or reused once its callback can run.
+     */
+    @VisibleForTesting
+    static final class PendingAckCompletions {
+        private final Consumer[] consumers;
+        private final Position[] positions;
+        private long[] batchAckedCounts;
+        private int size;
+
+        PendingAckCompletions(int capacity) {
+            consumers = new Consumer[capacity];
+            positions = new Position[capacity];
+        }
+
+        void add(Consumer consumer, Position position, boolean hasAckSet, long ackedCount) {
+            consumers[size] = consumer;
+            positions[size] = position;
+            if (hasAckSet) {
+                if (batchAckedCounts == null) {
+                    batchAckedCounts = new long[positions.length];
+                }
+                // Zero denotes a whole-entry ACK. Non-positive batch deltas must not remove the entry.
+                batchAckedCounts[size] = ackedCount > 0 ? ackedCount : -1;
+            }
+            size++;
+        }
+
+        int size() {
+            return size;
+        }
+
+        Consumer consumerAt(int index) {
+            return consumers[index];
+        }
+
+        Position positionAt(int index) {
+            return positions[index];
+        }
+
+        boolean hasAckSetAt(int index) {
+            return batchAckedCounts != null && batchAckedCounts[index] != 0;
+        }
+
+        long ackedCountAt(int index) {
+            return batchAckedCounts[index];
+        }
     }
 
-    private void applyPendingAckCompletions(List<PendingAckCompletion> pendingAckCompletions) {
-        for (PendingAckCompletion pendingAckCompletion : pendingAckCompletions) {
-            Consumer ackOwnerConsumer = pendingAckCompletion.consumer();
-            Position position = pendingAckCompletion.position();
+    private void applyPendingAckCompletions(PendingAckCompletions pendingAckCompletions) {
+        for (int i = 0; i < pendingAckCompletions.size(); i++) {
+            Consumer ackOwnerConsumer = pendingAckCompletions.consumerAt(i);
+            Position position = pendingAckCompletions.positionAt(i);
 
-            if (pendingAckCompletion.hasAckSet()) {
-                if (pendingAckCompletion.ackedCount() > 0) {
+            if (pendingAckCompletions.hasAckSetAt(i)) {
+                if (pendingAckCompletions.ackedCountAt(i) > 0) {
                     boolean updated = ackOwnerConsumer.updateRemainingUnacked(
-                            position.getLedgerId(), position.getEntryId(), (int) pendingAckCompletion.ackedCount());
+                            position.getLedgerId(), position.getEntryId(), (int) pendingAckCompletions.ackedCountAt(i));
                     if (updated) {
-                        addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) pendingAckCompletion.ackedCount());
+                        addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) pendingAckCompletions.ackedCountAt(i));
                     }
                 }
             } else {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
index 39d36e9820c6..a4b171615321 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.broker.service;
 import static java.util.Collections.emptyMap;
 import static org.apache.pulsar.client.api.MessageId.latest;
 import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Exclusive;
+import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Shared;
 import static org.apache.pulsar.common.api.proto.KeySharedMode.AUTO_SPLIT;
 import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -49,12 +50,13 @@ import org.testng.annotations.Test;
 public class ConsumerTest {
     private Consumer consumer;
     private Subscription subscription;
+    private ServerCnx cnx;
     private final ConsumerStatsImpl stats = new ConsumerStatsImpl();
 
     @BeforeMethod
     public void beforeMethod() {
         subscription = mock(Subscription.class);
-        ServerCnx cnx = mock(ServerCnx.class);
+        cnx = mock(ServerCnx.class);
         SocketAddress address = mock(SocketAddress.class);
         Topic topic = mock(Topic.class);
         BrokerService brokerService = mock(BrokerService.class);
@@ -120,4 +122,105 @@ public class ConsumerTest {
         persistence.complete(null);
         assertThat(result).isCompletedWithValue(null);
     }
+    @Test
+    public void testAckCommandsKeepIndependentCompletionState() {
+        Consumer receiver = new Consumer(subscription, Shared, "topic", 2, 0, "receiver", true, cnx,
+                "myrole-1", emptyMap(), false, null, latest, DEFAULT_CONSUMER_EPOCH);
+        when(subscription.getConsumers()).thenReturn(List.of(receiver));
+        receiver.getPendingAcks().addPendingAckIfAllowed(7, 1, 3, 0);
+        receiver.getPendingAcks().addPendingAckIfAllowed(7, 2, 5, 0);
+        CompletableFuture<Void> firstPersistence = new CompletableFuture<>();
+        CompletableFuture<Void> secondPersistence = new CompletableFuture<>();
+        when(subscription.acknowledgeMessageAsync(any(), eq(CommandAck.AckType.Individual), any()))
+                .thenReturn(firstPersistence, secondPersistence);
+        CommandAck ack = new CommandAck().setConsumerId(2).setAckType(CommandAck.AckType.Individual);
+        ack.addMessageId().setLedgerId(7).setEntryId(1);
+        try {
+            CompletableFuture<Void> first = receiver.messageAcked(ack, true);
+            ack.getMessageIdAt(0).setEntryId(2);
+            CompletableFuture<Void> second = receiver.messageAcked(ack, true);
+            // The parsed command can be reused while persistence callbacks are still pending.
+            ack.clear();
+            secondPersistence.complete(null);
+            assertThat(second).isCompletedWithValue(null);
+            assertThat(first).isNotDone();
+            assertThat(receiver.getPendingAcks().contains(7, 2)).isFalse();
+            assertThat(receiver.getPendingAcks().getRemainingUnacked(7, 1)).isEqualTo(3);
+            assertThat(receiver.getMessageAckCounter()).isEqualTo(5);
+            firstPersistence.complete(null);
+            assertThat(first).isCompletedWithValue(null);
+            assertThat(receiver.getPendingAcks().contains(7, 1)).isFalse();
+            assertThat(receiver.getMessageAckCounter()).isEqualTo(8);
+        } finally {
+            receiver.getPendingAcks().forEachAndClose((ledgerId, entryId, remaining, stickyKeyHash) -> { });
+        }
+    }
+
+    @DataProvider
+    public Object[][] completionOutcomes() {
+        return new Object[][] {
+                {true, true, 12L, 2}, {true, false, 12L, 2},
+                {false, true, 12L, 2}, {false, false, 12L, 2},
+                {true, true, 15L, 0}, {true, false, 15L, 0},
+                {false, true, 15L, 0}, {false, false, 15L, 0},
+                {true, true, 31L, -1}, {true, false, 31L, -1},
+                {false, true, 31L, -1}, {false, false, 31L, -1}
+        };
+    }
+
+    @Test(dataProvider = "completionOutcomes")
+    public void testMixedAckCompletionsKeepIndependentSnapshot(boolean requirePersistence, boolean success,
+                                                             long ackSet, int ackedDelta) {
+        when(subscription.getTopic().getBrokerService().getPulsar().getConfiguration()
+                .isAcknowledgmentAtBatchIndexLevelEnabled()).thenReturn(true);
+        Consumer receiver = new Consumer(subscription, Shared, "topic", 2, 0, "receiver", true, cnx,
+                "myrole-1", emptyMap(), false, null, latest, DEFAULT_CONSUMER_EPOCH);
+        Consumer other = new Consumer(subscription, Shared, "topic", 3, 0, "other", true, cnx,
+                "myrole-1", emptyMap(), false, null, latest, DEFAULT_CONSUMER_EPOCH);
+        when(subscription.getConsumers()).thenReturn(List.of(receiver, other));
+        receiver.getPendingAcks().addPendingAckIfAllowed(7, 1, 1, 0);
+        other.getPendingAcks().addPendingAckIfAllowed(7, 2, 4, 0);
+        receiver.getPendingAcks().addPendingAckIfAllowed(7, 3, 2, 0);
+        CompletableFuture<Void> persistence = new CompletableFuture<>();
+        when(subscription.acknowledgeMessageAsync(any(), eq(CommandAck.AckType.Individual), any()))
+                .thenAnswer(invocation -> {
+                    List<Position> positions = invocation.getArgument(0);
+                    positions.clear();
+                    return persistence;
+                });
+        CommandAck ack = new CommandAck().setConsumerId(2).setAckType(CommandAck.AckType.Individual);
+        ack.addMessageId().setLedgerId(7).setEntryId(1);
+        ack.addMessageId().setLedgerId(7).setEntryId(2).addAckSet(ackSet);
+        ack.addMessageId().setLedgerId(7).setEntryId(3);
+        try {
+            CompletableFuture<Void> result = receiver.messageAcked(ack, requirePersistence);
+            assertThat(receiver.getPendingAcks().getRemainingUnacked(7, 1)).isEqualTo(1);
+            assertThat(other.getPendingAcks().getRemainingUnacked(7, 2)).isEqualTo(4);
+            if (requirePersistence) {
+                assertThat(result).isNotDone();
+            } else {
+                assertThat(result).isCompletedWithValue(null);
+            }
+            if (success) {
+                persistence.complete(null);
+                assertThat(result).isCompletedWithValue(null);
+                assertThat(receiver.getPendingAcks().contains(7, 1)).isFalse();
+                assertThat(receiver.getPendingAcks().contains(7, 3)).isFalse();
+                assertThat(other.getPendingAcks().getRemainingUnacked(7, 2)).isEqualTo(4 - Math.max(0, ackedDelta));
+                assertThat(receiver.getMessageAckCounter()).isEqualTo(3 + ackedDelta);
+            } else {
+                persistence.completeExceptionally(new IllegalStateException("persistence failed"));
+                if (requirePersistence) {
+                    assertThat(result).isCompletedExceptionally();
+                }
+                assertThat(receiver.getPendingAcks().getRemainingUnacked(7, 1)).isEqualTo(1);
+                assertThat(receiver.getPendingAcks().getRemainingUnacked(7, 3)).isEqualTo(2);
+                assertThat(other.getPendingAcks().getRemainingUnacked(7, 2)).isEqualTo(4);
+            }
+        } finally {
+            receiver.getPendingAcks().forEachAndClose((ledgerId, entryId, remaining, stickyKeyHash) -> { });
+            other.getPendingAcks().forEachAndClose((ledgerId, entryId, remaining, stickyKeyHash) -> { });
+        }
+    }
+
 }