You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2026/09/15 16:40:22 UTC
(pulsar) branch master updated: [improve][broker] Store deferred acknowledgment completion state in arrays (#26592)
This is an automated email from the ASF dual-hosted git repository.
merlimat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 36651b1e5126 [improve][broker] Store deferred acknowledgment completion state in arrays (#26592)
36651b1e5126 is described below
commit 36651b1e512660e27d625c9c021c4897426fc43c
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Sep 15 19:40:16 2026 +0300
[improve][broker] Store deferred acknowledgment completion state in arrays (#26592)
---
.../service/PendingAckCompletionBenchmark.java | 105 +++++++++++++++++++++
.../org/apache/pulsar/broker/service/Consumer.java | 73 +++++++++++---
.../apache/pulsar/broker/service/ConsumerTest.java | 105 ++++++++++++++++++++-
3 files changed, 269 insertions(+), 14 deletions(-)
diff --git a/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAckCompletionBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAckCompletionBenchmark.java
new file mode 100644
index 000000000000..b2b3001ab2cb
--- /dev/null
+++ b/microbench/src/main/java/org/apache/pulsar/broker/service/PendingAckCompletionBenchmark.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/** Compares the previous record-list layout with the actual command-local completion snapshot. */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+public class PendingAckCompletionBenchmark {
+ @Param({"1", "100", "1000"})
+ public int entries;
+
+ @Param({"false", "true"})
+ public boolean batched;
+
+ @Param({"records", "arrays"})
+ public String representation;
+
+ private Position[] positions;
+ private Consumer owner;
+ private volatile Object retained;
+
+ @Setup
+ public void setup() {
+ owner = new Consumer("benchmark", 1000);
+ positions = new Position[entries];
+ for (int i = 0; i < entries; i++) {
+ positions[i] = PositionFactory.create(123, i);
+ }
+ }
+
+ @Benchmark
+ public long buildAndRead(Blackhole blackhole) {
+ long result = 0;
+ if ("records".equals(representation)) {
+ List<LegacyCompletion> completions = new ArrayList<>(entries);
+ for (int i = 0; i < entries; i++) {
+ completions.add(new LegacyCompletion(owner, positions[i], batched, i));
+ }
+ retained = completions;
+ for (LegacyCompletion completion : completions) {
+ blackhole.consume(completion.consumer());
+ result += completion.position().getEntryId();
+ if (completion.hasAckSet()) {
+ result += Math.max(0, completion.ackedCount());
+ }
+ }
+ } else {
+ Consumer.PendingAckCompletions completions = new Consumer.PendingAckCompletions(entries);
+ for (int i = 0; i < entries; i++) {
+ completions.add(owner, positions[i], batched, i);
+ }
+ retained = completions;
+ for (int i = 0; i < completions.size(); i++) {
+ blackhole.consume(completions.consumerAt(i));
+ result += completions.positionAt(i).getEntryId();
+ if (completions.hasAckSetAt(i)) {
+ result += Math.max(0, completions.ackedCountAt(i));
+ }
+ }
+ }
+ return result;
+ }
+
+ // Previous production record layout. Positions are prebuilt and owner calls are excluded.
+ private record LegacyCompletion(Consumer consumer, Position position, boolean hasAckSet, long ackedCount) {
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index abc6779e194a..db00accd401c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -610,8 +610,8 @@ public class Consumer {
// Non-txn path needs plain positions for acknowledgeMessageAsync.
List<Position> nonTxnPositions = hasTxn ? null : new ArrayList<>(ack.getMessageIdsCount());
// Deferred completions for non-txn (applied after persistence).
- List<PendingAckCompletion> pendingAckCompletions =
- hasTxn ? null : new ArrayList<>(ack.getMessageIdsCount());
+ PendingAckCompletions pendingAckCompletions =
+ hasTxn ? null : new PendingAckCompletions(ack.getMessageIdsCount());
long totalAckCount = 0;
for (int i = 0; i < ack.getMessageIdsCount(); i++) {
@@ -677,8 +677,7 @@ public class Consumer {
long ackedCount = computeAckedCount(msgId, position, ackOwnerConsumer, batchSize);
nonTxnPositions.add(position);
- pendingAckCompletions.add(new PendingAckCompletion(ackOwnerConsumer, position,
- hasAckSet, ackedCount));
+ pendingAckCompletions.add(ackOwnerConsumer, position, hasAckSet, ackedCount);
totalAckCount += ackedCount;
}
@@ -741,20 +740,68 @@ public class Consumer {
}
- private record PendingAckCompletion(Consumer consumer, Position position, boolean hasAckSet, long ackedCount) {
+ /**
+ * A command-local snapshot, independent of the position list passed to the subscription.
+ * Arrays avoid allocating a completion record per message ID and reduce traversal overhead for grouped ACKs.
+ * Built before starting persistence and never mutated or reused once its callback can run.
+ */
+ @VisibleForTesting
+ static final class PendingAckCompletions {
+ private final Consumer[] consumers;
+ private final Position[] positions;
+ private long[] batchAckedCounts;
+ private int size;
+
+ PendingAckCompletions(int capacity) {
+ consumers = new Consumer[capacity];
+ positions = new Position[capacity];
+ }
+
+ void add(Consumer consumer, Position position, boolean hasAckSet, long ackedCount) {
+ consumers[size] = consumer;
+ positions[size] = position;
+ if (hasAckSet) {
+ if (batchAckedCounts == null) {
+ batchAckedCounts = new long[positions.length];
+ }
+ // Zero denotes a whole-entry ACK. Non-positive batch deltas must not remove the entry.
+ batchAckedCounts[size] = ackedCount > 0 ? ackedCount : -1;
+ }
+ size++;
+ }
+
+ int size() {
+ return size;
+ }
+
+ Consumer consumerAt(int index) {
+ return consumers[index];
+ }
+
+ Position positionAt(int index) {
+ return positions[index];
+ }
+
+ boolean hasAckSetAt(int index) {
+ return batchAckedCounts != null && batchAckedCounts[index] != 0;
+ }
+
+ long ackedCountAt(int index) {
+ return batchAckedCounts[index];
+ }
}
- private void applyPendingAckCompletions(List<PendingAckCompletion> pendingAckCompletions) {
- for (PendingAckCompletion pendingAckCompletion : pendingAckCompletions) {
- Consumer ackOwnerConsumer = pendingAckCompletion.consumer();
- Position position = pendingAckCompletion.position();
+ private void applyPendingAckCompletions(PendingAckCompletions pendingAckCompletions) {
+ for (int i = 0; i < pendingAckCompletions.size(); i++) {
+ Consumer ackOwnerConsumer = pendingAckCompletions.consumerAt(i);
+ Position position = pendingAckCompletions.positionAt(i);
- if (pendingAckCompletion.hasAckSet()) {
- if (pendingAckCompletion.ackedCount() > 0) {
+ if (pendingAckCompletions.hasAckSetAt(i)) {
+ if (pendingAckCompletions.ackedCountAt(i) > 0) {
boolean updated = ackOwnerConsumer.updateRemainingUnacked(
- position.getLedgerId(), position.getEntryId(), (int) pendingAckCompletion.ackedCount());
+ position.getLedgerId(), position.getEntryId(), (int) pendingAckCompletions.ackedCountAt(i));
if (updated) {
- addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) pendingAckCompletion.ackedCount());
+ addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) pendingAckCompletions.ackedCountAt(i));
}
}
} else {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
index 39d36e9820c6..a4b171615321 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.broker.service;
import static java.util.Collections.emptyMap;
import static org.apache.pulsar.client.api.MessageId.latest;
import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Exclusive;
+import static org.apache.pulsar.common.api.proto.CommandSubscribe.SubType.Shared;
import static org.apache.pulsar.common.api.proto.KeySharedMode.AUTO_SPLIT;
import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH;
import static org.assertj.core.api.Assertions.assertThat;
@@ -49,12 +50,13 @@ import org.testng.annotations.Test;
public class ConsumerTest {
private Consumer consumer;
private Subscription subscription;
+ private ServerCnx cnx;
private final ConsumerStatsImpl stats = new ConsumerStatsImpl();
@BeforeMethod
public void beforeMethod() {
subscription = mock(Subscription.class);
- ServerCnx cnx = mock(ServerCnx.class);
+ cnx = mock(ServerCnx.class);
SocketAddress address = mock(SocketAddress.class);
Topic topic = mock(Topic.class);
BrokerService brokerService = mock(BrokerService.class);
@@ -120,4 +122,105 @@ public class ConsumerTest {
persistence.complete(null);
assertThat(result).isCompletedWithValue(null);
}
+ @Test
+ public void testAckCommandsKeepIndependentCompletionState() {
+ Consumer receiver = new Consumer(subscription, Shared, "topic", 2, 0, "receiver", true, cnx,
+ "myrole-1", emptyMap(), false, null, latest, DEFAULT_CONSUMER_EPOCH);
+ when(subscription.getConsumers()).thenReturn(List.of(receiver));
+ receiver.getPendingAcks().addPendingAckIfAllowed(7, 1, 3, 0);
+ receiver.getPendingAcks().addPendingAckIfAllowed(7, 2, 5, 0);
+ CompletableFuture<Void> firstPersistence = new CompletableFuture<>();
+ CompletableFuture<Void> secondPersistence = new CompletableFuture<>();
+ when(subscription.acknowledgeMessageAsync(any(), eq(CommandAck.AckType.Individual), any()))
+ .thenReturn(firstPersistence, secondPersistence);
+ CommandAck ack = new CommandAck().setConsumerId(2).setAckType(CommandAck.AckType.Individual);
+ ack.addMessageId().setLedgerId(7).setEntryId(1);
+ try {
+ CompletableFuture<Void> first = receiver.messageAcked(ack, true);
+ ack.getMessageIdAt(0).setEntryId(2);
+ CompletableFuture<Void> second = receiver.messageAcked(ack, true);
+ // The parsed command can be reused while persistence callbacks are still pending.
+ ack.clear();
+ secondPersistence.complete(null);
+ assertThat(second).isCompletedWithValue(null);
+ assertThat(first).isNotDone();
+ assertThat(receiver.getPendingAcks().contains(7, 2)).isFalse();
+ assertThat(receiver.getPendingAcks().getRemainingUnacked(7, 1)).isEqualTo(3);
+ assertThat(receiver.getMessageAckCounter()).isEqualTo(5);
+ firstPersistence.complete(null);
+ assertThat(first).isCompletedWithValue(null);
+ assertThat(receiver.getPendingAcks().contains(7, 1)).isFalse();
+ assertThat(receiver.getMessageAckCounter()).isEqualTo(8);
+ } finally {
+ receiver.getPendingAcks().forEachAndClose((ledgerId, entryId, remaining, stickyKeyHash) -> { });
+ }
+ }
+
+ @DataProvider
+ public Object[][] completionOutcomes() {
+ return new Object[][] {
+ {true, true, 12L, 2}, {true, false, 12L, 2},
+ {false, true, 12L, 2}, {false, false, 12L, 2},
+ {true, true, 15L, 0}, {true, false, 15L, 0},
+ {false, true, 15L, 0}, {false, false, 15L, 0},
+ {true, true, 31L, -1}, {true, false, 31L, -1},
+ {false, true, 31L, -1}, {false, false, 31L, -1}
+ };
+ }
+
+ @Test(dataProvider = "completionOutcomes")
+ public void testMixedAckCompletionsKeepIndependentSnapshot(boolean requirePersistence, boolean success,
+ long ackSet, int ackedDelta) {
+ when(subscription.getTopic().getBrokerService().getPulsar().getConfiguration()
+ .isAcknowledgmentAtBatchIndexLevelEnabled()).thenReturn(true);
+ Consumer receiver = new Consumer(subscription, Shared, "topic", 2, 0, "receiver", true, cnx,
+ "myrole-1", emptyMap(), false, null, latest, DEFAULT_CONSUMER_EPOCH);
+ Consumer other = new Consumer(subscription, Shared, "topic", 3, 0, "other", true, cnx,
+ "myrole-1", emptyMap(), false, null, latest, DEFAULT_CONSUMER_EPOCH);
+ when(subscription.getConsumers()).thenReturn(List.of(receiver, other));
+ receiver.getPendingAcks().addPendingAckIfAllowed(7, 1, 1, 0);
+ other.getPendingAcks().addPendingAckIfAllowed(7, 2, 4, 0);
+ receiver.getPendingAcks().addPendingAckIfAllowed(7, 3, 2, 0);
+ CompletableFuture<Void> persistence = new CompletableFuture<>();
+ when(subscription.acknowledgeMessageAsync(any(), eq(CommandAck.AckType.Individual), any()))
+ .thenAnswer(invocation -> {
+ List<Position> positions = invocation.getArgument(0);
+ positions.clear();
+ return persistence;
+ });
+ CommandAck ack = new CommandAck().setConsumerId(2).setAckType(CommandAck.AckType.Individual);
+ ack.addMessageId().setLedgerId(7).setEntryId(1);
+ ack.addMessageId().setLedgerId(7).setEntryId(2).addAckSet(ackSet);
+ ack.addMessageId().setLedgerId(7).setEntryId(3);
+ try {
+ CompletableFuture<Void> result = receiver.messageAcked(ack, requirePersistence);
+ assertThat(receiver.getPendingAcks().getRemainingUnacked(7, 1)).isEqualTo(1);
+ assertThat(other.getPendingAcks().getRemainingUnacked(7, 2)).isEqualTo(4);
+ if (requirePersistence) {
+ assertThat(result).isNotDone();
+ } else {
+ assertThat(result).isCompletedWithValue(null);
+ }
+ if (success) {
+ persistence.complete(null);
+ assertThat(result).isCompletedWithValue(null);
+ assertThat(receiver.getPendingAcks().contains(7, 1)).isFalse();
+ assertThat(receiver.getPendingAcks().contains(7, 3)).isFalse();
+ assertThat(other.getPendingAcks().getRemainingUnacked(7, 2)).isEqualTo(4 - Math.max(0, ackedDelta));
+ assertThat(receiver.getMessageAckCounter()).isEqualTo(3 + ackedDelta);
+ } else {
+ persistence.completeExceptionally(new IllegalStateException("persistence failed"));
+ if (requirePersistence) {
+ assertThat(result).isCompletedExceptionally();
+ }
+ assertThat(receiver.getPendingAcks().getRemainingUnacked(7, 1)).isEqualTo(1);
+ assertThat(receiver.getPendingAcks().getRemainingUnacked(7, 3)).isEqualTo(2);
+ assertThat(other.getPendingAcks().getRemainingUnacked(7, 2)).isEqualTo(4);
+ }
+ } finally {
+ receiver.getPendingAcks().forEachAndClose((ledgerId, entryId, remaining, stickyKeyHash) -> { });
+ other.getPendingAcks().forEachAndClose((ledgerId, entryId, remaining, stickyKeyHash) -> { });
+ }
+ }
+
}