You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/16 19:25:50 UTC
(pulsar) branch master updated: [improve][broker] Reduce the lookups and allocations of an individual ack (#26948)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new ad2f02417429 [improve][broker] Reduce the lookups and allocations of an individual ack (#26948)
ad2f02417429 is described below
commit ad2f02417429e90c1025255bc1b9267b94306406
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Oct 16 12:25:43 2026 -0700
[improve][broker] Reduce the lookups and allocations of an individual ack (#26948)
---
.../bookkeeper/mledger/impl/ManagedCursorImpl.java | 106 ++++++++++++---------
.../bookkeeper/mledger/impl/PositionRangeSet.java | 17 ++++
.../bookkeeper/mledger/impl/ManagedCursorTest.java | 47 +++++++++
.../mledger/impl/PositionRangeSetTest.java | 50 ++++++++++
.../org/apache/pulsar/broker/service/Consumer.java | 45 ++++-----
5 files changed, 193 insertions(+), 72 deletions(-)
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 3a096c01c021..15dd314cee65 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -106,7 +106,6 @@ import org.apache.bookkeeper.mledger.proto.PositionInfo;
import org.apache.bookkeeper.mledger.proto.StringProperty;
import org.apache.bookkeeper.mledger.util.ManagedLedgerUtils;
import org.apache.commons.lang3.mutable.MutableBoolean;
-import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats;
@@ -740,25 +739,22 @@ public class ManagedCursorImpl implements ManagedCursor {
}
}
- private List<LongListMap> buildLongPropertiesMap(Map<Long, long[]> properties) {
- if (properties.isEmpty()) {
- return Collections.emptyList();
- }
- List<LongListMap> longListMap = new ArrayList<>();
- MutableInt serializedSize = new MutableInt();
- properties.forEach((id, ranges) -> {
+ // Adds the ranges to the position info's range messages, which the reused position info keeps with their arrays,
+ // instead of building new messages and copying them in
+ private void addIndividualDeletedMessageRanges(PositionInfo pi, Map<Long, long[]> properties) {
+ int serializedSize = 0;
+ for (Map.Entry<Long, long[]> entry : properties.entrySet()) {
+ long[] ranges = entry.getValue();
if (ranges == null || ranges.length <= 0) {
- return;
+ continue;
}
- LongListMap lm = new LongListMap().setKey(id);
+ LongListMap lm = pi.addIndividualDeletedMessageRange().setKey(entry.getKey());
for (long range : ranges) {
lm.addValue(range);
}
- longListMap.add(lm);
- serializedSize.add(lm.getSerializedSize());
- });
- individualDeletedMessagesSerializedSize = serializedSize.toInteger();
- return longListMap;
+ serializedSize += lm.getSerializedSize();
+ }
+ individualDeletedMessagesSerializedSize = serializedSize;
}
private static Map<String, Long> recoverProperties(int count, IntFunction<LongProperty> accessor) {
@@ -2607,6 +2603,36 @@ public class ManagedCursorImpl implements ManagedCursor {
}
}
+ /**
+ * Adds an entry acknowledged as a whole to the individually deleted messages, unless it was deleted. Called with
+ * the write lock held.
+ *
+ * @return whether the entry was added, false when it was already deleted
+ */
+ private boolean deleteWholeEntry(Position position) {
+ long ledgerId = position.getLedgerId();
+ long entryId = position.getEntryId();
+ if (entryId >= 0 && entryId <= Integer.MAX_VALUE && position.compareTo(markDeletePosition) > 0) {
+ // The common case: check that it wasn't deleted and delete it in one lookup
+ return individualDeletedMessages.addIfAbsent(ledgerId, entryId);
+ }
+ if (internalIsMessageDeleted(position)) {
+ return false;
+ }
+ // Add a range (prev, pos] to the set. Adding the previous entry as an open limit to the range will
+ // make the RangeSet recognize the "continuity" between adjacent Positions.
+ // Before https://github.com/apache/pulsar/pull/21105 is merged, the range does not support crossing
+ // multi ledgers, so the first position's entryId maybe "-1".
+ if (entryId >= 0) {
+ // For entry zero, keep the lower bound in this ledger, as before.
+ individualDeletedMessages.addOpenClosed(ledgerId, entryId - 1, ledgerId, entryId);
+ } else {
+ Position previousPosition = ledger.getPreviousPosition(position);
+ individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(),
+ previousPosition.getEntryId(), ledgerId, entryId);
+ }
+ return true;
+ }
@Override
public void asyncDelete(Iterable<Position> positions, AsyncCallbacks.DeleteCallback callback, Object ctx) {
@@ -2639,36 +2665,28 @@ public class ManagedCursorImpl implements ManagedCursor {
return;
}
- if (internalIsMessageDeleted(position)) {
+ long[] ackSet = AckSetStateUtil.getAckSetArrayOrNull(position);
+ if (ackSet == null || ackSet.length == 0) {
if (batchDeletedIndexes != null) {
batchDeletedIndexes.remove(position);
}
- log.debug().attr("position", position).log("Position was already deleted");
+ if (deleteWholeEntry(position)) {
+ MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this);
+ log.debug().attr("deletedMessages", individualDeletedMessages)
+ .log("Individually deleted messages");
+ } else {
+ log.debug().attr("position", position).log("Position was already deleted");
+ }
continue;
}
- long[] ackSet = AckSetStateUtil.getAckSetArrayOrNull(position);
- if (ackSet == null || ackSet.length == 0) {
+ if (internalIsMessageDeleted(position)) {
if (batchDeletedIndexes != null) {
batchDeletedIndexes.remove(position);
}
- // Add a range (prev, pos] to the set. Adding the previous entry as an open limit to the range will
- // make the RangeSet recognize the "continuity" between adjacent Positions.
- // Before https://github.com/apache/pulsar/pull/21105 is merged, the range does not support crossing
- // multi ledgers, so the first position's entryId maybe "-1".
- long ledgerId = position.getLedgerId();
- long entryId = position.getEntryId();
- if (entryId >= 0) {
- // For entry zero, keep the lower bound in this ledger, as before.
- individualDeletedMessages.addOpenClosed(ledgerId, entryId - 1, ledgerId, entryId);
- } else {
- Position previousPosition = ledger.getPreviousPosition(position);
- individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(),
- previousPosition.getEntryId(), ledgerId, entryId);
- }
- MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this);
-
- log.debug().attr("deletedMessages", individualDeletedMessages).log("Individually deleted messages");
- } else if (batchDeletedIndexes != null) {
+ log.debug().attr("position", position).log("Position was already deleted");
+ continue;
+ }
+ if (batchDeletedIndexes != null) {
final var givenBitSet = BitSet.valueOf(ackSet);
// Check whether a bitmap for this batch already existed before computeIfAbsent,
// because a newly inserted bitmap must count as a change even though
@@ -3586,13 +3604,8 @@ public class ManagedCursorImpl implements ManagedCursor {
lock.readLock().unlock();
}
}
- List<LongListMap> longArrayRanges = null;
- List<MessageRange> messageRanges = null;
- if (internalRanges != null && !internalRanges.isEmpty()) {
- longArrayRanges = buildLongPropertiesMap(internalRanges);
- } else {
- messageRanges = buildIndividualDeletedMessageRanges();
- }
+ boolean persistLongArrays = internalRanges != null && !internalRanges.isEmpty();
+ List<MessageRange> messageRanges = persistLongArrays ? null : buildIndividualDeletedMessageRanges();
PositionInfo pi = reusablePositionInfo;
synchronized (pi) {
@@ -3601,8 +3614,9 @@ public class ManagedCursorImpl implements ManagedCursor {
.setEntryId(position.getEntryId())
.addAllBatchedEntryDeletionIndexInfos(batchedEntryDeletionIndexInfos)
.addAllProperties(propertiesList);
- if (longArrayRanges != null) {
- pi.addAllIndividualDeletedMessageRanges(longArrayRanges);
+ if (persistLongArrays) {
+ // the ranges are added in place, from the snapshot taken above
+ addIndividualDeletedMessageRanges(pi, internalRanges);
} else {
pi.addAllIndividualDeletedMessages(messageRanges);
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java
index 6d7900cbcd2f..2986e1fc98b6 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java
@@ -127,6 +127,23 @@ class PositionRangeSet implements LongPairRangeSet<Position> {
invalidateCaches();
}
+ /**
+ * Adds the entry unless it's present, in one lookup: the same as {@link #contains(long, long)} followed, when it
+ * returns false, by {@code addOpenClosed(ledgerId, entryId - 1, ledgerId, entryId)}, which marks no ledger dirty
+ * since the range is within one ledger. The caller checks that the entry id is from 0 to
+ * {@link Integer#MAX_VALUE}, the entry ids of the bitmaps.
+ *
+ * @return whether the entry was added, false when it was present
+ */
+ boolean addIfAbsent(long ledgerId, long entryId) {
+ LongBitmap rangeBitmap = rangeBitmapMap.computeIfAbsent(ledgerId, k -> LongBitmaps.create());
+ if (!rangeBitmap.checkedAdd(getSafeEntry(entryId))) {
+ return false;
+ }
+ invalidateCaches();
+ return true;
+ }
+
@Override
public boolean contains(long ledgerId, long entryId) {
LongBitmap rangeBitmap = rangeBitmapMap.get(ledgerId);
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
index 719fff4a31d7..2812cddfc88e 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
@@ -6728,6 +6728,53 @@ public class ManagedCursorTest extends MockedBookKeeperTestCase {
ml.delete();
}
+ @Test
+ public void testDeleteWholeEntryOnce() throws Exception {
+ ManagedLedgerConfig managedLedgerConfig = defaultConfig();
+ managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled(true);
+ ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("testDeleteWholeEntryOnce", managedLedgerConfig);
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ml.openCursor("c1");
+ List<Position> positions = new ArrayList<>();
+ for (int i = 0; i < 4; i++) {
+ positions.add(ml.addEntry(new byte[1]));
+ }
+ long consumed = cursor.getMessagesConsumedCounter();
+
+ // Deleting an entry twice counts it once and leaves the deleted messages as they were
+ cursor.delete(positions.get(2));
+ assertEquals(cursor.getMessagesConsumedCounter(), consumed + 1);
+ String deleted = cursor.getIndividuallyDeletedMessagesSet().toString();
+ cursor.delete(positions.get(2));
+ assertEquals(cursor.getMessagesConsumedCounter(), consumed + 1);
+ assertEquals(cursor.getIndividuallyDeletedMessagesSet().toString(), deleted);
+
+ // A batch acknowledged in part, and then as a whole
+ Position batch = positions.get(1);
+ cursor.delete(AckSetStateUtil.createPositionWithAckSet(batch.getLedgerId(), batch.getEntryId(),
+ new long[]{0b1110L}));
+ assertNotNull(cursor.getBatchPositionAckSet(batch));
+ assertFalse(cursor.isMessageDeleted(batch));
+ cursor.delete(batch);
+ assertNull(cursor.getBatchPositionAckSet(batch));
+ assertTrue(cursor.isMessageDeleted(batch));
+ assertEquals(cursor.getMessagesConsumedCounter(), consumed + 2);
+
+ // An entry at or before the mark-delete position was deleted already
+ cursor.markDelete(positions.get(0));
+ assertEquals(cursor.getMarkDeletedPosition(), positions.get(2));
+ long consumedBeforeDeletingAgain = cursor.getMessagesConsumedCounter();
+ cursor.delete(positions.get(1));
+ cursor.delete(positions.get(2));
+ assertEquals(cursor.getMessagesConsumedCounter(), consumedBeforeDeletingAgain);
+ assertTrue(cursor.getIndividuallyDeletedMessagesSet().isEmpty());
+
+ // In one call: an entry twice and one before the mark-delete position count once
+ cursor.delete(List.of(positions.get(3), positions.get(3), positions.get(1)));
+ assertEquals(cursor.getMessagesConsumedCounter(), consumedBeforeDeletingAgain + 1);
+ assertEquals(cursor.getMarkDeletedPosition(), positions.get(3));
+ ml.delete();
+ }
+
@Test
public void testEstimateEntryCountBySize() throws Exception {
final String mlName = "ml-" + UUID.randomUUID().toString().replaceAll("-", "");
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java
index dc9636f3b60c..a671b80c433b 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java
@@ -37,6 +37,7 @@ import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.pulsar.common.util.collections.LongPairRangeSet.LongPairConsumer;
import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet;
import org.roaringbitmap.RoaringBitSet;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
/**
@@ -83,6 +84,55 @@ public class PositionRangeSetTest {
}
}
+ @DataProvider
+ public static Object[][] multiEntry() {
+ return new Object[][] {{true}, {false}};
+ }
+
+ @Test(dataProvider = "multiEntry")
+ public void addIfAbsentMatchesContainsThenAddOpenClosed(boolean enableMultiEntry) {
+ Random random = new Random(1);
+ PositionRangeSet expected = new PositionRangeSet(CONSUMER, enableMultiEntry);
+ PositionRangeSet actual = new PositionRangeSet(CONSUMER, enableMultiEntry);
+ // dirty ledgers from a range across ledgers, which adding single entries keeps: a range within one ledger
+ // marks no ledger dirty
+ expected.addOpenClosed(1, 5, 3, 7);
+ actual.addOpenClosed(1, 5, 3, 7);
+ long[] boundaryEntryIds = {0, 1, Integer.MAX_VALUE - 1, Integer.MAX_VALUE};
+ for (int i = 0; i < 5_000; i++) {
+ long ledgerId = 1 + random.nextInt(4);
+ long entryId = i % 50 == 0 ? boundaryEntryIds[(i / 50) % boundaryEntryIds.length] : random.nextInt(300);
+ boolean wasAbsent = !expected.contains(ledgerId, entryId);
+ if (wasAbsent) {
+ expected.addOpenClosed(ledgerId, entryId - 1, ledgerId, entryId);
+ }
+ // the cached size and string are recomputed after an added entry, and kept for a present one
+ int sizeBefore = actual.size();
+ String stringBefore = actual.toString();
+ assertEquals(actual.addIfAbsent(ledgerId, entryId), wasAbsent);
+ assertEquals(actual.size(), expected.size());
+ assertEquals(actual.toString(), expected.toString());
+ if (!wasAbsent) {
+ assertEquals(actual.size(), sizeBefore);
+ assertEquals(actual.toString(), stringBefore);
+ }
+ if (i % 500 == 0) {
+ // ranges acknowledged at once and removed up to a mark-delete position, as the cursor does
+ long ledger = 1 + random.nextInt(4);
+ expected.addOpenClosed(ledger, 10, ledger, 40);
+ actual.addOpenClosed(ledger, 10, ledger, 40);
+ long removeUpTo = random.nextInt(200);
+ expected.removeAtMost(1, removeUpTo);
+ actual.removeAtMost(1, removeUpTo);
+ }
+ }
+ assertEquals(actual.asRanges(), expected.asRanges());
+ assertEquals(actual.size(), expected.size());
+ for (long ledgerId = 0; ledgerId <= 5; ledgerId++) {
+ assertEquals(actual.isDirtyLedgers(ledgerId), expected.isDirtyLedgers(ledgerId));
+ }
+ }
+
@Test
public void testDirtyLedgerDisabledWhenMultiEntryOff() {
PositionRangeSet rangeSet = new PositionRangeSet(CONSUMER, false);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index a75aaaaa6e22..f95f91734682 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -645,18 +645,14 @@ public class Consumer {
for (int i = 0; i < ack.getMessageIdsCount(); i++) {
MessageIdData msgId = ack.getMessageIdAt(i);
boolean hasAckSet = msgId.getAckSetsCount() > 0;
- ObjectIntPair<Consumer> ackOwnerConsumerAndBatchSize =
- getAckOwnerConsumerAndBatchSize(msgId.getLedgerId(), msgId.getEntryId());
-
- if (hasTxn && ackOwnerConsumerAndBatchSize == null) {
- log.warn()
- .attr("ledgerId", msgId.getLedgerId())
- .attr("entryId", msgId.getEntryId())
- .log("Acknowledging message that was already deleted");
- continue;
- }
-
- Consumer ackOwnerConsumer = ackOwnerConsumerAndBatchSize.left();
+ // the common case, this consumer's own pending ack, without allocating a pair
+ int ownRemainingUnacked = Subscription.isIndividualAckMode(subType)
+ ? getPendingAcks().getRemainingUnacked(msgId.getLedgerId(), msgId.getEntryId())
+ : PENDING_ACK_NOT_FOUND;
+ ObjectIntPair<Consumer> ackOwnerConsumerAndBatchSize = ownRemainingUnacked != PENDING_ACK_NOT_FOUND
+ ? null : getAckOwnerConsumerAndBatchSize(msgId.getLedgerId(), msgId.getEntryId());
+ Consumer ackOwnerConsumer =
+ ackOwnerConsumerAndBatchSize != null ? ackOwnerConsumerAndBatchSize.left() : this;
if (hasTxn) {
// Transactional: use batch size from message ID (not from pendingAcks, which may not
@@ -700,7 +696,8 @@ public class Consumer {
} else {
// Non-transactional: build position and compute acked count, defer state updates.
Position position = buildPosition(msgId);
- int batchSize = ackOwnerConsumerAndBatchSize.rightInt();
+ int batchSize = ackOwnerConsumerAndBatchSize != null
+ ? ackOwnerConsumerAndBatchSize.rightInt() : ownRemainingUnacked;
long ackedCount = computeAckedCount(msgId, position, ackOwnerConsumer, batchSize);
nonTxnPositions.add(position);
@@ -917,25 +914,21 @@ public class Consumer {
}
/**
- * Retrieves the acknowledgment owner consumer and batch size for the specified ledgerId and entryId.
+ * Retrieves the acknowledgment owner consumer and batch size for the specified ledgerId and entryId, among the
+ * other consumers of the subscription: the caller has already looked up this consumer's pending acks.
*
* @param ledgerId The ID of the ledger.
* @param entryId The ID of the entry.
- * @return Pair<Consumer, BatchSize>
+ * @return Pair<Consumer, BatchSize>, this consumer and 1 when no other consumer has the pending ack
*/
private ObjectIntPair<Consumer> getAckOwnerConsumerAndBatchSize(long ledgerId, long entryId) {
if (Subscription.isIndividualAckMode(subType)) {
- int remainingUnacked = getPendingAcks().getRemainingUnacked(ledgerId, entryId);
- if (remainingUnacked != PENDING_ACK_NOT_FOUND) {
- return ObjectIntPair.of(this, remainingUnacked);
- } else {
- // If there are more consumers, this step will consume more CPU, and it should be optimized later.
- for (Consumer consumer : subscription.getConsumers()) {
- if (consumer != this) {
- remainingUnacked = consumer.getPendingAcks().getRemainingUnacked(ledgerId, entryId);
- if (remainingUnacked != PENDING_ACK_NOT_FOUND) {
- return ObjectIntPair.of(consumer, remainingUnacked);
- }
+ // If there are more consumers, this step will consume more CPU, and it should be optimized later.
+ for (Consumer consumer : subscription.getConsumers()) {
+ if (consumer != this) {
+ int remainingUnacked = consumer.getPendingAcks().getRemainingUnacked(ledgerId, entryId);
+ if (remainingUnacked != PENDING_ACK_NOT_FOUND) {
+ return ObjectIntPair.of(consumer, remainingUnacked);
}
}
}