You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/16 19:25:50 UTC

(pulsar) branch master updated: [improve][broker] Reduce the lookups and allocations of an individual ack (#26948)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new ad2f02417429 [improve][broker] Reduce the lookups and allocations of an individual ack (#26948)
ad2f02417429 is described below

commit ad2f02417429e90c1025255bc1b9267b94306406
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Oct 16 12:25:43 2026 -0700

    [improve][broker] Reduce the lookups and allocations of an individual ack (#26948)
---
 .../bookkeeper/mledger/impl/ManagedCursorImpl.java | 106 ++++++++++++---------
 .../bookkeeper/mledger/impl/PositionRangeSet.java  |  17 ++++
 .../bookkeeper/mledger/impl/ManagedCursorTest.java |  47 +++++++++
 .../mledger/impl/PositionRangeSetTest.java         |  50 ++++++++++
 .../org/apache/pulsar/broker/service/Consumer.java |  45 ++++-----
 5 files changed, 193 insertions(+), 72 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 3a096c01c021..15dd314cee65 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -106,7 +106,6 @@ import org.apache.bookkeeper.mledger.proto.PositionInfo;
 import org.apache.bookkeeper.mledger.proto.StringProperty;
 import org.apache.bookkeeper.mledger.util.ManagedLedgerUtils;
 import org.apache.commons.lang3.mutable.MutableBoolean;
-import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.commons.lang3.mutable.MutableLong;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats;
@@ -740,25 +739,22 @@ public class ManagedCursorImpl implements ManagedCursor {
         }
     }
 
-    private List<LongListMap> buildLongPropertiesMap(Map<Long, long[]> properties) {
-        if (properties.isEmpty()) {
-            return Collections.emptyList();
-        }
-        List<LongListMap> longListMap = new ArrayList<>();
-        MutableInt serializedSize = new MutableInt();
-        properties.forEach((id, ranges) -> {
+    // Adds the ranges to the position info's range messages, which the reused position info keeps with their arrays,
+    // instead of building new messages and copying them in
+    private void addIndividualDeletedMessageRanges(PositionInfo pi, Map<Long, long[]> properties) {
+        int serializedSize = 0;
+        for (Map.Entry<Long, long[]> entry : properties.entrySet()) {
+            long[] ranges = entry.getValue();
             if (ranges == null || ranges.length <= 0) {
-                return;
+                continue;
             }
-            LongListMap lm = new LongListMap().setKey(id);
+            LongListMap lm = pi.addIndividualDeletedMessageRange().setKey(entry.getKey());
             for (long range : ranges) {
                 lm.addValue(range);
             }
-            longListMap.add(lm);
-            serializedSize.add(lm.getSerializedSize());
-        });
-        individualDeletedMessagesSerializedSize = serializedSize.toInteger();
-        return longListMap;
+            serializedSize += lm.getSerializedSize();
+        }
+        individualDeletedMessagesSerializedSize = serializedSize;
     }
 
     private static Map<String, Long> recoverProperties(int count, IntFunction<LongProperty> accessor) {
@@ -2607,6 +2603,36 @@ public class ManagedCursorImpl implements ManagedCursor {
         }
     }
 
+    /**
+     * Adds an entry acknowledged as a whole to the individually deleted messages, unless it was deleted. Called with
+     * the write lock held.
+     *
+     * @return whether the entry was added, false when it was already deleted
+     */
+    private boolean deleteWholeEntry(Position position) {
+        long ledgerId = position.getLedgerId();
+        long entryId = position.getEntryId();
+        if (entryId >= 0 && entryId <= Integer.MAX_VALUE && position.compareTo(markDeletePosition) > 0) {
+            // The common case: check that it wasn't deleted and delete it in one lookup
+            return individualDeletedMessages.addIfAbsent(ledgerId, entryId);
+        }
+        if (internalIsMessageDeleted(position)) {
+            return false;
+        }
+        // Add a range (prev, pos] to the set. Adding the previous entry as an open limit to the range will
+        // make the RangeSet recognize the "continuity" between adjacent Positions.
+        // Before https://github.com/apache/pulsar/pull/21105 is merged, the range does not support crossing
+        // multi ledgers, so the first position's entryId maybe "-1".
+        if (entryId >= 0) {
+            // For entry zero, keep the lower bound in this ledger, as before.
+            individualDeletedMessages.addOpenClosed(ledgerId, entryId - 1, ledgerId, entryId);
+        } else {
+            Position previousPosition = ledger.getPreviousPosition(position);
+            individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(),
+                    previousPosition.getEntryId(), ledgerId, entryId);
+        }
+        return true;
+    }
 
     @Override
     public void asyncDelete(Iterable<Position> positions, AsyncCallbacks.DeleteCallback callback, Object ctx) {
@@ -2639,36 +2665,28 @@ public class ManagedCursorImpl implements ManagedCursor {
                     return;
                 }
 
-                if (internalIsMessageDeleted(position)) {
+                long[] ackSet = AckSetStateUtil.getAckSetArrayOrNull(position);
+                if (ackSet == null || ackSet.length == 0) {
                     if (batchDeletedIndexes != null) {
                         batchDeletedIndexes.remove(position);
                     }
-                    log.debug().attr("position", position).log("Position was already deleted");
+                    if (deleteWholeEntry(position)) {
+                        MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this);
+                        log.debug().attr("deletedMessages", individualDeletedMessages)
+                                .log("Individually deleted messages");
+                    } else {
+                        log.debug().attr("position", position).log("Position was already deleted");
+                    }
                     continue;
                 }
-                long[] ackSet = AckSetStateUtil.getAckSetArrayOrNull(position);
-                if (ackSet == null || ackSet.length == 0) {
+                if (internalIsMessageDeleted(position)) {
                     if (batchDeletedIndexes != null) {
                         batchDeletedIndexes.remove(position);
                     }
-                    // Add a range (prev, pos] to the set. Adding the previous entry as an open limit to the range will
-                    // make the RangeSet recognize the "continuity" between adjacent Positions.
-                    // Before https://github.com/apache/pulsar/pull/21105 is merged, the range does not support crossing
-                    // multi ledgers, so the first position's entryId maybe "-1".
-                    long ledgerId = position.getLedgerId();
-                    long entryId = position.getEntryId();
-                    if (entryId >= 0) {
-                        // For entry zero, keep the lower bound in this ledger, as before.
-                        individualDeletedMessages.addOpenClosed(ledgerId, entryId - 1, ledgerId, entryId);
-                    } else {
-                        Position previousPosition = ledger.getPreviousPosition(position);
-                        individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(),
-                                previousPosition.getEntryId(), ledgerId, entryId);
-                    }
-                    MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this);
-
-                    log.debug().attr("deletedMessages", individualDeletedMessages).log("Individually deleted messages");
-                } else if (batchDeletedIndexes != null) {
+                    log.debug().attr("position", position).log("Position was already deleted");
+                    continue;
+                }
+                if (batchDeletedIndexes != null) {
                     final var givenBitSet = BitSet.valueOf(ackSet);
                     // Check whether a bitmap for this batch already existed before computeIfAbsent,
                     // because a newly inserted bitmap must count as a change even though
@@ -3586,13 +3604,8 @@ public class ManagedCursorImpl implements ManagedCursor {
                 lock.readLock().unlock();
             }
         }
-        List<LongListMap> longArrayRanges = null;
-        List<MessageRange> messageRanges = null;
-        if (internalRanges != null && !internalRanges.isEmpty()) {
-            longArrayRanges = buildLongPropertiesMap(internalRanges);
-        } else {
-            messageRanges = buildIndividualDeletedMessageRanges();
-        }
+        boolean persistLongArrays = internalRanges != null && !internalRanges.isEmpty();
+        List<MessageRange> messageRanges = persistLongArrays ? null : buildIndividualDeletedMessageRanges();
 
         PositionInfo pi = reusablePositionInfo;
         synchronized (pi) {
@@ -3601,8 +3614,9 @@ public class ManagedCursorImpl implements ManagedCursor {
                     .setEntryId(position.getEntryId())
                     .addAllBatchedEntryDeletionIndexInfos(batchedEntryDeletionIndexInfos)
                     .addAllProperties(propertiesList);
-            if (longArrayRanges != null) {
-                pi.addAllIndividualDeletedMessageRanges(longArrayRanges);
+            if (persistLongArrays) {
+                // the ranges are added in place, from the snapshot taken above
+                addIndividualDeletedMessageRanges(pi, internalRanges);
             } else {
                 pi.addAllIndividualDeletedMessages(messageRanges);
             }
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java
index 6d7900cbcd2f..2986e1fc98b6 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java
@@ -127,6 +127,23 @@ class PositionRangeSet implements LongPairRangeSet<Position> {
         invalidateCaches();
     }
 
+    /**
+     * Adds the entry unless it's present, in one lookup: the same as {@link #contains(long, long)} followed, when it
+     * returns false, by {@code addOpenClosed(ledgerId, entryId - 1, ledgerId, entryId)}, which marks no ledger dirty
+     * since the range is within one ledger. The caller checks that the entry id is from 0 to
+     * {@link Integer#MAX_VALUE}, the entry ids of the bitmaps.
+     *
+     * @return whether the entry was added, false when it was present
+     */
+    boolean addIfAbsent(long ledgerId, long entryId) {
+        LongBitmap rangeBitmap = rangeBitmapMap.computeIfAbsent(ledgerId, k -> LongBitmaps.create());
+        if (!rangeBitmap.checkedAdd(getSafeEntry(entryId))) {
+            return false;
+        }
+        invalidateCaches();
+        return true;
+    }
+
     @Override
     public boolean contains(long ledgerId, long entryId) {
         LongBitmap rangeBitmap = rangeBitmapMap.get(ledgerId);
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
index 719fff4a31d7..2812cddfc88e 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
@@ -6728,6 +6728,53 @@ public class ManagedCursorTest extends MockedBookKeeperTestCase {
         ml.delete();
     }
 
+    @Test
+    public void testDeleteWholeEntryOnce() throws Exception {
+        ManagedLedgerConfig managedLedgerConfig = defaultConfig();
+        managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled(true);
+        ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("testDeleteWholeEntryOnce", managedLedgerConfig);
+        ManagedCursorImpl cursor = (ManagedCursorImpl) ml.openCursor("c1");
+        List<Position> positions = new ArrayList<>();
+        for (int i = 0; i < 4; i++) {
+            positions.add(ml.addEntry(new byte[1]));
+        }
+        long consumed = cursor.getMessagesConsumedCounter();
+
+        // Deleting an entry twice counts it once and leaves the deleted messages as they were
+        cursor.delete(positions.get(2));
+        assertEquals(cursor.getMessagesConsumedCounter(), consumed + 1);
+        String deleted = cursor.getIndividuallyDeletedMessagesSet().toString();
+        cursor.delete(positions.get(2));
+        assertEquals(cursor.getMessagesConsumedCounter(), consumed + 1);
+        assertEquals(cursor.getIndividuallyDeletedMessagesSet().toString(), deleted);
+
+        // A batch acknowledged in part, and then as a whole
+        Position batch = positions.get(1);
+        cursor.delete(AckSetStateUtil.createPositionWithAckSet(batch.getLedgerId(), batch.getEntryId(),
+                new long[]{0b1110L}));
+        assertNotNull(cursor.getBatchPositionAckSet(batch));
+        assertFalse(cursor.isMessageDeleted(batch));
+        cursor.delete(batch);
+        assertNull(cursor.getBatchPositionAckSet(batch));
+        assertTrue(cursor.isMessageDeleted(batch));
+        assertEquals(cursor.getMessagesConsumedCounter(), consumed + 2);
+
+        // An entry at or before the mark-delete position was deleted already
+        cursor.markDelete(positions.get(0));
+        assertEquals(cursor.getMarkDeletedPosition(), positions.get(2));
+        long consumedBeforeDeletingAgain = cursor.getMessagesConsumedCounter();
+        cursor.delete(positions.get(1));
+        cursor.delete(positions.get(2));
+        assertEquals(cursor.getMessagesConsumedCounter(), consumedBeforeDeletingAgain);
+        assertTrue(cursor.getIndividuallyDeletedMessagesSet().isEmpty());
+
+        // In one call: an entry twice and one before the mark-delete position count once
+        cursor.delete(List.of(positions.get(3), positions.get(3), positions.get(1)));
+        assertEquals(cursor.getMessagesConsumedCounter(), consumedBeforeDeletingAgain + 1);
+        assertEquals(cursor.getMarkDeletedPosition(), positions.get(3));
+        ml.delete();
+    }
+
     @Test
     public void testEstimateEntryCountBySize() throws Exception {
         final String mlName = "ml-" + UUID.randomUUID().toString().replaceAll("-", "");
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java
index dc9636f3b60c..a671b80c433b 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java
@@ -37,6 +37,7 @@ import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.pulsar.common.util.collections.LongPairRangeSet.LongPairConsumer;
 import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet;
 import org.roaringbitmap.RoaringBitSet;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 /**
@@ -83,6 +84,55 @@ public class PositionRangeSetTest {
         }
     }
 
+    @DataProvider
+    public static Object[][] multiEntry() {
+        return new Object[][] {{true}, {false}};
+    }
+
+    @Test(dataProvider = "multiEntry")
+    public void addIfAbsentMatchesContainsThenAddOpenClosed(boolean enableMultiEntry) {
+        Random random = new Random(1);
+        PositionRangeSet expected = new PositionRangeSet(CONSUMER, enableMultiEntry);
+        PositionRangeSet actual = new PositionRangeSet(CONSUMER, enableMultiEntry);
+        // dirty ledgers from a range across ledgers, which adding single entries keeps: a range within one ledger
+        // marks no ledger dirty
+        expected.addOpenClosed(1, 5, 3, 7);
+        actual.addOpenClosed(1, 5, 3, 7);
+        long[] boundaryEntryIds = {0, 1, Integer.MAX_VALUE - 1, Integer.MAX_VALUE};
+        for (int i = 0; i < 5_000; i++) {
+            long ledgerId = 1 + random.nextInt(4);
+            long entryId = i % 50 == 0 ? boundaryEntryIds[(i / 50) % boundaryEntryIds.length] : random.nextInt(300);
+            boolean wasAbsent = !expected.contains(ledgerId, entryId);
+            if (wasAbsent) {
+                expected.addOpenClosed(ledgerId, entryId - 1, ledgerId, entryId);
+            }
+            // the cached size and string are recomputed after an added entry, and kept for a present one
+            int sizeBefore = actual.size();
+            String stringBefore = actual.toString();
+            assertEquals(actual.addIfAbsent(ledgerId, entryId), wasAbsent);
+            assertEquals(actual.size(), expected.size());
+            assertEquals(actual.toString(), expected.toString());
+            if (!wasAbsent) {
+                assertEquals(actual.size(), sizeBefore);
+                assertEquals(actual.toString(), stringBefore);
+            }
+            if (i % 500 == 0) {
+                // ranges acknowledged at once and removed up to a mark-delete position, as the cursor does
+                long ledger = 1 + random.nextInt(4);
+                expected.addOpenClosed(ledger, 10, ledger, 40);
+                actual.addOpenClosed(ledger, 10, ledger, 40);
+                long removeUpTo = random.nextInt(200);
+                expected.removeAtMost(1, removeUpTo);
+                actual.removeAtMost(1, removeUpTo);
+            }
+        }
+        assertEquals(actual.asRanges(), expected.asRanges());
+        assertEquals(actual.size(), expected.size());
+        for (long ledgerId = 0; ledgerId <= 5; ledgerId++) {
+            assertEquals(actual.isDirtyLedgers(ledgerId), expected.isDirtyLedgers(ledgerId));
+        }
+    }
+
     @Test
     public void testDirtyLedgerDisabledWhenMultiEntryOff() {
         PositionRangeSet rangeSet = new PositionRangeSet(CONSUMER, false);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index a75aaaaa6e22..f95f91734682 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -645,18 +645,14 @@ public class Consumer {
         for (int i = 0; i < ack.getMessageIdsCount(); i++) {
             MessageIdData msgId = ack.getMessageIdAt(i);
             boolean hasAckSet = msgId.getAckSetsCount() > 0;
-            ObjectIntPair<Consumer> ackOwnerConsumerAndBatchSize =
-                    getAckOwnerConsumerAndBatchSize(msgId.getLedgerId(), msgId.getEntryId());
-
-            if (hasTxn && ackOwnerConsumerAndBatchSize == null) {
-                log.warn()
-                        .attr("ledgerId", msgId.getLedgerId())
-                        .attr("entryId", msgId.getEntryId())
-                        .log("Acknowledging message that was already deleted");
-                continue;
-            }
-
-            Consumer ackOwnerConsumer = ackOwnerConsumerAndBatchSize.left();
+            // the common case, this consumer's own pending ack, without allocating a pair
+            int ownRemainingUnacked = Subscription.isIndividualAckMode(subType)
+                    ? getPendingAcks().getRemainingUnacked(msgId.getLedgerId(), msgId.getEntryId())
+                    : PENDING_ACK_NOT_FOUND;
+            ObjectIntPair<Consumer> ackOwnerConsumerAndBatchSize = ownRemainingUnacked != PENDING_ACK_NOT_FOUND
+                    ? null : getAckOwnerConsumerAndBatchSize(msgId.getLedgerId(), msgId.getEntryId());
+            Consumer ackOwnerConsumer =
+                    ackOwnerConsumerAndBatchSize != null ? ackOwnerConsumerAndBatchSize.left() : this;
 
             if (hasTxn) {
                 // Transactional: use batch size from message ID (not from pendingAcks, which may not
@@ -700,7 +696,8 @@ public class Consumer {
             } else {
                 // Non-transactional: build position and compute acked count, defer state updates.
                 Position position = buildPosition(msgId);
-                int batchSize = ackOwnerConsumerAndBatchSize.rightInt();
+                int batchSize = ackOwnerConsumerAndBatchSize != null
+                        ? ackOwnerConsumerAndBatchSize.rightInt() : ownRemainingUnacked;
                 long ackedCount = computeAckedCount(msgId, position, ackOwnerConsumer, batchSize);
 
                 nonTxnPositions.add(position);
@@ -917,25 +914,21 @@ public class Consumer {
     }
 
     /**
-     * Retrieves the acknowledgment owner consumer and batch size for the specified ledgerId and entryId.
+     * Retrieves the acknowledgment owner consumer and batch size for the specified ledgerId and entryId, among the
+     * other consumers of the subscription: the caller has already looked up this consumer's pending acks.
      *
      * @param ledgerId The ID of the ledger.
      * @param entryId The ID of the entry.
-     * @return Pair<Consumer, BatchSize>
+     * @return Pair<Consumer, BatchSize>, this consumer and 1 when no other consumer has the pending ack
      */
     private ObjectIntPair<Consumer> getAckOwnerConsumerAndBatchSize(long ledgerId, long entryId) {
         if (Subscription.isIndividualAckMode(subType)) {
-            int remainingUnacked = getPendingAcks().getRemainingUnacked(ledgerId, entryId);
-            if (remainingUnacked != PENDING_ACK_NOT_FOUND) {
-                return ObjectIntPair.of(this, remainingUnacked);
-            } else {
-                // If there are more consumers, this step will consume more CPU, and it should be optimized later.
-                for (Consumer consumer : subscription.getConsumers()) {
-                    if (consumer != this) {
-                        remainingUnacked = consumer.getPendingAcks().getRemainingUnacked(ledgerId, entryId);
-                        if (remainingUnacked != PENDING_ACK_NOT_FOUND) {
-                            return ObjectIntPair.of(consumer, remainingUnacked);
-                        }
+            // If there are more consumers, this step will consume more CPU, and it should be optimized later.
+            for (Consumer consumer : subscription.getConsumers()) {
+                if (consumer != this) {
+                    int remainingUnacked = consumer.getPendingAcks().getRemainingUnacked(ledgerId, entryId);
+                    if (remainingUnacked != PENDING_ACK_NOT_FOUND) {
+                        return ObjectIntPair.of(consumer, remainingUnacked);
                     }
                 }
             }