You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/09/23 12:36:51 UTC
(pulsar) 01/21: [fix] Fix ManagedLedgerImpl's pendingAddEntries leak. (#25240)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 43a90395bfedda8a0c07cf3d750d0549aee345db
Author: zzb <48...@users.noreply.github.com>
AuthorDate: Thu May 14 20:53:51 2026 +0800
[fix] Fix ManagedLedgerImpl's pendingAddEntries leak. (#25240)
Co-authored-by: zhaizhibo <zh...@kuaishou.com>
(cherry picked from commit de56ab517f2762f3315e6f207f7f60e363580b9e)
---
.../bookkeeper/mledger/impl/ManagedLedgerImpl.java | 67 +++++++++++-----
.../bookkeeper/mledger/impl/ManagedLedgerTest.java | 88 ++++++++++++++++++++++
.../broker/service/persistent/PersistentTopic.java | 6 +-
3 files changed, 142 insertions(+), 19 deletions(-)
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 8874c79cf4a5..ffd7283198c5 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -1560,6 +1560,7 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
factory.close(this);
STATE_UPDATER.set(this, State.Closed);
+ clearPendingAddEntries(new ManagedLedgerAlreadyClosedException("Managed ledger is closed"));
cancelScheduledTasks();
LedgerHandle lh = currentLedger;
@@ -1660,19 +1661,37 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
}
ledgersStat = stat;
synchronized (ManagedLedgerImpl.this) {
- LedgerHandle originalCurrentLedger = currentLedger;
- ledgers.put(lh.getId(), newLedger);
- currentLedger = lh;
- currentLedgerTimeoutTriggered = new AtomicBoolean();
- currentLedgerEntries = 0;
- currentLedgerSize = 0;
- updateLedgersIdsComplete(originalCurrentLedger);
- mbean.addLedgerSwitchLatencySample(System.currentTimeMillis()
- - lastLedgerCreationInitiationTimestamp, TimeUnit.MILLISECONDS);
- // May need to update the cursor position
- maybeUpdateCursorBeforeTrimmingConsumedLedger();
+ try {
+ State state = STATE_UPDATER.get(ManagedLedgerImpl.this);
+ if (state == State.Closed || state.isFenced()) {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Skip ledger update after create complete, ledger is closed"
+ + " or fenced", name);
+ }
+ lh.closeAsync().exceptionally(e -> {
+ if (e != null) {
+ log.error("[{}] Failed to close ledger {}: {}", name, lh.getId(),
+ e.getMessage());
+ }
+ return null;
+ });
+ } else {
+ LedgerHandle originalCurrentLedger = currentLedger;
+ ledgers.put(lh.getId(), newLedger);
+ currentLedger = lh;
+ currentLedgerTimeoutTriggered = new AtomicBoolean();
+ currentLedgerEntries = 0;
+ currentLedgerSize = 0;
+ updateLedgersIdsComplete(originalCurrentLedger);
+ mbean.addLedgerSwitchLatencySample(System.currentTimeMillis()
+ - lastLedgerCreationInitiationTimestamp, TimeUnit.MILLISECONDS);
+ // May need to update the cursor position
+ maybeUpdateCursorBeforeTrimmingConsumedLedger();
+ }
+ } finally {
+ metadataMutex.unlock();
+ }
}
- metadataMutex.unlock();
}
@Override
@@ -1853,6 +1872,9 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
// The managed ledger was closed during the write operation
clearPendingAddEntries(new ManagedLedgerAlreadyClosedException("Managed ledger was already closed"));
return;
+ } else if (state.isFenced()) {
+ clearPendingAddEntries(new ManagedLedgerFencedException("Managed ledger is fenced"));
+ return;
} else {
// In case we get multiple write errors for different outstanding write request, we should close the ledger
// just once
@@ -1981,9 +2003,9 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
return managedLedgerInterceptor;
}
- void clearPendingAddEntries(ManagedLedgerException e) {
- while (!pendingAddEntries.isEmpty()) {
- OpAddEntry op = pendingAddEntries.poll();
+ synchronized void clearPendingAddEntries(ManagedLedgerException e) {
+ OpAddEntry op;
+ while ((op = pendingAddEntries.poll()) != null) {
op.failed(e);
}
}
@@ -4271,7 +4293,8 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
return activeCursors.get(cursor.getName()) != null;
}
- private boolean currentLedgerIsFull() {
+ @VisibleForTesting
+ protected boolean currentLedgerIsFull() {
if (!factory.isMetadataServiceAvailable()) {
// We don't want to trigger metadata operations if we already know that we're currently disconnected
return false;
@@ -4360,12 +4383,20 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
@VisibleForTesting
public synchronized void setFenced() {
log.info("{} Moving to Fenced state", name);
- STATE_UPDATER.set(this, State.Fenced);
+ State prev = STATE_UPDATER.getAndSet(this, State.Fenced);
+ if (prev != State.Fenced) {
+ clearPendingAddEntries(new ManagedLedgerFencedException("ManagedLedger "
+ + name + " is fenced"));
+ }
}
synchronized void setFencedForDeletion() {
log.info("{} Moving to FencedForDeletion state", name);
- STATE_UPDATER.set(this, State.FencedForDeletion);
+ State prev = STATE_UPDATER.getAndSet(this, State.FencedForDeletion);
+ if (prev != State.FencedForDeletion) {
+ clearPendingAddEntries(new ManagedLedgerFencedException("ManagedLedger "
+ + name + " is fenced"));
+ }
}
MetaStore getStore() {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
index eef9b8a1e8e2..0359b119b581 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
@@ -784,6 +784,94 @@ public class ManagedLedgerTest extends MockedBookKeeperTestCase {
fail("Should have thrown an exception in the above line");
}
+ @Test(timeOut = 30000)
+ public void testCloseManagedLedgerAfterRollover() throws Exception {
+ ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
+ config.setMaxCacheSize(0);
+ @Cleanup("shutdown")
+ ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, config);
+ ManagedLedgerImpl realLedger = (ManagedLedgerImpl) factory.open("my_test_ledger");
+ ManagedLedgerImpl ledger = Mockito.spy(realLedger);
+ AtomicBoolean onlyOnce = new AtomicBoolean(false);
+ when(ledger.currentLedgerIsFull()).thenAnswer(invocation -> onlyOnce.compareAndSet(false, true));
+ OpAddEntry realOp = OpAddEntry.createNoRetainBuffer(ledger,
+ ByteBufAllocator.DEFAULT.buffer(128), null, null, new AtomicBoolean());
+ OpAddEntry op = spy(realOp);
+ CountDownLatch createLatch = new CountDownLatch(1);
+ CountDownLatch closeLatch = new CountDownLatch(1);
+ doAnswer(invocationOnMock -> {
+ // Simulate that before the rollover is completed, new write requests arrive,
+ // and after these write requests are added to pendingAddEntries, the ledger is closed.
+ log.info("before add, ledger state: " + ledger.getState());
+ for (int i = 0; i < 10; ++i) {
+ ledger.internalAsyncAddEntry(OpAddEntry.createNoRetainBuffer(ledger,
+ ByteBufAllocator.DEFAULT.buffer(128), null, null, new AtomicBoolean()));
+ }
+ ledger.asyncClose(new CloseCallback() {
+ @Override
+ public void closeComplete(Object ctx) {
+ log.info("closeComplete finished, ledger state: " + ledger.getState());
+ closeLatch.countDown();
+ }
+
+ @Override
+ public void closeFailed(ManagedLedgerException exception, Object ctx) {
+ log.info("closeFailed, ex: " + exception.getMessage() + ", state: " + ledger.getState());
+ closeLatch.countDown();
+ }
+ }, null);
+ log.info("after add, ledger state: " + ledger.getState());
+ return invocationOnMock.callRealMethod();
+ }).when(ledger).asyncCreateLedger(any(), any(), any(), any(), any());
+ doAnswer(invocationOnMock -> {
+ Object o = invocationOnMock.callRealMethod();
+ log.info("createComplete finished, state: " + ledger.getState());
+ ledger.executor.execute(createLatch::countDown);
+ return o;
+ }).when(ledger).createComplete(anyInt(), any(), any());
+ ledger.internalAsyncAddEntry(op);
+ createLatch.await();
+ closeLatch.await();
+ Assert.assertEquals(ledger.pendingAddEntries.size(), 0);
+ }
+
+ @Test(timeOut = 20000)
+ public void testFencedManagedLedgerAfterAdd() throws Exception {
+ @Cleanup("shutdown")
+ ManagedLedgerFactory factory1 = new ManagedLedgerFactoryImpl(metadataStore, bkc);
+ ManagedLedgerImpl realLedger = (ManagedLedgerImpl) factory1.open("my_test_ledger");
+ ManagedLedgerImpl ledger = spy(realLedger);
+
+ int sendNum = 10;
+ CountDownLatch sendLatch = new CountDownLatch(sendNum);
+ CountDownLatch fencedLatch = new CountDownLatch(1);
+ doAnswer(invocationOnMock -> {
+ stopBookKeeper();
+ stopMetadataStore();
+ for (int i = 0; i < sendNum; ++i) {
+ ledger.internalAsyncAddEntry(OpAddEntry.createNoRetainBuffer(ledger,
+ ByteBufAllocator.DEFAULT.buffer(128), new AddEntryCallback() {
+ @Override
+ public void addComplete(Position position, ByteBuf entryData, Object ctx) {
+ sendLatch.countDown();
+ }
+
+ @Override
+ public void addFailed(ManagedLedgerException exception, Object ctx) {
+ sendLatch.countDown();
+ }
+ }, null, new AtomicBoolean()));
+ }
+ Object o = invocationOnMock.callRealMethod();
+ fencedLatch.countDown();
+ return o;
+ }).when(ledger).setFenced();
+ ledger.setFenced();
+ fencedLatch.await();
+ sendLatch.await();
+ assertEquals(ledger.pendingAddEntries.size(), 0);
+ }
+
@Test(timeOut = 20000)
public void deleteAndReopen() throws Exception {
ManagedLedger ledger = factory.open("my_test_ledger");
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 2f737b7a5380..94b7b3346042 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -767,18 +767,22 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
Instead, we will rely on the service unit state channel's bundle(topic) transfer protocol.
At the end of the transfer protocol, at Owned state, the source broker should close the topic properly.
*/
+ PublishContext callback = (PublishContext) ctx;
if (transferring) {
if (log.isDebugEnabled()) {
log.debug("[{}] Failed to persist msg in store: {} while transferring.",
topic, exception.getMessage(), exception);
}
+ callback.completed(new TopicClosedException(exception), -1, -1);
+ decrementPendingWriteOpsAndCheck();
return;
}
- PublishContext callback = (PublishContext) ctx;
if (exception instanceof ManagedLedgerFencedException) {
// If the managed ledger has been fenced, we cannot continue using it. We need to close and reopen
close();
+ callback.completed(new TopicFencedException(exception.getMessage()), -1, -1);
+ decrementPendingWriteOpsAndCheck();
} else {
// fence topic when failed to write a message to BK
fence();