You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2026/10/21 02:27:55 UTC
(pulsar) branch master updated: [fix][broker] Fix managed-ledger scheduler thread stalls when estimating offline backlog (#26972)
This is an automated email from the ASF dual-hosted git repository.
codelipenghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 20b4390179a8 [fix][broker] Fix managed-ledger scheduler thread stalls when estimating offline backlog (#26972)
20b4390179a8 is described below
commit 20b4390179a89b3b00817de1280ea9bd23ec98b2
Author: void-ptr974 <vo...@protonmail.com>
AuthorDate: Wed Oct 21 10:27:49 2026 +0800
[fix][broker] Fix managed-ledger scheduler thread stalls when estimating offline backlog (#26972)
---
.../bookkeeper/mledger/ManagedLedgerFactory.java | 42 ++
.../mledger/impl/ManagedLedgerFactoryImpl.java | 767 +++++++++++----------
.../mledger/impl/ManagedLedgerOfflineBacklog.java | 21 +
.../mledger/ManagedLedgerFactoryBacklogTest.java | 171 +++++
.../mledger/impl/ManagedLedgerBkTest.java | 15 +-
.../ManagedLedgerOfflineBacklogRecoveryTest.java | 206 ++++++
.../impl/ManagedLedgerOfflineBacklogTest.java | 614 +++++++++++++++++
.../broker/admin/impl/PersistentTopicsBase.java | 23 +-
.../admin/impl/PersistentTopicsBacklogTest.java | 138 ++++
9 files changed, 1613 insertions(+), 384 deletions(-)
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java
index b9adbe3b353f..1840c02bc1db 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java
@@ -19,7 +19,10 @@
package org.apache.bookkeeper.mledger;
import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.Executor;
import java.util.function.Supplier;
import org.apache.bookkeeper.common.annotation.InterfaceAudience;
import org.apache.bookkeeper.common.annotation.InterfaceStability;
@@ -270,5 +273,44 @@ public interface ManagedLedgerFactory {
void estimateUnloadedTopicBacklog(PersistentOfflineTopicStats offlineTopicStats, TopicName topicName,
boolean accurate, Object ctx) throws Exception;
+ /**
+ * Asynchronously estimate backlog without loading the topic. Implementations should override this method
+ * with non-blocking metadata and ledger reads. The default preserves compatibility with synchronous
+ * implementations by running them on the supplied executor.
+ *
+ * @param blockingExecutor executor for legacy blocking implementations
+ * @return a future completed after all statistics have been collected
+ */
+ default CompletableFuture<Void> estimateUnloadedTopicBacklogAsync(PersistentOfflineTopicStats offlineTopicStats,
+ TopicName topicName, boolean accurate, Object ctx, Executor blockingExecutor) {
+ try {
+ Objects.requireNonNull(offlineTopicStats, "offlineTopicStats");
+ Objects.requireNonNull(topicName, "topicName");
+ Objects.requireNonNull(blockingExecutor, "blockingExecutor");
+ PersistentOfflineTopicStats collectedStats = new PersistentOfflineTopicStats(
+ offlineTopicStats.topicName, offlineTopicStats.brokerName);
+ return CompletableFuture.runAsync(() -> {
+ try {
+ estimateUnloadedTopicBacklog(collectedStats, topicName, accurate, ctx);
+ } catch (Exception e) {
+ throw new CompletionException(e);
+ }
+ }, blockingExecutor).thenRun(() -> {
+ offlineTopicStats.storageSize = collectedStats.storageSize;
+ offlineTopicStats.totalMessages = collectedStats.totalMessages;
+ offlineTopicStats.messageBacklog = collectedStats.messageBacklog;
+ offlineTopicStats.statGeneratedAt.setTime(collectedStats.statGeneratedAt.getTime());
+ offlineTopicStats.dataLedgerDetails.clear();
+ collectedStats.dataLedgerDetails.forEach(ledger -> offlineTopicStats.addLedgerDetails(
+ ledger.entries, ledger.timestamp, ledger.size, ledger.ledgerId));
+ offlineTopicStats.cursorDetails.clear();
+ collectedStats.cursorDetails.forEach((name, cursor) -> offlineTopicStats.addCursorDetails(
+ name, cursor.cursorBacklog, cursor.cursorLedgerId));
+ });
+ } catch (Exception e) {
+ return CompletableFuture.failedFuture(e);
+ }
+ }
+
ManagedLedgerFactoryConfig getConfig();
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
index bfef9949129e..d351e51e4795 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
@@ -31,35 +31,35 @@ import io.opentelemetry.api.OpenTelemetry;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
-import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
+import java.util.Objects;
import java.util.TreeMap;
import java.util.UUID;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
-import java.util.function.BiConsumer;
+import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import lombok.CustomLog;
import lombok.Getter;
-import org.apache.bookkeeper.client.AsyncCallback;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.BookKeeper;
-import org.apache.bookkeeper.client.BookKeeperAdmin;
-import org.apache.bookkeeper.client.LedgerEntry;
import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.client.api.LedgerEntries;
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.common.util.OrderedScheduler;
import org.apache.bookkeeper.conf.ClientConfiguration;
@@ -73,6 +73,7 @@ import org.apache.bookkeeper.mledger.ManagedLedger;
import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
import org.apache.bookkeeper.mledger.ManagedLedgerException;
import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
+import org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException;
import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
import org.apache.bookkeeper.mledger.ManagedLedgerFactoryMXBean;
@@ -1297,373 +1298,370 @@ public class ManagedLedgerFactoryImpl implements ManagedLedgerFactory {
return bookkeeperFactory.get();
}
+ private record CursorBacklog(String cursorName, long ledgerId, long backlog, boolean requiresRecovery) {
+ }
+
+ private record OfflineTopicBacklog(NavigableMap<Long, LedgerInfo> ledgers, List<CursorBacklog> cursors) {
+ }
+
@Override
public void estimateUnloadedTopicBacklog(PersistentOfflineTopicStats offlineTopicStats,
- TopicName topicName, boolean accurate, Object ctx)
- throws Exception {
- String managedLedgerName = topicName.getPersistenceNamingEncoding();
- long numberOfEntries = 0;
- long totalSize = 0;
- BookKeeper.DigestType digestType = (BookKeeper.DigestType) ((List) ctx).get(0);
- byte[] password = (byte[]) ((List) ctx).get(1);
- NavigableMap<Long, LedgerInfo> ledgers =
- getManagedLedgersInfo(topicName, accurate, digestType, password);
- for (LedgerInfo ls : ledgers.values()) {
- numberOfEntries += ls.getEntries();
- totalSize += ls.getSize();
- if (accurate) {
- offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId());
- }
+ TopicName topicName, boolean accurate, Object ctx) throws Exception {
+ List<?> credentials = (List<?>) ctx;
+ BookKeeper.DigestType digestType = (BookKeeper.DigestType) credentials.get(0);
+ byte[] password = (byte[]) credentials.get(1);
+ CompletableFuture<OfflineTopicBacklog> computation = new CompletableFuture<>();
+ if (!accurate) {
+ computation.orTimeout(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
}
- offlineTopicStats.totalMessages = numberOfEntries;
- offlineTopicStats.storageSize = totalSize;
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("numberOfEntries", numberOfEntries)
- .attr("totalSize", totalSize)
- .log("Total number of entries and size");
+ FutureUtil.completeAfter(computation,
+ readOfflineTopicBacklogAsync(topicName, digestType, password, computation));
+ OfflineTopicBacklog result = computation.get();
+ // A synchronous caller can perform the rare unconfirmed cursor reads on its own thread.
+ List<CursorBacklog> cursors = accurate
+ ? recoverCursorBacklogs(result.cursors(), result.ledgers(), digestType, password) : result.cursors();
+ publishOfflineTopicBacklog(offlineTopicStats, result.ledgers(), cursors, accurate);
+ }
- // calculate per cursor message backlog
- calculateCursorBacklogs(topicName, ledgers, offlineTopicStats, accurate, digestType, password);
- offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis());
+ @Override
+ public CompletableFuture<Void> estimateUnloadedTopicBacklogAsync(PersistentOfflineTopicStats offlineTopicStats,
+ TopicName topicName, boolean accurate, Object ctx, Executor blockingExecutor) {
+ return estimateUnloadedTopicBacklogAsync(offlineTopicStats, topicName, accurate, ctx, blockingExecutor,
+ META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
}
- private NavigableMap<Long, LedgerInfo> getManagedLedgersInfo(
- final TopicName topicName, boolean accurate, BookKeeper.DigestType digestType, byte[] password)
- throws Exception {
- final NavigableMap<Long, LedgerInfo> ledgers = new ConcurrentSkipListMap<>();
+ @VisibleForTesting
+ CompletableFuture<Void> estimateUnloadedTopicBacklogAsync(PersistentOfflineTopicStats offlineTopicStats,
+ TopicName topicName, boolean accurate, Object ctx, Executor blockingExecutor,
+ long timeout, TimeUnit timeoutUnit) {
+ try {
+ Objects.requireNonNull(offlineTopicStats, "offlineTopicStats");
+ Objects.requireNonNull(topicName, "topicName");
+ Objects.requireNonNull(blockingExecutor, "blockingExecutor");
+ List<?> credentials = (List<?>) ctx;
+ BookKeeper.DigestType digestType = (BookKeeper.DigestType) credentials.get(0);
+ byte[] password = (byte[]) credentials.get(1);
+ CompletableFuture<OfflineTopicBacklog> computation = new CompletableFuture<>();
+ if (!accurate) {
+ computation.orTimeout(timeout, timeoutUnit);
+ }
+ FutureUtil.completeAfter(computation,
+ readOfflineTopicBacklogAsync(topicName, digestType, password, computation).thenCompose(result -> {
+ checkOfflineBacklogRequest(computation);
+ if (!accurate || result.cursors().stream().noneMatch(CursorBacklog::requiresRecovery)) {
+ return CompletableFuture.completedFuture(result);
+ }
+ // Unconfirmed reads wait synchronously and must stay off metadata and BK callback threads.
+ return getBookKeeper().thenCompose(bk -> CompletableFuture.supplyAsync(() -> {
+ checkOfflineBacklogRequest(computation);
+ return new OfflineTopicBacklog(result.ledgers(), recoverCursorBacklogs(
+ bk, result.cursors(), result.ledgers(), digestType, password));
+ }, blockingExecutor));
+ }));
+ // Only publish after timeout has been resolved. Late callbacks still release their resources.
+ CompletableFuture<Void> published = computation.thenAccept(result -> publishOfflineTopicBacklog(
+ offlineTopicStats, result.ledgers(), result.cursors(), accurate));
+ published.whenComplete((ignored, error) -> {
+ if (published.isCancelled()) {
+ computation.cancel(false);
+ }
+ });
+ return published;
+ } catch (Exception e) {
+ return CompletableFuture.failedFuture(e);
+ }
+ }
- String managedLedgerName = topicName.getPersistenceNamingEncoding();
- MetaStore store = getMetaStore();
+ private static void checkOfflineBacklogRequest(CompletableFuture<?> request) {
+ if (request.isDone()) {
+ throw new CancellationException("Offline backlog estimation has completed");
+ }
+ }
- final CountDownLatch mlMetaCounter = new CountDownLatch(1);
- store.getManagedLedgerInfo(managedLedgerName, false /* createIfMissing */,
- new MetaStore.MetaStoreCallback<ManagedLedgerInfo>() {
- @Override
- public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
- for (int i = 0; i < mlInfo.getLedgerInfosCount(); i++) {
- LedgerInfo ls = mlInfo.getLedgerInfoAt(i);
- ledgers.put(ls.getLedgerId(), ls);
- }
+ private CompletableFuture<OfflineTopicBacklog> readOfflineTopicBacklogAsync(TopicName topicName,
+ BookKeeper.DigestType digestType, byte[] password, CompletableFuture<?> request) {
+ return getManagedLedgersInfoAsync(topicName, digestType, password, request)
+ .thenCompose(ledgers -> readCursorBacklogsAsync(topicName, ledgers, digestType, password, request)
+ .thenApply(cursors -> new OfflineTopicBacklog(ledgers, cursors)));
+ }
- // find no of entries in last ledger
- if (!ledgers.isEmpty()) {
- final long id = ledgers.lastKey();
- BiConsumer<Integer, ReadHandle> opencb = (rc, lh) -> {
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("ledgerId", id)
- .attr("result", BKException.getMessage(rc))
- .log("Opened ledger");
- if (rc == BKException.Code.OK) {
- LedgerInfo info =
- new LedgerInfo()
- .setLedgerId(id)
- .setEntries(lh.getLastAddConfirmed() + 1)
- .setSize(lh.getLength()).setTimestamp(System.currentTimeMillis());
- ledgers.put(id, info);
- mlMetaCounter.countDown();
- } else if (Errors.isNoSuchLedgerExistsException(rc)) {
- log.warn().attr("managedLedger", managedLedgerName)
- .attr("ledgerId", ledgers.lastKey())
- .log("Ledger not found");
- ledgers.remove(ledgers.lastKey());
- mlMetaCounter.countDown();
- } else {
- log.error().attr("managedLedger", managedLedgerName)
- .attr("ledgerId", id)
- .attr("result", BKException.getMessage(rc))
- .log("Failed to open ledger");
- mlMetaCounter.countDown();
- }
- };
-
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("ledgerId", id).log("Opening ledger");
- getBookKeeper()
- .thenCompose(bk -> bk.newOpenLedgerOp()
- .withRecovery(false)
- .withLedgerId(id)
- .withDigestType(digestType.toApiDigestType())
- .withPassword(password)
- .withLoggerContext(
- log.with().attr("managedLedger", managedLedgerName).build())
- .execute())
- .whenComplete((rh, ex) -> {
- if (ex != null) {
- log.warn().attr("managedLedger", managedLedgerName)
- .attr("ledgerId", id)
- .exception(ex)
- .log("Failed to open ledger");
- opencb.accept(BKException.getExceptionCode(ex), null);
- } else {
- opencb.accept(BKException.Code.OK, rh);
- }
- });
- } else {
- log.warn().attr("managedLedger", managedLedgerName).log("Ledger list empty");
- mlMetaCounter.countDown();
+ private CompletableFuture<NavigableMap<Long, LedgerInfo>> getManagedLedgersInfoAsync(
+ TopicName topicName, BookKeeper.DigestType digestType, byte[] password, CompletableFuture<?> request) {
+ CompletableFuture<ManagedLedgerInfo> metadata = new CompletableFuture<>();
+ try {
+ checkOfflineBacklogRequest(request);
+ getMetaStore().getManagedLedgerInfo(topicName.getPersistenceNamingEncoding(), false,
+ new MetaStoreCallback<ManagedLedgerInfo>() {
+ @Override
+ public void operationComplete(ManagedLedgerInfo info, Stat stat) {
+ metadata.complete(info);
}
- }
- @Override
- public void operationFailed(ManagedLedgerException.MetaStoreException e) {
- log.warn().attr("managedLedger", managedLedgerName).exception(e)
- .log("Unable to obtain managed ledger metadata");
- mlMetaCounter.countDown();
+ @Override
+ public void operationFailed(MetaStoreException e) {
+ if (e instanceof MetadataNotFoundException) {
+ // Preserve the previous behavior for missing managed-ledger metadata: publish an empty
+ // snapshot so an offline backlog request returns zero instead of failing.
+ metadata.complete(new ManagedLedgerInfo());
+ } else {
+ metadata.completeExceptionally(e);
+ }
+ }
+ });
+ } catch (Exception e) {
+ metadata.completeExceptionally(e);
+ }
+ return metadata.thenCompose(info -> {
+ checkOfflineBacklogRequest(request);
+ NavigableMap<Long, LedgerInfo> ledgers = new TreeMap<>();
+ for (int i = 0; i < info.getLedgerInfosCount(); i++) {
+ LedgerInfo ledger = info.getLedgerInfoAt(i);
+ ledgers.put(ledger.getLedgerId(), ledger);
+ }
+ if (ledgers.isEmpty()) {
+ return CompletableFuture.completedFuture(ledgers);
+ }
+ long ledgerId = ledgers.lastKey();
+ return getBookKeeper().thenCompose(bk -> {
+ checkOfflineBacklogRequest(request);
+ return bk.newOpenLedgerOp()
+ .withRecovery(false)
+ .withLedgerId(ledgerId)
+ .withDigestType(digestType.toApiDigestType())
+ .withPassword(password)
+ .withLoggerContext(log.with().attr("managedLedger",
+ topicName.getPersistenceNamingEncoding()).build())
+ .execute();
+ }).handle((handle, error) -> {
+ if (error != null) {
+ if (Errors.isNoSuchLedgerExistsException(BKException.getExceptionCode(error))) {
+ ledgers.remove(ledgerId);
+ return CompletableFuture.completedFuture(ledgers);
}
- });
+ return CompletableFuture.<NavigableMap<Long, LedgerInfo>>failedFuture(error);
+ }
+ CompletableFuture<NavigableMap<Long, LedgerInfo>> result;
+ try {
+ ledgers.put(ledgerId, new LedgerInfo().setLedgerId(ledgerId)
+ .setEntries(handle.getLastAddConfirmed() + 1).setSize(handle.getLength())
+ .setTimestamp(System.currentTimeMillis()));
+ result = CompletableFuture.completedFuture(ledgers);
+ } catch (Exception e) {
+ result = CompletableFuture.failedFuture(e);
+ }
+ return closeLedgerAfter(result, handle);
+ }).thenCompose(result -> result);
+ });
+ }
+ public void calculateCursorBacklogs(TopicName topicName, NavigableMap<Long, LedgerInfo> ledgers,
+ PersistentOfflineTopicStats offlineTopicStats, boolean accurate,
+ BookKeeper.DigestType digestType, byte[] password) throws Exception {
+ if (ledgers.isEmpty()) {
+ return;
+ }
+ NavigableMap<Long, LedgerInfo> ledgerSnapshot = new TreeMap<>(ledgers);
+ CompletableFuture<List<CursorBacklog>> computation = new CompletableFuture<>();
+ if (!accurate) {
+ computation.orTimeout(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+ FutureUtil.completeAfter(computation,
+ readCursorBacklogsAsync(topicName, ledgerSnapshot, digestType, password, computation));
+ List<CursorBacklog> cursors = computation.get();
if (accurate) {
- // block until however long it takes for operation to complete
- mlMetaCounter.await();
- } else {
- mlMetaCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ cursors = recoverCursorBacklogs(cursors, ledgerSnapshot, digestType, password);
}
- return ledgers;
+ publishCursorBacklogs(offlineTopicStats, cursors);
}
- public void calculateCursorBacklogs(final TopicName topicName,
- final NavigableMap<Long, LedgerInfo> ledgers,
- final PersistentOfflineTopicStats offlineTopicStats, boolean accurate,
- BookKeeper.DigestType digestType, byte[] password) throws Exception {
+ private CompletableFuture<List<CursorBacklog>> readCursorBacklogsAsync(TopicName topicName,
+ NavigableMap<Long, LedgerInfo> ledgers, BookKeeper.DigestType digestType, byte[] password,
+ CompletableFuture<?> request) {
if (ledgers.isEmpty()) {
- return;
+ return CompletableFuture.completedFuture(List.of());
}
- String managedLedgerName = topicName.getPersistenceNamingEncoding();
- MetaStore store = getMetaStore();
- BookKeeper bk = getBookKeeper().get();
- final CountDownLatch allCursorsCounter = new CountDownLatch(1);
- final long errorInReadingCursor = -1;
- final var ledgerRetryMap = new ConcurrentHashMap<String, Long>();
-
- final LedgerInfo ledgerInfo = ledgers.lastEntry().getValue();
- final Position lastLedgerPosition =
- PositionFactory.create(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1);
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("lastLedgerPosition", lastLedgerPosition)
- .log("Last ledger position");
-
- store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback<List<String>>() {
- @Override
- public void operationComplete(List<String> cursors, Stat v) {
- // Load existing cursors
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("numCursors", cursors.size())
- .log("Found cursors");
-
- if (cursors.isEmpty()) {
- allCursorsCounter.countDown();
- return;
+ CompletableFuture<List<String>> cursorNames = new CompletableFuture<>();
+ try {
+ checkOfflineBacklogRequest(request);
+ getMetaStore().getCursors(topicName.getPersistenceNamingEncoding(),
+ new MetaStoreCallback<List<String>>() {
+ @Override
+ public void operationComplete(List<String> cursors, Stat stat) {
+ cursorNames.complete(cursors);
+ }
+
+ @Override
+ public void operationFailed(MetaStoreException e) {
+ cursorNames.completeExceptionally(e);
+ }
+ });
+ } catch (Exception e) {
+ cursorNames.completeExceptionally(e);
+ }
+ return cursorNames.thenCompose(names -> {
+ checkOfflineBacklogRequest(request);
+ if (names.isEmpty()) {
+ return CompletableFuture.completedFuture(List.of());
+ }
+ return getBookKeeper().thenCompose(bk -> {
+ checkOfflineBacklogRequest(request);
+ List<CompletableFuture<CursorBacklog>> cursors = new ArrayList<>(names.size());
+ for (String cursor : names) {
+ if (request.isDone()) {
+ break;
+ }
+ cursors.add(readCursorBacklogAsync(topicName.getPersistenceNamingEncoding(), cursor,
+ ledgers, bk, digestType, password, request));
}
+ return CompletableFuture.allOf(cursors.toArray(CompletableFuture[]::new))
+ .thenApply(__ -> cursors.stream().map(cursor -> cursor.getNow(null)).toList());
+ });
+ });
+ }
- final CountDownLatch cursorCounter = new CountDownLatch(cursors.size());
-
- for (final String cursorName : cursors) {
- // determine subscription position from cursor ledger
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .log("Loading cursor");
-
- AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> {
- long ledgerId = lh.getId();
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("ledgerId", ledgerId)
- .attr("cursor", cursorName)
- .attr("rc", rc)
- .log("Opened cursor ledger");
- if (rc != BKException.Code.OK) {
- log.warn().attr("managedLedger", managedLedgerName)
- .attr("ledgerId", ledgerId)
- .attr("cursor", cursorName)
- .attr("result", BKException.getMessage(rc))
- .log("Error opening metadata ledger for cursor");
- cursorCounter.countDown();
- return;
+ private CompletableFuture<CursorBacklog> readCursorBacklogAsync(String managedLedgerName, String cursorName,
+ NavigableMap<Long, LedgerInfo> ledgers, BookKeeper bk,
+ BookKeeper.DigestType digestType, byte[] password, CompletableFuture<?> request) {
+ CompletableFuture<ManagedCursorInfo> metadata = new CompletableFuture<>();
+ try {
+ checkOfflineBacklogRequest(request);
+ getMetaStore().asyncGetCursorInfo(managedLedgerName, cursorName,
+ new MetaStoreCallback<ManagedCursorInfo>() {
+ @Override
+ public void operationComplete(ManagedCursorInfo info, Stat stat) {
+ metadata.complete(info);
}
- long lac = lh.getLastAddConfirmed();
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .attr("lac", lac)
- .attr("ledgerId", ledgerId)
- .log("Cursor LAC read from ledger");
-
- if (lac == LedgerHandle.INVALID_ENTRY_ID) {
- // save the ledger id and cursor to retry outside of this call back
- // since we are trying to read the same cursor ledger, we will block until
- // this current callback completes, since an attempt to read the entry
- // will block behind this current operation to complete
- ledgerRetryMap.put(cursorName, ledgerId);
- log.info().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .attr("lac", lac)
- .attr("ledgerId", ledgerId)
- .log("Cursor LAC read from ledger");
- cursorCounter.countDown();
- return;
+
+ @Override
+ public void operationFailed(MetaStoreException e) {
+ metadata.completeExceptionally(e);
}
- final long entryId = lac;
- // read last acked message position for subscription
- lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() {
- @Override
- public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> seq,
- Object ctx) {
- try {
- log.debug().attr("rc", rc).attr("entryId", entryId).log("readComplete");
- if (rc != BKException.Code.OK) {
- log.warn().attr("managedLedger", managedLedgerName)
- .attr("ledgerId", ledgerId)
- .attr("cursor", cursorName)
- .attr("result", BKException.getMessage(rc))
- .log("Error reading from metadata ledger for cursor");
- // indicate that this cursor should be excluded
- offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor,
- lh.getId());
- } else {
- LedgerEntry entry = seq.nextElement();
- PositionInfo positionInfo;
- try {
- positionInfo = new PositionInfo();
- positionInfo.parseFrom(entry.getEntry());
- } catch (Exception e) {
- log.warn().attr("managedLedger", managedLedgerName)
- .attr("ledgerId", ledgerId)
- .attr("cursor", cursorName)
- .exception(e)
- .log("Error reading position from metadata ledger for cursor");
- offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor,
- lh.getId());
- return;
- }
- final Position lastAckedMessagePosition =
- PositionFactory.create(positionInfo.getLedgerId(),
- positionInfo.getEntryId());
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .attr("markDeletePosition", lastAckedMessagePosition)
- .attr("lastLedgerPosition", lastLedgerPosition)
- .log("Cursor mark-delete read last ledger position");
- // calculate cursor backlog
- Range<Position> range = Range.openClosed(lastAckedMessagePosition,
- lastLedgerPosition);
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .attr("range", range)
- .log("Calculating backlog for cursor");
- long cursorBacklog = getNumberOfEntries(range, ledgers);
- offlineTopicStats.messageBacklog += cursorBacklog;
- offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId());
+ });
+ } catch (Exception e) {
+ metadata.completeExceptionally(e);
+ }
+ return metadata.thenCompose(info -> {
+ checkOfflineBacklogRequest(request);
+ long ledgerId = info.getCursorsLedgerId();
+ if (ledgerId == -1) {
+ Position position = PositionFactory.create(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId());
+ return CompletableFuture.completedFuture(cursorBacklog(cursorName, ledgerId, position, ledgers));
+ }
+ return bk.newOpenLedgerOp()
+ .withRecovery(false)
+ .withLedgerId(ledgerId)
+ .withDigestType(digestType.toApiDigestType())
+ .withPassword(password)
+ .withLoggerContext(log.with().attr("managedLedger", managedLedgerName).build())
+ .execute().thenCompose(handle -> {
+ CompletableFuture<CursorBacklog> result;
+ try {
+ checkOfflineBacklogRequest(request);
+ long entryId = handle.getLastAddConfirmed();
+ if (entryId == LedgerHandle.INVALID_ENTRY_ID) {
+ result = CompletableFuture.completedFuture(
+ new CursorBacklog(cursorName, ledgerId, -1, true));
+ } else {
+ result = handle.readAsync(entryId, entryId).thenApply(entries -> {
+ try (LedgerEntries ledgerEntries = entries) {
+ checkOfflineBacklogRequest(request);
+ PositionInfo positionInfo = new PositionInfo();
+ positionInfo.parseFrom(ledgerEntries.getEntry(entryId).getEntryBytes());
+ Position position = PositionFactory.create(
+ positionInfo.getLedgerId(), positionInfo.getEntryId());
+ return cursorBacklog(cursorName, ledgerId, position, ledgers);
}
- } finally {
- cursorCounter.countDown();
- }
+ });
}
- }, null);
+ } catch (Exception e) {
+ result = CompletableFuture.failedFuture(e);
+ }
+ return closeLedgerAfter(result, handle);
+ });
+ });
+ }
- }; // end of cursor meta read callback
-
- store.asyncGetCursorInfo(managedLedgerName, cursorName,
- new MetaStore.MetaStoreCallback<ManagedCursorInfo>() {
- @Override
- public void operationComplete(ManagedCursorInfo info,
- Stat stat) {
- long cursorLedgerId = info.getCursorsLedgerId();
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .attr("cursorLedgerId", cursorLedgerId)
- .log("Cursor meta-data read ledger id");
- if (cursorLedgerId != -1) {
- var openLog = log.with().attr("managedLedger", managedLedgerName).build();
- bk.newOpenLedgerOp()
- .withRecovery(false)
- .withLedgerId(cursorLedgerId)
- .withDigestType(digestType.toApiDigestType())
- .withPassword(password)
- .withLoggerContext(openLog)
- .execute()
- .whenComplete((rh, ex) -> ManagedLedgerImpl.completeOpenCallback(
- openLog, cursorLedgerId, cursorLedgerOpenCb, rh, ex));
- } else {
- Position lastAckedMessagePosition = PositionFactory.create(
- info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId());
- Range<Position> range = Range.openClosed(lastAckedMessagePosition,
- lastLedgerPosition);
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .attr("range", range)
- .log("Calculating backlog for cursor");
- long cursorBacklog = getNumberOfEntries(range, ledgers);
- offlineTopicStats.messageBacklog += cursorBacklog;
- offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId);
- cursorCounter.countDown();
- }
+ private <T> CompletableFuture<T> closeLedgerAfter(CompletableFuture<T> operation, ReadHandle handle) {
+ return operation.handle((value, error) -> {
+ CompletableFuture<Void> close;
+ try {
+ close = handle.closeAsync();
+ } catch (Exception e) {
+ close = CompletableFuture.failedFuture(e);
+ }
+ return close.handle((ignored, closeError) -> {
+ if (error != null) {
+ throw new CompletionException(error);
+ }
+ if (closeError != null) {
+ throw new CompletionException(closeError);
+ }
+ return value;
+ });
+ }).thenCompose(result -> result);
+ }
- }
+ private CursorBacklog cursorBacklog(String cursorName, long ledgerId, Position position,
+ NavigableMap<Long, LedgerInfo> ledgers) {
+ LedgerInfo lastLedger = ledgers.lastEntry().getValue();
+ Position lastPosition = PositionFactory.create(lastLedger.getLedgerId(), lastLedger.getEntries() - 1);
+ long backlog = position.compareTo(lastPosition) >= 0 ? 0
+ : getNumberOfEntries(Range.openClosed(position, lastPosition), ledgers);
+ return new CursorBacklog(cursorName, ledgerId, backlog, false);
+ }
- @Override
- public void operationFailed(ManagedLedgerException.MetaStoreException e) {
- log.warn().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .exception(e)
- .log("Unable to obtain cursor ledger for cursor");
- cursorCounter.countDown();
- }
- });
- } // for every cursor find backlog
- try {
- if (accurate) {
- cursorCounter.await();
- } else {
- cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
- }
- } catch (Exception e) {
- log.warn().attr("managedLedger", managedLedgerName)
- .exception(e)
- .log("Error reading subscription positions");
- } finally {
- allCursorsCounter.countDown();
- }
+ private List<CursorBacklog> recoverCursorBacklogs(List<CursorBacklog> cursors,
+ NavigableMap<Long, LedgerInfo> ledgers,
+ BookKeeper.DigestType digestType, byte[] password) throws Exception {
+ if (cursors.stream().noneMatch(CursorBacklog::requiresRecovery)) {
+ return cursors;
+ }
+ return recoverCursorBacklogs(getBookKeeper().get(), cursors, ledgers, digestType, password);
+ }
+
+ private List<CursorBacklog> recoverCursorBacklogs(BookKeeper bk, List<CursorBacklog> cursors,
+ NavigableMap<Long, LedgerInfo> ledgers, BookKeeper.DigestType digestType, byte[] password) {
+ List<CursorBacklog> recovered = new ArrayList<>(cursors.size());
+ for (CursorBacklog cursor : cursors) {
+ if (!cursor.requiresRecovery()) {
+ recovered.add(cursor);
+ continue;
}
+ Position position = tryGetMDPosition(bk, cursor.ledgerId(), cursor.cursorName(), digestType, password);
+ if (position == null) {
+ throw new CompletionException(new ManagedLedgerException(
+ "Unable to determine position for cursor " + cursor.cursorName()));
+ }
+ recovered.add(cursorBacklog(cursor.cursorName(), cursor.ledgerId(), position, ledgers));
+ }
+ return recovered;
+ }
- @Override
- public void operationFailed(ManagedLedgerException.MetaStoreException e) {
- log.warn().attr("managedLedger", managedLedgerName).exception(e).log("Failed to get the cursors list");
- allCursorsCounter.countDown();
+ private void publishOfflineTopicBacklog(PersistentOfflineTopicStats stats,
+ NavigableMap<Long, LedgerInfo> ledgers, List<CursorBacklog> cursors, boolean accurate) {
+ stats.totalMessages = 0;
+ stats.storageSize = 0;
+ stats.dataLedgerDetails.clear();
+ for (LedgerInfo ledger : ledgers.values()) {
+ stats.totalMessages += ledger.getEntries();
+ stats.storageSize += ledger.getSize();
+ if (accurate) {
+ stats.addLedgerDetails(ledger.getEntries(), ledger.getTimestamp(),
+ ledger.getSize(), ledger.getLedgerId());
}
- });
- if (accurate) {
- allCursorsCounter.await();
- } else {
- allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
}
+ stats.messageBacklog = 0;
+ stats.cursorDetails.clear();
+ publishCursorBacklogs(stats, cursors);
+ stats.statGeneratedAt.setTime(System.currentTimeMillis());
+ }
- // go through ledgers where LAC was -1
- if (accurate && ledgerRetryMap.size() > 0) {
- ledgerRetryMap.forEach((cursorName, ledgerId) -> {
- log.debug().attr("cursor", cursorName)
- .attr("ledgerId", ledgerId)
- .log("Trying to obtain mark-delete position from BkAdmin");
- Position lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName);
- if (lastAckedMessagePosition == null) {
- log.warn().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .attr("ledgerId", ledgerId)
- .log("Unable to determine cursor position");
- } else {
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .attr("ledgerId", ledgerId)
- .attr("position", lastAckedMessagePosition)
- .log("Cursor read from ledger using bk admin");
- // calculate cursor backlog
- Range<Position> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
- log.debug().attr("managedLedger", managedLedgerName)
- .attr("cursor", cursorName)
- .attr("range", range)
- .log("Calculating backlog for cursor");
- long cursorBacklog = getNumberOfEntries(range, ledgers);
- offlineTopicStats.messageBacklog += cursorBacklog;
- offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId);
- }
- });
+ private void publishCursorBacklogs(PersistentOfflineTopicStats stats, List<CursorBacklog> cursors) {
+ for (CursorBacklog cursor : cursors) {
+ if (!cursor.requiresRecovery()) {
+ stats.messageBacklog += cursor.backlog();
+ }
+ stats.addCursorDetails(cursor.cursorName(), cursor.backlog(), cursor.ledgerId());
}
}
@@ -1706,47 +1704,84 @@ public class ManagedLedgerFactoryImpl implements ManagedLedgerFactory {
}
- private Position tryGetMDPosition(BookKeeper bookKeeper, long ledgerId, String cursorName) {
- BookKeeperAdmin bookKeeperAdmin = null;
- long lastEntry = LedgerHandle.INVALID_ENTRY_ID;
+ @VisibleForTesting
+ Position tryGetMDPosition(BookKeeper bookKeeper, long ledgerId, String cursorName,
+ BookKeeper.DigestType digestType, byte[] password) {
+ ReadHandle readHandle = null;
Position lastAckedMessagePosition = null;
try {
- bookKeeperAdmin = new BookKeeperAdmin(bookKeeper);
- for (LedgerEntry ledgerEntry : bookKeeperAdmin.readEntries(ledgerId, 0, lastEntry)) {
- lastEntry = ledgerEntry.getEntryId();
- log.debug().attr("entryId", lastEntry)
- .attr("ledgerId", ledgerId)
- .attr("cursor", cursorName)
- .log("Read entry from ledger for cursor");
- PositionInfo positionInfo = new PositionInfo();
- positionInfo.parseFrom(ledgerEntry.getEntry());
- lastAckedMessagePosition =
- PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId());
- log.debug().attr("cursor", cursorName)
- .attr("position", lastAckedMessagePosition)
- .log("Cursor read position");
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException();
+ }
+ readHandle = awaitOrClose(bookKeeper.newOpenLedgerOp()
+ .withRecovery(false)
+ .withLedgerId(ledgerId)
+ .withDigestType(digestType.toApiDigestType())
+ .withPassword(password)
+ .withLoggerContext(log.with().attr("cursor", cursorName).build())
+ .execute(), this::closeBacklogRecoveryHandle);
+ // As with BookKeeperAdmin's iterator, recheck LAC after every read: an open cursor ledger
+ // can reveal a newer LAC while reading entries.
+ for (long entryId = 0; entryId <= readHandle.getLastAddConfirmed(); entryId++) {
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException();
+ }
+ try (LedgerEntries entries = awaitOrClose(readHandle.readUnconfirmedAsync(entryId, entryId),
+ LedgerEntries::close)) {
+ PositionInfo positionInfo = new PositionInfo();
+ positionInfo.parseFrom(entries.getEntry(entryId).getEntryBytes());
+ lastAckedMessagePosition =
+ PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId());
+ } catch (ExecutionException e) {
+ if (BKException.getExceptionCode(e) == BKException.Code.NoSuchEntryException) {
+ break;
+ }
+ throw e;
+ }
}
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ lastAckedMessagePosition = null;
} catch (Exception e) {
log.warn().attr("ledgerId", ledgerId)
.attr("cursor", cursorName)
.exception(e)
.log("Unable to determine LAC");
+ lastAckedMessagePosition = null;
} finally {
- if (bookKeeperAdmin != null) {
- try {
- bookKeeperAdmin.close();
- } catch (Exception e) {
- log.warn().attr("ledgerId", ledgerId)
- .attr("cursor", cursorName)
- .exception(e)
- .log("Unable to close bk admin");
- }
+ if (readHandle != null) {
+ closeBacklogRecoveryHandle(readHandle);
}
-
}
return lastAckedMessagePosition;
}
+ private void closeBacklogRecoveryHandle(ReadHandle handle) {
+ CompletableFuture<Void> closing;
+ try {
+ closing = handle.closeAsync();
+ } catch (Exception e) {
+ closing = CompletableFuture.failedFuture(e);
+ }
+ closing.exceptionally(error -> {
+ log.warn().attr("ledgerId", handle.getId())
+ .exception(error).log("Unable to close cursor read handle");
+ return null;
+ });
+ }
+
+ private static <T> T awaitOrClose(CompletableFuture<T> future, Consumer<T> close)
+ throws InterruptedException, ExecutionException {
+ try {
+ return future.get();
+ } catch (InterruptedException e) {
+ // Interrupting the wait does not cancel BookKeeper IO. Release its eventual result,
+ // which the interrupted caller can no longer receive.
+ future.thenAccept(close);
+ throw e;
+ }
+ }
+
private static final int META_READ_TIMEOUT_SECONDS = 60;
/**
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java
index 0ce5e0a355d8..920b05d5f6b8 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java
@@ -23,6 +23,8 @@ import com.google.common.collect.Range;
import java.util.ArrayList;
import java.util.List;
import java.util.NavigableMap;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.api.DigestType;
import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
@@ -110,4 +112,23 @@ public class ManagedLedgerOfflineBacklog {
return offlineTopicStats;
}
+
+ public CompletableFuture<PersistentOfflineTopicStats> estimateUnloadedTopicBacklogAsync(
+ ManagedLedgerFactory factory, TopicName topicName, Executor blockingExecutor) {
+ try {
+ PersistentOfflineTopicStats offlineTopicStats = new PersistentOfflineTopicStats(
+ topicName.getPersistenceNamingEncoding(), brokerName);
+ Object ctx = null;
+ if (factory instanceof ManagedLedgerFactoryImpl) {
+ List<Object> credentials = new ArrayList<>();
+ credentials.add(digestType);
+ credentials.add(password);
+ ctx = credentials;
+ }
+ return factory.estimateUnloadedTopicBacklogAsync(offlineTopicStats, topicName, accurate, ctx,
+ blockingExecutor).thenApply(__ -> offlineTopicStats);
+ } catch (Exception e) {
+ return CompletableFuture.failedFuture(e);
+ }
+ }
}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryBacklogTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryBacklogTest.java
new file mode 100644
index 000000000000..cd40cfddd0c8
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryBacklogTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyNoInteractions;
+import java.util.ArrayDeque;
+import java.util.Queue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.bookkeeper.client.api.DigestType;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerOfflineBacklog;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
+import org.testng.annotations.Test;
+
+public class ManagedLedgerFactoryBacklogTest {
+
+ @Test
+ public void testLegacyImplementationUsesSuppliedExecutorAndPublishesSnapshot() throws Exception {
+ ManagedLedgerFactory factory = legacyFactory();
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/topic");
+ PersistentOfflineTopicStats stats = new PersistentOfflineTopicStats("topic", "broker");
+ Queue<Runnable> tasks = new ArrayDeque<>();
+ AtomicReference<PersistentOfflineTopicStats> collected = new AtomicReference<>();
+ Object context = new Object();
+ doAnswer(invocation -> {
+ PersistentOfflineTopicStats value = invocation.getArgument(0);
+ collected.set(value);
+ value.totalMessages = 5;
+ value.storageSize = 50;
+ value.messageBacklog = 3;
+ value.addLedgerDetails(5, 10, 50, 1);
+ value.addCursorDetails("subscription", 3, 2);
+ value.statGeneratedAt.setTime(10);
+ return null;
+ }).when(factory).estimateUnloadedTopicBacklog(any(), eq(topicName), eq(true), eq(context));
+
+ CompletableFuture<Void> future = factory.estimateUnloadedTopicBacklogAsync(
+ stats, topicName, true, context, tasks::add);
+ assertThat(future).isNotDone();
+ assertThat(collected.get()).isNull();
+ tasks.remove().run();
+ future.get(5, TimeUnit.SECONDS);
+
+ assertThat(stats.totalMessages).isEqualTo(5);
+ assertThat(stats.storageSize).isEqualTo(50);
+ assertThat(stats.messageBacklog).isEqualTo(3);
+ assertThat(stats.dataLedgerDetails).hasSize(1);
+ assertThat(stats.cursorDetails.get("subscription").cursorBacklog).isEqualTo(3);
+ assertThat(stats.statGeneratedAt.getTime()).isEqualTo(10);
+
+ collected.get().dataLedgerDetails.get(0).entries = 100;
+ collected.get().cursorDetails.get("subscription").cursorBacklog = 100;
+ collected.get().statGeneratedAt.setTime(100);
+ collected.get().messageBacklog = 100;
+ assertThat(stats.dataLedgerDetails.get(0).entries).isEqualTo(5);
+ assertThat(stats.cursorDetails.get("subscription").cursorBacklog).isEqualTo(3);
+ assertThat(stats.statGeneratedAt.getTime()).isEqualTo(10);
+ assertThat(stats.messageBacklog).isEqualTo(3);
+ }
+
+ @Test
+ public void testLegacyFailureDoesNotPublishPartialStats() throws Exception {
+ ManagedLedgerFactory factory = legacyFactory();
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/topic");
+ PersistentOfflineTopicStats stats = new PersistentOfflineTopicStats("topic", "broker");
+ Exception failure = new Exception("metadata read failed");
+ doAnswer(invocation -> {
+ PersistentOfflineTopicStats value = invocation.getArgument(0);
+ value.messageBacklog = 3;
+ value.addCursorDetails("subscription", 3, 2);
+ throw failure;
+ }).when(factory).estimateUnloadedTopicBacklog(any(), eq(topicName), eq(false), isNull());
+
+ CompletableFuture<Void> future = factory.estimateUnloadedTopicBacklogAsync(
+ stats, topicName, false, null, Runnable::run);
+ assertThatThrownBy(() -> future.get(5, TimeUnit.SECONDS)).hasCause(failure);
+ assertThat(stats.messageBacklog).isZero();
+ assertThat(stats.cursorDetails).isEmpty();
+ }
+
+ @Test
+ public void testLegacySubmissionAndValidationFailuresCompleteFuture() {
+ ManagedLedgerFactory factory = legacyFactory();
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/topic");
+ PersistentOfflineTopicStats stats = new PersistentOfflineTopicStats("topic", "broker");
+ RejectedExecutionException rejection = new RejectedExecutionException("executor stopped");
+ Executor executor = command -> {
+ throw rejection;
+ };
+ CompletableFuture<Void> rejected = factory.estimateUnloadedTopicBacklogAsync(
+ stats, topicName, false, null, executor);
+ assertThatThrownBy(() -> rejected.get(5, TimeUnit.SECONDS)).hasCause(rejection);
+ assertThat(factory.estimateUnloadedTopicBacklogAsync(null, topicName, false, null, executor))
+ .isCompletedExceptionally();
+ assertThat(factory.estimateUnloadedTopicBacklogAsync(stats, null, false, null, executor))
+ .isCompletedExceptionally();
+ assertThat(factory.estimateUnloadedTopicBacklogAsync(stats, topicName, false, null, null))
+ .isCompletedExceptionally();
+ }
+
+ @Test
+ public void testAsyncHelperPreservesLegacyContext() throws Exception {
+ ManagedLedgerFactory factory = legacyFactory();
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/topic");
+ Queue<Runnable> tasks = new ArrayDeque<>();
+ ManagedLedgerOfflineBacklog backlog = new ManagedLedgerOfflineBacklog(
+ DigestType.CRC32C, new byte[0], "broker", false);
+ doAnswer(invocation -> {
+ PersistentOfflineTopicStats stats = invocation.getArgument(0);
+ stats.messageBacklog = 3;
+ return null;
+ }).when(factory).estimateUnloadedTopicBacklog(any(), eq(topicName), eq(false), isNull());
+
+ CompletableFuture<PersistentOfflineTopicStats> future = backlog.estimateUnloadedTopicBacklogAsync(
+ factory, topicName, tasks::add);
+ assertThat(future).isNotDone();
+ tasks.remove().run();
+ PersistentOfflineTopicStats stats = future.get(5, TimeUnit.SECONDS);
+ assertThat(stats.messageBacklog).isEqualTo(3);
+ assertThat(stats.topicName).isEqualTo(topicName.getPersistenceNamingEncoding());
+ assertThat(stats.brokerName).isEqualTo("broker");
+ }
+
+ @Test
+ public void testAsyncHelperReportsInvalidArgumentsThroughFuture() {
+ ManagedLedgerFactory factory = mock(ManagedLedgerFactory.class);
+ ManagedLedgerOfflineBacklog backlog = new ManagedLedgerOfflineBacklog(
+ DigestType.CRC32C, new byte[0], "broker", false);
+ assertThat(backlog.estimateUnloadedTopicBacklogAsync(factory, null, Runnable::run))
+ .isCompletedExceptionally();
+ assertThat(backlog.estimateUnloadedTopicBacklogAsync(null,
+ TopicName.get("persistent://tenant/namespace/topic"), Runnable::run)).isCompletedExceptionally();
+ verifyNoInteractions(factory);
+ }
+
+ private static ManagedLedgerFactory legacyFactory() {
+ ManagedLedgerFactory factory = mock(ManagedLedgerFactory.class);
+ doCallRealMethod().when(factory).estimateUnloadedTopicBacklogAsync(any(), any(), anyBoolean(),
+ any(), any());
+ return factory;
+ }
+}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
index c3f6355d56d6..27dd6e966bf5 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
@@ -72,6 +72,7 @@ import org.apache.bookkeeper.mledger.util.ThrowableToStringUtil;
import org.apache.bookkeeper.net.BookieId;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
import org.apache.pulsar.common.util.FutureUtil;
import org.awaitility.Awaitility;
@@ -873,11 +874,12 @@ public class ManagedLedgerBkTest extends BookKeeperClusterTestCase {
@Cleanup("shutdown")
ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf);
ManagedLedgerConfig config = defaultConfig();
- config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1);
- ManagedLedger ledger = factory.open("property/namespace/my-ledger", config);
+ config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setDigestType(DigestType.CRC32);
+ TopicName topicName = TopicName.get("persistent://property/namespace/my-ledger");
+ ManagedLedger ledger = factory.open(topicName.getPersistenceNamingEncoding(), config);
ManagedCursor cursor = ledger.openCursor("c1");
- int num = 1;
+ int num = 3;
for (int i = 0; i < num; i++) {
String entry = "entry-" + i;
@@ -891,9 +893,12 @@ public class ManagedLedgerBkTest extends BookKeeperClusterTestCase {
ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(
DigestType.CRC32, "".getBytes(StandardCharsets.UTF_8), "", false);
- PersistentOfflineTopicStats offlineTopicStats = offlineTopicBacklog.getEstimatedUnloadedTopicBacklog(
- (ManagedLedgerFactoryImpl) factory, "property/namespace/my-ledger");
+ PersistentOfflineTopicStats offlineTopicStats = offlineTopicBacklog.estimateUnloadedTopicBacklog(
+ factory, topicName);
assertNotNull(offlineTopicStats);
+ assertEquals(offlineTopicStats.totalMessages, num);
+ assertEquals(offlineTopicStats.messageBacklog, num);
+ assertEquals(offlineTopicStats.cursorDetails.get("c1").cursorBacklog, num);
}
@Test(timeOut = 20000)
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklogRecoveryTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklogRecoveryTest.java
new file mode 100644
index 000000000000..7b5664b9edfd
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklogRecoveryTest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.api.DigestType;
+import org.apache.bookkeeper.client.api.LedgerEntries;
+import org.apache.bookkeeper.client.api.OpenBuilder;
+import org.apache.bookkeeper.client.api.ReadHandle;
+import org.apache.bookkeeper.client.impl.LedgerEntriesImpl;
+import org.apache.bookkeeper.client.impl.LedgerEntryImpl;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.apache.bookkeeper.mledger.proto.PositionInfo;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class ManagedLedgerOfflineBacklogRecoveryTest {
+ private static final long CURSOR_LEDGER_ID = 17;
+ private static final byte[] PASSWORD = "cursor-password".getBytes(StandardCharsets.UTF_8);
+ private ManagedLedgerFactoryImpl factory;
+ private BookKeeper bookKeeper;
+ private OpenBuilder builder;
+ private ReadHandle handle;
+
+ @BeforeMethod
+ public void setUp() {
+ factory = mock(ManagedLedgerFactoryImpl.class, CALLS_REAL_METHODS);
+ bookKeeper = mock(BookKeeper.class);
+ builder = mock(OpenBuilder.class, RETURNS_SELF);
+ handle = mock(ReadHandle.class);
+ when(bookKeeper.newOpenLedgerOp()).thenReturn(builder);
+ when(builder.execute()).thenReturn(CompletableFuture.completedFuture(handle));
+ when(handle.getId()).thenReturn(CURSOR_LEDGER_ID);
+ when(handle.closeAsync()).thenReturn(CompletableFuture.completedFuture(null));
+ }
+
+ @Test
+ public void testRecoveryReadsPositionsAndClosesResources() {
+ ByteBuf first = positionBuffer(3, 4);
+ ByteBuf last = positionBuffer(3, 8);
+ when(handle.getLastAddConfirmed()).thenReturn(1L);
+ when(handle.readUnconfirmedAsync(0, 0)).thenReturn(CompletableFuture.completedFuture(entries(0, first)));
+ when(handle.readUnconfirmedAsync(1, 1)).thenReturn(CompletableFuture.completedFuture(entries(1, last)));
+
+ assertThat(recover()).isEqualTo(PositionFactory.create(3, 8));
+
+ assertThat(first.refCnt()).isZero();
+ assertThat(last.refCnt()).isZero();
+ verify(handle).closeAsync();
+ verify(builder).withRecovery(false);
+ verify(builder).withLedgerId(CURSOR_LEDGER_ID);
+ verify(builder).withDigestType(DigestType.CRC32C);
+ verify(builder).withPassword(PASSWORD);
+ }
+
+ @Test
+ public void testSynchronousCloseFailurePreservesRecoveredPosition() {
+ ByteBuf buffer = positionBuffer(3, 4);
+ when(handle.getLastAddConfirmed()).thenReturn(0L);
+ when(handle.readUnconfirmedAsync(0, 0)).thenReturn(CompletableFuture.completedFuture(entries(0, buffer)));
+ when(handle.closeAsync()).thenThrow(new IllegalStateException("injected close failure"));
+
+ assertThat(recover()).isEqualTo(PositionFactory.create(3, 4));
+
+ assertThat(buffer.refCnt()).isZero();
+ verify(handle).closeAsync();
+ }
+
+ @Test
+ public void testMalformedPositionDoesNotReturnPartialRecovery() {
+ ByteBuf first = positionBuffer(3, 4);
+ ByteBuf malformed = Unpooled.wrappedBuffer(new byte[] {(byte) 0xff});
+ when(handle.getLastAddConfirmed()).thenReturn(1L);
+ when(handle.readUnconfirmedAsync(0, 0)).thenReturn(CompletableFuture.completedFuture(entries(0, first)));
+ when(handle.readUnconfirmedAsync(1, 1))
+ .thenReturn(CompletableFuture.completedFuture(entries(1, malformed)));
+
+ assertThat(recover()).isNull();
+
+ assertThat(first.refCnt()).isZero();
+ assertThat(malformed.refCnt()).isZero();
+ verify(handle).closeAsync();
+ }
+
+ @Test
+ public void testNoSuchEntryEndsRecoveryAndClosesHandle() {
+ ByteBuf first = positionBuffer(3, 4);
+ when(handle.getLastAddConfirmed()).thenReturn(1L);
+ when(handle.readUnconfirmedAsync(0, 0)).thenReturn(CompletableFuture.completedFuture(entries(0, first)));
+ when(handle.readUnconfirmedAsync(1, 1)).thenReturn(CompletableFuture.failedFuture(
+ BKException.create(BKException.Code.NoSuchEntryException)));
+
+ assertThat(recover()).isEqualTo(PositionFactory.create(3, 4));
+
+ assertThat(first.refCnt()).isZero();
+ verify(handle).closeAsync();
+ }
+
+ @DataProvider(name = "interruptedOperation")
+ public Object[][] interruptedOperation() {
+ return new Object[][] {{"open"}, {"read"}};
+ }
+
+ @Test(dataProvider = "interruptedOperation", timeOut = 15000)
+ public void testInterruptedRecoveryClosesLateResult(String operation) throws Exception {
+ CompletableFuture<ReadHandle> open = new CompletableFuture<>();
+ CompletableFuture<LedgerEntries> read = new CompletableFuture<>();
+ CountDownLatch operationStarted = new CountDownLatch(1);
+ ByteBuf buffer = positionBuffer(3, 4);
+ LedgerEntries entries = entries(0, buffer);
+ if ("open".equals(operation)) {
+ when(builder.execute()).thenAnswer(invocation -> {
+ operationStarted.countDown();
+ return open;
+ });
+ } else {
+ when(handle.getLastAddConfirmed()).thenReturn(0L);
+ when(handle.readUnconfirmedAsync(0, 0)).thenAnswer(invocation -> {
+ operationStarted.countDown();
+ return read;
+ });
+ }
+ CompletableFuture<Position> result = new CompletableFuture<>();
+ AtomicBoolean interrupted = new AtomicBoolean();
+ Thread recoveryThread = new Thread(() -> {
+ Position position = recover();
+ interrupted.set(Thread.currentThread().isInterrupted());
+ result.complete(position);
+ }, "offline-backlog-recovery");
+ recoveryThread.start();
+ try {
+ assertThat(operationStarted.await(5, TimeUnit.SECONDS)).isTrue();
+ recoveryThread.interrupt();
+ assertThat(result.get(5, TimeUnit.SECONDS)).isNull();
+ assertThat(interrupted).isTrue();
+ if ("open".equals(operation)) {
+ verify(handle, never()).closeAsync();
+ open.complete(handle);
+ verify(handle, never()).readUnconfirmedAsync(0, 0);
+ } else {
+ assertThat(buffer.refCnt()).isEqualTo(1);
+ read.complete(entries);
+ assertThat(buffer.refCnt()).isZero();
+ }
+ verify(handle).closeAsync();
+ } finally {
+ recoveryThread.interrupt();
+ open.complete(handle);
+ read.complete(entries);
+ recoveryThread.join(TimeUnit.SECONDS.toMillis(5));
+ if (buffer.refCnt() > 0) {
+ entries.close();
+ }
+ }
+ assertThat(recoveryThread.isAlive()).isFalse();
+ }
+
+ private Position recover() {
+ return factory.tryGetMDPosition(bookKeeper, CURSOR_LEDGER_ID, "subscription",
+ BookKeeper.DigestType.CRC32C, PASSWORD);
+ }
+
+ private static ByteBuf positionBuffer(long ledgerId, long entryId) {
+ return Unpooled.wrappedBuffer(new PositionInfo().setLedgerId(ledgerId).setEntryId(entryId).toByteArray());
+ }
+
+ private static LedgerEntries entries(long entryId, ByteBuf buffer) {
+ return LedgerEntriesImpl.create(new ArrayList<>(List.of(
+ LedgerEntryImpl.create(CURSOR_LEDGER_ID, entryId, buffer.readableBytes(), buffer))));
+ }
+}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklogTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklogTest.java
new file mode 100644
index 000000000000..1925caae0efc
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklogTest.java
@@ -0,0 +1,614 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.NavigableMap;
+import java.util.Optional;
+import java.util.TreeMap;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.client.api.DigestType;
+import org.apache.bookkeeper.client.api.LedgerEntries;
+import org.apache.bookkeeper.client.api.OpenBuilder;
+import org.apache.bookkeeper.client.api.ReadHandle;
+import org.apache.bookkeeper.mledger.ManagedLedger;
+import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
+import org.apache.bookkeeper.mledger.proto.ManagedCursorInfo;
+import org.apache.bookkeeper.mledger.proto.ManagedLedgerInfo;
+import org.apache.bookkeeper.mledger.proto.PositionInfo;
+import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.Stat;
+import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore.OperationType;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class ManagedLedgerOfflineBacklogTest extends MockedBookKeeperTestCase {
+
+ @DataProvider(name = "accuracy")
+ public Object[][] accuracy() {
+ return new Object[][] {{true}, {false}};
+ }
+
+ @Test(dataProvider = "accuracy", timeOut = 15000)
+ public void testCursorMetadataCallbacksDoNotBlockTheirExecutor(boolean accurate) throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-backlog");
+ String managedLedgerName = topicName.getPersistenceNamingEncoding();
+ createOfflineTopic(topicName);
+
+ Thread metadataWorker = factory.getScheduledExecutor().chooseThread(managedLedgerName)
+ .submit(Thread::currentThread).get(5, TimeUnit.SECONDS);
+ ManagedLedgerOfflineBacklog backlog =
+ new ManagedLedgerOfflineBacklog(DigestType.CRC32, new byte[0], "broker", accurate);
+ CompletableFuture<PersistentOfflineTopicStats> result = CompletableFuture.supplyAsync(() -> {
+ try {
+ return backlog.estimateUnloadedTopicBacklog(factory, topicName);
+ } catch (Exception e) {
+ throw new CompletionException(e);
+ }
+ }, cachedExecutor);
+ try {
+ PersistentOfflineTopicStats stats = result.get(5, TimeUnit.SECONDS);
+ assertThat(stats.totalMessages).isEqualTo(3);
+ assertThat(stats.messageBacklog).isEqualTo(3);
+ assertThat(stats.cursorDetails).containsOnlyKeys("subscription");
+ assertThat(stats.cursorDetails.get("subscription").cursorBacklog).isEqualTo(3);
+ factory.getScheduledExecutor().chooseThread(managedLedgerName)
+ .submit(() -> null).get(5, TimeUnit.SECONDS);
+ } finally {
+ if (!result.isDone()) {
+ // The unfixed code waits for a cursor callback queued behind itself on this worker.
+ // Interrupt only on failure so the regression also terminates against that version.
+ metadataWorker.interrupt();
+ result.handle((stats, error) -> null).get(5, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ @Test(dataProvider = "accuracy", timeOut = 15000)
+ public void testAsyncEstimateCanStartOnTheMetadataCallbackThread(boolean accurate) throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-async");
+ createOfflineTopic(topicName);
+ CompletableFuture<PersistentOfflineTopicStats> result = new CompletableFuture<>();
+ CompletableFuture<Void> heartbeat = new CompletableFuture<>();
+ factory.getMetaStore().getCursors(topicName.getPersistenceNamingEncoding(), new MetaStoreCallback<>() {
+ @Override
+ public void operationComplete(List<String> cursors, Stat stat) {
+ ManagedLedgerOfflineBacklog backlog =
+ new ManagedLedgerOfflineBacklog(DigestType.CRC32, new byte[0], "broker", accurate);
+ backlog.estimateUnloadedTopicBacklogAsync(factory, topicName, cachedExecutor)
+ .whenComplete((stats, error) -> {
+ if (error == null) {
+ result.complete(stats);
+ } else {
+ result.completeExceptionally(error);
+ }
+ });
+ factory.getScheduledExecutor().executeOrdered(topicName.getPersistenceNamingEncoding(),
+ () -> heartbeat.complete(null));
+ }
+
+ @Override
+ public void operationFailed(ManagedLedgerException.MetaStoreException error) {
+ result.completeExceptionally(error);
+ }
+ });
+ heartbeat.get(5, TimeUnit.SECONDS);
+ PersistentOfflineTopicStats stats = result.get(5, TimeUnit.SECONDS);
+ assertThat(stats.totalMessages).isEqualTo(3);
+ assertThat(stats.messageBacklog).isEqualTo(3);
+ assertThat(stats.dataLedgerDetails).hasSize(accurate ? 1 : 0);
+ }
+
+ @Test(timeOut = 15000)
+ public void testMixedMetadataAndLedgerCursorsAreAggregated() throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-many-cursors");
+ Position lastPosition = createOfflineTopic(topicName);
+ writeCursorInfo(topicName, "subscription", -1, lastPosition.getLedgerId(), -1);
+ writeCursorInfo(topicName, "metadata-cursor", -1, lastPosition.getLedgerId(), 0);
+ CountDownLatch readsReady = new CountDownLatch(12);
+ CompletableFuture<Void> releaseReads = new CompletableFuture<>();
+ List<LedgerEntries> observedEntries = new CopyOnWriteArrayList<>();
+ for (int i = 0; i < 12; i++) {
+ LedgerHandle cursorLedger = bkc.createLedger(BookKeeper.DigestType.CRC32, new byte[0]);
+ cursorLedger.addEntry(new PositionInfo().setLedgerId(lastPosition.getLedgerId())
+ .setEntryId(i % 3 - 1).toByteArray());
+ cursorLedger.close();
+ writeCursorInfo(topicName, "ledger-cursor-" + i, cursorLedger.getId(), -1, -1);
+ }
+ bkc.setReadHandleInterceptor((ledgerId, first, last, entries) -> {
+ LedgerEntries observed = spy(entries);
+ observedEntries.add(observed);
+ readsReady.countDown();
+ return releaseReads.thenApplyAsync(ignored -> observed, cachedExecutor);
+ });
+ ManagedLedgerOfflineBacklog backlog =
+ new ManagedLedgerOfflineBacklog(DigestType.CRC32, new byte[0], "broker", true);
+ CompletableFuture<PersistentOfflineTopicStats> result =
+ backlog.estimateUnloadedTopicBacklogAsync(factory, topicName, cachedExecutor);
+ try {
+ assertThat(readsReady.await(5, TimeUnit.SECONDS)).isTrue();
+ assertThat(result.isDone()).isFalse();
+ } finally {
+ releaseReads.complete(null);
+ }
+ PersistentOfflineTopicStats stats = result.get(5, TimeUnit.SECONDS);
+ assertThat(stats.totalMessages).isEqualTo(3);
+ assertThat(stats.messageBacklog).isEqualTo(29);
+ assertThat(stats.cursorDetails).hasSize(14);
+ assertThat(observedEntries).hasSize(12);
+ observedEntries.forEach(entries -> verify(entries).close());
+ assertThat(stats.cursorDetails.get("subscription").cursorBacklog).isEqualTo(3);
+ assertThat(stats.cursorDetails.get("metadata-cursor").cursorBacklog).isEqualTo(2);
+ for (int i = 0; i < 12; i++) {
+ assertThat(stats.cursorDetails.get("ledger-cursor-" + i).cursorBacklog).isEqualTo(3 - i % 3);
+ }
+ }
+
+ @Test(timeOut = 15000)
+ public void testCalculateCursorBacklogsPreservesExistingStats() throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-existing-stats");
+ Position lastPosition = createOfflineTopic(topicName);
+ NavigableMap<Long, ManagedLedgerInfo.LedgerInfo> ledgers = new TreeMap<>();
+ ledgers.put(lastPosition.getLedgerId(), new ManagedLedgerInfo.LedgerInfo()
+ .setLedgerId(lastPosition.getLedgerId()).setEntries(3).setSize(21));
+ PersistentOfflineTopicStats stats = new PersistentOfflineTopicStats(topicName.toString(), "broker");
+ stats.messageBacklog = 5;
+ stats.addCursorDetails("existing", 2, 99);
+
+ factory.calculateCursorBacklogs(topicName, ledgers, stats, false, BookKeeper.DigestType.CRC32, new byte[0]);
+
+ assertThat(stats.messageBacklog).isEqualTo(8);
+ assertThat(stats.cursorDetails).containsOnlyKeys("existing", "subscription");
+ assertThat(stats.cursorDetails.get("existing").cursorBacklog).isEqualTo(2);
+ assertThat(stats.cursorDetails.get("existing").cursorLedgerId).isEqualTo(99);
+ assertThat(stats.cursorDetails.get("subscription").cursorBacklog).isEqualTo(3);
+ }
+
+ @Test(timeOut = 15000)
+ public void testTimeoutBeforeManagedLedgerMetadataDoesNotStartMoreReads() throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-metadata-timeout");
+ createOfflineTopic(topicName);
+ MetaStore actualStore = factory.getMetaStore();
+ MetaStore delayedStore = spy(actualStore);
+ BookKeeper observedBookKeeper = spy(bkc);
+ ManagedLedgerFactoryImpl estimatorFactory = spy(factory);
+ doReturn(delayedStore).when(estimatorFactory).getMetaStore();
+ doReturn(CompletableFuture.completedFuture(observedBookKeeper)).when(estimatorFactory).getBookKeeper();
+ CompletableFuture<Runnable> delayedCompletion = new CompletableFuture<>();
+ doAnswer(invocation -> {
+ MetaStoreCallback<ManagedLedgerInfo> callback = invocation.getArgument(2);
+ actualStore.getManagedLedgerInfo(topicName.getPersistenceNamingEncoding(), false,
+ new MetaStoreCallback<>() {
+ @Override
+ public void operationComplete(ManagedLedgerInfo info, Stat stat) {
+ delayedCompletion.complete(() -> callback.operationComplete(info, stat));
+ }
+
+ @Override
+ public void operationFailed(ManagedLedgerException.MetaStoreException error) {
+ delayedCompletion.complete(() -> callback.operationFailed(error));
+ }
+ });
+ return null;
+ }).when(delayedStore).getManagedLedgerInfo(eq(topicName.getPersistenceNamingEncoding()), eq(false), any());
+ PersistentOfflineTopicStats stats = new PersistentOfflineTopicStats(topicName.toString(), "broker");
+ stats.totalMessages = 17;
+ stats.storageSize = 19;
+ stats.messageBacklog = 23;
+ long generatedAt = stats.statGeneratedAt.getTime();
+ CompletableFuture<Void> result = estimatorFactory.estimateUnloadedTopicBacklogAsync(stats, topicName, false,
+ List.of(BookKeeper.DigestType.CRC32, new byte[0]), cachedExecutor, 100, TimeUnit.MILLISECONDS);
+ Runnable completeMetadata = delayedCompletion.get(5, TimeUnit.SECONDS);
+ assertThatThrownBy(() -> result.get(5, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class).hasCauseInstanceOf(TimeoutException.class);
+
+ factory.getScheduledExecutor().chooseThread(topicName.getPersistenceNamingEncoding())
+ .submit(completeMetadata).get(5, TimeUnit.SECONDS);
+ factory.getScheduledExecutor().chooseThread(topicName.getPersistenceNamingEncoding())
+ .submit(() -> null).get(5, TimeUnit.SECONDS);
+
+ verify(observedBookKeeper, never()).newOpenLedgerOp();
+ verify(delayedStore, never()).getCursors(any(), any());
+ assertStatsUnchanged(stats, generatedAt);
+ }
+
+ @Test(timeOut = 15000)
+ public void testTimeoutDoesNotPublishOrMutatePartialStats() throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-timeout");
+ Position lastPosition = createOfflineTopic(topicName);
+ LedgerHandle cursorLedger = bkc.createLedger(BookKeeper.DigestType.CRC32, new byte[0]);
+ cursorLedger.addEntry(new PositionInfo().setLedgerId(lastPosition.getLedgerId())
+ .setEntryId(-1).toByteArray());
+ cursorLedger.close();
+ writeCursorInfo(topicName, "subscription", cursorLedger.getId(), -1, -1);
+ MetaStore actualStore = factory.getMetaStore();
+ MetaStore delayedStore = spy(actualStore);
+ CompletableFuture<ReadHandle> dataHandle = new CompletableFuture<>();
+ ManagedLedgerFactoryImpl estimatorFactory = observeReadHandle(lastPosition.getLedgerId(), dataHandle);
+ BookKeeper observedBookKeeper = estimatorFactory.getBookKeeper().get(5, TimeUnit.SECONDS);
+ doReturn(delayedStore).when(estimatorFactory).getMetaStore();
+ CompletableFuture<Runnable> delayedCompletion = new CompletableFuture<>();
+ doAnswer(invocation -> {
+ MetaStoreCallback<ManagedCursorInfo> callback = invocation.getArgument(2);
+ actualStore.asyncGetCursorInfo(topicName.getPersistenceNamingEncoding(), "subscription",
+ new MetaStoreCallback<>() {
+ @Override
+ public void operationComplete(ManagedCursorInfo info, Stat stat) {
+ delayedCompletion.complete(() -> callback.operationComplete(info, stat));
+ }
+
+ @Override
+ public void operationFailed(ManagedLedgerException.MetaStoreException error) {
+ delayedCompletion.complete(() -> callback.operationFailed(error));
+ }
+ });
+ return null;
+ }).when(delayedStore).asyncGetCursorInfo(eq(topicName.getPersistenceNamingEncoding()),
+ eq("subscription"), any());
+ PersistentOfflineTopicStats stats = new PersistentOfflineTopicStats(topicName.toString(), "broker");
+ stats.totalMessages = 17;
+ stats.storageSize = 19;
+ stats.messageBacklog = 23;
+ long generatedAt = stats.statGeneratedAt.getTime();
+ CompletableFuture<Void> result = estimatorFactory.estimateUnloadedTopicBacklogAsync(stats, topicName, false,
+ List.of(BookKeeper.DigestType.CRC32, new byte[0]), cachedExecutor, 1, TimeUnit.SECONDS);
+ Runnable completeCursor = delayedCompletion.get(5, TimeUnit.SECONDS);
+ assertThatThrownBy(() -> result.get(5, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class).hasCauseInstanceOf(TimeoutException.class);
+ assertStatsUnchanged(stats, generatedAt);
+ factory.getScheduledExecutor().chooseThread(topicName.getPersistenceNamingEncoding())
+ .submit(completeCursor).get(5, TimeUnit.SECONDS);
+ factory.getScheduledExecutor().chooseThread(topicName.getPersistenceNamingEncoding())
+ .submit(() -> null).get(5, TimeUnit.SECONDS);
+ verify(observedBookKeeper).newOpenLedgerOp();
+ verify(dataHandle.get(5, TimeUnit.SECONDS)).closeAsync();
+ assertStatsUnchanged(stats, generatedAt);
+ }
+
+ @Test(timeOut = 15000)
+ public void testTimeoutDuringCursorReadClosesLateResources() throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-cursor-read-timeout");
+ Position lastPosition = createOfflineTopic(topicName);
+ LedgerHandle cursorLedger = bkc.createLedger(BookKeeper.DigestType.CRC32, new byte[0]);
+ cursorLedger.addEntry(new PositionInfo().setLedgerId(lastPosition.getLedgerId())
+ .setEntryId(-1).toByteArray());
+ cursorLedger.close();
+ writeCursorInfo(topicName, "subscription", cursorLedger.getId(), -1, -1);
+ CompletableFuture<ReadHandle> observedHandle = new CompletableFuture<>();
+ ManagedLedgerFactoryImpl estimatorFactory = observeReadHandle(cursorLedger.getId(), observedHandle);
+ CompletableFuture<LedgerEntries> observedEntries = new CompletableFuture<>();
+ CompletableFuture<Void> releaseRead = new CompletableFuture<>();
+ bkc.setReadHandleInterceptor((ledgerId, first, last, entries) -> {
+ if (ledgerId != cursorLedger.getId()) {
+ return CompletableFuture.completedFuture(entries);
+ }
+ LedgerEntries observed = spy(entries);
+ observedEntries.complete(observed);
+ return releaseRead.thenApply(__ -> observed);
+ });
+ PersistentOfflineTopicStats stats = new PersistentOfflineTopicStats(topicName.toString(), "broker");
+ stats.totalMessages = 17;
+ stats.storageSize = 19;
+ stats.messageBacklog = 23;
+ long generatedAt = stats.statGeneratedAt.getTime();
+ CompletableFuture<Void> result = estimatorFactory.estimateUnloadedTopicBacklogAsync(stats, topicName, false,
+ List.of(BookKeeper.DigestType.CRC32, new byte[0]), cachedExecutor, 1, TimeUnit.SECONDS);
+ LedgerEntries entries = observedEntries.get(5, TimeUnit.SECONDS);
+ ReadHandle handle = observedHandle.get(5, TimeUnit.SECONDS);
+ try {
+ assertThatThrownBy(() -> result.get(5, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class).hasCauseInstanceOf(TimeoutException.class);
+ verify(entries, never()).close();
+ verify(handle, never()).closeAsync();
+ assertStatsUnchanged(stats, generatedAt);
+ } finally {
+ releaseRead.complete(null);
+ }
+
+ verify(entries, timeout(5000)).close();
+ verify(handle, timeout(5000)).closeAsync();
+ assertStatsUnchanged(stats, generatedAt);
+ }
+
+ @DataProvider(name = "cursorLedgerFailures")
+ public Object[][] cursorLedgerFailures() {
+ return new Object[][] {{"open"}, {"read"}, {"parse"}};
+ }
+
+ @Test(dataProvider = "cursorLedgerFailures", timeOut = 15000)
+ public void testCursorLedgerFailuresCompleteTheFuture(String failure) throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-failure");
+ Position lastPosition = createOfflineTopic(topicName);
+ LedgerHandle cursorLedger = bkc.createLedger(BookKeeper.DigestType.CRC32, new byte[0]);
+ cursorLedger.addEntry("parse".equals(failure) ? new byte[] {(byte) 0xff}
+ : new PositionInfo().setLedgerId(lastPosition.getLedgerId()).setEntryId(-1).toByteArray());
+ cursorLedger.close();
+ writeCursorInfo(topicName, "subscription", cursorLedger.getId(), -1, -1);
+ CompletableFuture<ReadHandle> observedHandle = new CompletableFuture<>();
+ CompletableFuture<LedgerEntries> observedEntries = new CompletableFuture<>();
+ ManagedLedgerFactoryImpl estimatorFactory = factory;
+ if ("open".equals(failure)) {
+ bkc.deleteLedger(cursorLedger.getId());
+ } else {
+ estimatorFactory = observeReadHandle(cursorLedger.getId(), observedHandle);
+ bkc.setReadHandleInterceptor((ledgerId, first, last, entries) -> {
+ LedgerEntries observed = spy(entries);
+ observedEntries.complete(observed);
+ if ("read".equals(failure)) {
+ observed.close();
+ return CompletableFuture.failedFuture(BKException.create(BKException.Code.ReadException));
+ }
+ return CompletableFuture.completedFuture(observed);
+ });
+ }
+ ManagedLedgerOfflineBacklog backlog =
+ new ManagedLedgerOfflineBacklog(DigestType.CRC32, new byte[0], "broker", true);
+ CompletableFuture<PersistentOfflineTopicStats> result =
+ backlog.estimateUnloadedTopicBacklogAsync(estimatorFactory, topicName, cachedExecutor);
+ assertThatThrownBy(() -> result.get(5, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class).cause().isNotInstanceOf(TimeoutException.class);
+ if (!"open".equals(failure)) {
+ verify(observedHandle.get(5, TimeUnit.SECONDS)).closeAsync();
+ verify(observedEntries.get(5, TimeUnit.SECONDS)).close();
+ }
+ factory.getScheduledExecutor().chooseThread(topicName.getPersistenceNamingEncoding())
+ .submit(() -> null).get(5, TimeUnit.SECONDS);
+ }
+
+ @Test(timeOut = 15000)
+ public void testCursorFailureWaitsForOtherCursorCleanup() throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-wait-for-cursors");
+ Position lastPosition = createOfflineTopic(topicName);
+ LedgerHandle slowCursorLedger = bkc.createLedger(BookKeeper.DigestType.CRC32, new byte[0]);
+ slowCursorLedger.addEntry(new PositionInfo().setLedgerId(lastPosition.getLedgerId())
+ .setEntryId(-1).toByteArray());
+ slowCursorLedger.close();
+ writeCursorInfo(topicName, "slow-subscription", slowCursorLedger.getId(), -1, -1);
+
+ MetaStore actualStore = factory.getMetaStore();
+ MetaStore delayedStore = spy(actualStore);
+ CompletableFuture<MetaStoreCallback<ManagedCursorInfo>> failureCallback = new CompletableFuture<>();
+ doAnswer(invocation -> {
+ String cursorName = invocation.getArgument(1);
+ MetaStoreCallback<ManagedCursorInfo> callback = invocation.getArgument(2);
+ if ("subscription".equals(cursorName)) {
+ failureCallback.complete(callback);
+ } else {
+ actualStore.asyncGetCursorInfo(invocation.getArgument(0), cursorName, callback);
+ }
+ return null;
+ }).when(delayedStore).asyncGetCursorInfo(eq(topicName.getPersistenceNamingEncoding()), any(), any());
+
+ CompletableFuture<ReadHandle> observedHandle = new CompletableFuture<>();
+ ManagedLedgerFactoryImpl estimatorFactory = observeReadHandle(slowCursorLedger.getId(), observedHandle);
+ doReturn(delayedStore).when(estimatorFactory).getMetaStore();
+ CompletableFuture<LedgerEntries> observedEntries = new CompletableFuture<>();
+ CompletableFuture<Void> releaseSlowRead = new CompletableFuture<>();
+ bkc.setReadHandleInterceptor((ledgerId, first, last, entries) -> {
+ if (ledgerId != slowCursorLedger.getId()) {
+ return CompletableFuture.completedFuture(entries);
+ }
+ LedgerEntries observed = spy(entries);
+ observedEntries.complete(observed);
+ return releaseSlowRead.thenApply(__ -> observed);
+ });
+
+ ManagedLedgerOfflineBacklog backlog =
+ new ManagedLedgerOfflineBacklog(DigestType.CRC32, new byte[0], "broker", false);
+ CompletableFuture<PersistentOfflineTopicStats> result =
+ backlog.estimateUnloadedTopicBacklogAsync(estimatorFactory, topicName, cachedExecutor);
+ MetaStoreCallback<ManagedCursorInfo> callback = failureCallback.get(5, TimeUnit.SECONDS);
+ LedgerEntries entries = observedEntries.get(5, TimeUnit.SECONDS);
+ ReadHandle handle = observedHandle.get(5, TimeUnit.SECONDS);
+ callback.operationFailed(new ManagedLedgerException.MetaStoreException("injected cursor failure"));
+ try {
+ assertThat(result).isNotDone();
+ verify(handle, never()).closeAsync();
+ } finally {
+ releaseSlowRead.complete(null);
+ }
+
+ assertThatThrownBy(() -> result.get(5, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class)
+ .hasRootCauseInstanceOf(ManagedLedgerException.MetaStoreException.class);
+ verify(entries).close();
+ verify(handle).closeAsync();
+ }
+
+ @Test(dataProvider = "accuracy", timeOut = 15000)
+ public void testUnknownCursorLacUsesTheBlockingExecutorOnlyForAccurateStats(boolean accurate) throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-unknown-lac");
+ createOfflineTopic(topicName);
+ LedgerHandle cursorLedger = bkc.createLedger(BookKeeper.DigestType.CRC32, new byte[0]);
+ cursorLedger.close();
+ writeCursorInfo(topicName, "subscription", cursorLedger.getId(), -1, -1);
+ CompletableFuture<ReadHandle> observedHandle = new CompletableFuture<>();
+ ManagedLedgerFactoryImpl estimatorFactory = observeReadHandle(cursorLedger.getId(), observedHandle);
+ AtomicInteger blockingTasks = new AtomicInteger();
+ Executor blockingExecutor = task -> {
+ blockingTasks.incrementAndGet();
+ cachedExecutor.execute(task);
+ };
+ ManagedLedgerOfflineBacklog backlog =
+ new ManagedLedgerOfflineBacklog(DigestType.CRC32, new byte[0], "broker", accurate);
+ CompletableFuture<PersistentOfflineTopicStats> result =
+ backlog.estimateUnloadedTopicBacklogAsync(estimatorFactory, topicName, blockingExecutor);
+ if (accurate) {
+ assertThatThrownBy(() -> result.get(5, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class)
+ .hasRootCauseInstanceOf(ManagedLedgerException.class);
+ assertThat(blockingTasks.get()).isEqualTo(1);
+ } else {
+ PersistentOfflineTopicStats stats = result.get(5, TimeUnit.SECONDS);
+ assertThat(stats.totalMessages).isEqualTo(3);
+ assertThat(stats.messageBacklog).isZero();
+ assertThat(stats.cursorDetails.get("subscription").cursorBacklog).isEqualTo(-1);
+ assertThat(blockingTasks.get()).isZero();
+ }
+ verify(observedHandle.get(5, TimeUnit.SECONDS)).closeAsync();
+ factory.getScheduledExecutor().chooseThread(topicName.getPersistenceNamingEncoding())
+ .submit(() -> null).get(5, TimeUnit.SECONDS);
+ }
+
+ @Test(dataProvider = "accuracy", timeOut = 15000)
+ public void testMissingManagedLedgerReturnsZeroStats(boolean accurate) throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-missing");
+ ManagedLedgerOfflineBacklog backlog =
+ new ManagedLedgerOfflineBacklog(DigestType.CRC32, new byte[0], "broker", accurate);
+
+ PersistentOfflineTopicStats stats = backlog
+ .estimateUnloadedTopicBacklogAsync(factory, topicName, cachedExecutor).get(5, TimeUnit.SECONDS);
+
+ assertThat(stats.storageSize).isZero();
+ assertThat(stats.totalMessages).isZero();
+ assertThat(stats.messageBacklog).isZero();
+ assertThat(stats.dataLedgerDetails).isEmpty();
+ assertThat(stats.cursorDetails).isEmpty();
+ assertThat(metadataStore.exists("/managed-ledgers/" + topicName.getPersistenceNamingEncoding())
+ .get(5, TimeUnit.SECONDS)).isFalse();
+ }
+
+ @Test(dataProvider = "accuracy", timeOut = 15000)
+ public void testEmptyLedgerMetadataAndCursorList(boolean accurate) throws Exception {
+ TopicName emptyTopic = TopicName.get("persistent://tenant/namespace/offline-empty-ledgers");
+ metadataStore.put("/managed-ledgers/" + emptyTopic.getPersistenceNamingEncoding(),
+ new ManagedLedgerInfo().toByteArray(), Optional.empty()).get(5, TimeUnit.SECONDS);
+ ManagedLedgerOfflineBacklog backlog =
+ new ManagedLedgerOfflineBacklog(DigestType.CRC32, new byte[0], "broker", accurate);
+ PersistentOfflineTopicStats emptyStats = backlog
+ .estimateUnloadedTopicBacklogAsync(factory, emptyTopic, cachedExecutor).get(5, TimeUnit.SECONDS);
+ assertThat(emptyStats.totalMessages).isZero();
+ assertThat(emptyStats.messageBacklog).isZero();
+ assertThat(emptyStats.cursorDetails).isEmpty();
+
+ TopicName noCursorsTopic = TopicName.get("persistent://tenant/namespace/offline-empty-cursors");
+ createOfflineTopic(noCursorsTopic);
+ metadataStore.delete("/managed-ledgers/" + noCursorsTopic.getPersistenceNamingEncoding() + "/subscription",
+ Optional.empty()).get(5, TimeUnit.SECONDS);
+ PersistentOfflineTopicStats noCursorStats = backlog
+ .estimateUnloadedTopicBacklogAsync(factory, noCursorsTopic, cachedExecutor).get(5, TimeUnit.SECONDS);
+ assertThat(noCursorStats.totalMessages).isEqualTo(3);
+ assertThat(noCursorStats.messageBacklog).isZero();
+ assertThat(noCursorStats.cursorDetails).isEmpty();
+ }
+
+ @DataProvider(name = "metadataFailures")
+ public Object[][] metadataFailures() {
+ return new Object[][] {{OperationType.GET, false}, {OperationType.GET_CHILDREN, false},
+ {OperationType.GET, true}};
+ }
+
+ @Test(dataProvider = "metadataFailures", timeOut = 15000)
+ public void testMetadataFailuresCompleteTheFuture(OperationType operation, boolean cursorInfo) throws Exception {
+ TopicName topicName = TopicName.get("persistent://tenant/namespace/offline-metadata-failure");
+ createOfflineTopic(topicName);
+ String failurePath = "/managed-ledgers/" + topicName.getPersistenceNamingEncoding()
+ + (cursorInfo ? "/subscription" : "");
+ metadataStore.failConditional(new MetadataStoreException("injected metadata failure"),
+ (actualOperation, path) -> actualOperation == operation && path.equals(failurePath));
+ ManagedLedgerOfflineBacklog backlog =
+ new ManagedLedgerOfflineBacklog(DigestType.CRC32, new byte[0], "broker", true);
+ CompletableFuture<PersistentOfflineTopicStats> result =
+ backlog.estimateUnloadedTopicBacklogAsync(factory, topicName, cachedExecutor);
+ assertThatThrownBy(() -> result.get(5, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class).hasRootCauseInstanceOf(MetadataStoreException.class);
+ }
+
+ @SuppressWarnings("unchecked")
+ private ManagedLedgerFactoryImpl observeReadHandle(long ledgerId, CompletableFuture<ReadHandle> observedHandle) {
+ BookKeeper observedBookKeeper = spy(bkc);
+ doAnswer(invocation -> {
+ OpenBuilder builder = spy(bkc.newOpenLedgerOp());
+ doAnswer(executeInvocation -> {
+ CompletableFuture<ReadHandle> opened =
+ (CompletableFuture<ReadHandle>) executeInvocation.callRealMethod();
+ return opened.thenApply(handle -> {
+ if (handle.getId() != ledgerId) {
+ return handle;
+ }
+ ReadHandle observed = spy(handle);
+ observedHandle.complete(observed);
+ return observed;
+ });
+ }).when(builder).execute();
+ return builder;
+ }).when(observedBookKeeper).newOpenLedgerOp();
+ ManagedLedgerFactoryImpl estimatorFactory = spy(factory);
+ doReturn(CompletableFuture.completedFuture(observedBookKeeper)).when(estimatorFactory).getBookKeeper();
+ return estimatorFactory;
+ }
+
+ private Position createOfflineTopic(TopicName topicName) throws Exception {
+ ManagedLedger ledger = factory.open(topicName.getPersistenceNamingEncoding(),
+ new ManagedLedgerConfig().setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1)
+ .setDigestType(DigestType.CRC32));
+ ledger.openCursor("subscription");
+ Position lastPosition = null;
+ for (int i = 0; i < 3; i++) {
+ lastPosition = ledger.addEntry(("entry-" + i).getBytes(StandardCharsets.UTF_8));
+ }
+ ledger.close();
+ return lastPosition;
+ }
+
+ private void writeCursorInfo(TopicName topicName, String cursorName, long cursorLedgerId,
+ long markDeleteLedgerId, long markDeleteEntryId) throws Exception {
+ ManagedCursorInfo info = new ManagedCursorInfo().setCursorsLedgerId(cursorLedgerId)
+ .setMarkDeleteLedgerId(markDeleteLedgerId).setMarkDeleteEntryId(markDeleteEntryId);
+ metadataStore.put("/managed-ledgers/" + topicName.getPersistenceNamingEncoding() + "/" + cursorName,
+ info.toByteArray(), Optional.empty()).get(5, TimeUnit.SECONDS);
+ }
+
+ private void assertStatsUnchanged(PersistentOfflineTopicStats stats, long generatedAt) {
+ assertThat(stats.totalMessages).isEqualTo(17);
+ assertThat(stats.storageSize).isEqualTo(19);
+ assertThat(stats.messageBacklog).isEqualTo(23);
+ assertThat(stats.cursorDetails).isEmpty();
+ assertThat(stats.dataLedgerDetails).isEmpty();
+ assertThat(stats.statGeneratedAt.getTime()).isEqualTo(generatedAt);
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 453a4d7ff867..f7111bd7dc4e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -3539,19 +3539,16 @@ public class PersistentTopicsBase extends AdminResource {
ManagedLedgerOfflineBacklog offlineTopicBacklog =
new ManagedLedgerOfflineBacklog(config.getDigestType(), config.getPassword(),
pulsar().getAdvertisedAddress(), false);
- try {
- PersistentOfflineTopicStats estimateOfflineTopicStats =
- offlineTopicBacklog.estimateUnloadedTopicBacklog(
- pulsar().getBrokerService()
- .getManagedLedgerFactoryForTopic(topicName,
- config.getStorageClassName()),
- topicName);
- pulsar().getBrokerService()
- .cacheOfflineTopicStats(topicName, estimateOfflineTopicStats);
- return CompletableFuture.completedFuture(estimateOfflineTopicStats);
- } catch (Exception e) {
- throw new RestException(e);
- }
+ return offlineTopicBacklog.estimateUnloadedTopicBacklogAsync(
+ pulsar().getBrokerService()
+ .getManagedLedgerFactoryForTopic(topicName,
+ config.getStorageClassName()),
+ topicName, pulsar().getExecutor())
+ .thenApply(estimateOfflineTopicStats -> {
+ pulsar().getBrokerService()
+ .cacheOfflineTopicStats(topicName, estimateOfflineTopicStats);
+ return estimateOfflineTopicStats;
+ });
});
});
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBacklogTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBacklogTest.java
new file mode 100644
index 000000000000..eebbe11c1b91
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBacklogTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin.impl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
+import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.resources.NamespaceResources;
+import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
+import org.apache.pulsar.common.policies.data.Policies;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker-admin")
+public class PersistentTopicsBacklogTest {
+
+ @Test
+ public void testBacklogRequestWaitsForAsyncEstimateBeforeCaching() throws Exception {
+ BacklogTopics topics = new BacklogTopics();
+ CompletableFuture<Void> estimate = new CompletableFuture<>();
+ AtomicReference<PersistentOfflineTopicStats> collected = new AtomicReference<>();
+ doAnswer(invocation -> {
+ collected.set(invocation.getArgument(0));
+ return estimate;
+ }).when(topics.factory).estimateUnloadedTopicBacklogAsync(any(), eq(topics.backlogTopicName), eq(false),
+ isNull(), eq(topics.executor));
+
+ CompletableFuture<PersistentOfflineTopicStats> response = topics.internalGetBacklogAsync(false);
+ assertThat(response).isNotDone();
+ verify(topics.brokerService, never()).cacheOfflineTopicStats(any(), any());
+ verify(topics.factory, never()).estimateUnloadedTopicBacklog(any(), any(), anyBoolean(), any());
+
+ collected.get().messageBacklog = 3;
+ estimate.complete(null);
+ PersistentOfflineTopicStats stats = response.get(5, TimeUnit.SECONDS);
+ assertThat(stats.messageBacklog).isEqualTo(3);
+ verify(topics.brokerService).cacheOfflineTopicStats(topics.backlogTopicName, stats);
+ }
+
+ @Test
+ public void testFailedBacklogEstimateIsNotCached() {
+ BacklogTopics topics = new BacklogTopics();
+ CompletableFuture<Void> estimate = new CompletableFuture<>();
+ when(topics.factory.estimateUnloadedTopicBacklogAsync(any(), eq(topics.backlogTopicName), eq(false),
+ isNull(), eq(topics.executor))).thenReturn(estimate);
+
+ CompletableFuture<PersistentOfflineTopicStats> response = topics.internalGetBacklogAsync(false);
+ TimeoutException failure = new TimeoutException("metadata read timed out");
+ estimate.completeExceptionally(failure);
+ assertThatThrownBy(() -> response.get(5, TimeUnit.SECONDS)).hasCause(failure);
+ verify(topics.brokerService, never()).cacheOfflineTopicStats(any(), any());
+ }
+
+ @Test
+ public void testCachedBacklogSkipsEstimation() throws Exception {
+ BacklogTopics topics = new BacklogTopics();
+ PersistentOfflineTopicStats cached = new PersistentOfflineTopicStats("topic", "broker");
+ when(topics.brokerService.getOfflineTopicStat(topics.backlogTopicName)).thenReturn(cached);
+
+ assertThat(topics.internalGetBacklogAsync(false).get(5, TimeUnit.SECONDS)).isSameAs(cached);
+ verifyNoInteractions(topics.factory);
+ verify(topics.brokerService, never()).getManagedLedgerConfig(any());
+ }
+
+ private static class BacklogTopics extends PersistentTopicsBase {
+ private final TopicName backlogTopicName = TopicName.get("persistent://tenant/namespace/topic");
+ private final PulsarService pulsarService = mock(PulsarService.class);
+ private final BrokerService brokerService = mock(BrokerService.class);
+ private final NamespaceResources namespaceResources = mock(NamespaceResources.class);
+ private final ManagedLedgerFactory factory = mock(ManagedLedgerFactory.class);
+ private final ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
+
+ private BacklogTopics() {
+ topicName = backlogTopicName;
+ namespaceName = topicName.getNamespaceObject();
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ when(pulsarService.getBrokerService()).thenReturn(brokerService);
+ when(pulsarService.getExecutor()).thenReturn(executor);
+ when(namespaceResources.getPoliciesAsync(namespaceName))
+ .thenReturn(CompletableFuture.completedFuture(Optional.of(new Policies())));
+ when(brokerService.getManagedLedgerConfig(topicName))
+ .thenReturn(CompletableFuture.completedFuture(config));
+ when(brokerService.getManagedLedgerFactoryForTopic(topicName, config.getStorageClassName()))
+ .thenReturn(factory);
+ }
+
+ @Override
+ protected PulsarService pulsar() {
+ return pulsarService;
+ }
+
+ @Override
+ protected NamespaceResources namespaceResources() {
+ return namespaceResources;
+ }
+
+ @Override
+ protected CompletableFuture<Void> validateGlobalNamespaceOwnershipAsync(NamespaceName namespace) {
+ return CompletableFuture.completedFuture(null);
+ }
+ }
+}