You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by yu...@apache.org on 2025/05/27 09:39:17 UTC
(pulsar) branch master updated: [improve][ml] Offload ledgers without check ledger length (#24344)
This is an automated email from the ASF dual-hosted git repository.
yubiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 3ccea6a5984 [improve][ml] Offload ledgers without check ledger length (#24344)
3ccea6a5984 is described below
commit 3ccea6a598495f18bde39916409e3e29eefed900
Author: fengyubiao <yu...@streamnative.io>
AuthorDate: Tue May 27 17:39:09 2025 +0800
[improve][ml] Offload ledgers without check ledger length (#24344)
Co-authored-by: Lari Hotari <lh...@users.noreply.github.com>
---
.../offload/filesystem/impl/FileSystemManagedLedgerOffloader.java | 7 ++++++-
.../offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java | 7 ++++++-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
index 2431e61c91c..40ae52d7b79 100644
--- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
+++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
@@ -194,11 +194,16 @@ public class FileSystemManagedLedgerOffloader implements LedgerOffloader {
@Override
public void run() {
- if (readHandle.getLength() == 0 || !readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) {
+ if (!readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) {
promise.completeExceptionally(
new IllegalArgumentException("An empty or open ledger should never be offloaded"));
return;
}
+ if (readHandle.getLength() <= 0) {
+ log.warn("Ledger [{}] has zero length, but it contains {} entries. "
+ + " Attempting to offload ledger since it contains entries.", readHandle.getId(),
+ readHandle.getLastAddConfirmed() + 1);
+ }
long ledgerId = readHandle.getId();
final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME);
String storagePath = getStoragePath(storageBasePath, managedLedgerName);
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
index 5ee62591b42..33bbc49ee22 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
@@ -204,11 +204,16 @@ public class BlobStoreManagedLedgerOffloader implements LedgerOffloader {
final BlobStore writeBlobStore = getBlobStore(config.getBlobStoreLocation());
log.info("offload {} uuid {} extraMetadata {} to {} {}", readHandle.getId(), uuid, extraMetadata,
config.getBlobStoreLocation(), writeBlobStore);
- if (readHandle.getLength() == 0 || !readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) {
+ if (!readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) {
promise.completeExceptionally(
new IllegalArgumentException("An empty or open ledger should never be offloaded"));
return;
}
+ if (readHandle.getLength() <= 0) {
+ log.warn("[{}] Ledger [{}] has zero length, but it contains {} entries."
+ + " Attempting to offload ledger since it contains entries.", topicName, readHandle.getId(),
+ readHandle.getLastAddConfirmed() + 1);
+ }
OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create()
.withLedgerMetadata(readHandle.getLedgerMetadata())
.withDataBlockHeaderLength(BlockAwareSegmentInputStreamImpl.getHeaderSize());