You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by yu...@apache.org on 2025/05/27 09:39:17 UTC

(pulsar) branch master updated: [improve][ml] Offload ledgers without check ledger length (#24344)

This is an automated email from the ASF dual-hosted git repository.

yubiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 3ccea6a5984 [improve][ml] Offload ledgers without check ledger length (#24344)
3ccea6a5984 is described below

commit 3ccea6a598495f18bde39916409e3e29eefed900
Author: fengyubiao <yu...@streamnative.io>
AuthorDate: Tue May 27 17:39:09 2025 +0800

    [improve][ml] Offload ledgers without check ledger length (#24344)
    
    Co-authored-by: Lari Hotari <lh...@users.noreply.github.com>
---
 .../offload/filesystem/impl/FileSystemManagedLedgerOffloader.java  | 7 ++++++-
 .../offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java       | 7 ++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
index 2431e61c91c..40ae52d7b79 100644
--- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
+++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java
@@ -194,11 +194,16 @@ public class FileSystemManagedLedgerOffloader implements LedgerOffloader {
 
         @Override
         public void run() {
-            if (readHandle.getLength() == 0 || !readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) {
+            if (!readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) {
                 promise.completeExceptionally(
                         new IllegalArgumentException("An empty or open ledger should never be offloaded"));
                 return;
             }
+            if (readHandle.getLength() <= 0) {
+                log.warn("Ledger [{}] has zero length, but it contains {} entries. "
+                    + " Attempting to offload ledger since it contains entries.", readHandle.getId(),
+                    readHandle.getLastAddConfirmed() + 1);
+            }
             long ledgerId = readHandle.getId();
             final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME);
             String storagePath = getStoragePath(storageBasePath, managedLedgerName);
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
index 5ee62591b42..33bbc49ee22 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java
@@ -204,11 +204,16 @@ public class BlobStoreManagedLedgerOffloader implements LedgerOffloader {
             final BlobStore writeBlobStore = getBlobStore(config.getBlobStoreLocation());
             log.info("offload {} uuid {} extraMetadata {} to {} {}", readHandle.getId(), uuid, extraMetadata,
                 config.getBlobStoreLocation(), writeBlobStore);
-            if (readHandle.getLength() == 0 || !readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) {
+            if (!readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) {
                 promise.completeExceptionally(
                         new IllegalArgumentException("An empty or open ledger should never be offloaded"));
                 return;
             }
+            if (readHandle.getLength() <= 0) {
+                log.warn("[{}] Ledger [{}] has zero length, but it contains {} entries."
+                    + " Attempting to offload ledger since it contains entries.", topicName, readHandle.getId(),
+                    readHandle.getLastAddConfirmed() + 1);
+            }
             OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create()
                 .withLedgerMetadata(readHandle.getLedgerMetadata())
                 .withDataBlockHeaderLength(BlockAwareSegmentInputStreamImpl.getHeaderSize());