You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/13 14:25:41 UTC

(pulsar) 02/02: Address review: serialize cache publication with close, recheck the cache before fetching, don't prefetch oversized chunks

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch lh-improve-offload-read-chunk-cache
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 0d517ef6076dd5fbdd706df96244ebf28a874dc8
Author: Lari Hotari <lh...@apache.org>
AuthorDate: Tue Oct 13 16:17:33 2026 +0300

    Address review: serialize cache publication with close, recheck the cache before fetching, don't prefetch oversized chunks
    
    Assisted-by: Claude Code (claude-opus-5-5)
---
 conf/broker.conf                                   |  3 +-
 .../mledger/OffloadedReadCacheConfig.java          |  3 +-
 .../apache/pulsar/broker/ServiceConfiguration.java |  3 +-
 .../jcloud/impl/OffloadedDataChunkCache.java       | 68 +++++++++++++++++-----
 .../jcloud/impl/OffloadedDataChunkCacheTest.java   | 52 ++++++++++++++++-
 5 files changed, 110 insertions(+), 19 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index a9aa4ac3f16b..45b92da55c75 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -2217,7 +2217,8 @@ managedLedgerOffloadReadThreads=2
 
 # The most offloaded data, in MB, that the broker keeps in direct memory for reads of offloaded ledgers: chunks of the
 # read buffer size that the reads of every offloaded ledger share, so that cursors reading the same ledger at different
-# positions don't fetch the same data again. A read of data that is being fetched waits for that fetch. 0 keeps no data
+# positions don't fetch the same data again. A read of data that is being fetched waits for that fetch. 0 keeps no data.
+# The chunks that reads are using and the fetches in progress come on top of it
 managedLedgerOffloadReadCacheSizeInMB=64
 
 # How long, in seconds, the cache of offloaded data keeps data that no read has used
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OffloadedReadCacheConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OffloadedReadCacheConfig.java
index fd54c011d33a..e7dce0610e4a 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OffloadedReadCacheConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OffloadedReadCacheConfig.java
@@ -26,7 +26,8 @@ import org.apache.bookkeeper.common.annotation.InterfaceStability.Evolving;
  * creates: the cache of offloaded data that their read handles share.
  *
  * @param cacheSizeBytes the most bytes of offloaded data that the cache keeps; 0 keeps none, and only reads of the
- *                       same data at the same time share it
+ *                       same data at the same time share it. The data that reads are using and the fetches in
+ *                       progress come on top of it
  * @param cacheExpirySeconds how long the cache keeps data that no read has used
  * @param prefetchChunks how many chunks after the one that a sequential read moves on to are fetched in the
  *                       background; 0 fetches none
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index f0f3a9270fb1..0685491402e6 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -4147,7 +4147,8 @@ public class ServiceConfiguration implements PulsarConfiguration {
             doc = "The most offloaded data, in MB, that the broker keeps in direct memory for reads of offloaded"
                     + " ledgers: chunks of the read buffer size that the reads of every offloaded ledger share, so"
                     + " that cursors reading the same ledger at different positions don't fetch the same data again."
-                    + " A read of data that is being fetched waits for that fetch. 0 keeps no data"
+                    + " A read of data that is being fetched waits for that fetch. 0 keeps no data. The chunks that"
+                    + " reads are using and the fetches in progress come on top of it"
     )
     private long managedLedgerOffloadReadCacheSizeInMB = 64;
 
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCache.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCache.java
index 0ec62e21b725..2d402065f52d 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCache.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCache.java
@@ -110,10 +110,15 @@ public class OffloadedDataChunkCache implements AutoCloseable {
      */
     private static final class PendingFetch {
         private final CompletableFuture<Chunk> future = new CompletableFuture<>();
+        private final Thread starter = Thread.currentThread();
         // Guarded by this
         private int waiters;
         private boolean done;
         private Chunk chunk;
+
+        boolean startedBy(Thread thread) {
+            return starter == thread;
+        }
     }
 
     private final long maxSizeBytes;
@@ -122,7 +127,8 @@ public class OffloadedDataChunkCache implements AutoCloseable {
     private final ConcurrentHashMap<Key, PendingFetch> pendingFetches = new ConcurrentHashMap<>();
     private final ExecutorService fetchExecutor;
     private final ScheduledExecutorService cleanupExecutor;
-    private volatile boolean closed;
+    // Guarded by this
+    private boolean closed;
     private final LongAdder hits = new LongAdder();
     private final LongAdder joinedFetches = new LongAdder();
     private final LongAdder fetches = new LongAdder();
@@ -174,7 +180,8 @@ public class OffloadedDataChunkCache implements AutoCloseable {
                     return cached;
                 }
                 pending = startFetch(key, fetcher);
-            } else {
+            }
+            if (!pending.startedBy(Thread.currentThread())) {
                 joinedFetches.increment();
             }
             synchronized (pending) {
@@ -196,7 +203,9 @@ public class OffloadedDataChunkCache implements AutoCloseable {
 
     /** Fetches the chunk in the background unless it is cached or being fetched. */
     void prefetch(Key key, Fetcher fetcher) {
-        if (prefetchChunks == 0 || chunks.getIfPresent(key) != null || pendingFetches.containsKey(key)) {
+        // A chunk larger than the budget would be dropped at once, before a read uses it
+        if (prefetchChunks == 0 || key.length() > maxSizeBytes || chunks.getIfPresent(key) != null
+                || pendingFetches.containsKey(key)) {
             return;
         }
         startFetch(key, fetcher);
@@ -212,12 +221,18 @@ public class OffloadedDataChunkCache implements AutoCloseable {
         if (existing != null) {
             return existing;
         }
+        // A fetch that completed since the caller's last look at the cache has cached the chunk
+        Chunk cached = chunks.getIfPresent(key);
+        if (cached != null && cached.tryRetain()) {
+            complete(key, created, cached, null, false);
+            return created;
+        }
         fetches.increment();
         try {
             fetchExecutor.execute(() -> fetch(key, fetcher, created));
         } catch (RuntimeException e) {
             // Such as a rejection after close
-            complete(key, created, null, e);
+            complete(key, created, null, e, false);
         }
         return created;
     }
@@ -227,13 +242,17 @@ public class OffloadedDataChunkCache implements AutoCloseable {
         try {
             buffer = fetcher.fetch(key);
         } catch (Throwable e) {
-            complete(key, pending, null, e);
+            complete(key, pending, null, e, false);
             return;
         }
-        complete(key, pending, new Chunk(buffer), null);
+        complete(key, pending, new Chunk(buffer), null, true);
     }
 
-    private void complete(Key key, PendingFetch pending, Chunk chunk, Throwable failure) {
+    /**
+     * Completes the fetch with the chunk, with a reference that it releases, or with the failure. A fetched chunk is
+     * {@code publish}ed into the cache.
+     */
+    private void complete(Key key, PendingFetch pending, Chunk chunk, Throwable failure, boolean publish) {
         synchronized (pending) {
             if (pending.done) {
                 // close() has failed it already
@@ -249,10 +268,15 @@ public class OffloadedDataChunkCache implements AutoCloseable {
                 chunk.retain(pending.waiters);
             }
         }
-        if (chunk != null && maxSizeBytes > 0 && !closed) {
-            chunk.retain(1);
-            // Into the cache before it stops being pending, see acquire()
-            chunks.put(key, chunk);
+        if (chunk != null && publish && maxSizeBytes > 0) {
+            // With close(), so that no chunk is put into the cache after close() has emptied it
+            synchronized (this) {
+                if (!closed) {
+                    chunk.retain(1);
+                    // Into the cache before it stops being pending, see acquire()
+                    chunks.put(key, chunk);
+                }
+            }
         }
         pendingFetches.remove(key, pending);
         if (chunk != null) {
@@ -301,6 +325,18 @@ public class OffloadedDataChunkCache implements AutoCloseable {
         return fetches.sum();
     }
 
+    /** The reads waiting for the fetch of the chunk, or -1 when it isn't being fetched. */
+    @VisibleForTesting
+    int waiters(Key key) {
+        PendingFetch pending = pendingFetches.get(key);
+        if (pending == null) {
+            return -1;
+        }
+        synchronized (pending) {
+            return pending.waiters;
+        }
+    }
+
     @VisibleForTesting
     long cachedBytes() {
         return chunks.asMap().keySet().stream().mapToLong(Key::length).sum();
@@ -312,12 +348,16 @@ public class OffloadedDataChunkCache implements AutoCloseable {
 
     @Override
     public void close() {
-        closed = true;
+        synchronized (this) {
+            closed = true;
+        }
         cleanupExecutor.shutdownNow();
         fetchExecutor.shutdownNow();
         // The reads waiting for fetches that won't run fail instead of waiting forever
         pendingFetches.forEach((key, pending) -> complete(key, pending, null,
-                new IOException("The offloaded data chunk cache is closed")));
-        chunks.invalidateAll();
+                new IOException("The offloaded data chunk cache is closed"), false));
+        synchronized (this) {
+            chunks.invalidateAll();
+        }
     }
 }
diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCacheTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCacheTest.java
index 757ecf2c41b5..cf57182c4b65 100644
--- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCacheTest.java
+++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCacheTest.java
@@ -81,8 +81,8 @@ public class OffloadedDataChunkCacheTest extends BlobStoreTestBase {
             reads.add(readers.submit(() -> cache.acquire(KEY, fetcher)));
         }
         assertThat(fetching.await(10, TimeUnit.SECONDS)).isTrue();
-        // every read has joined the fetch before it completes
-        Awaitility.await().untilAsserted(() -> assertThat(cache.joinedFetches()).isEqualTo(7));
+        // every read waits for the fetch before it completes
+        Awaitility.await().untilAsserted(() -> assertThat(cache.waiters(KEY)).isEqualTo(8));
         release.countDown();
         OffloadedDataChunkCache.Chunk first = reads.get(0).get(10, TimeUnit.SECONDS);
         for (Future<OffloadedDataChunkCache.Chunk> read : reads) {
@@ -143,6 +143,54 @@ public class OffloadedDataChunkCacheTest extends BlobStoreTestBase {
         assertThat(cache.fetches()).isEqualTo(2);
     }
 
+    @Test
+    public void releasesTheBuffersOfExpiredChunks() throws Exception {
+        @Cleanup
+        OffloadedDataChunkCache cache = new OffloadedDataChunkCache(1024, 1, 0, 1);
+        OffloadedDataChunkCache.Chunk chunk = cache.acquire(KEY, OffloadedDataChunkCacheTest::bytes);
+        chunk.release();
+        assertThat(chunk.references()).isEqualTo(1);
+        // the cleanup drops it once no read has used it for the expiry
+        Awaitility.await().atMost(10, TimeUnit.SECONDS)
+                .untilAsserted(() -> assertThat(chunk.buffer().refCnt()).isZero());
+        assertThat(cache.cachedBytes()).isZero();
+    }
+
+    @Test
+    public void anInterruptedReadReleasesItsReferenceWhenTheFetchCompletes() throws Exception {
+        @Cleanup
+        OffloadedDataChunkCache cache = new OffloadedDataChunkCache(1024, 60, 0, 1);
+        CountDownLatch release = new CountDownLatch(1);
+        OffloadedDataChunkCache.Fetcher fetcher = key -> {
+            try {
+                release.await();
+            } catch (InterruptedException e) {
+                throw new IOException(e);
+            }
+            return bytes(key);
+        };
+        @Cleanup("shutdownNow")
+        ExecutorService readers = Executors.newSingleThreadExecutor();
+        Future<OffloadedDataChunkCache.Chunk> read = readers.submit(() -> cache.acquire(KEY, fetcher));
+        Awaitility.await().untilAsserted(() -> assertThat(cache.waiters(KEY)).isEqualTo(1));
+        read.cancel(true);
+        release.countDown();
+        Awaitility.await().untilAsserted(() -> assertThat(cache.waiters(KEY)).isEqualTo(-1));
+        OffloadedDataChunkCache.Chunk chunk = cache.acquire(KEY, fetcher);
+        // the cache's reference and this read's: the interrupted read's has been released
+        Awaitility.await().untilAsserted(() -> assertThat(chunk.references()).isEqualTo(2));
+        chunk.release();
+        assertThat(cache.fetches()).isEqualTo(1);
+    }
+
+    @Test
+    public void doesNotPrefetchAChunkLargerThanTheBudget() throws Exception {
+        @Cleanup
+        OffloadedDataChunkCache cache = new OffloadedDataChunkCache(8, 60, 1, 1);
+        cache.prefetch(KEY, OffloadedDataChunkCacheTest::bytes);
+        assertThat(cache.fetches()).isZero();
+    }
+
     @Test
     public void withoutABudgetKeepsNoChunk() throws Exception {
         @Cleanup