You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/13 14:25:41 UTC
(pulsar) 02/02: Address review: serialize cache publication with close, recheck the cache before fetching, don't prefetch oversized chunks
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch lh-improve-offload-read-chunk-cache
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 0d517ef6076dd5fbdd706df96244ebf28a874dc8
Author: Lari Hotari <lh...@apache.org>
AuthorDate: Tue Oct 13 16:17:33 2026 +0300
Address review: serialize cache publication with close, recheck the cache before fetching, don't prefetch oversized chunks
Assisted-by: Claude Code (claude-opus-5-5)
---
conf/broker.conf | 3 +-
.../mledger/OffloadedReadCacheConfig.java | 3 +-
.../apache/pulsar/broker/ServiceConfiguration.java | 3 +-
.../jcloud/impl/OffloadedDataChunkCache.java | 68 +++++++++++++++++-----
.../jcloud/impl/OffloadedDataChunkCacheTest.java | 52 ++++++++++++++++-
5 files changed, 110 insertions(+), 19 deletions(-)
diff --git a/conf/broker.conf b/conf/broker.conf
index a9aa4ac3f16b..45b92da55c75 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -2217,7 +2217,8 @@ managedLedgerOffloadReadThreads=2
# The most offloaded data, in MB, that the broker keeps in direct memory for reads of offloaded ledgers: chunks of the
# read buffer size that the reads of every offloaded ledger share, so that cursors reading the same ledger at different
-# positions don't fetch the same data again. A read of data that is being fetched waits for that fetch. 0 keeps no data
+# positions don't fetch the same data again. A read of data that is being fetched waits for that fetch. 0 keeps no data.
+# The chunks that reads are using and the fetches in progress come on top of it
managedLedgerOffloadReadCacheSizeInMB=64
# How long, in seconds, the cache of offloaded data keeps data that no read has used
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OffloadedReadCacheConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OffloadedReadCacheConfig.java
index fd54c011d33a..e7dce0610e4a 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OffloadedReadCacheConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OffloadedReadCacheConfig.java
@@ -26,7 +26,8 @@ import org.apache.bookkeeper.common.annotation.InterfaceStability.Evolving;
* creates: the cache of offloaded data that their read handles share.
*
* @param cacheSizeBytes the most bytes of offloaded data that the cache keeps; 0 keeps none, and only reads of the
- * same data at the same time share it
+ * same data at the same time share it. The data that reads are using and the fetches in
+ * progress come on top of it
* @param cacheExpirySeconds how long the cache keeps data that no read has used
* @param prefetchChunks how many chunks after the one that a sequential read moves on to are fetched in the
* background; 0 fetches none
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index f0f3a9270fb1..0685491402e6 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -4147,7 +4147,8 @@ public class ServiceConfiguration implements PulsarConfiguration {
doc = "The most offloaded data, in MB, that the broker keeps in direct memory for reads of offloaded"
+ " ledgers: chunks of the read buffer size that the reads of every offloaded ledger share, so"
+ " that cursors reading the same ledger at different positions don't fetch the same data again."
- + " A read of data that is being fetched waits for that fetch. 0 keeps no data"
+ + " A read of data that is being fetched waits for that fetch. 0 keeps no data. The chunks that"
+ + " reads are using and the fetches in progress come on top of it"
)
private long managedLedgerOffloadReadCacheSizeInMB = 64;
diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCache.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCache.java
index 0ec62e21b725..2d402065f52d 100644
--- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCache.java
+++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCache.java
@@ -110,10 +110,15 @@ public class OffloadedDataChunkCache implements AutoCloseable {
*/
private static final class PendingFetch {
private final CompletableFuture<Chunk> future = new CompletableFuture<>();
+ private final Thread starter = Thread.currentThread();
// Guarded by this
private int waiters;
private boolean done;
private Chunk chunk;
+
+ boolean startedBy(Thread thread) {
+ return starter == thread;
+ }
}
private final long maxSizeBytes;
@@ -122,7 +127,8 @@ public class OffloadedDataChunkCache implements AutoCloseable {
private final ConcurrentHashMap<Key, PendingFetch> pendingFetches = new ConcurrentHashMap<>();
private final ExecutorService fetchExecutor;
private final ScheduledExecutorService cleanupExecutor;
- private volatile boolean closed;
+ // Guarded by this
+ private boolean closed;
private final LongAdder hits = new LongAdder();
private final LongAdder joinedFetches = new LongAdder();
private final LongAdder fetches = new LongAdder();
@@ -174,7 +180,8 @@ public class OffloadedDataChunkCache implements AutoCloseable {
return cached;
}
pending = startFetch(key, fetcher);
- } else {
+ }
+ if (!pending.startedBy(Thread.currentThread())) {
joinedFetches.increment();
}
synchronized (pending) {
@@ -196,7 +203,9 @@ public class OffloadedDataChunkCache implements AutoCloseable {
/** Fetches the chunk in the background unless it is cached or being fetched. */
void prefetch(Key key, Fetcher fetcher) {
- if (prefetchChunks == 0 || chunks.getIfPresent(key) != null || pendingFetches.containsKey(key)) {
+ // A chunk larger than the budget would be dropped at once, before a read uses it
+ if (prefetchChunks == 0 || key.length() > maxSizeBytes || chunks.getIfPresent(key) != null
+ || pendingFetches.containsKey(key)) {
return;
}
startFetch(key, fetcher);
@@ -212,12 +221,18 @@ public class OffloadedDataChunkCache implements AutoCloseable {
if (existing != null) {
return existing;
}
+ // A fetch that completed since the caller's last look at the cache has cached the chunk
+ Chunk cached = chunks.getIfPresent(key);
+ if (cached != null && cached.tryRetain()) {
+ complete(key, created, cached, null, false);
+ return created;
+ }
fetches.increment();
try {
fetchExecutor.execute(() -> fetch(key, fetcher, created));
} catch (RuntimeException e) {
// Such as a rejection after close
- complete(key, created, null, e);
+ complete(key, created, null, e, false);
}
return created;
}
@@ -227,13 +242,17 @@ public class OffloadedDataChunkCache implements AutoCloseable {
try {
buffer = fetcher.fetch(key);
} catch (Throwable e) {
- complete(key, pending, null, e);
+ complete(key, pending, null, e, false);
return;
}
- complete(key, pending, new Chunk(buffer), null);
+ complete(key, pending, new Chunk(buffer), null, true);
}
- private void complete(Key key, PendingFetch pending, Chunk chunk, Throwable failure) {
+ /**
+ * Completes the fetch with the chunk, with a reference that it releases, or with the failure. A fetched chunk is
+ * {@code publish}ed into the cache.
+ */
+ private void complete(Key key, PendingFetch pending, Chunk chunk, Throwable failure, boolean publish) {
synchronized (pending) {
if (pending.done) {
// close() has failed it already
@@ -249,10 +268,15 @@ public class OffloadedDataChunkCache implements AutoCloseable {
chunk.retain(pending.waiters);
}
}
- if (chunk != null && maxSizeBytes > 0 && !closed) {
- chunk.retain(1);
- // Into the cache before it stops being pending, see acquire()
- chunks.put(key, chunk);
+ if (chunk != null && publish && maxSizeBytes > 0) {
+ // With close(), so that no chunk is put into the cache after close() has emptied it
+ synchronized (this) {
+ if (!closed) {
+ chunk.retain(1);
+ // Into the cache before it stops being pending, see acquire()
+ chunks.put(key, chunk);
+ }
+ }
}
pendingFetches.remove(key, pending);
if (chunk != null) {
@@ -301,6 +325,18 @@ public class OffloadedDataChunkCache implements AutoCloseable {
return fetches.sum();
}
+ /** The reads waiting for the fetch of the chunk, or -1 when it isn't being fetched. */
+ @VisibleForTesting
+ int waiters(Key key) {
+ PendingFetch pending = pendingFetches.get(key);
+ if (pending == null) {
+ return -1;
+ }
+ synchronized (pending) {
+ return pending.waiters;
+ }
+ }
+
@VisibleForTesting
long cachedBytes() {
return chunks.asMap().keySet().stream().mapToLong(Key::length).sum();
@@ -312,12 +348,16 @@ public class OffloadedDataChunkCache implements AutoCloseable {
@Override
public void close() {
- closed = true;
+ synchronized (this) {
+ closed = true;
+ }
cleanupExecutor.shutdownNow();
fetchExecutor.shutdownNow();
// The reads waiting for fetches that won't run fail instead of waiting forever
pendingFetches.forEach((key, pending) -> complete(key, pending, null,
- new IOException("The offloaded data chunk cache is closed")));
- chunks.invalidateAll();
+ new IOException("The offloaded data chunk cache is closed"), false));
+ synchronized (this) {
+ chunks.invalidateAll();
+ }
}
}
diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCacheTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCacheTest.java
index 757ecf2c41b5..cf57182c4b65 100644
--- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCacheTest.java
+++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadedDataChunkCacheTest.java
@@ -81,8 +81,8 @@ public class OffloadedDataChunkCacheTest extends BlobStoreTestBase {
reads.add(readers.submit(() -> cache.acquire(KEY, fetcher)));
}
assertThat(fetching.await(10, TimeUnit.SECONDS)).isTrue();
- // every read has joined the fetch before it completes
- Awaitility.await().untilAsserted(() -> assertThat(cache.joinedFetches()).isEqualTo(7));
+ // every read waits for the fetch before it completes
+ Awaitility.await().untilAsserted(() -> assertThat(cache.waiters(KEY)).isEqualTo(8));
release.countDown();
OffloadedDataChunkCache.Chunk first = reads.get(0).get(10, TimeUnit.SECONDS);
for (Future<OffloadedDataChunkCache.Chunk> read : reads) {
@@ -143,6 +143,54 @@ public class OffloadedDataChunkCacheTest extends BlobStoreTestBase {
assertThat(cache.fetches()).isEqualTo(2);
}
+ @Test
+ public void releasesTheBuffersOfExpiredChunks() throws Exception {
+ @Cleanup
+ OffloadedDataChunkCache cache = new OffloadedDataChunkCache(1024, 1, 0, 1);
+ OffloadedDataChunkCache.Chunk chunk = cache.acquire(KEY, OffloadedDataChunkCacheTest::bytes);
+ chunk.release();
+ assertThat(chunk.references()).isEqualTo(1);
+ // the cleanup drops it once no read has used it for the expiry
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)
+ .untilAsserted(() -> assertThat(chunk.buffer().refCnt()).isZero());
+ assertThat(cache.cachedBytes()).isZero();
+ }
+
+ @Test
+ public void anInterruptedReadReleasesItsReferenceWhenTheFetchCompletes() throws Exception {
+ @Cleanup
+ OffloadedDataChunkCache cache = new OffloadedDataChunkCache(1024, 60, 0, 1);
+ CountDownLatch release = new CountDownLatch(1);
+ OffloadedDataChunkCache.Fetcher fetcher = key -> {
+ try {
+ release.await();
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ return bytes(key);
+ };
+ @Cleanup("shutdownNow")
+ ExecutorService readers = Executors.newSingleThreadExecutor();
+ Future<OffloadedDataChunkCache.Chunk> read = readers.submit(() -> cache.acquire(KEY, fetcher));
+ Awaitility.await().untilAsserted(() -> assertThat(cache.waiters(KEY)).isEqualTo(1));
+ read.cancel(true);
+ release.countDown();
+ Awaitility.await().untilAsserted(() -> assertThat(cache.waiters(KEY)).isEqualTo(-1));
+ OffloadedDataChunkCache.Chunk chunk = cache.acquire(KEY, fetcher);
+ // the cache's reference and this read's: the interrupted read's has been released
+ Awaitility.await().untilAsserted(() -> assertThat(chunk.references()).isEqualTo(2));
+ chunk.release();
+ assertThat(cache.fetches()).isEqualTo(1);
+ }
+
+ @Test
+ public void doesNotPrefetchAChunkLargerThanTheBudget() throws Exception {
+ @Cleanup
+ OffloadedDataChunkCache cache = new OffloadedDataChunkCache(8, 60, 1, 1);
+ cache.prefetch(KEY, OffloadedDataChunkCacheTest::bytes);
+ assertThat(cache.fetches()).isZero();
+ }
+
@Test
public void withoutABudgetKeepsNoChunk() throws Exception {
@Cleanup