You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2026/10/07 08:51:25 UTC

(hbase) branch master updated: HBASE-30454 BucketCache persistence recovery downgrades MULTI blocks to SINGLE (#8741)

This is an automated email from the ASF dual-hosted git repository.

wchevreuil pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
     new c1f529d4a15 HBASE-30454 BucketCache persistence recovery downgrades MULTI blocks to SINGLE (#8741)
c1f529d4a15 is described below

commit c1f529d4a157f288f8212549f36e7a823945b9d9
Author: Julian Chandra Sutadi <ju...@gmail.com>
AuthorDate: Wed Oct 7 15:51:18 2026 +0700

    HBASE-30454 BucketCache persistence recovery downgrades MULTI blocks to SINGLE (#8741)
    
    Signed-off-by: Wellington Chevreuil <wc...@apache.org>
---
 .../hadoop/hbase/io/hfile/bucket/BucketEntry.java  | 11 +++--
 .../hbase/io/hfile/bucket/BucketProtoUtils.java    | 16 ++++++-
 .../bucket/TestRecoveryPersistentBucketCache.java  | 50 ++++++++++++++++++++++
 3 files changed, 71 insertions(+), 6 deletions(-)

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketEntry.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketEntry.java
index 6ee95371734..a3adcdd0d77 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketEntry.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketEntry.java
@@ -102,12 +102,15 @@ public class BucketEntry implements HBaseReferenceCounted {
    */
   BucketEntry(long offset, int length, int onDiskSizeWithHeader, long accessCounter,
     boolean inMemory, Function<BucketEntry, Recycler> createRecycler, ByteBuffAllocator allocator) {
-    this(offset, length, onDiskSizeWithHeader, accessCounter, System.nanoTime(), inMemory,
-      createRecycler, allocator);
+    this(offset, length, onDiskSizeWithHeader, accessCounter, System.nanoTime(),
+      inMemory ? BlockPriority.MEMORY : BlockPriority.SINGLE, createRecycler, allocator);
   }
 
+  /**
+   * @param priority the priority of the entry, e.g. the one recovered from a persisted backing map
+   */
   BucketEntry(long offset, int length, int onDiskSizeWithHeader, long accessCounter,
-    long cachedTime, boolean inMemory, Function<BucketEntry, Recycler> createRecycler,
+    long cachedTime, BlockPriority priority, Function<BucketEntry, Recycler> createRecycler,
     ByteBuffAllocator allocator) {
     if (createRecycler == null) {
       throw new IllegalArgumentException("createRecycler could not be null!");
@@ -117,7 +120,7 @@ public class BucketEntry implements HBaseReferenceCounted {
     this.onDiskSizeWithHeader = onDiskSizeWithHeader;
     this.accessCounter = accessCounter;
     this.cachedTime = cachedTime;
-    this.priority = inMemory ? BlockPriority.MEMORY : BlockPriority.SINGLE;
+    this.priority = priority;
     this.refCnt = RefCnt.create(createRecycler.apply(this));
     this.markedAsEvicted = new AtomicBoolean(false);
     this.allocator = allocator;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
index 1488d949def..c180a64f5c9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
@@ -188,8 +188,7 @@ final class BucketProtoUtils {
       // which created by RpcServer elegantly.
       BucketEntry value = new BucketEntry(protoValue.getOffset(), protoValue.getLength(),
         protoValue.getDiskSizeWithHeader(), protoValue.getAccessCounter(),
-        protoValue.getCachedTime(),
-        protoValue.getPriority() == BucketCacheProtos.BlockPriority.memory, createRecycler,
+        protoValue.getCachedTime(), fromPb(protoValue.getPriority()), createRecycler,
         ByteBuffAllocator.HEAP);
       // This is the deserializer that we stored
       int oldIndex = protoValue.getDeserialiserIndex();
@@ -243,6 +242,19 @@ final class BucketProtoUtils {
     }
   }
 
+  private static BlockPriority fromPb(BucketCacheProtos.BlockPriority p) {
+    switch (p) {
+      case multi:
+        return BlockPriority.MULTI;
+      case memory:
+        return BlockPriority.MEMORY;
+      case single:
+        return BlockPriority.SINGLE;
+      default:
+        throw new Error("Unrecognized BlockPriority.");
+    }
+  }
+
   static Map<String, BucketCacheProtos.RegionFileSizeMap>
     toCachedPB(Map<String, Pair<String, Long>> prefetchedHfileNames) {
     Map<String, BucketCacheProtos.RegionFileSizeMap> tmpMap = new HashMap<>();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestRecoveryPersistentBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestRecoveryPersistentBucketCache.java
index adc072b8d50..ccffffe879a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestRecoveryPersistentBucketCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestRecoveryPersistentBucketCache.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtil;
 import org.apache.hadoop.hbase.Waiter;
 import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.io.hfile.BlockPriority;
 import org.apache.hadoop.hbase.io.hfile.CacheTestUtils;
 import org.apache.hadoop.hbase.io.hfile.Cacheable;
 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
@@ -158,6 +159,55 @@ public class TestRecoveryPersistentBucketCache {
     TEST_UTIL.cleanupTestDir();
   }
 
+  @Test
+  public void testBlockPriorityAfterRecovery() throws Exception {
+    HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
+    Path testDir = TEST_UTIL.getDataTestDir();
+    TEST_UTIL.getTestFileSystem().mkdirs(testDir);
+    Configuration conf = HBaseConfiguration.create();
+    // Disables the persister thread by setting its interval to MAX_VALUE
+    conf.setLong(BUCKETCACHE_PERSIST_INTERVAL_KEY, Long.MAX_VALUE);
+    int[] bucketSizes = new int[] { 8 * 1024 + 1024 };
+    BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
+      8192, bucketSizes, writeThreads, writerQLen, testDir + "/bucket.persistence",
+      DEFAULT_ERROR_TOLERATION_DURATION, conf);
+    assertTrue(bucketCache.waitForCacheInitialization(10000));
+
+    CacheTestUtils.HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(8192, 3);
+    String[] names = CacheTestUtils.getHFileNames(blocks);
+
+    // first block is only cached, so it stays SINGLE
+    cacheAndWaitUntilFlushedToBucket(bucketCache, blocks[0].getBlockName(), blocks[0].getBlock());
+    // second block is read after being cached, so it is promoted to MULTI
+    cacheAndWaitUntilFlushedToBucket(bucketCache, blocks[1].getBlockName(), blocks[1].getBlock());
+    bucketCache.getBlock(blocks[1].getBlockName(), false, false, false);
+    // third block is an in-memory block, so it is MEMORY
+    bucketCache.cacheBlock(blocks[2].getBlockName(), blocks[2].getBlock(), true);
+    waitUntilFlushedToBucket(bucketCache, blocks[2].getBlockName());
+
+    assertEquals(BlockPriority.SINGLE,
+      bucketCache.backingMap.get(blocks[0].getBlockName()).getPriority());
+    assertEquals(BlockPriority.MULTI,
+      bucketCache.backingMap.get(blocks[1].getBlockName()).getPriority());
+    assertEquals(BlockPriority.MEMORY,
+      bucketCache.backingMap.get(blocks[2].getBlockName()).getPriority());
+
+    // saves the current state of the cache
+    bucketCache.persistToFile();
+
+    BucketCache newBucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
+      8192, bucketSizes, writeThreads, writerQLen, testDir + "/bucket.persistence",
+      DEFAULT_ERROR_TOLERATION_DURATION, conf);
+    assertTrue(newBucketCache.waitForCacheInitialization(10000));
+    BlockCacheKey[] newKeys = CacheTestUtils.regenerateKeys(blocks, names);
+
+    // the recovered blocks should keep the priority they had before persisting
+    assertEquals(BlockPriority.SINGLE, newBucketCache.backingMap.get(newKeys[0]).getPriority());
+    assertEquals(BlockPriority.MULTI, newBucketCache.backingMap.get(newKeys[1]).getPriority());
+    assertEquals(BlockPriority.MEMORY, newBucketCache.backingMap.get(newKeys[2]).getPriority());
+    TEST_UTIL.cleanupTestDir();
+  }
+
   @Test
   public void testValidateCacheInitialization() throws Exception {
     HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();