You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2024/10/02 11:49:52 UTC

(hbase) branch branch-2.6 updated: HBASE-28890 RefCnt Leak error when caching index blocks at write time (#6317) (#6328)

This is an automated email from the ASF dual-hosted git repository.

wchevreuil pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.6 by this push:
     new 1efd92e7c49 HBASE-28890 RefCnt Leak error when caching index blocks at write time (#6317) (#6328)
1efd92e7c49 is described below

commit 1efd92e7c49fb6be3f7523f243fa0dd42f43c344
Author: Wellington Ramos Chevreuil <wc...@apache.org>
AuthorDate: Wed Oct 2 12:08:47 2024 +0100

    HBASE-28890 RefCnt Leak error when caching index blocks at write time (#6317) (#6328)
    
    Signed-off-by: Tak Lon (Stephen) Wu <ta...@apache.org>
    Reviewed-by: Vaibhav Joshi <vj...@cloudera.com>
---
 .../java/org/apache/hadoop/hbase/nio/RefCnt.java   |   2 +-
 .../hadoop/hbase/io/hfile/HFileBlockIndex.java     |   3 +
 .../hadoop/hbase/io/hfile/TestHFileBlockIndex.java | 102 +++++++++++++++------
 3 files changed, 79 insertions(+), 28 deletions(-)

diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/RefCnt.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/RefCnt.java
index 59b96674ec3..d30a2f0ffda 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/RefCnt.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/RefCnt.java
@@ -35,7 +35,7 @@ import org.apache.hbase.thirdparty.io.netty.util.ResourceLeakTracker;
 @InterfaceAudience.Private
 public class RefCnt extends AbstractReferenceCounted {
 
-  private static final ResourceLeakDetector<RefCnt> detector =
+  public static final ResourceLeakDetector<RefCnt> detector =
     ResourceLeakDetectorFactory.instance().newResourceLeakDetector(RefCnt.class);
   private final Recycler recycler;
   private final ResourceLeakTracker<RefCnt> leak;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index 592c19c866c..0aa9e7820d8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -1072,6 +1072,9 @@ public class HFileBlockIndex {
             HFileBlock blockForCaching = blockWriter.getBlockForCaching(cacheConf);
             cache.cacheBlock(new BlockCacheKey(nameForCaching, rootLevelIndexPos, true,
               blockForCaching.getBlockType()), blockForCaching);
+            // Index blocks always go to LRU, which then converts any off-heap buffer to on-heap,
+            // so we need to release any off-heap buffers now to avoid leak
+            blockForCaching.release();
           });
         }
       }
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index 2872065a43e..12e662ee4e7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -33,6 +33,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -59,6 +60,7 @@ import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader;
 import org.apache.hadoop.hbase.io.hfile.NoOpIndexBlockEncoder.NoOpEncodedSeeker;
 import org.apache.hadoop.hbase.nio.ByteBuff;
 import org.apache.hadoop.hbase.nio.MultiByteBuff;
+import org.apache.hadoop.hbase.nio.RefCnt;
 import org.apache.hadoop.hbase.testclassification.IOTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -74,6 +76,8 @@ import org.junit.runners.Parameterized.Parameters;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hbase.thirdparty.io.netty.util.ResourceLeakDetector;
+
 @RunWith(Parameterized.class)
 @Category({ IOTests.class, MediumTests.class })
 public class TestHFileBlockIndex {
@@ -140,6 +144,76 @@ public class TestHFileBlockIndex {
     testBlockIndexInternals(true);
   }
 
+  private void writeDataBlocksAndCreateIndex(HFileBlock.Writer hbw, FSDataOutputStream outputStream,
+    HFileBlockIndex.BlockIndexWriter biw) throws IOException {
+    for (int i = 0; i < NUM_DATA_BLOCKS; ++i) {
+      hbw.startWriting(BlockType.DATA).write(Bytes.toBytes(String.valueOf(RNG.nextInt(1000))));
+      long blockOffset = outputStream.getPos();
+      hbw.writeHeaderAndData(outputStream);
+
+      byte[] firstKey = null;
+      byte[] family = Bytes.toBytes("f");
+      byte[] qualifier = Bytes.toBytes("q");
+      for (int j = 0; j < 16; ++j) {
+        byte[] k = new KeyValue(RandomKeyValueUtil.randomOrderedKey(RNG, i * 16 + j), family,
+          qualifier, EnvironmentEdgeManager.currentTime(), KeyValue.Type.Put).getKey();
+        keys.add(k);
+        if (j == 8) {
+          firstKey = k;
+        }
+      }
+      assertTrue(firstKey != null);
+      if (firstKeyInFile == null) {
+        firstKeyInFile = firstKey;
+      }
+      biw.addEntry(firstKey, blockOffset, hbw.getOnDiskSizeWithHeader());
+
+      writeInlineBlocks(hbw, outputStream, biw, false);
+    }
+    writeInlineBlocks(hbw, outputStream, biw, true);
+    rootIndexOffset = biw.writeIndexBlocks(outputStream);
+    outputStream.close();
+  }
+
+  @Test
+  public void testBlockIndexWithOffHeapBuffer() throws Exception {
+    ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID);
+    path = new Path(TEST_UTIL.getDataTestDir(), "block_index_testBlockIndexWithOffHeapBuffer");
+    assertEquals(0, keys.size());
+    HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(true)
+      .withIncludesMvcc(includesMemstoreTS).withIncludesTags(true).withCompression(compr)
+      .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM).build();
+    ByteBuffAllocator allocator = ByteBuffAllocator.create(TEST_UTIL.getConfiguration(), true);
+    HFileBlock.Writer hbw = new HFileBlock.Writer(TEST_UTIL.getConfiguration(), null, meta,
+      allocator, meta.getBlocksize());
+    FSDataOutputStream outputStream = fs.create(path);
+
+    final AtomicInteger counter = new AtomicInteger();
+    RefCnt.detector.setLeakListener(new ResourceLeakDetector.LeakListener() {
+      @Override
+      public void onLeak(String s, String s1) {
+        counter.incrementAndGet();
+      }
+    });
+
+    long maxSize = NUM_DATA_BLOCKS * 1000;
+    long blockSize = 1000;
+    LruBlockCache cache = new LruBlockCache(maxSize, blockSize);
+    CacheConfig cacheConfig = new CacheConfig(TEST_UTIL.getConfiguration(), null, cache, allocator);
+
+    HFileBlockIndex.BlockIndexWriter biw =
+      new HFileBlockIndex.BlockIndexWriter(hbw, cacheConfig, path.getName(), null);
+
+    writeDataBlocksAndCreateIndex(hbw, outputStream, biw);
+
+    System.gc();
+    Thread.sleep(1000);
+
+    allocator.allocate(128 * 1024).release();
+
+    assertEquals(0, counter.get());
+  }
+
   private void clear() throws IOException {
     keys.clear();
     firstKeyInFile = null;
@@ -269,33 +343,7 @@ public class TestHFileBlockIndex {
     FSDataOutputStream outputStream = fs.create(path);
     HFileBlockIndex.BlockIndexWriter biw =
       new HFileBlockIndex.BlockIndexWriter(hbw, null, null, null);
-    for (int i = 0; i < NUM_DATA_BLOCKS; ++i) {
-      hbw.startWriting(BlockType.DATA).write(Bytes.toBytes(String.valueOf(RNG.nextInt(1000))));
-      long blockOffset = outputStream.getPos();
-      hbw.writeHeaderAndData(outputStream);
-
-      byte[] firstKey = null;
-      byte[] family = Bytes.toBytes("f");
-      byte[] qualifier = Bytes.toBytes("q");
-      for (int j = 0; j < 16; ++j) {
-        byte[] k = new KeyValue(RandomKeyValueUtil.randomOrderedKey(RNG, i * 16 + j), family,
-          qualifier, EnvironmentEdgeManager.currentTime(), KeyValue.Type.Put).getKey();
-        keys.add(k);
-        if (j == 8) {
-          firstKey = k;
-        }
-      }
-      assertTrue(firstKey != null);
-      if (firstKeyInFile == null) {
-        firstKeyInFile = firstKey;
-      }
-      biw.addEntry(firstKey, blockOffset, hbw.getOnDiskSizeWithHeader());
-
-      writeInlineBlocks(hbw, outputStream, biw, false);
-    }
-    writeInlineBlocks(hbw, outputStream, biw, true);
-    rootIndexOffset = biw.writeIndexBlocks(outputStream);
-    outputStream.close();
+    writeDataBlocksAndCreateIndex(hbw, outputStream, biw);
 
     numLevels = biw.getNumLevels();
     numRootEntries = biw.getNumRootEntries();