You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2024/10/02 11:49:52 UTC
(hbase) branch branch-2.6 updated: HBASE-28890 RefCnt Leak error when caching index blocks at write time (#6317) (#6328)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new 1efd92e7c49 HBASE-28890 RefCnt Leak error when caching index blocks at write time (#6317) (#6328)
1efd92e7c49 is described below
commit 1efd92e7c49fb6be3f7523f243fa0dd42f43c344
Author: Wellington Ramos Chevreuil <wc...@apache.org>
AuthorDate: Wed Oct 2 12:08:47 2024 +0100
HBASE-28890 RefCnt Leak error when caching index blocks at write time (#6317) (#6328)
Signed-off-by: Tak Lon (Stephen) Wu <ta...@apache.org>
Reviewed-by: Vaibhav Joshi <vj...@cloudera.com>
---
.../java/org/apache/hadoop/hbase/nio/RefCnt.java | 2 +-
.../hadoop/hbase/io/hfile/HFileBlockIndex.java | 3 +
.../hadoop/hbase/io/hfile/TestHFileBlockIndex.java | 102 +++++++++++++++------
3 files changed, 79 insertions(+), 28 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/RefCnt.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/RefCnt.java
index 59b96674ec3..d30a2f0ffda 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/RefCnt.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/RefCnt.java
@@ -35,7 +35,7 @@ import org.apache.hbase.thirdparty.io.netty.util.ResourceLeakTracker;
@InterfaceAudience.Private
public class RefCnt extends AbstractReferenceCounted {
- private static final ResourceLeakDetector<RefCnt> detector =
+ public static final ResourceLeakDetector<RefCnt> detector =
ResourceLeakDetectorFactory.instance().newResourceLeakDetector(RefCnt.class);
private final Recycler recycler;
private final ResourceLeakTracker<RefCnt> leak;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index 592c19c866c..0aa9e7820d8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -1072,6 +1072,9 @@ public class HFileBlockIndex {
HFileBlock blockForCaching = blockWriter.getBlockForCaching(cacheConf);
cache.cacheBlock(new BlockCacheKey(nameForCaching, rootLevelIndexPos, true,
blockForCaching.getBlockType()), blockForCaching);
+ // Index blocks always go to LRU, which then converts any off-heap buffer to on-heap,
+ // so we need to release any off-heap buffers now to avoid leak
+ blockForCaching.release();
});
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index 2872065a43e..12e662ee4e7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -33,6 +33,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -59,6 +60,7 @@ import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader;
import org.apache.hadoop.hbase.io.hfile.NoOpIndexBlockEncoder.NoOpEncodedSeeker;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.MultiByteBuff;
+import org.apache.hadoop.hbase.nio.RefCnt;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -74,6 +76,8 @@ import org.junit.runners.Parameterized.Parameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.io.netty.util.ResourceLeakDetector;
+
@RunWith(Parameterized.class)
@Category({ IOTests.class, MediumTests.class })
public class TestHFileBlockIndex {
@@ -140,6 +144,76 @@ public class TestHFileBlockIndex {
testBlockIndexInternals(true);
}
+ private void writeDataBlocksAndCreateIndex(HFileBlock.Writer hbw, FSDataOutputStream outputStream,
+ HFileBlockIndex.BlockIndexWriter biw) throws IOException {
+ for (int i = 0; i < NUM_DATA_BLOCKS; ++i) {
+ hbw.startWriting(BlockType.DATA).write(Bytes.toBytes(String.valueOf(RNG.nextInt(1000))));
+ long blockOffset = outputStream.getPos();
+ hbw.writeHeaderAndData(outputStream);
+
+ byte[] firstKey = null;
+ byte[] family = Bytes.toBytes("f");
+ byte[] qualifier = Bytes.toBytes("q");
+ for (int j = 0; j < 16; ++j) {
+ byte[] k = new KeyValue(RandomKeyValueUtil.randomOrderedKey(RNG, i * 16 + j), family,
+ qualifier, EnvironmentEdgeManager.currentTime(), KeyValue.Type.Put).getKey();
+ keys.add(k);
+ if (j == 8) {
+ firstKey = k;
+ }
+ }
+ assertTrue(firstKey != null);
+ if (firstKeyInFile == null) {
+ firstKeyInFile = firstKey;
+ }
+ biw.addEntry(firstKey, blockOffset, hbw.getOnDiskSizeWithHeader());
+
+ writeInlineBlocks(hbw, outputStream, biw, false);
+ }
+ writeInlineBlocks(hbw, outputStream, biw, true);
+ rootIndexOffset = biw.writeIndexBlocks(outputStream);
+ outputStream.close();
+ }
+
+ @Test
+ public void testBlockIndexWithOffHeapBuffer() throws Exception {
+ ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID);
+ path = new Path(TEST_UTIL.getDataTestDir(), "block_index_testBlockIndexWithOffHeapBuffer");
+ assertEquals(0, keys.size());
+ HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(true)
+ .withIncludesMvcc(includesMemstoreTS).withIncludesTags(true).withCompression(compr)
+ .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM).build();
+ ByteBuffAllocator allocator = ByteBuffAllocator.create(TEST_UTIL.getConfiguration(), true);
+ HFileBlock.Writer hbw = new HFileBlock.Writer(TEST_UTIL.getConfiguration(), null, meta,
+ allocator, meta.getBlocksize());
+ FSDataOutputStream outputStream = fs.create(path);
+
+ final AtomicInteger counter = new AtomicInteger();
+ RefCnt.detector.setLeakListener(new ResourceLeakDetector.LeakListener() {
+ @Override
+ public void onLeak(String s, String s1) {
+ counter.incrementAndGet();
+ }
+ });
+
+ long maxSize = NUM_DATA_BLOCKS * 1000;
+ long blockSize = 1000;
+ LruBlockCache cache = new LruBlockCache(maxSize, blockSize);
+ CacheConfig cacheConfig = new CacheConfig(TEST_UTIL.getConfiguration(), null, cache, allocator);
+
+ HFileBlockIndex.BlockIndexWriter biw =
+ new HFileBlockIndex.BlockIndexWriter(hbw, cacheConfig, path.getName(), null);
+
+ writeDataBlocksAndCreateIndex(hbw, outputStream, biw);
+
+ System.gc();
+ Thread.sleep(1000);
+
+ allocator.allocate(128 * 1024).release();
+
+ assertEquals(0, counter.get());
+ }
+
private void clear() throws IOException {
keys.clear();
firstKeyInFile = null;
@@ -269,33 +343,7 @@ public class TestHFileBlockIndex {
FSDataOutputStream outputStream = fs.create(path);
HFileBlockIndex.BlockIndexWriter biw =
new HFileBlockIndex.BlockIndexWriter(hbw, null, null, null);
- for (int i = 0; i < NUM_DATA_BLOCKS; ++i) {
- hbw.startWriting(BlockType.DATA).write(Bytes.toBytes(String.valueOf(RNG.nextInt(1000))));
- long blockOffset = outputStream.getPos();
- hbw.writeHeaderAndData(outputStream);
-
- byte[] firstKey = null;
- byte[] family = Bytes.toBytes("f");
- byte[] qualifier = Bytes.toBytes("q");
- for (int j = 0; j < 16; ++j) {
- byte[] k = new KeyValue(RandomKeyValueUtil.randomOrderedKey(RNG, i * 16 + j), family,
- qualifier, EnvironmentEdgeManager.currentTime(), KeyValue.Type.Put).getKey();
- keys.add(k);
- if (j == 8) {
- firstKey = k;
- }
- }
- assertTrue(firstKey != null);
- if (firstKeyInFile == null) {
- firstKeyInFile = firstKey;
- }
- biw.addEntry(firstKey, blockOffset, hbw.getOnDiskSizeWithHeader());
-
- writeInlineBlocks(hbw, outputStream, biw, false);
- }
- writeInlineBlocks(hbw, outputStream, biw, true);
- rootIndexOffset = biw.writeIndexBlocks(outputStream);
- outputStream.close();
+ writeDataBlocksAndCreateIndex(hbw, outputStream, biw);
numLevels = biw.getNumLevels();
numRootEntries = biw.getNumRootEntries();