You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2024/10/01 20:28:07 UTC
(hbase) branch master updated: HBASE-28893 RefCnt Leak error when closing a HalfStoreFileReader (#6329)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 24c7a3f862d HBASE-28893 RefCnt Leak error when closing a HalfStoreFileReader (#6329)
24c7a3f862d is described below
commit 24c7a3f862df420d1424a334e9465714acb12cf5
Author: Wellington Ramos Chevreuil <wc...@apache.org>
AuthorDate: Tue Oct 1 21:28:01 2024 +0100
HBASE-28893 RefCnt Leak error when closing a HalfStoreFileReader (#6329)
Signed-off-by: Istvan Toth <st...@apache.org>
---
.../hadoop/hbase/io/HalfStoreFileReader.java | 1 +
.../hadoop/hbase/io/TestHalfStoreFileReader.java | 37 ++++++++++++++++++++--
2 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index 55a73802004..9d99904e213 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -376,6 +376,7 @@ public class HalfStoreFileReader extends StoreFileReader {
reference, referred, top, numEvictedReferred, numEvictedReference);
});
}
+ s.close();
reader.close(false);
} else {
reader.close(evictOnClose);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
index 0954ece74a3..3ae54057208 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.io;
+import static org.apache.hadoop.hbase.io.ByteBuffAllocator.BUFFER_SIZE_KEY;
+import static org.apache.hadoop.hbase.io.hfile.CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -24,6 +26,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -36,6 +39,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
@@ -43,6 +47,8 @@ import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.io.hfile.ReaderContext;
import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder;
+import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
+import org.apache.hadoop.hbase.nio.RefCnt;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.testclassification.IOTests;
@@ -54,6 +60,8 @@ import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.apache.hbase.thirdparty.io.netty.util.ResourceLeakDetector;
+
@Category({ IOTests.class, SmallTests.class })
public class TestHalfStoreFileReader {
@@ -84,7 +92,8 @@ public class TestHalfStoreFileReader {
* top of the file while we are at it.
*/
@Test
- public void testHalfScanAndReseek() throws IOException {
+ public void testHalfScanAndReseek() throws Exception {
+ ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID);
Configuration conf = TEST_UTIL.getConfiguration();
FileSystem fs = FileSystem.get(conf);
String root_dir = TEST_UTIL.getDataTestDir().toString();
@@ -93,8 +102,25 @@ public class TestHalfStoreFileReader {
Path splitAPath = new Path(new Path(root_dir, "splita"), "CF");
Path splitBPath = new Path(new Path(root_dir, "splitb"), "CF");
Path filePath = StoreFileWriter.getUniqueFile(fs, parentPath);
+ String ioEngineName = "file:" + TEST_UTIL.getDataTestDir() + "/bucketNoRecycler.cache";
+ BucketCache bucketCache = new BucketCache(ioEngineName, 32 * 1024 * 1024, 1024,
+ new int[] { 4 * 1024, 8 * 1024, 64 * 1024, 96 * 1024 }, 1, 1, null);
+ conf.setBoolean(CACHE_BLOCKS_ON_WRITE_KEY, true);
+ conf.setInt(BUFFER_SIZE_KEY, 1024);
+ ByteBuffAllocator allocator = ByteBuffAllocator.create(conf, true);
+
+ final AtomicInteger counter = new AtomicInteger();
+ RefCnt.detector.setLeakListener(new ResourceLeakDetector.LeakListener() {
+ @Override
+ public void onLeak(String s, String s1) {
+ counter.incrementAndGet();
+ }
+ });
+
+ ColumnFamilyDescriptorBuilder cfBuilder =
+ ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("CF"));
+ CacheConfig cacheConf = new CacheConfig(conf, cfBuilder.build(), bucketCache, allocator);
- CacheConfig cacheConf = new CacheConfig(conf);
HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
HFile.Writer w =
HFile.getWriterFactory(conf, cacheConf).withPath(fs, filePath).withFileContext(meta).create();
@@ -122,10 +148,12 @@ public class TestHalfStoreFileReader {
doTestOfScanAndReseek(splitFileB, fs, top, cacheConf);
r.close();
+
+ assertEquals(0, counter.get());
}
private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, CacheConfig cacheConf)
- throws IOException {
+ throws Exception {
Path referencePath = StoreFileInfo.getReferredToFile(p);
FSDataInputStreamWrapper in = new FSDataInputStreamWrapper(fs, referencePath, false, 0);
FileStatus status = fs.getFileStatus(referencePath);
@@ -158,6 +186,9 @@ public class TestHalfStoreFileReader {
}
halfreader.close(true);
+
+ System.gc();
+ Thread.sleep(1000);
}
// Tests the scanner on an HFile that is backed by HalfStoreFiles