You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hbase.apache.org by "longping_jie (Jira)" <ji...@apache.org> on 2022/09/18 04:48:00 UTC
[jira] [Created] (HBASE-27375) Suspected memory leak caused by major compaction
longping_jie created HBASE-27375:
------------------------------------
Summary: Suspected memory leak caused by major compaction
Key: HBASE-27375
URL: https://issues.apache.org/jira/browse/HBASE-27375
Project: HBase
Issue Type: Bug
Affects Versions: 2.2.6
Environment: hbase 2.2.6
Reporter: longping_jie
Attachments: image-2022-09-18-12-46-08-478.png
There are 49G Hfiles in a certain region, and major compact has been unsuccessful, and the Old area in the heap memory continues to fill up, causing frequent Full GCs and causing the RegionServer process to OOM. After analyzing the dump file of heap memory using MAT, we found that:
!image-2022-09-18-12-46-08-478.png!
The full stack involved is as follows:
regionserver/node71:60020-longCompactions-3
at org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSums(IILjava/nio/ByteBuffer;ILjava/nio/ByteBuffer;IILjava/lang/String;JZ)V (Native Method)
at org.apache.hadoop.util.NativeCrc32.verifyChunkedSums(IILjava/nio/ByteBuffer;Ljava/nio/ByteBuffer;Ljava/lang/String;J)V (NativeCrc32.java:63)
at org.apache.hadoop.util.DataChecksum.verifyChunkedSums(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;Ljava/lang/String;J)V (DataChecksum.java:380)
at org.apache.hadoop.hbase.io.hfile.ChecksumUtil.verifyChunkedSums(Lorg/apache/hadoop/util/DataChecksum;Lorg/apache/hadoop/hbase/nio/ByteBuff;Lorg/apache/hadoop/hbase/nio/ByteBuff;Ljava/lang/String;)Z (ChecksumUtil.java:104)
at org.apache.hadoop.hbase.io.hfile.ChecksumUtil.validateChecksum(Lorg/apache/hadoop/hbase/nio/ByteBuff;Ljava/lang/String;JI)Z (ChecksumUtil.java:191)
at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.validateChecksum(JLorg/apache/hadoop/hbase/nio/ByteBuff;I)Z (HFileBlock.java:1842)
at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(Lorg/apache/hadoop/fs/FSDataInputStream;JJZZZZ)Lorg/apache/hadoop/hbase/io/hfile/HFileBlock; (HFileBlock.java:1777)
at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(JJZZZ)Lorg/apache/hadoop/hbase/io/hfile/HFileBlock; (HFileBlock.java:1570)
at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(JJZZZZLorg/apache/hadoop/hbase/io/hfile/BlockType;Lorg/apache/hadoop/hbase/io/encoding/DataBlockEncoding;)Lorg/apache/hadoop/hbase/io/hfile/HFileBlock; (HFileReaderImpl.java:1519)
at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock()Lorg/apache/hadoop/hbase/io/hfile/HFileBlock; (HFileReaderImpl.java:928)
at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.isNextBlock()Z (HFileReaderImpl.java:1061)
at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.positionForNextBlock()Z (HFileReaderImpl.java:1055)
at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl._next()Z (HFileReaderImpl.java:1073)
at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.next()Z (HFileReaderImpl.java:1094)
at org.apache.hadoop.hbase.io.HalfStoreFileReader$1.next()Z (HalfStoreFileReader.java:169)
at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(Lorg/apache/hadoop/hbase/io/hfile/HFileScanner;Lorg/apache/hadoop/hbase/Cell;)Z (StoreFileScanner.java:351)
at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(Lorg/apache/hadoop/hbase/Cell;)Z (StoreFileScanner.java:244)
at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(Lorg/apache/hadoop/hbase/regionserver/KeyValueScanner;Lorg/apache/hadoop/hbase/Cell;Z)Z (NonLazyKeyValueScanner.java:55)
at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(ZLorg/apache/hadoop/hbase/Cell;ZZ)Z (KeyValueHeap.java:324)
at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(Lorg/apache/hadoop/hbase/Cell;)Z (KeyValueHeap.java:267)
at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(Lorg/apache/hadoop/hbase/Cell;)Z (StoreScanner.java:1099)
at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(Lorg/apache/hadoop/hbase/Cell;)Z (StoreScanner.java:1088)
at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(Lorg/apache/hadoop/hbase/Cell;)V (StoreScanner.java:823)
at org.apache.hadoop.hbase.regionserver.StoreScanner.next(Ljava/util/List;Lorg/apache/hadoop/hbase/regionserver/ScannerContext;)Z (StoreScanner.java:730)
at org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$FileDetails;Lorg/apache/hadoop/hbase/regionserver/InternalScanner;Lorg/apache/hadoop/hbase/regionserver/CellSink;JZLorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;ZI)Z (Compactor.java:387)
at org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionRequestImpl;Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$InternalScannerFactory;Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$CellSinkFactory;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List; (Compactor.java:326)
at org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionRequestImpl;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List; (DefaultCompactor.java:65)
at org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List; (DefaultStoreEngine.java:126)
at org.apache.hadoop.hbase.regionserver.HStore.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionContext;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List; (HStore.java:1469)
at org.apache.hadoop.hbase.regionserver.HRegion.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionContext;Lorg/apache/hadoop/hbase/regionserver/HStore;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Z (HRegion.java:2264)
at org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(Lorg/apache/hadoop/hbase/security/User;)V (CompactSplit.java:624)
at org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run()V (CompactSplit.java:666)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V (ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run()V (ThreadPoolExecutor.java:617)
at java.lang.Thread.run()V (Thread.java:748)
--
This message was sent by Atlassian Jira
(v8.20.10#820010)