You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hbase.apache.org by "kangTwang (Jira)" <ji...@apache.org> on 2021/03/12 08:28:00 UTC
[jira] [Created] (HBASE-25661) Unable rename error occurred in
AbstractFSWAL, causing regionserver to crash
kangTwang created HBASE-25661:
---------------------------------
Summary: Unable rename error occurred in AbstractFSWAL, causing regionserver to crash
Key: HBASE-25661
URL: https://issues.apache.org/jira/browse/HBASE-25661
Project: HBase
Issue Type: Bug
Components: API
Affects Versions: 2.1.0
Reporter: kangTwang
Fix For: 2.1.0
The error is as follows：
[ERROR] - org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631) - Cache flush failed for region test_2,00000000000000000293601280,1614762174258.030beae347d51a5fb6782f6cb025f763.
[ERROR] - org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631) - Cache flush failed for region test_2,00000000000000000293601280,1614762174258.030beae347d51a5fb6782f6cb025f763.java.io.IOException: WAL has been closed at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doShutdown(AsyncFSWAL.java:698) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.shutdown(AbstractFSWAL.java:817) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.LogRoller.abort(LogRoller.java:143) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:201) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834) ~[?:?][17:11:59:664] [INFO] - org.apache.hadoop.hbase.regionserver.HRegion.logFatLineOnFlush(HRegion.java:2636) - Flushing 1/1 column families, dataSize=127.76 MB heapSize=136.99 MB[17:11:59:665] [WARN] - org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2652) - Received unexpected exception trying to write ABORT_FLUSH marker to WAL:java.io.IOException: Cannot append; log is closed, regionName = test_2,00000000000000000377487360,1614762174258.146bbdf3caa203124cd039e48dd3e344. at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85) at org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2649) at org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2599) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439) at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361) at java.base/java.lang.Thread.run(Thread.java:834)
[17:11:59:665] [ERROR] - org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631) - Cache flush failed for region test_2,00000000000000000377487360,1614762174258.146bbdf3caa203124cd039e48dd3e344.java.io.IOException: Cannot append; log is closed, regionName = test_2,00000000000000000377487360,1614762174258.146bbdf3caa203124cd039e48dd3e344. at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2588) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834) ~[?:?][17:11:59:665] [INFO] - org.apache.hadoop.hbase.regionserver.HRegion.logFatLineOnFlush(HRegion.java:2636) - Flushing 1/1 column families, dataSize=126.24 MB heapSize=135.36 MB[17:11:59:665] [WARN] - org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2652) - Received unexpected exception trying to write ABORT_FLUSH marker to WAL:java.io.IOException: Cannot append; log is closed, regionName = test_2,00000000000000000587202560,1614762174258.c958379b0ac4a008cb4d75223683377a. at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85) at org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2649) at org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2599) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439) at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361) at java.base/java.lang.Thread.run(Thread.java:834)
[17:11:59:665] [ERROR] - org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631) - Cache flush failed for region test_2,00000000000000000587202560,1614762174258.c958379b0ac4a008cb4d75223683377a.java.io.IOException: Cannot append; log is closed, regionName = test_2,00000000000000000587202560,1614762174258.c958379b0ac4a008cb4d75223683377a. at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2588) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834) ~[?:?][17:11:59:666] [WARN] - org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2652) - Received unexpected exception trying to write ABORT_FLUSH marker to WAL:java.io.IOException: Cannot append; log is closed, regionName = test_2,00000000000000000125829120,1614762174258.b14f5ec6e23e1390b2daf0f83be84ee0. at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85) at org.apache.hadoop.hbase.regionserver.HRegion.doAbortFlushToWAL(HRegion.java:2649) at org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2599) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439) at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361) at java.base/java.lang.Thread.run(Thread.java:834)
[17:11:59:666] [ERROR] - org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:631) - Cache flush failed for region test_2,00000000000000000125829120,1614762174258.b14f5ec6e23e1390b2daf0f83be84ee0.java.io.IOException: Cannot append; log is closed, regionName = test_2,00000000000000000125829120,1614762174258.b14f5ec6e23e1390b2daf0f83be84ee0. at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:962) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:563) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullAppendTransaction(WALUtil.java:156) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeFlushMarker(WALUtil.java:85) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2588) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2465) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2439) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2329) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:612) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:581) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$1000(MemStoreFlusher.java:68) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:361) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834) ~[?:?][17:11:59:813] [ERROR] - org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2376) - ***** ABORTING region server node017,16020,1614750431707: IOE in log roller *****java.io.IOException: Unable to rename hdfs://buc-001/hbase/WALs/node017,16020,1614750431707/node017%2C16020%2C1614750431707.1614762525711 to hdfs://buc-001/hbase/oldWALs/node017%2C16020%2C1614750431707.1614762525711 at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archiveLogFile(AbstractFSWAL.java:651) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.cleanOldLogs(AbstractFSWAL.java:627) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:767) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:190) ~[hbase-server-2.1.0-cdh6.3.0.jar:?] at java.lang.Thread.run(Thread.java:834) ~[?:?]
Has anyone encountered a similar problem？？？
--
This message was sent by Atlassian Jira
(v8.3.4#803005)