You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hbase.apache.org by "Michael Stack (Jira)" <ji...@apache.org> on 2021/03/12 05:42:00 UTC
[jira] [Resolved] (HBASE-25570) On largish cluster, "CleanerChore:
Could not delete dir..." makes master log unreadable
[ https://issues.apache.org/jira/browse/HBASE-25570?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Michael Stack resolved HBASE-25570.
-----------------------------------
Fix Version/s: 2.4.2
3.0.0-alpha-1
Hadoop Flags: Reviewed
Assignee: Michael Stack
Resolution: Fixed
Pushed to branch-2.4+. Thanks for the reviews [~shahrs87] and [~zhangduo]
> On largish cluster, "CleanerChore: Could not delete dir..." makes master log unreadable
> ---------------------------------------------------------------------------------------
>
> Key: HBASE-25570
> URL: https://issues.apache.org/jira/browse/HBASE-25570
> Project: HBase
> Issue Type: Improvement
> Reporter: Michael Stack
> Assignee: Michael Stack
> Priority: Major
> Fix For: 3.0.0-alpha-1, 2.4.2
>
>
> Looking at a running master on a largish cluster, the master log is full of spew to this effect:
> {{
> 2021-02-10 23:16:49,330 INFO [dir-scan-pool4-thread-206391] cleaner.CleanerChore: Could not delete dir under hdfs://web2-dfs:8020/hbase/archive/data/default/crawl_graph. might be transient; we'll retry. if it keeps happening, use following exception when asking on mailing list.
> org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.fs.PathIsNotEmptyDirectoryException): `/hbase/archive/data/default/crawl_graph is non empty': Directory is not empty
> at org.apache.hadoop.hdfs.server.namenode.FSDirDeleteOp.delete(FSDirDeleteOp.java:116)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.delete(FSNamesystem.java:2962)
> at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.delete(NameNodeRpcServer.java:1092)
> at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.delete(ClientNamenodeProtocolServerSideTranslatorPB.java:681)
> at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
> at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:523)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:991)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:869)
> at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:815)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1875)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2675)
> at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1499)
> at org.apache.hadoop.ipc.Client.call(Client.java:1445)
> at org.apache.hadoop.ipc.Client.call(Client.java:1355)
> at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:228)
> at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:116)
> at com.sun.proxy.$Proxy19.delete(Unknown Source)
> at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.delete(ClientNamenodeProtocolTranslatorPB.java:621)
> at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
> at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
> at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
> at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
> at com.sun.proxy.$Proxy20.delete(Unknown Source)
> at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:372)
> at com.sun.proxy.$Proxy21.delete(Unknown Source)
> at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:372)
> at com.sun.proxy.$Proxy21.delete(Unknown Source)
> at org.apache.hadoop.hdfs.DFSClient.delete(DFSClient.java:1572)
> at org.apache.hadoop.hdfs.DistributedFileSystem$19.doCall(DistributedFileSystem.java:880)
> at org.apache.hadoop.hdfs.DistributedFileSystem$19.doCall(DistributedFileSystem.java:877)
> at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
> at org.apache.hadoop.hdfs.DistributedFileSystem.delete(DistributedFileSystem.java:887)
> at org.apache.hadoop.hbase.master.cleaner.CleanerChore.lambda$null$4(CleanerChore.java:441)
> at org.apache.hadoop.hbase.master.cleaner.CleanerChore.deleteAction(CleanerChore.java:466)
> at org.apache.hadoop.hbase.master.cleaner.CleanerChore.lambda$traverseAndDelete$5(CleanerChore.java:441)
> at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:68)
> at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774)
> at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750)
> at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)
> at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1975)
> at org.apache.hadoop.hbase.master.cleaner.CleanerChore.lambda$traverseAndDelete$5(CleanerChore.java:443)
> at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:68)
> at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774)
> at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750)
> at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)
> at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1975)
> at org.apache.hadoop.hbase.master.cleaner.CleanerChore.lambda$traverseAndDelete$5(CleanerChore.java:443)
> at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:68)
> at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774)
> at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:792)
> at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2153)
> at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:61)
> at org.apache.hadoop.hbase.master.cleaner.CleanerChore.traverseAndDelete(CleanerChore.java:427)
> at org.apache.hadoop.hbase.master.cleaner.CleanerChore.lambda$null$2(CleanerChore.java:420)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)}}
> It goes on and on.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)