You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-dev@hadoop.apache.org by "Tsz Wo (Nicholas), SZE (JIRA)" <ji...@apache.org> on 2010/09/16 00:58:34 UTC
[jira] Resolved: (HDFS-198)
org.apache.hadoop.dfs.LeaseExpiredException during dfs write
[ https://issues.apache.org/jira/browse/HDFS-198?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Tsz Wo (Nicholas), SZE resolved HDFS-198.
-----------------------------------------
Resolution: Not A Problem
I believe this issue went stale. Closing.
> org.apache.hadoop.dfs.LeaseExpiredException during dfs write
> ------------------------------------------------------------
>
> Key: HDFS-198
> URL: https://issues.apache.org/jira/browse/HDFS-198
> Project: Hadoop HDFS
> Issue Type: Bug
> Reporter: Runping Qi
>
> Many long running cpu intensive map tasks failed due to org.apache.hadoop.dfs.LeaseExpiredException.
> Here is except from the log:
> 2008-10-26 11:54:17,282 INFO org.apache.hadoop.dfs.DFSClient: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
> at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
> at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
> at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
> at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
> at org.apache.hadoop.ipc.Client.call(Client.java:557)
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
> 2008-10-26 11:54:17,282 WARN org.apache.hadoop.dfs.DFSClient: NotReplicatedYetException sleeping /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 retries left 2
> 2008-10-26 11:54:18,886 INFO org.apache.hadoop.dfs.DFSClient: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
> at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
> at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
> at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
> at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
> at org.apache.hadoop.ipc.Client.call(Client.java:557)
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
> 2008-10-26 11:54:18,886 WARN org.apache.hadoop.dfs.DFSClient: NotReplicatedYetException sleeping /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 retries left 1
> 2008-10-26 11:54:22,090 WARN org.apache.hadoop.dfs.DFSClient: DataStreamer Exception: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
> at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
> at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
> at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
> at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
> at org.apache.hadoop.ipc.Client.call(Client.java:557)
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
> 2008-10-26 11:54:22,090 WARN org.apache.hadoop.dfs.DFSClient: Error Recovery for block null bad datanode[0]
> 2008-10-26 11:54:22,219 WARN org.apache.hadoop.mapred.TaskTracker: Error running child
> java.io.IOException: Could not get block locations. Aborting...
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2081)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1300(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1818)
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.