You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-dev@hadoop.apache.org by "Runping Qi (JIRA)" <ji...@apache.org> on 2008/10/27 19:14:44 UTC
[jira] Created: (HADOOP-4524)
org.apache.hadoop.dfs.LeaseExpiredException during dfs write
org.apache.hadoop.dfs.LeaseExpiredException during dfs write
------------------------------------------------------------
Key: HADOOP-4524
URL: https://issues.apache.org/jira/browse/HADOOP-4524
Project: Hadoop Core
Issue Type: Bug
Components: dfs
Affects Versions: 0.17.2
Reporter: Runping Qi
Many long running cpu intensive map tasks failed due to org.apache.hadoop.dfs.LeaseExpiredException.
Here is except from the log:
2008-10-26 11:54:17,282 INFO org.apache.hadoop.dfs.DFSClient: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
at org.apache.hadoop.ipc.Client.call(Client.java:557)
at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
2008-10-26 11:54:17,282 WARN org.apache.hadoop.dfs.DFSClient: NotReplicatedYetException sleeping /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 retries left 2
2008-10-26 11:54:18,886 INFO org.apache.hadoop.dfs.DFSClient: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
at org.apache.hadoop.ipc.Client.call(Client.java:557)
at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
2008-10-26 11:54:18,886 WARN org.apache.hadoop.dfs.DFSClient: NotReplicatedYetException sleeping /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 retries left 1
2008-10-26 11:54:22,090 WARN org.apache.hadoop.dfs.DFSClient: DataStreamer Exception: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
at org.apache.hadoop.ipc.Client.call(Client.java:557)
at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
2008-10-26 11:54:22,090 WARN org.apache.hadoop.dfs.DFSClient: Error Recovery for block null bad datanode[0]
2008-10-26 11:54:22,219 WARN org.apache.hadoop.mapred.TaskTracker: Error running child
java.io.IOException: Could not get block locations. Aborting...
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2081)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1300(DFSClient.java:1702)
at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1818)
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Updated: (HADOOP-4524)
org.apache.hadoop.dfs.LeaseExpiredException during dfs write
Posted by "Rusty Burchfield (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HADOOP-4524?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Rusty Burchfield updated HADOOP-4524:
-------------------------------------
Affects Version/s: 0.19.0
> org.apache.hadoop.dfs.LeaseExpiredException during dfs write
> ------------------------------------------------------------
>
> Key: HADOOP-4524
> URL: https://issues.apache.org/jira/browse/HADOOP-4524
> Project: Hadoop Core
> Issue Type: Bug
> Components: dfs
> Affects Versions: 0.17.2, 0.19.0
> Reporter: Runping Qi
>
> Many long running cpu intensive map tasks failed due to org.apache.hadoop.dfs.LeaseExpiredException.
> Here is except from the log:
> 2008-10-26 11:54:17,282 INFO org.apache.hadoop.dfs.DFSClient: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
> at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
> at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
> at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
> at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
> at org.apache.hadoop.ipc.Client.call(Client.java:557)
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
> 2008-10-26 11:54:17,282 WARN org.apache.hadoop.dfs.DFSClient: NotReplicatedYetException sleeping /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 retries left 2
> 2008-10-26 11:54:18,886 INFO org.apache.hadoop.dfs.DFSClient: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
> at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
> at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
> at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
> at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
> at org.apache.hadoop.ipc.Client.call(Client.java:557)
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
> 2008-10-26 11:54:18,886 WARN org.apache.hadoop.dfs.DFSClient: NotReplicatedYetException sleeping /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 retries left 1
> 2008-10-26 11:54:22,090 WARN org.apache.hadoop.dfs.DFSClient: DataStreamer Exception: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
> at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
> at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
> at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
> at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
> at org.apache.hadoop.ipc.Client.call(Client.java:557)
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
> 2008-10-26 11:54:22,090 WARN org.apache.hadoop.dfs.DFSClient: Error Recovery for block null bad datanode[0]
> 2008-10-26 11:54:22,219 WARN org.apache.hadoop.mapred.TaskTracker: Error running child
> java.io.IOException: Could not get block locations. Aborting...
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2081)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1300(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1818)
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Commented: (HADOOP-4524)
org.apache.hadoop.dfs.LeaseExpiredException during dfs write
Posted by "Chris K Wensel (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HADOOP-4524?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=12658670#action_12658670 ]
Chris K Wensel commented on HADOOP-4524:
----------------------------------------
Has a workaround for this been found?
> org.apache.hadoop.dfs.LeaseExpiredException during dfs write
> ------------------------------------------------------------
>
> Key: HADOOP-4524
> URL: https://issues.apache.org/jira/browse/HADOOP-4524
> Project: Hadoop Core
> Issue Type: Bug
> Components: dfs
> Affects Versions: 0.17.2
> Reporter: Runping Qi
>
> Many long running cpu intensive map tasks failed due to org.apache.hadoop.dfs.LeaseExpiredException.
> Here is except from the log:
> 2008-10-26 11:54:17,282 INFO org.apache.hadoop.dfs.DFSClient: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
> at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
> at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
> at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
> at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
> at org.apache.hadoop.ipc.Client.call(Client.java:557)
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
> 2008-10-26 11:54:17,282 WARN org.apache.hadoop.dfs.DFSClient: NotReplicatedYetException sleeping /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 retries left 2
> 2008-10-26 11:54:18,886 INFO org.apache.hadoop.dfs.DFSClient: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
> at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
> at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
> at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
> at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
> at org.apache.hadoop.ipc.Client.call(Client.java:557)
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
> 2008-10-26 11:54:18,886 WARN org.apache.hadoop.dfs.DFSClient: NotReplicatedYetException sleeping /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 retries left 1
> 2008-10-26 11:54:22,090 WARN org.apache.hadoop.dfs.DFSClient: DataStreamer Exception: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.dfs.LeaseExpiredException: No lease on /xxx/_temporary/_task_200810232126_0001_m_000033_0/part-00033 File does not exist. [Lease. Holder: 44 46 53 43 6c 69 65 6e 74 5f 74 61 73 6b 5f 32 30 30 38 31 30 32 33 32 31 32 36 5f 30 30 30 31 5f 6d 5f 30 30 30 30 33 33 5f 30, heldlocks: 0, pendingcreates: 1]
> at org.apache.hadoop.dfs.FSNamesystem.checkLease(FSNamesystem.java:1194)
> at org.apache.hadoop.dfs.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1125)
> at org.apache.hadoop.dfs.NameNode.addBlock(NameNode.java:300)
> at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:446)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:896)
> at org.apache.hadoop.ipc.Client.call(Client.java:557)
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:212)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
> at org.apache.hadoop.dfs.$Proxy1.addBlock(Unknown Source)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:2335)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:2220)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1700(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1842)
> 2008-10-26 11:54:22,090 WARN org.apache.hadoop.dfs.DFSClient: Error Recovery for block null bad datanode[0]
> 2008-10-26 11:54:22,219 WARN org.apache.hadoop.mapred.TaskTracker: Error running child
> java.io.IOException: Could not get block locations. Aborting...
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2081)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream.access$1300(DFSClient.java:1702)
> at org.apache.hadoop.dfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:1818)
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.