You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by om...@apache.org on 2011/03/08 06:52:38 UTC
svn commit: r1079178 - in /hadoop/mapreduce/branches/yahoo-merge:
CHANGES.txt src/java/org/apache/hadoop/mapreduce/Job.java
src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
src/webapps/job/jobdetailshistory.jsp
Author: omalley
Date: Tue Mar 8 05:52:38 2011
New Revision: 1079178
URL: http://svn.apache.org/viewvc?rev=1079178&view=rev
Log:
commit e3b491c7ffe6611550473522f80aab5a6aea619d
Author: Mahadev Konar <ma...@yahoo-inc.com>
Date: Mon Oct 25 18:02:59 2010 -0700
Please make it clear why the job failed (Krishna Ramachandran via mahadev) from
Modified:
hadoop/mapreduce/branches/yahoo-merge/CHANGES.txt
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp
Modified: hadoop/mapreduce/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/CHANGES.txt?rev=1079178&r1=1079177&r2=1079178&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/yahoo-merge/CHANGES.txt Tue Mar 8 05:52:38 2011
@@ -7,6 +7,7 @@ Trunk (unreleased changes)
NEW FEATURES
IMPROVEMENTS
+ Bug-2367384 Please make it clear why the job failed (Krishna Ramachandran via mahadev) from http://bug.corp.yahoo.com/attachment.cgi?id=1169543
OPTIMIZATIONS
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java?rev=1079178&r1=1079177&r2=1079178&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java Tue Mar 8 05:52:38 2011
@@ -122,7 +122,7 @@ public class Job extends JobContextImpl
private JobStatus status;
private long statustime;
private Cluster cluster;
-
+
@Deprecated
public Job() throws IOException {
this(new Configuration());
@@ -360,8 +360,11 @@ public class Job extends JobContextImpl
@Override
public String toString() {
ensureState(JobState.RUNNING);
+ String reasonforFailure = " ";
try {
updateStatus();
+ if (status.getState().equals(JobStatus.State.FAILED))
+ reasonforFailure = getTaskFailureEventString();
} catch (IOException e) {
} catch (InterruptedException ie) {
}
@@ -378,10 +381,34 @@ public class Job extends JobContextImpl
sb.append(status.getState()).append("\n");
sb.append("history URL: ");
sb.append(status.getHistoryFile()).append("\n");
- sb.append("retired: ").append(status.isRetired());
+ sb.append("retired: ").append(status.isRetired()).append("\n");
+ sb.append("reason for failure: ").append(reasonforFailure);
return sb.toString();
}
-
+
+ /**
+ * @return taskid which caused job failure
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ String getTaskFailureEventString() throws IOException,
+ InterruptedException {
+ int failCount = 1;
+ TaskCompletionEvent lastEvent = null;
+ for (TaskCompletionEvent event : cluster.getClient().getTaskCompletionEvents(
+ status.getJobID(), 0, 10)) {
+ if (event.getStatus().equals(TaskCompletionEvent.Status.FAILED)) {
+ failCount++;
+ lastEvent = event;
+ }
+ }
+ String[] taskAttemptID = lastEvent.getTaskAttemptId().toString().split("_", 2);
+ String taskID = taskAttemptID[1].substring(0, taskAttemptID[1].length()-2);
+ return (" task " + taskID + " failed " +
+ failCount + " times " + "For details check tasktracker at: " +
+ lastEvent.getTaskTrackerHttp());
+ }
+
/**
* Get the information of the current state of the tasks of a job.
*
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1079178&r1=1079177&r2=1079178&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java Tue Mar 8 05:52:38 2011
@@ -262,6 +262,8 @@ public class JobHistoryParser {
taskInfo.finishTime = event.getFinishTime();
taskInfo.error = event.getError();
taskInfo.failedDueToAttemptId = event.getFailedAttemptID();
+ info.errorInfo = "Task " + taskInfo.taskId +" failed " +
+ taskInfo.attemptsMap.size() + " times ";
}
private void handleTaskStartedEvent(TaskStartedEvent event) {
@@ -321,6 +323,7 @@ public class JobHistoryParser {
* The class where job information is aggregated into after parsing
*/
public static class JobInfo {
+ String errorInfo = "None";
long submitTime;
long finishTime;
JobID jobid;
@@ -406,6 +409,7 @@ public class JobHistoryParser {
public long getFinishedReduces() { return finishedReduces; }
/** Get the job status */
public String getJobStatus() { return jobStatus; }
+ public String getErrorInfo() { return errorInfo; }
/** Get the counters for the job */
public Counters getTotalCounters() { return totalCounters; }
/** Get the map counters for the job */
Modified: hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp?rev=1079178&r1=1079177&r2=1079178&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp Tue Mar 8 05:52:38 2011
@@ -45,6 +45,7 @@
<%! static SimpleDateFormat dateFormat = new SimpleDateFormat("d-MMM-yyyy HH:mm:ss") ; %>
<%
String logFile = request.getParameter("logFile");
+ String reasonforFailure = " ";
final Path jobFile = new Path(logFile);
String jobid = JobHistory.getJobIDFromHistoryFilePath(jobFile).toString();
@@ -55,6 +56,8 @@
if (job == null) {
return;
}
+ if (job.getJobStatus().equals("FAILED"))
+ reasonforFailure = job.getErrorInfo();
%>
<html>
@@ -78,6 +81,7 @@
<b>Launched At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLaunchTime(), job.getSubmitTime()) %><br/>
<b>Finished At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getFinishTime(), job.getLaunchTime()) %><br/>
<b>Status: </b> <%= ((job.getJobStatus()) == null ? "Incomplete" :job.getJobStatus()) %><br/>
+<b>ReasonForFailure: </b> <%=reasonforFailure %><br/>
<%
HistoryViewer.SummarizedJob sj = new HistoryViewer.SummarizedJob(job);
%>