You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2008/03/17 06:19:57 UTC
svn commit: r637723 [2/2] - in /hadoop/core/trunk: ./ conf/ docs/
src/docs/src/documentation/content/xdocs/
src/java/org/apache/hadoop/mapred/ src/webapps/history/ src/webapps/job/
Added: hadoop/core/trunk/src/webapps/job/analysejobhistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/analysejobhistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/analysejobhistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/analysejobhistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,249 @@
+<%@ page
+ contentType="text/html; charset=UTF-8"
+ import="javax.servlet.http.*"
+ import="java.io.*"
+ import="java.util.*"
+ import="org.apache.hadoop.mapred.*"
+ import="org.apache.hadoop.util.*"
+ import="java.text.SimpleDateFormat"
+ import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<jsp:include page="loadhistory.jsp">
+ <jsp:param name="jobid" value="<%=request.getParameter("jobid") %>"/>
+ <jsp:param name="logFile" value="<%=request.getParameter("logFile") %>"/>
+</jsp:include>
+<%! private static SimpleDateFormat dateFormat
+ = new SimpleDateFormat("d/MM HH:mm:ss") ;
+%>
+<html><body>
+<%
+ String jobid = request.getParameter("jobid");
+ String logFile = request.getParameter("logFile");
+ String numTasks = request.getParameter("numTasks");
+ int showTasks = 10 ;
+ if (numTasks != null) {
+ showTasks = Integer.parseInt(numTasks);
+ }
+ JobInfo job = (JobInfo)request.getSession().getAttribute("job");
+%>
+<h2>Hadoop Job <a href="jobdetailshistory.jsp?jobid=<%=jobid%>&&logFile=<%=logFile %>"><%=jobid %> </a></h2>
+<b>User : </b> <%=job.get(Keys.USER) %><br/>
+<b>JobName : </b> <%=job.get(Keys.JOBNAME) %><br/>
+<b>JobConf : </b> <%=job.get(Keys.JOBCONF) %><br/>
+<b>Submitted At : </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.SUBMIT_TIME), 0 ) %><br/>
+<b>Launched At : </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.LAUNCH_TIME), job.getLong(Keys.SUBMIT_TIME)) %><br/>
+<b>Finished At : </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME)) %><br/>
+<b>Status : </b> <%= ((job.get(Keys.JOB_STATUS) == null)?"Incomplete" :job.get(Keys.JOB_STATUS)) %><br/>
+<hr/>
+<center>
+<%
+ if (!Values.SUCCESS.name().equals(job.get(Keys.JOB_STATUS))) {
+ out.print("<h3>No Analysis available as job did not finish</h3>");
+ return;
+ }
+ Map<String, JobHistory.Task> tasks = job.getAllTasks();
+ int finishedMaps = job.getInt(Keys.FINISHED_MAPS) ;
+ int finishedReduces = job.getInt(Keys.FINISHED_REDUCES) ;
+ JobHistory.Task [] mapTasks = new JobHistory.Task[finishedMaps];
+ JobHistory.Task [] reduceTasks = new JobHistory.Task[finishedReduces];
+ int mapIndex = 0 , reduceIndex=0;
+ long avgMapTime = 0;
+ long avgReduceTime = 0;
+ long avgShuffleTime = 0;
+
+ for (JobHistory.Task task : tasks.values()) {
+ Map<String, TaskAttempt> attempts = task.getTaskAttempts();
+ for (JobHistory.TaskAttempt attempt : attempts.values()) {
+ if (attempt.get(Keys.TASK_STATUS).equals(Values.SUCCESS.name())) {
+ long avgFinishTime = (attempt.getLong(Keys.FINISH_TIME) -
+ attempt.getLong(Keys.START_TIME));
+ if (Values.MAP.name().equals(task.get(Keys.TASK_TYPE))) {
+ mapTasks[mapIndex++] = attempt ;
+ avgMapTime += avgFinishTime;
+ } else {
+ reduceTasks[reduceIndex++] = attempt;
+ avgShuffleTime += (attempt.getLong(Keys.SHUFFLE_FINISHED) -
+ attempt.getLong(Keys.START_TIME));
+ avgReduceTime += (attempt.getLong(Keys.FINISH_TIME) -
+ attempt.getLong(Keys.SHUFFLE_FINISHED));
+ }
+ break;
+ }
+ }
+ }
+
+ if (finishedMaps > 0) {
+ avgMapTime /= finishedMaps;
+ }
+ if (finishedReduces > 0) {
+ avgReduceTime /= finishedReduces;
+ avgShuffleTime /= finishedReduces;
+ }
+ Comparator<JobHistory.Task> cMap = new Comparator<JobHistory.Task>(){
+ public int compare(JobHistory.Task t1, JobHistory.Task t2){
+ long l1 = t1.getLong(Keys.FINISH_TIME) - t1.getLong(Keys.START_TIME);
+ long l2 = t2.getLong(Keys.FINISH_TIME) - t2.getLong(Keys.START_TIME);
+ return (l2<l1 ? -1 : (l2==l1 ? 0 : 1));
+ }
+ };
+ Comparator<JobHistory.Task> cShuffle = new Comparator<JobHistory.Task>(){
+ public int compare(JobHistory.Task t1, JobHistory.Task t2){
+ long l1 = t1.getLong(Keys.SHUFFLE_FINISHED) -
+ t1.getLong(Keys.START_TIME);
+ long l2 = t2.getLong(Keys.SHUFFLE_FINISHED) -
+ t2.getLong(Keys.START_TIME);
+ return (l2<l1 ? -1 : (l2==l1 ? 0 : 1));
+ }
+ };
+ Arrays.sort(mapTasks, cMap);
+ JobHistory.Task minMap = mapTasks[mapTasks.length-1] ;
+%>
+
+<h3>Time taken by best performing Map task
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>&taskid=<%=minMap.get(Keys.TASKID)%>">
+<%=minMap.get(Keys.TASKID) %></a> : <%=StringUtils.formatTimeDiff(minMap.getLong(Keys.FINISH_TIME), minMap.getLong(Keys.START_TIME) ) %></h3>
+<h3>Average time taken by Map tasks:
+<%=StringUtils.formatTimeDiff(avgMapTime, 0) %></h3>
+<h3>Worse performing map tasks</h3>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Time taken</td></tr>
+<%
+ for (int i=0;i<showTasks && i<mapTasks.length; i++) {
+%>
+ <tr>
+ <td><a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>&taskid=<%=mapTasks[i].get(Keys.TASKID)%>">
+ <%=mapTasks[i].get(Keys.TASKID) %></a></td>
+ <td><%=StringUtils.formatTimeDiff(mapTasks[i].getLong(Keys.FINISH_TIME), mapTasks[i].getLong(Keys.START_TIME)) %></td>
+ </tr>
+<%
+ }
+%>
+</table>
+<%
+ Comparator<JobHistory.Task> cFinishMapRed =
+ new Comparator<JobHistory.Task>() {
+ public int compare(JobHistory.Task t1, JobHistory.Task t2){
+ long l1 = t1.getLong(Keys.FINISH_TIME);
+ long l2 = t2.getLong(Keys.FINISH_TIME);
+ return (l2<l1 ? -1 : (l2==l1 ? 0 : 1));
+ }
+ };
+ Arrays.sort(mapTasks, cFinishMapRed);
+ JobHistory.Task lastMap = mapTasks[0] ;
+%>
+
+<h3>The last Map task
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>
+&taskid=<%=lastMap.get(Keys.TASKID)%>"><%=lastMap.get(Keys.TASKID) %></a>
+finished at (relative to the Job launch time):
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat,
+ lastMap.getLong(Keys.FINISH_TIME),
+ job.getLong(Keys.LAUNCH_TIME) ) %></h3>
+<hr/>
+
+<%
+ if (reduceTasks.length <= 0) return;
+ Arrays.sort(reduceTasks, cShuffle);
+ JobHistory.Task minShuffle = reduceTasks[reduceTasks.length-1] ;
+%>
+<h3>Time taken by best performing shuffle
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>
+&taskid=<%=minShuffle.get(Keys.TASKID)%>"><%=minShuffle.get(Keys.TASKID)%></a> :
+<%=StringUtils.formatTimeDiff(minShuffle.getLong(Keys.SHUFFLE_FINISHED),
+ minShuffle.getLong(Keys.START_TIME) ) %></h3>
+<h3>Average time taken by Shuffle:
+<%=StringUtils.formatTimeDiff(avgShuffleTime, 0) %></h3>
+<h3>Worse performing Shuffle(s)</h3>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Time taken</td></tr>
+<%
+ for (int i=0;i<showTasks && i<reduceTasks.length; i++) {
+%>
+ <tr>
+ <td><a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=
+ <%=logFile%>&taskid=<%=reduceTasks[i].get(Keys.TASKID)%>">
+ <%=reduceTasks[i].get(Keys.TASKID) %></a></td>
+ <td><%=
+ StringUtils.formatTimeDiff(
+ reduceTasks[i].getLong(Keys.SHUFFLE_FINISHED),
+ reduceTasks[i].getLong(Keys.START_TIME)) %>
+ </td>
+ </tr>
+<%
+ }
+%>
+</table>
+<%
+ Comparator<JobHistory.Task> cFinishShuffle =
+ new Comparator<JobHistory.Task>() {
+ public int compare(JobHistory.Task t1, JobHistory.Task t2){
+ long l1 = t1.getLong(Keys.SHUFFLE_FINISHED);
+ long l2 = t2.getLong(Keys.SHUFFLE_FINISHED);
+ return (l2<l1 ? -1 : (l2==l1 ? 0 : 1));
+ }
+ };
+ Arrays.sort(reduceTasks, cFinishShuffle);
+ JobHistory.Task lastShuffle = reduceTasks[0] ;
+%>
+
+<h3>The last Shuffle
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>
+&taskid=<%=lastShuffle.get(Keys.TASKID)%>"><%=lastShuffle.get(Keys.TASKID)%>
+</a> finished at (relative to the Job launch time):
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat,
+ lastShuffle.getLong(Keys.SHUFFLE_FINISHED),
+ job.getLong(Keys.LAUNCH_TIME) ) %></h3>
+
+<%
+ Comparator<JobHistory.Task> cReduce = new Comparator<JobHistory.Task>(){
+ public int compare(JobHistory.Task t1, JobHistory.Task t2){
+ long l1 = t1.getLong(Keys.FINISH_TIME) -
+ t1.getLong(Keys.SHUFFLE_FINISHED);
+ long l2 = t2.getLong(Keys.FINISH_TIME) -
+ t2.getLong(Keys.SHUFFLE_FINISHED);
+ return (l2<l1 ? -1 : (l2==l1 ? 0 : 1));
+ }
+ };
+ Arrays.sort(reduceTasks, cReduce);
+ JobHistory.Task minReduce = reduceTasks[reduceTasks.length-1] ;
+%>
+<hr/>
+<h3>Time taken by best performing Reduce task :
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>&taskid=<%=minReduce.get(Keys.TASKID)%>">
+<%=minReduce.get(Keys.TASKID) %></a> :
+<%=StringUtils.formatTimeDiff(minReduce.getLong(Keys.FINISH_TIME),
+ minReduce.getLong(Keys.SHUFFLE_FINISHED) ) %></h3>
+
+<h3>Average time taken by Reduce tasks:
+<%=StringUtils.formatTimeDiff(avgReduceTime, 0) %></h3>
+<h3>Worse performing reduce tasks</h3>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Time taken</td></tr>
+<%
+ for (int i=0;i<showTasks && i<reduceTasks.length; i++) {
+%>
+ <tr>
+ <td><a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>&taskid=<%=reduceTasks[i].get(Keys.TASKID)%>">
+ <%=reduceTasks[i].get(Keys.TASKID) %></a></td>
+ <td><%=StringUtils.formatTimeDiff(
+ reduceTasks[i].getLong(Keys.FINISH_TIME),
+ reduceTasks[i].getLong(Keys.SHUFFLE_FINISHED)) %></td>
+ </tr>
+<%
+ }
+%>
+</table>
+<%
+ Arrays.sort(reduceTasks, cFinishMapRed);
+ JobHistory.Task lastReduce = reduceTasks[0] ;
+%>
+
+<h3>The last Reduce task
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile%>
+&taskid=<%=lastReduce.get(Keys.TASKID)%>"><%=lastReduce.get(Keys.TASKID)%>
+</a> finished at (relative to the Job launch time):
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat,
+ lastReduce.getLong(Keys.FINISH_TIME),
+ job.getLong(Keys.LAUNCH_TIME) ) %></h3>
+</center>
+</body></html>
\ No newline at end of file
Added: hadoop/core/trunk/src/webapps/job/jobconf_history.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobconf_history.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobconf_history.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/jobconf_history.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,56 @@
+<%@ page
+ contentType="text/html; charset=UTF-8"
+ import="javax.servlet.*"
+ import="javax.servlet.http.*"
+ import="java.io.*"
+ import="java.net.URL"
+ import="org.apache.hadoop.mapred.*"
+ import="org.apache.hadoop.fs.*"
+ import="org.apache.hadoop.util.*"
+%>
+
+
+<%
+ JobTracker tracker = (JobTracker) application.getAttribute("job.tracker");
+ String jobId = request.getParameter("jobid");
+ if (jobId == null) {
+ out.println("<h2>Missing 'jobid' for fetching job configuration!</h2>");
+ return;
+ }
+%>
+
+<html>
+
+<title>Job Configuration: JobId - <%= jobId %></title>
+
+<body>
+<h2>Job Configuration: JobId - <%= jobId %></h2><br>
+
+<%
+ Path logDir = new Path(request.getParameter("jobLogDir"));
+ Path jobFilePath = new Path(logDir,
+ request.getParameter("jobUniqueString") + "_conf.xml");
+ FileSystem fs = (FileSystem)request.getSession().getAttribute("fs");
+ FSDataInputStream jobFile = null;
+ try {
+ jobFile = fs.open(jobFilePath);
+ JobConf jobConf = new JobConf(jobFilePath);
+ XMLUtils.transform(
+ jobConf.getConfResourceAsInputStream("webapps/static/jobconf.xsl"),
+ jobFile, out);
+ } catch (Exception e) {
+ out.println("Failed to retreive job configuration for job '" + jobId + "!");
+ out.println(e);
+ } finally {
+ if (jobFile != null) {
+ try {
+ jobFile.close();
+ } catch (IOException e) {}
+ }
+ }
+%>
+
+<br>
+<%
+out.println(ServletUtil.htmlFooter());
+%>
Added: hadoop/core/trunk/src/webapps/job/jobdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobdetailshistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobdetailshistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/jobdetailshistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,192 @@
+<%@ page
+ contentType="text/html; charset=UTF-8"
+ import="javax.servlet.http.*"
+ import="java.io.*"
+ import="java.util.*"
+ import="org.apache.hadoop.fs.*"
+ import="org.apache.hadoop.mapred.*"
+ import="org.apache.hadoop.util.*"
+ import="java.text.SimpleDateFormat"
+ import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<jsp:include page="loadhistory.jsp">
+ <jsp:param name="jobid" value="<%=request.getParameter("jobid") %>"/>
+ <jsp:param name="logFile" value="<%=request.getParameter("logFile") %>"/>
+</jsp:include>
+<%! static SimpleDateFormat dateFormat = new SimpleDateFormat("d-MMM-yyyy HH:mm:ss") ; %>
+<%
+ String jobid = request.getParameter("jobid");
+ String logFile = request.getParameter("logFile");
+
+ Path jobFile = new Path(logFile);
+ String[] jobDetails = jobFile.getName().split("_");
+ String jobUniqueString = jobDetails[0] + "_" +jobDetails[1] + "_" + jobid ;
+
+ JobInfo job = (JobInfo)request.getSession().getAttribute("job");
+ FileSystem fs = (FileSystem)request.getSession().getAttribute("fs");
+%>
+<html><body>
+<h2>Hadoop Job <%=jobid %> on <a href="jobhistory.jsp">History Viewer</a></h2>
+
+<b>User: </b> <%=job.get(Keys.USER) %><br/>
+<b>JobName: </b> <%=job.get(Keys.JOBNAME) %><br/>
+<b>JobConf: </b> <a href="jobconf_history.jsp?jobid=<%=jobid%>&jobLogDir=<%=new Path(logFile).getParent().toString()%>&jobUniqueString=<%=jobUniqueString%>">
+ <%=job.get(Keys.JOBCONF) %></a><br/>
+<b>Submitted At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.SUBMIT_TIME), 0 ) %><br/>
+<b>Launched At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.LAUNCH_TIME), job.getLong(Keys.SUBMIT_TIME)) %><br/>
+<b>Finished At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME)) %><br/>
+<b>Status: </b> <%= ((job.get(Keys.JOB_STATUS) == "")?"Incomplete" :job.get(Keys.JOB_STATUS)) %><br/>
+<%
+ Map<String, JobHistory.Task> tasks = job.getAllTasks();
+ int totalMaps = 0 ;
+ int totalReduces = 0;
+ int numFailedMaps = 0;
+ int numKilledMaps = 0;
+ int numFailedReduces = 0 ;
+ int numKilledReduces = 0;
+
+ long mapStarted = 0 ;
+ long mapFinished = 0 ;
+ long reduceStarted = 0 ;
+ long reduceFinished = 0;
+
+ Map <String,String> allHosts = new TreeMap<String,String>();
+ for (JobHistory.Task task : tasks.values()) {
+ Map<String, TaskAttempt> attempts = task.getTaskAttempts();
+ allHosts.put(task.get(Keys.HOSTNAME), "");
+ for (TaskAttempt attempt : attempts.values()) {
+ long startTime = attempt.getLong(Keys.START_TIME) ;
+ long finishTime = attempt.getLong(Keys.FINISH_TIME) ;
+ if (Values.MAP.name().equals(task.get(Keys.TASK_TYPE))){
+ if (mapStarted==0 || mapStarted > startTime ) {
+ mapStarted = startTime;
+ }
+ if (mapFinished < finishTime ) {
+ mapFinished = finishTime ;
+ }
+ totalMaps++;
+ if (Values.FAILED.name().equals(attempt.get(Keys.TASK_STATUS))) {
+ numFailedMaps++;
+ } else if (Values.KILLED.name().equals(attempt.get(Keys.TASK_STATUS))) {
+ numKilledMaps++;
+ }
+ } else {
+ if (reduceStarted==0||reduceStarted > startTime) {
+ reduceStarted = startTime ;
+ }
+ if (reduceFinished < finishTime) {
+ reduceFinished = finishTime;
+ }
+ totalReduces++;
+ if (Values.FAILED.name().equals(attempt.get(Keys.TASK_STATUS))) {
+ numFailedReduces++;
+ } else if (Values.KILLED.name().equals(attempt.get(Keys.TASK_STATUS))) {
+ numKilledReduces++;
+ }
+ }
+ }
+ }
+%>
+<b><a href="analysejobhistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile %>">Analyse This Job</a></b>
+<hr/>
+<center>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr>
+<td>Kind</td><td>Total Tasks(successful+failed+killed)</td><td>Successful tasks</td><td>Failed tasks</td><td>Killed tasks</td><td>Start Time</td><td>Finish Time</td>
+</tr>
+<tr>
+<td>Map</td>
+ <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile %>&taskType=<%=Values.MAP.name() %>&status=all">
+ <%=totalMaps %></a></td>
+ <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile %>&taskType=<%=Values.MAP.name() %>&status=<%=Values.SUCCESS %>">
+ <%=job.getInt(Keys.FINISHED_MAPS) %></a></td>
+ <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile %>&taskType=<%=Values.MAP.name() %>&status=<%=Values.FAILED %>">
+ <%=numFailedMaps %></a></td>
+ <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile %>&taskType=<%=Values.MAP.name() %>&status=<%=Values.KILLED %>">
+ <%=numKilledMaps %></a></td>
+ <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, mapStarted, 0) %></td>
+ <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, mapFinished, mapStarted) %></td>
+</tr>
+<tr>
+<td>Reduce</td>
+ <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile %>&taskType=<%=Values.REDUCE.name() %>&status=all">
+ <%=totalReduces%></a></td>
+ <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile %>&taskType=<%=Values.REDUCE.name() %>&status=<%=Values.SUCCESS %>">
+ <%=job.getInt(Keys.FINISHED_REDUCES)%></a></td>
+ <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile %>&taskType=<%=Values.REDUCE.name() %>&status=<%=Values.FAILED %>">
+ <%=numFailedReduces%></a></td>
+ <td><a href="jobtaskshistory.jsp?jobid=<%=jobid %>&logFile=<%=logFile %>&taskType=<%=Values.REDUCE.name() %>&status=<%=Values.KILLED %>">
+ <%=numKilledReduces%></a></td>
+ <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, reduceStarted, 0) %></td>
+ <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, reduceFinished, reduceStarted) %></td>
+</tr>
+</table>
+
+<br/>
+ <%
+ DefaultJobHistoryParser.FailedOnNodesFilter filter =
+ new DefaultJobHistoryParser.FailedOnNodesFilter();
+ JobHistory.parseHistoryFromFS(logFile, filter, fs);
+ Map<String, Set<String>> badNodes = filter.getValues();
+ if (badNodes.size() > 0) {
+ %>
+<h3>Failed tasks attempts by nodes </h3>
+<table border="1">
+<tr><td>Hostname</td><td>Failed Tasks</td></tr>
+ <%
+ for (Map.Entry<String, Set<String>> entry : badNodes.entrySet()) {
+ String node = entry.getKey();
+ Set<String> failedTasks = entry.getValue();
+%>
+ <tr>
+ <td><%=node %></td>
+ <td>
+<%
+ for (String t : failedTasks) {
+%>
+ <a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile %>&taskid=<%=t %>"><%=t %></a>,
+<%
+ }
+%>
+ </td>
+ </tr>
+<%
+ }
+ }
+ %>
+</table>
+<br/>
+ <%
+ DefaultJobHistoryParser.KilledOnNodesFilter killedFilter =
+ new DefaultJobHistoryParser.KilledOnNodesFilter();
+ JobHistory.parseHistoryFromFS(logFile, filter, fs);
+ badNodes = killedFilter.getValues();
+ if (badNodes.size() > 0) {
+ %>
+<h3>Killed tasks attempts by nodes </h3>
+<table border="1">
+<tr><td>Hostname</td><td>Killed Tasks</td></tr>
+ <%
+ for (Map.Entry<String, Set<String>> entry : badNodes.entrySet()) {
+ String node = entry.getKey();
+ Set<String> killedTasks = entry.getValue();
+%>
+ <tr>
+ <td><%=node %></td>
+ <td>
+<%
+ for (String t : killedTasks) {
+%>
+ <a href="taskdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=logFile %>&taskid=<%=t %>"><%=t %></a>,
+<%
+ }
+%>
+ </td>
+ </tr>
+<%
+ }
+ }
+%>
+</table>
+</center>
+</body></html>
Added: hadoop/core/trunk/src/webapps/job/jobhistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobhistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobhistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/jobhistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,105 @@
+<%@ page
+ contentType="text/html; charset=UTF-8"
+ import="java.io.*"
+ import="java.util.*"
+ import="org.apache.hadoop.mapred.*"
+ import="org.apache.hadoop.util.*"
+ import="org.apache.hadoop.fs.*"
+ import="javax.servlet.jsp.*"
+ import="java.text.SimpleDateFormat"
+ import="org.apache.hadoop.mapred.*"
+ import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<%!
+ private static SimpleDateFormat dateFormat =
+ new SimpleDateFormat("d/MM HH:mm:ss");
+%>
+<html>
+<head>
+<title>Hadoop Map/Reduce Administration</title>
+<link rel="stylesheet" type="text/css" href="/static/hadoop.css">
+</head>
+<body>
+<h1>Hadoop Map/Reduce History Viewer</h1>
+<hr>
+<h2>Available History </h2>
+<%
+ PathFilter jobLogFileFilter = new PathFilter() {
+ public boolean accept(Path path) {
+ return !(path.getName().endsWith(".xml"));
+ }
+ };
+
+ FileSystem fs = (FileSystem) application.getAttribute("fileSys");
+ String historyLogDir = (String) application.getAttribute("historyLogDir");
+ if (fs == null) {
+ out.println("Null file system. May be namenode is in safemode!");
+ return;
+ }
+ Path[] jobFiles = fs.listPaths(new Path(historyLogDir), jobLogFileFilter);
+ if (null == jobFiles ) {
+ out.println("NULL files !!!");
+ return ;
+ }
+
+ // sort the files on creation time.
+ Arrays.sort(jobFiles, new Comparator<Path>() {
+ public int compare(Path p1, Path p2) {
+ String[] split1 = p1.getName().split("_");
+ String[] split2 = p2.getName().split("_");
+
+ // compare job tracker start time
+ int res = new Date(Long.parseLong(split1[1])).compareTo(
+ new Date(Long.parseLong(split2[1])));
+ if (res == 0) {
+ res = new Date(Long.parseLong(split1[3])).compareTo(
+ new Date(Long.parseLong(split2[3])));
+ }
+ if (res == 0) {
+ Long l1 = Long.parseLong(split1[4]);
+ res = l1.compareTo(Long.parseLong(split2[4]));
+ }
+ return res;
+ }
+ });
+
+ out.print("<table align=center border=2 cellpadding=\"5\" cellspacing=\"2\">");
+ out.print("<tr><td align=\"center\" colspan=\"9\"><b>Available Jobs </b></td></tr>\n");
+ out.print("<tr>");
+ out.print("<td>Job tracker Host Name</td>" +
+ "<td>Job tracker Start time</td>" +
+ "<td>Job Id</td><td>Name</td><td>User</td>") ;
+ out.print("</tr>");
+ for (Path jobFile: jobFiles) {
+ String[] jobDetails = jobFile.getName().split("_");
+ String trackerHostName = jobDetails[0];
+ String trackerStartTime = jobDetails[1];
+ String jobId = jobDetails[2] + "_" +jobDetails[3] + "_" + jobDetails[4] ;
+ String user = jobDetails[5];
+ String jobName = jobDetails[6];
+%>
+<center>
+<%
+ printJob(trackerHostName, trackerStartTime, jobId,
+ jobName, user, jobFile.toString(), out) ;
+%>
+</center>
+<%
+ } // end while trackers
+%>
+<%!
+ private void printJob(String trackerHostName, String trackerid,
+ String jobId, String jobName,
+ String user, String logFile, JspWriter out)
+ throws IOException {
+ out.print("<tr>");
+ out.print("<td>" + trackerHostName + "</td>");
+ out.print("<td>" + new Date(Long.parseLong(trackerid)) + "</td>");
+ out.print("<td>" + "<a href=\"jobdetailshistory.jsp?jobid=" + jobId +
+ "&logFile=" + logFile + "\">" + jobId + "</a></td>");
+ out.print("<td>" + jobName + "</td>");
+ out.print("<td>" + user + "</td>");
+ out.print("</tr>");
+ }
+%>
+</body></html>
Added: hadoop/core/trunk/src/webapps/job/jobtaskshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobtaskshistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobtaskshistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/jobtaskshistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,68 @@
+<%@ page
+ contentType="text/html; charset=UTF-8"
+ import="javax.servlet.http.*"
+ import="java.io.*"
+ import="java.util.*"
+ import="org.apache.hadoop.mapred.*"
+ import="org.apache.hadoop.util.*"
+ import="java.text.SimpleDateFormat"
+ import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<jsp:include page="loadhistory.jsp">
+ <jsp:param name="jobid" value="<%=request.getParameter("jobid") %>"/>
+ <jsp:param name="logFile" value="<%=request.getParameter("logFile") %>"/>
+</jsp:include>
+<%!
+ private static SimpleDateFormat dateFormat =
+ new SimpleDateFormat("d/MM HH:mm:ss") ;
+%>
+
+<%
+ String jobid = request.getParameter("jobid");
+ String logFile = request.getParameter("logFile");
+ String taskStatus = request.getParameter("status");
+ String taskType = request.getParameter("taskType");
+
+ JobHistory.JobInfo job = (JobHistory.JobInfo)request.
+ getSession().getAttribute("job");
+ Map<String, JobHistory.Task> tasks = job.getAllTasks();
+%>
+<html>
+<body>
+<h2><%=taskStatus%> <%=taskType %> task list for <a href="jobdetailshistory.jsp?jobid=<%=jobid%>&&logFile=<%=logFile %>"><%=jobid %> </a></h2>
+<center>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Start Time</td><td>Finish Time<br/></td><td>Error</td></tr>
+<%
+ for (JobHistory.Task task : tasks.values()) {
+ if (taskType.equals(task.get(Keys.TASK_TYPE))){
+ Map <String, TaskAttempt> taskAttempts = task.getTaskAttempts();
+ for (JobHistory.TaskAttempt taskAttempt : taskAttempts.values()) {
+ if (taskStatus.equals(taskAttempt.get(Keys.TASK_STATUS)) ||
+ taskStatus.equals("all")){
+ printTask(jobid, logFile, taskAttempt, out);
+ }
+ }
+ }
+ }
+%>
+</table>
+<%!
+ private void printTask(String jobid, String trackerId,
+ JobHistory.TaskAttempt attempt, JspWriter out) throws IOException{
+ out.print("<tr>");
+ out.print("<td>" + "<a href=\"taskdetailshistory.jsp?jobid=" + jobid +
+ "&logFile="+ trackerId +"&taskid="+attempt.get(Keys.TASKID)+"\">" +
+ attempt.get(Keys.TASKID) + "</a></td>");
+ out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat,
+ attempt.getLong(Keys.START_TIME), 0 ) + "</td>");
+ out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat,
+ attempt.getLong(Keys.FINISH_TIME),
+ attempt.getLong(Keys.START_TIME) ) + "</td>");
+ out.print("<td>" + attempt.get(Keys.ERROR) + "</td>");
+ out.print("</tr>");
+ }
+%>
+</center>
+</body>
+</html>
Modified: hadoop/core/trunk/src/webapps/job/jobtracker.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/jobtracker.jsp?rev=637723&r1=637722&r2=637723&view=diff
==============================================================================
--- hadoop/core/trunk/src/webapps/job/jobtracker.jsp (original)
+++ hadoop/core/trunk/src/webapps/job/jobtracker.jsp Sun Mar 16 22:19:52 2008
@@ -134,7 +134,7 @@
<hr>
<h2>Local logs</h2>
-<a href="logs/">Log</a> directory, <a href="http://<%=tracker.getHistoryAddress()%>">
+<a href="logs/">Log</a> directory, <a href="jobhistory.jsp">
Job Tracker History</a>
<%
Added: hadoop/core/trunk/src/webapps/job/loadhistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/loadhistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/loadhistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/loadhistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,36 @@
+<%@ page
+ contentType="text/html; charset=UTF-8"
+ import="java.io.*"
+ import="java.util.*"
+ import="org.apache.hadoop.mapred.*"
+ import="org.apache.hadoop.fs.*"
+ import="org.apache.hadoop.util.*"
+ import="javax.servlet.jsp.*"
+ import="java.text.SimpleDateFormat"
+ import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<%
+ PathFilter jobLogFileFilter = new PathFilter() {
+ public boolean accept(Path path) {
+ return !(path.getName().endsWith(".xml"));
+ }
+ };
+
+ FileSystem fs = (FileSystem) application.getAttribute("fileSys");
+ String jobId = (String)request.getParameter("jobid");
+ JobHistory.JobInfo job = (JobHistory.JobInfo)
+ request.getSession().getAttribute("job");
+ if (null != job && (!jobId.equals(job.get(Keys.JOBID)))) {
+ // remove jobInfo from session, keep only one job in session at a time
+ request.getSession().removeAttribute("job");
+ job = null ;
+ }
+
+ if (null == job) {
+ String jobLogFile = (String)request.getParameter("logFile");
+ job = new JobHistory.JobInfo(jobId);
+ DefaultJobHistoryParser.parseJobTasks(jobLogFile, job, fs) ;
+ request.getSession().setAttribute("job", job);
+ request.getSession().setAttribute("fs", fs);
+ }
+%>
Added: hadoop/core/trunk/src/webapps/job/taskdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/taskdetailshistory.jsp?rev=637723&view=auto
==============================================================================
--- hadoop/core/trunk/src/webapps/job/taskdetailshistory.jsp (added)
+++ hadoop/core/trunk/src/webapps/job/taskdetailshistory.jsp Sun Mar 16 22:19:52 2008
@@ -0,0 +1,74 @@
+<%@ page
+ contentType="text/html; charset=UTF-8"
+ import="javax.servlet.http.*"
+ import="java.io.*"
+ import="java.util.*"
+ import="org.apache.hadoop.mapred.*"
+ import="org.apache.hadoop.util.*"
+ import="java.text.SimpleDateFormat"
+ import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<jsp:include page="loadhistory.jsp">
+ <jsp:param name="jobid" value="<%=request.getParameter("jobid") %>"/>
+ <jsp:param name="jobTrackerId" value="<%=request.getParameter("jobTrackerId") %>"/>
+</jsp:include>
+<%! private static SimpleDateFormat dateFormat = new SimpleDateFormat("d/MM HH:mm:ss") ; %>
+
+<%
+ String jobid = request.getParameter("jobid");
+ String logFile = request.getParameter("logFile");
+ String taskid = request.getParameter("taskid");
+ JobHistory.JobInfo job = (JobHistory.JobInfo)
+ request.getSession().getAttribute("job");
+ JobHistory.Task task = job.getAllTasks().get(taskid);
+%>
+<html>
+<body>
+<h2><%=taskid %> attempts for <a href="jobdetailshistory.jsp?jobid=<%=jobid%>&&logFile=<%=logFile %>"> <%=jobid %> </a></h2>
+<center>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Start Time</td>
+<%
+ if (Values.REDUCE.name().equals(task.get(Keys.TASK_TYPE))) {
+%>
+ <td>Shuffle Finished</td><td>Sort Finished</td>
+<%
+ }
+%>
+<td>Finish Time</td><td>Host</td><td>Error</td></tr>
+<%
+ for (JobHistory.TaskAttempt attempt : task.getTaskAttempts().values()) {
+ printTaskAttempt(attempt, task.get(Keys.TASK_TYPE), out);
+ }
+%>
+</table>
+<%!
+ private void printTaskAttempt(JobHistory.TaskAttempt taskAttempt,
+ String type, JspWriter out)
+ throws IOException {
+ out.print("<tr>");
+ out.print("<td>" + taskAttempt.get(Keys.TASK_ATTEMPT_ID) + "</td>");
+ out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat,
+ taskAttempt.getLong(Keys.START_TIME), 0 ) + "</td>");
+ if (Values.REDUCE.name().equals(type)) {
+ JobHistory.ReduceAttempt reduceAttempt =
+ (JobHistory.ReduceAttempt)taskAttempt;
+ out.print("<td>" +
+ StringUtils.getFormattedTimeWithDiff(dateFormat,
+ reduceAttempt.getLong(Keys.SHUFFLE_FINISHED),
+ reduceAttempt.getLong(Keys.START_TIME)) + "</td>");
+ out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat,
+ reduceAttempt.getLong(Keys.SORT_FINISHED),
+ reduceAttempt.getLong(Keys.SHUFFLE_FINISHED)) + "</td>");
+ }
+ out.print("<td>"+ StringUtils.getFormattedTimeWithDiff(dateFormat,
+ taskAttempt.getLong(Keys.FINISH_TIME),
+ taskAttempt.getLong(Keys.START_TIME) ) + "</td>");
+ out.print("<td>" + taskAttempt.get(Keys.HOSTNAME) + "</td>");
+ out.print("<td>" + taskAttempt.get(Keys.ERROR) + "</td>");
+ out.print("</tr>");
+ }
+%>
+</center>
+</body>
+</html>