You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2007/03/23 22:01:09 UTC
svn commit: r521900 - in /lucene/hadoop/trunk: CHANGES.txt
src/webapps/job/analysejobhistory.jsp
Author: cutting
Date: Fri Mar 23 14:01:08 2007
New Revision: 521900
URL: http://svn.apache.org/viewvc?view=rev&rev=521900
Log:
HADOOP-1101. In web ui, split shuffle statisitics from reduce statistics, and add some averages. Contributed by Devaraj.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/webapps/job/analysejobhistory.jsp
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=521900&r1=521899&r2=521900
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Mar 23 14:01:08 2007
@@ -9,6 +9,9 @@
2. HADOOP-1063. Fix a race condition in MiniDFSCluster test code.
(Hairong Kuang via cutting)
+ 3. HADOOP-1101. In web ui, split shuffle statistics from reduce
+ statistics, and add some task averages. (Devaraj Das via cutting)
+
Release 0.12.2 - 2007-23-17
Modified: lucene/hadoop/trunk/src/webapps/job/analysejobhistory.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/analysejobhistory.jsp?view=diff&rev=521900&r1=521899&r2=521900
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/analysejobhistory.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/analysejobhistory.jsp Fri Mar 23 14:01:08 2007
@@ -48,33 +48,67 @@
JobHistory.Task [] mapTasks = new JobHistory.Task[finishedMaps];
JobHistory.Task [] reduceTasks = new JobHistory.Task[finishedReduces];
int mapIndex = 0 , reduceIndex=0;
+ long avgMapTime = 0;
+ long avgReduceTime = 0;
+ long avgShuffleTime = 0;
for( JobHistory.Task task : tasks.values() ) {
+ long avgFinishTime = (task.getLong(Keys.FINISH_TIME) -
+ task.getLong(Keys.START_TIME));
if( Values.MAP.name().equals(task.get(Keys.TASK_TYPE)) ){
mapTasks[mapIndex++] = task ;
- }else{
- reduceTasks[reduceIndex++] = task;
+ avgMapTime += avgFinishTime;
+ }else{
+ Map<String, TaskAttempt> attempts = task.getTaskAttempts();
+ for (JobHistory.TaskAttempt attempt : attempts.values()) {
+ if (attempt.get(Keys.TASK_STATUS).equals(Values.SUCCESS.name())) {
+ reduceTasks[reduceIndex++] = attempt;
+ avgShuffleTime += (attempt.getLong(Keys.SHUFFLE_FINISHED) -
+ attempt.getLong(Keys.START_TIME));
+ avgReduceTime += (attempt.getLong(Keys.FINISH_TIME) -
+ attempt.getLong(Keys.SHUFFLE_FINISHED));
+ break;
+ }
+ }
}
}
- Comparator<JobHistory.Task> c = new Comparator<JobHistory.Task>(){
+ if (finishedMaps > 0) {
+ avgMapTime /= finishedMaps;
+ }
+ if (finishedReduces > 0) {
+ avgReduceTime /= finishedReduces;
+ avgShuffleTime /= finishedReduces;
+ }
+ Comparator<JobHistory.Task> cMap = new Comparator<JobHistory.Task>(){
public int compare(JobHistory.Task t1, JobHistory.Task t2){
Long l1 = new Long(t1.getLong(Keys.FINISH_TIME) - t1.getLong(Keys.START_TIME));
Long l2 = new Long(t2.getLong(Keys.FINISH_TIME) - t2.getLong(Keys.START_TIME)) ;
return l2.compareTo(l1);
}
};
- Arrays.sort(mapTasks, c);
- Arrays.sort(reduceTasks, c);
+ Comparator<JobHistory.Task> cShuffle = new Comparator<JobHistory.Task>(){
+ public int compare(JobHistory.Task t1, JobHistory.Task t2){
+ Long l1 = new Long(t1.getLong(Keys.SHUFFLE_FINISHED) -
+ t1.getLong(Keys.START_TIME));
+ Long l2 = new Long(t2.getLong(Keys.SHUFFLE_FINISHED) -
+ t2.getLong(Keys.START_TIME)) ;
+ return l2.compareTo(l1);
+ }
+ };
+ Arrays.sort(mapTasks, cMap);
+ Arrays.sort(reduceTasks, cShuffle);
JobHistory.Task minMap = mapTasks[mapTasks.length-1] ;
- JobHistory.Task minReduce = reduceTasks[reduceTasks.length-1] ;
+ JobHistory.Task minShuffle = reduceTasks[reduceTasks.length-1] ;
%>
<h3>Time taken by best performing Map task
<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>&taskid=<%=minMap.get(Keys.TASKID)%>">
<%=minMap.get(Keys.TASKID) %></a> : <%=StringUtils.formatTimeDiff(minMap.getLong(Keys.FINISH_TIME), minMap.getLong(Keys.START_TIME) ) %></h3>
+<h3>Average time taken by Map tasks:
+<%=StringUtils.formatTimeDiff(avgMapTime, 0) %></h3>
<h3>Worse performing map tasks</h3>
<table border="2" cellpadding="5" cellspacing="2">
<tr><td>Task Id</td><td>Time taken</td></tr>
@@ -90,10 +124,96 @@
}
%>
</table>
+<%
+ Comparator<JobHistory.Task> cFinishMapRed =
+ new Comparator<JobHistory.Task>() {
+ public int compare(JobHistory.Task t1, JobHistory.Task t2){
+ Long l1 = new Long(t1.getLong(Keys.FINISH_TIME));
+ Long l2 = new Long(t2.getLong(Keys.FINISH_TIME));
+ return l2.compareTo(l1);
+ }
+ };
+ Arrays.sort(mapTasks, cFinishMapRed);
+ JobHistory.Task lastMap = mapTasks[0] ;
+%>
+<h3>The last Map task
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>
+&taskid=<%=lastMap.get(Keys.TASKID)%>"><%=lastMap.get(Keys.TASKID) %></a>
+finished at (relative to the Job launch time):
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat,
+ lastMap.getLong(Keys.FINISH_TIME),
+ job.getLong(Keys.LAUNCH_TIME) ) %></h3>
+<hr/>
+<h3>Time taken by best performing shuffle
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>
+&taskid=<%=minShuffle.get(Keys.TASKID)%>"><%=minShuffle.get(Keys.TASKID)%></a> :
+<%=StringUtils.formatTimeDiff(minShuffle.getLong(Keys.SHUFFLE_FINISHED),
+ minShuffle.getLong(Keys.START_TIME) ) %></h3>
+<h3>Average time taken by Shuffle:
+<%=StringUtils.formatTimeDiff(avgShuffleTime, 0) %></h3>
+<h3>Worse performing Shuffle(s)</h3>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Time taken</td></tr>
+<%
+ for( int i=0;i<showTasks && i<reduceTasks.length; i++){
+%>
+ <tr>
+ <td><a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=
+ <%=jobTrackerId%>&taskid=<%=reduceTasks[i].get(Keys.TASKID)%>">
+ <%=reduceTasks[i].get(Keys.TASKID) %></a></td>
+ <td><%=
+ StringUtils.formatTimeDiff(
+ reduceTasks[i].getLong(Keys.SHUFFLE_FINISHED),
+ reduceTasks[i].getLong(Keys.START_TIME)) %>
+ </td>
+ </tr>
+<%
+ }
+%>
+</table>
+<%
+ Comparator<JobHistory.Task> cFinishShuffle =
+ new Comparator<JobHistory.Task>() {
+ public int compare(JobHistory.Task t1, JobHistory.Task t2){
+ Long l1 = new Long(t1.getLong(Keys.SHUFFLE_FINISHED));
+ Long l2 = new Long(t2.getLong(Keys.SHUFFLE_FINISHED));
+ return l2.compareTo(l1);
+ }
+ };
+ Arrays.sort(reduceTasks, cFinishShuffle);
+ JobHistory.Task lastShuffle = reduceTasks[0] ;
+%>
+
+<h3>The last Shuffle
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>
+&taskid=<%=lastShuffle.get(Keys.TASKID)%>"><%=lastShuffle.get(Keys.TASKID)%>
+</a> finished at (relative to the Job launch time):
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat,
+ lastShuffle.getLong(Keys.SHUFFLE_FINISHED),
+ job.getLong(Keys.LAUNCH_TIME) ) %></h3>
+
+<%
+ Comparator<JobHistory.Task> cReduce = new Comparator<JobHistory.Task>(){
+ public int compare(JobHistory.Task t1, JobHistory.Task t2){
+ Long l1 = new Long(t1.getLong(Keys.FINISH_TIME) -
+ t1.getLong(Keys.SHUFFLE_FINISHED));
+ Long l2 = new Long(t2.getLong(Keys.FINISH_TIME) -
+ t2.getLong(Keys.SHUFFLE_FINISHED));
+ return l2.compareTo(l1);
+ }
+ };
+ Arrays.sort(reduceTasks, cReduce);
+ JobHistory.Task minReduce = reduceTasks[reduceTasks.length-1] ;
+%>
+<hr/>
<h3>Time taken by best performing Reduce task :
<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>&taskid=<%=minReduce.get(Keys.TASKID)%>">
-<%=minReduce.get(Keys.TASKID) %></a> : <%=StringUtils.formatTimeDiff(minReduce.getLong(Keys.FINISH_TIME), minReduce.getLong(Keys.START_TIME) ) %></h3>
+<%=minReduce.get(Keys.TASKID) %></a> :
+<%=StringUtils.formatTimeDiff(minReduce.getLong(Keys.FINISH_TIME),
+ minReduce.getLong(Keys.SHUFFLE_FINISHED) ) %></h3>
+<h3>Average time taken by Reduce tasks:
+<%=StringUtils.formatTimeDiff(avgReduceTime, 0) %></h3>
<h3>Worse performing reduce tasks</h3>
<table border="2" cellpadding="5" cellspacing="2">
<tr><td>Task Id</td><td>Time taken</td></tr>
@@ -103,11 +223,25 @@
<tr>
<td><a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>&taskid=<%=reduceTasks[i].get(Keys.TASKID)%>">
<%=reduceTasks[i].get(Keys.TASKID) %></a></td>
- <td><%=StringUtils.formatTimeDiff(reduceTasks[i].getLong(Keys.FINISH_TIME), reduceTasks[i].getLong(Keys.START_TIME)) %></td>
+ <td><%=StringUtils.formatTimeDiff(
+ reduceTasks[i].getLong(Keys.FINISH_TIME),
+ reduceTasks[i].getLong(Keys.SHUFFLE_FINISHED)) %></td>
</tr>
<%
}
%>
</table>
+<%
+ Arrays.sort(reduceTasks, cFinishMapRed);
+ JobHistory.Task lastReduce = reduceTasks[0] ;
+%>
+
+<h3>The last Reduce task
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>
+&taskid=<%=lastReduce.get(Keys.TASKID)%>"><%=lastReduce.get(Keys.TASKID)%>
+</a> finished at (relative to the Job launch time):
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat,
+ lastReduce.getLong(Keys.FINISH_TIME),
+ job.getLong(Keys.LAUNCH_TIME) ) %></h3>
</center>
</body></html>