You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 04:30:41 UTC
svn commit: r1077017 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
mapred/org/apache/hadoop/mapred/ test/org/apache/hadoop/mapred/ webapps/job/
Author: omalley
Date: Fri Mar 4 03:30:40 2011
New Revision: 1077017
URL: http://svn.apache.org/viewvc?rev=1077017&view=rev
Log:
commit 0d4d9249af5dd6d20a2bf9dc7fc104c421dfb7a9
Author: Hemanth Yamijala <yh...@apache.org>
Date: Thu Oct 15 11:37:22 2009 +0530
MAPREDUCE:277 from https://issues.apache.org/jira/secure/attachment/12421419/patch-277-0.20.txt
+++ b/YAHOO-CHANGES.txt
+65. MAPREDUCE-277. Makes job history counters available on the job history
+ viewers. (Jothi Padmanabhan via ddas)
+
Added:
hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/taskstatshistory.jsp
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/HistoryViewer.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistory.java
hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetailshistory.jsp
hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/taskdetailshistory.jsp
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/HistoryViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/HistoryViewer.java?rev=1077017&r1=1077016&r2=1077017&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/HistoryViewer.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/HistoryViewer.java Fri Mar 4 03:30:40 2011
@@ -19,10 +19,14 @@
package org.apache.hadoop.mapred;
import java.io.IOException;
+import java.text.DecimalFormat;
+import java.text.Format;
+import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
+import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -32,6 +36,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.mapred.Counters.Counter;
+import org.apache.hadoop.mapred.Counters.Group;
import org.apache.hadoop.mapred.DefaultJobHistoryParser.*;
import org.apache.hadoop.mapred.JobHistory.*;
import org.apache.hadoop.util.StringUtils;
@@ -89,7 +95,7 @@ class HistoryViewer {
}
}
- public void print() throws IOException{
+ public void print() throws IOException {
printJobDetails();
printTaskSummary();
printJobAnalysis();
@@ -117,7 +123,7 @@ class HistoryViewer {
printFailedAttempts(filter);
}
- private void printJobDetails() {
+ private void printJobDetails() throws IOException {
StringBuffer jobDetails = new StringBuffer();
jobDetails.append("\nHadoop job: " ).append(jobId);
jobDetails.append("\n=====================================");
@@ -140,10 +146,58 @@ class HistoryViewer {
job.getLong(Keys.LAUNCH_TIME)));
jobDetails.append("\nStatus: ").append(((job.get(Keys.JOB_STATUS) == "") ?
"Incomplete" :job.get(Keys.JOB_STATUS)));
+ try {
+ printCounters(jobDetails, job);
+ } catch (ParseException p) {
+ throw new IOException(p);
+ }
jobDetails.append("\n=====================================");
System.out.println(jobDetails.toString());
}
+ private void printCounters(StringBuffer buff, JobInfo job)
+ throws ParseException {
+ Counters mapCounters =
+ Counters.fromEscapedCompactString(job.get(Keys.MAP_COUNTERS));
+ Counters reduceCounters =
+ Counters.fromEscapedCompactString(job.get(Keys.REDUCE_COUNTERS));
+ Counters totalCounters =
+ Counters.fromEscapedCompactString(job.get(Keys.COUNTERS));
+
+ // Killed jobs might not have counters
+ if (totalCounters == null) {
+ return;
+ }
+ buff.append("\nCounters: \n\n");
+ buff.append(String.format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|",
+ "Group Name",
+ "Counter name",
+ "Map Value",
+ "Reduce Value",
+ "Total Value"));
+ buff.append("\n------------------------------------------"+
+ "---------------------------------------------");
+ for (String groupName : totalCounters.getGroupNames()) {
+ Group totalGroup = totalCounters.getGroup(groupName);
+ Group mapGroup = mapCounters.getGroup(groupName);
+ Group reduceGroup = reduceCounters.getGroup(groupName);
+ Format decimal = new DecimalFormat();
+ Iterator<Counter> ctrItr = totalGroup.iterator();
+ while (ctrItr.hasNext()) {
+ Counter counter = ctrItr.next();
+ String name = counter.getDisplayName();
+ String mapValue = decimal.format(mapGroup.getCounter(name));
+ String reduceValue = decimal.format(reduceGroup.getCounter(name));
+ String totalValue = decimal.format(counter.getValue());
+ buff.append(
+ String.format("\n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s",
+ totalGroup.getDisplayName(),
+ counter.getDisplayName(),
+ mapValue, reduceValue, totalValue));
+ }
+ }
+ }
+
private void printTasks(String taskType, String taskStatus) {
Map<String, JobHistory.Task> tasks = job.getAllTasks();
StringBuffer taskList = new StringBuffer();
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=1077017&r1=1077016&r2=1077017&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java Fri Mar 4 03:30:40 2011
@@ -266,7 +266,7 @@ public class JobHistory {
FINISHED_MAPS, FINISHED_REDUCES, JOB_STATUS, TASKID, HOSTNAME, TASK_TYPE,
ERROR, TASK_ATTEMPT_ID, TASK_STATUS, COPY_PHASE, SORT_PHASE, REDUCE_PHASE,
SHUFFLE_FINISHED, SORT_FINISHED, COUNTERS, SPLITS, JOB_PRIORITY, HTTP_PORT,
- TRACKER_NAME, STATE_STRING, VERSION
+ TRACKER_NAME, STATE_STRING, VERSION, MAP_COUNTERS, REDUCE_COUNTERS
}
/**
@@ -1334,7 +1334,9 @@ public class JobHistory {
public static void logFinished(JobID jobId, long finishTime,
int finishedMaps, int finishedReduces,
int failedMaps, int failedReduces,
- Counters counters){
+ Counters mapCounters,
+ Counters reduceCounters,
+ Counters counters) {
if (!disableHistory){
// close job file for this job
ArrayList<PrintWriter> writer = fileManager.getWriters(jobId);
@@ -1345,6 +1347,7 @@ public class JobHistory {
Keys.JOB_STATUS, Keys.FINISHED_MAPS,
Keys.FINISHED_REDUCES,
Keys.FAILED_MAPS, Keys.FAILED_REDUCES,
+ Keys.MAP_COUNTERS, Keys.REDUCE_COUNTERS,
Keys.COUNTERS},
new String[] {jobId.toString(), Long.toString(finishTime),
Values.SUCCESS.name(),
@@ -1352,6 +1355,8 @@ public class JobHistory {
String.valueOf(finishedReduces),
String.valueOf(failedMaps),
String.valueOf(failedReduces),
+ mapCounters.makeEscapedCompactString(),
+ reduceCounters.makeEscapedCompactString(),
counters.makeEscapedCompactString()});
for (PrintWriter out : writer) {
out.close();
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1077017&r1=1077016&r2=1077017&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Fri Mar 4 03:30:40 2011
@@ -2338,7 +2338,8 @@ class JobInProgress {
JobHistory.JobInfo.logFinished(this.status.getJobID(), finishTime,
this.finishedMapTasks,
this.finishedReduceTasks, failedMapTasks,
- failedReduceTasks, getCounters());
+ failedReduceTasks, getMapCounters(),
+ getReduceCounters(), getCounters());
// Note that finalize will close the job history handles which garbage collect
// might try to finalize
garbageCollect();
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistory.java?rev=1077017&r1=1077016&r2=1077017&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistory.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistory.java Fri Mar 4 03:30:40 2011
@@ -555,6 +555,14 @@ public class TestJobHistory extends Test
assertTrue("Counters of job obtained from history file did not " +
"match the expected value",
c.makeEscapedCompactString().equals(values.get(Keys.COUNTERS)));
+ Counters m = jip.getMapCounters();
+ assertTrue("Map Counters of job obtained from history file did not " +
+ "match the expected value", m.makeEscapedCompactString().
+ equals(values.get(Keys.MAP_COUNTERS)));
+ Counters r = jip.getReduceCounters();
+ assertTrue("Reduce Counters of job obtained from history file did not " +
+ "match the expected value", r.makeEscapedCompactString().
+ equals(values.get(Keys.REDUCE_COUNTERS)));
// Validate number of total maps, total reduces, finished maps,
// finished reduces, failed maps, failed recudes
Modified: hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetailshistory.jsp?rev=1077017&r1=1077016&r2=1077017&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetailshistory.jsp (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetailshistory.jsp Fri Mar 4 03:30:40 2011
@@ -6,7 +6,7 @@
import="org.apache.hadoop.fs.*"
import="org.apache.hadoop.mapred.*"
import="org.apache.hadoop.util.*"
- import="java.text.SimpleDateFormat"
+ import="java.text.*"
import="org.apache.hadoop.mapred.JobHistory.*"
%>
@@ -23,7 +23,14 @@
FileSystem fs = (FileSystem) application.getAttribute("fileSys");
JobInfo job = JSPUtil.getJobInfo(request, fs);
%>
-<html><body>
+
+<html>
+<head>
+<title>Hadoop Job <%=jobid%> on History Viewer</title>
+<link rel="stylesheet" type="text/css" href="/static/hadoop.css">
+</head>
+<body>
+
<h2>Hadoop Job <%=jobid %> on <a href="jobhistory.jsp">History Viewer</a></h2>
<b>User: </b> <%=job.get(Keys.USER) %><br/>
@@ -188,6 +195,68 @@
</tr>
</table>
+<br>
+<br>
+
+<table border=2 cellpadding="5" cellspacing="2">
+ <tr>
+ <th><br/></th>
+ <th>Counter</th>
+ <th>Map</th>
+ <th>Reduce</th>
+ <th>Total</th>
+</tr>
+
+<%
+
+ Counters totalCounters =
+ Counters.fromEscapedCompactString(job.get(Keys.COUNTERS));
+ Counters mapCounters =
+ Counters.fromEscapedCompactString(job.get(Keys.MAP_COUNTERS));
+ Counters reduceCounters =
+ Counters.fromEscapedCompactString(job.get(Keys.REDUCE_COUNTERS));
+
+ if (totalCounters != null) {
+ for (String groupName : totalCounters.getGroupNames()) {
+ Counters.Group totalGroup = totalCounters.getGroup(groupName);
+ Counters.Group mapGroup = mapCounters.getGroup(groupName);
+ Counters.Group reduceGroup = reduceCounters.getGroup(groupName);
+
+ Format decimal = new DecimalFormat();
+
+ boolean isFirst = true;
+ Iterator<Counters.Counter> ctrItr = totalGroup.iterator();
+ while(ctrItr.hasNext()) {
+ Counters.Counter counter = ctrItr.next();
+ String name = counter.getDisplayName();
+ String mapValue =
+ decimal.format(mapGroup.getCounter(name));
+ String reduceValue =
+ decimal.format(reduceGroup.getCounter(name));
+ String totalValue = decimal.format(counter.getCounter());
+%>
+ <tr>
+<%
+ if (isFirst) {
+ isFirst = false;
+%>
+ <td rowspan="<%=totalGroup.size()%>"><%=totalGroup.getDisplayName()%></td>
+<%
+ }
+%>
+ <td><%=counter.getDisplayName()%></td>
+ <td align="right"><%=mapValue%></td>
+ <td align="right"><%=reduceValue%></td>
+ <td align="right"><%=totalValue%></td>
+ </tr>
+<%
+ }
+ }
+ }
+%>
+</table>
+<br>
+
<br/>
<%
DefaultJobHistoryParser.FailedOnNodesFilter filter =
@@ -222,6 +291,7 @@
%>
</table>
<br/>
+
<%
DefaultJobHistoryParser.KilledOnNodesFilter killedFilter =
new DefaultJobHistoryParser.KilledOnNodesFilter();
Modified: hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/taskdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/taskdetailshistory.jsp?rev=1077017&r1=1077016&r2=1077017&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/taskdetailshistory.jsp (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/taskdetailshistory.jsp Fri Mar 4 03:30:40 2011
@@ -35,10 +35,11 @@
<%
}
%>
-<td>Finish Time</td><td>Host</td><td>Error</td><td>Task Logs</td></tr>
+<td>Finish Time</td><td>Host</td><td>Error</td><td>Task Logs</td>
+<td>Counters</td></tr>
<%
for (JobHistory.TaskAttempt attempt : task.getTaskAttempts().values()) {
- printTaskAttempt(attempt, type, out);
+ printTaskAttempt(attempt, type, out, encodedLogFileName);
}
%>
</table>
@@ -60,8 +61,9 @@
%>
<%!
private void printTaskAttempt(JobHistory.TaskAttempt taskAttempt,
- String type, JspWriter out)
- throws IOException {
+ String type, JspWriter out,
+ String logFile)
+ throws Exception {
out.print("<tr>");
out.print("<td>" + taskAttempt.get(Keys.TASK_ATTEMPT_ID) + "</td>");
out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat,
@@ -97,6 +99,22 @@
out.print("n/a");
}
out.print("</td>");
+ Counters counters =
+ Counters.fromEscapedCompactString(taskAttempt.get(Keys.COUNTERS));
+ if (counters != null) {
+ TaskAttemptID attemptId =
+ TaskAttemptID.forName(taskAttempt.get(Keys.TASK_ATTEMPT_ID));
+ TaskID taskId = attemptId.getTaskID();
+ org.apache.hadoop.mapreduce.JobID jobId = taskId.getJobID();
+ out.print("<td>"
+ + "<a href=\"/taskstatshistory.jsp?jobid=" + jobId
+ + "&taskid=" + taskId
+ + "&attemptid=" + attemptId
+ + "&logFile=" + logFile + "\">"
+ + counters.size() + "</a></td>");
+ } else {
+ out.print("<td></td>");
+ }
out.print("</tr>");
}
%>
Added: hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/taskstatshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/taskstatshistory.jsp?rev=1077017&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/taskstatshistory.jsp (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/taskstatshistory.jsp Fri Mar 4 03:30:40 2011
@@ -0,0 +1,106 @@
+<%
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page
+ contentType="text/html; charset=UTF-8"
+ import="javax.servlet.http.*"
+ import="java.io.*"
+ import="java.util.*"
+ import="org.apache.hadoop.mapred.*"
+ import="org.apache.hadoop.fs.*"
+ import="org.apache.hadoop.util.*"
+ import="java.text.*"
+ import="org.apache.hadoop.mapred.JobHistory.*"
+%>
+<%! private static SimpleDateFormat dateFormat = new SimpleDateFormat("d/MM HH:mm:ss") ;
+ private static final long serialVersionUID = 1L;
+%>
+
+<%
+ String jobid = request.getParameter("jobid");
+ String attemptid = request.getParameter("attemptid");
+ String taskid = request.getParameter("taskid");
+ String logFile = request.getParameter("logFile");
+ String encodedLogFileName =
+ JobHistory.JobInfo.encodeJobHistoryFilePath(logFile);
+
+ Format decimal = new DecimalFormat();
+
+ FileSystem fs = (FileSystem) application.getAttribute("fileSys");
+ JobHistory.JobInfo job = JSPUtil.getJobInfo(request, fs);
+
+ JobHistory.Task task = job.getAllTasks().get(taskid);
+ JobHistory.TaskAttempt attempt = task.getTaskAttempts().get(attemptid);
+
+ Counters counters =
+ Counters.fromEscapedCompactString(attempt.get(Keys.COUNTERS));
+%>
+
+<html>
+ <head>
+ <title>Counters for <%=attemptid%></title>
+ </head>
+<body>
+<h1>Counters for <%=attemptid%></h1>
+
+<hr>
+
+<%
+ if (counters == null) {
+%>
+ <h3>No counter information found for this attempt</h3>
+<%
+ } else {
+%>
+ <table>
+<%
+ for (String groupName : counters.getGroupNames()) {
+ Counters.Group group = counters.getGroup(groupName);
+ String displayGroupName = group.getDisplayName();
+%>
+ <tr>
+ <td colspan="3"><br/><b><%=displayGroupName%></b></td>
+ </tr>
+<%
+ Iterator<Counters.Counter> ctrItr = group.iterator();
+ while(ctrItr.hasNext()) {
+ Counters.Counter counter = ctrItr.next();
+ String displayCounterName = counter.getDisplayName();
+ long value = counter.getCounter();
+%>
+ <tr>
+ <td width="50"></td>
+ <td><%=displayCounterName%></td>
+ <td align="right"><%=decimal.format(value)%></td>
+ </tr>
+<%
+ }
+ }
+%>
+ </table>
+<%
+ }
+%>
+
+<hr>
+<a href="jobdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=encodedLogFileName%>">Go back to the job</a><br>
+<a href="jobtracker.jsp">Go back to JobTracker</a><br>
+<%
+out.println(ServletUtil.htmlFooter());
+%>