You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by mc...@apache.org on 2005/06/30 21:08:40 UTC
svn commit: r208663 - in /lucene/nutch/branches/mapred/src:
java/org/apache/nutch/mapred/JobSubmissionProtocol.java
java/org/apache/nutch/mapred/JobTracker.java
java/org/apache/nutch/mapred/LocalJobRunner.java
webapps/jobtracker/jobdetails.jsp
Author: mc
Date: Thu Jun 30 12:08:38 2005
New Revision: 208663
URL: http://svn.apache.org/viewcvs?rev=208663&view=rev
Log:
Supply diagnostic info to HTML interface and job client
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobSubmissionProtocol.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/LocalJobRunner.java
lucene/nutch/branches/mapred/src/webapps/jobtracker/jobdetails.jsp
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobSubmissionProtocol.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobSubmissionProtocol.java?rev=208663&r1=208662&r2=208663&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobSubmissionProtocol.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobSubmissionProtocol.java Thu Jun 30 12:08:38 2005
@@ -17,6 +17,7 @@
package org.apache.nutch.mapred;
import java.io.*;
+import java.util.*;
import org.apache.nutch.io.*;
/**
@@ -45,6 +46,12 @@
* Grab a handle to a job that is already known to the JobTracker
*/
public JobStatus getJobStatus(String jobid);
+
+ /**
+ * Grab a bunch of info on the tasks that make up the job
+ */
+ public Vector[] getMapTaskReport(String jobid);
+ public Vector[] getReduceTaskReport(String jobid);
/**
* A MapReduce system always operates on a single filesystem. This
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java?rev=208663&r1=208662&r2=208663&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java Thu Jun 30 12:08:38 2005
@@ -443,6 +443,53 @@
return null;
}
}
+ public Vector[] getMapTaskReport(String jobid) {
+ JobInProgress job = (JobInProgress) jobs.get(jobid);
+ if (job == null) {
+ return new Vector[0];
+ } else {
+ Vector reports = new Vector();
+ TreeMap completeMapTasks = job.getCompleteMapTasks();
+ for (Iterator it = completeMapTasks.keySet().iterator(); it.hasNext(); ) {
+ String taskid = (String) it.next();
+ reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid)));
+ }
+ TreeMap incompleteMapTasks = job.getIncompleteMapTasks();
+ for (Iterator it = incompleteMapTasks.keySet().iterator(); it.hasNext(); ) {
+ String taskid = (String) it.next();
+ reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid)));
+ }
+ return (Vector[]) reports.toArray(new Vector[reports.size()]);
+ }
+ }
+
+ public Vector[] getReduceTaskReport(String jobid) {
+ JobInProgress job = (JobInProgress) jobs.get(jobid);
+ if (job == null) {
+ return new Vector[0];
+ } else {
+ Vector reports = new Vector();
+ TreeMap completeReduceTasks = job.getCompleteReduceTasks();
+ for (Iterator it = completeReduceTasks.keySet().iterator(); it.hasNext(); ) {
+ String taskid = (String) it.next();
+ reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid)));
+ }
+ TreeMap incompleteReduceTasks = job.getIncompleteReduceTasks();
+ for (Iterator it = incompleteReduceTasks.keySet().iterator(); it.hasNext(); ) {
+ String taskid = (String) it.next();
+ reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid)));
+ }
+ return (Vector[]) reports.toArray(new Vector[reports.size()]);
+ }
+ }
+
+ Vector generateSingleReport(String taskid, TaskStatus status, String diagInfo) {
+ Vector report = new Vector();
+ report.add(taskid);
+ report.add("" + status.getProgress());
+ report.add(diagInfo);
+ return report;
+ }
public String getFilesystemName() throws IOException {
return fs.getName();
@@ -483,6 +530,7 @@
TreeMap incompleteReduceTasks = new TreeMap();
TreeMap completeReduceTasks = new TreeMap();
TreeMap taskFailures = new TreeMap();
+ TreeMap taskDiagnosticData = new TreeMap();
// Info for user; useless for JobTracker
int numMapTasks = 0;
@@ -712,6 +760,18 @@
//////////////////////////
// A number of accessors
//////////////////////////
+ TreeMap getCompleteMapTasks() {
+ return completeMapTasks;
+ }
+ TreeMap getIncompleteMapTasks() {
+ return incompleteMapTasks;
+ }
+ TreeMap getCompleteReduceTasks() {
+ return completeReduceTasks;
+ }
+ TreeMap getIncompleteReduceTasks() {
+ return incompleteReduceTasks;
+ }
public JobProfile getProfile() {
return profile;
}
@@ -740,6 +800,14 @@
}
return t;
}
+ public String getTaskDiagnosticInfo(String taskid) {
+ StringBuffer buf = (StringBuffer) taskDiagnosticData.get(taskid);
+ if (buf == null) {
+ return "";
+ } else {
+ return buf.toString().trim();
+ }
+ }
public float completedRatio() {
return (0.5f * status.mapProgress()) + (0.5f * status.reduceProgress());
}
@@ -778,7 +846,12 @@
String diagInfo = status.getDiagnosticInfo();
if (diagInfo != null && diagInfo.length() > 0) {
- LOG.info("Task '" + taskid + "' has reported diagnostic info:\n" + diagInfo);
+ StringBuffer buf = (StringBuffer) taskDiagnosticData.get(taskid);
+ if (buf == null) {
+ buf = new StringBuffer();
+ taskDiagnosticData.put(taskid, buf);
+ }
+ buf.append(diagInfo);
}
if (incompleteMapTasks.get(taskid) != null ||
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/LocalJobRunner.java?rev=208663&r1=208662&r2=208663&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/LocalJobRunner.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/LocalJobRunner.java Thu Jun 30 12:08:38 2005
@@ -146,6 +146,13 @@
return new JobProfile(id, job.file, "http://localhost:8080/");
}
+ public Vector[] getMapTaskReport(String id) {
+ return new Vector[0];
+ }
+ public Vector[] getReduceTaskReport(String id) {
+ return new Vector[0];
+ }
+
public JobStatus getJobStatus(String id) {
Job job = (Job)jobs.get(id);
return job.status;
Modified: lucene/nutch/branches/mapred/src/webapps/jobtracker/jobdetails.jsp
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/webapps/jobtracker/jobdetails.jsp?rev=208663&r1=208662&r2=208663&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/webapps/jobtracker/jobdetails.jsp (original)
+++ lucene/nutch/branches/mapred/src/webapps/jobtracker/jobdetails.jsp Thu Jun 30 12:08:38 2005
@@ -12,6 +12,9 @@
JobTracker.JobInProgress job = (JobTracker.JobInProgress) tracker.getJob(jobid);
JobProfile profile = (job != null) ? (job.getProfile()) : null;
JobStatus status = (job != null) ? (job.getStatus()) : null;
+
+ Vector mapTaskReports[] = job.getMapTaskReport(jobid);
+ Vector reduceTaskReports[] = job.getReduceTaskReport(jobid);
%>
<html>
@@ -35,10 +38,13 @@
<h2>Map Tasks</h2>
<center>
<table border=2 cellpadding="5" cellspacing="2">
- <tr><td align="center" colspan="4">Map Tasks</td></tr>
+ <tr><td align="center" colspan="4">Map Task Id</td><td>Pct Complete</td><td>Diagnostic Text</td></tr>
<%
-
+ for (int i = 0; i < mapTaskReports.length; i++) {
+ Vector v = mapTaskReports[i];
+ out.print("<tr><td>" + v.elementAt(0) + "</td><td>" + v.elementAt(1) + "</td><td>" + v.elementAt(2) + "</td></tr>\n");
+ }
%>
</table>
</center>
@@ -46,6 +52,18 @@
<h2>Reduce Tasks</h2>
+ <center>
+ <table border=2 cellpadding="5" cellspacing="2">
+ <tr><td align="center" colspan="4">Reduce Task Id</td><td>Pct Complete</td><td>Diagnostic Text</td></tr>
+
+ <%
+ for (int i = 0; i < reduceTaskReports.length; i++) {
+ Vector v = reduceTaskReports[i];
+ out.print("<tr><td>" + v.elementAt(0) + "</td><td>" + v.elementAt(1) + "</td><td>" + v.elementAt(2) + "</td></tr>\n");
+ }
+ %>
+ </table>
+ </center>
<hr>