You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by mc...@apache.org on 2005/09/08 21:28:26 UTC
svn commit: r279603 - in /lucene/nutch/branches/mapred/src:
java/org/apache/nutch/mapred/JobTracker.java
java/org/apache/nutch/mapred/TaskStatus.java
java/org/apache/nutch/mapred/TaskTracker.java
webapps/jobtracker/jobdetails.jsp
Author: mc
Date: Thu Sep 8 12:28:24 2005
New Revision: 279603
URL: http://svn.apache.org/viewcvs?rev=279603&view=rev
Log:
Add the task state string to the web interface. This
lets you track current progress through a map or reduce
job.
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskStatus.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java
lucene/nutch/branches/mapred/src/webapps/jobtracker/jobdetails.jsp
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java?rev=279603&r1=279602&r2=279603&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java Thu Sep 8 12:28:24 2005
@@ -461,12 +461,12 @@
TreeMap completeMapTasks = job.getCompleteMapTasks();
for (Iterator it = completeMapTasks.keySet().iterator(); it.hasNext(); ) {
String taskid = (String) it.next();
- reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid)));
+ reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
}
TreeMap incompleteMapTasks = job.getIncompleteMapTasks();
for (Iterator it = incompleteMapTasks.keySet().iterator(); it.hasNext(); ) {
String taskid = (String) it.next();
- reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid)));
+ reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
}
return (Vector[]) reports.toArray(new Vector[reports.size()]);
}
@@ -481,12 +481,12 @@
TreeMap completeReduceTasks = job.getCompleteReduceTasks();
for (Iterator it = completeReduceTasks.keySet().iterator(); it.hasNext(); ) {
String taskid = (String) it.next();
- reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid)));
+ reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
}
TreeMap incompleteReduceTasks = job.getIncompleteReduceTasks();
for (Iterator it = incompleteReduceTasks.keySet().iterator(); it.hasNext(); ) {
String taskid = (String) it.next();
- reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid)));
+ reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
}
return (Vector[]) reports.toArray(new Vector[reports.size()]);
}
@@ -496,10 +496,11 @@
// (See InterTrackerProtocol section for getFilesystemName())
//////////////////////////////////////////////////////////////
- Vector generateSingleReport(String taskid, TaskStatus status, Vector diagInfo) {
+ Vector generateSingleReport(String taskid, TaskStatus status, Vector diagInfo, String stateString) {
Vector report = new Vector();
report.add(taskid);
report.add("" + status.getProgress());
+ report.add(stateString);
report.addAll(diagInfo);
return report;
}
@@ -554,6 +555,7 @@
TreeMap completeReduceTasks = new TreeMap();
TreeMap taskFailures = new TreeMap();
TreeMap taskDiagnosticData = new TreeMap();
+ TreeMap taskStateStrings = new TreeMap();
// Info for user; useless for JobTracker
int numMapTasks = 0;
@@ -840,6 +842,9 @@
return v;
}
}
+ public String getTaskStateString(String taskid) {
+ return (String) taskStateStrings.get(taskid);
+ }
public float completedRatio() {
return (0.5f * status.mapProgress()) + (0.5f * status.reduceProgress());
}
@@ -886,6 +891,8 @@
v.add(diagInfo);
}
+ taskStateStrings.put(taskid, status.getStateString());
+
if (incompleteMapTasks.get(taskid) != null ||
completeMapTasks.get(taskid) != null) {
totalReportedMapProgress += delta;
@@ -919,7 +926,7 @@
} else if (incompleteReduceTasks.get(taskid) != null) {
attemptedReduceExecutions++;
}
- updateTaskStatus(taskid, new TaskStatus(taskid, 0.0f, TaskStatus.UNASSIGNED, ""));
+ updateTaskStatus(taskid, new TaskStatus(taskid, 0.0f, TaskStatus.UNASSIGNED, "", ""));
unassignedTasks.add(taskid);
}
}
@@ -986,7 +993,7 @@
// Move task status to RUNNING
JobInProgress job = (JobInProgress) jobs.get((String) taskToJobMap.get(taskid));
- job.updateTaskStatus(taskid, new TaskStatus(taskid, 0.0f, TaskStatus.RUNNING, ""));
+ job.updateTaskStatus(taskid, new TaskStatus(taskid, 0.0f, TaskStatus.RUNNING, "", ""));
// Remember where we are running it
TreeSet taskset = (TreeSet) trackerToTaskMap.get(taskTracker);
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskStatus.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskStatus.java?rev=279603&r1=279602&r2=279603&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskStatus.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskStatus.java Thu Sep 8 12:28:24 2005
@@ -37,14 +37,16 @@
private float progress;
private int runState;
private String diagnosticInfo;
+ private String stateString;
public TaskStatus() {}
- public TaskStatus(String taskid, float progress, int runState, String diagnosticInfo) {
+ public TaskStatus(String taskid, float progress, int runState, String diagnosticInfo, String stateString) {
this.taskid = taskid;
this.progress = progress;
this.runState = runState;
this.diagnosticInfo = diagnosticInfo;
+ this.stateString = stateString;
}
public String getTaskId() { return taskid; }
@@ -54,6 +56,8 @@
public void setRunState(int runState) { this.runState = runState; }
public String getDiagnosticInfo() { return diagnosticInfo; }
public void setDiagnosticInfo(String info) { this.diagnosticInfo = info; }
+ public String getStateString() { return stateString; }
+ public void setStateString(String stateString) { this.stateString = stateString; }
//////////////////////////////////////////////
// Writable
@@ -63,6 +67,7 @@
out.writeFloat(progress);
out.writeInt(runState);
UTF8.writeString(out, diagnosticInfo);
+ UTF8.writeString(out, stateString);
}
public void readFields(DataInput in) throws IOException {
@@ -70,5 +75,6 @@
this.progress = in.readFloat();
this.runState = in.readInt();
this.diagnosticInfo = UTF8.readString(in);
+ this.stateString = UTF8.readString(in);
}
}
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java?rev=279603&r1=279602&r2=279603&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java Thu Sep 8 12:28:24 2005
@@ -299,6 +299,7 @@
File localTaskDir;
float progress;
int runstate;
+ String stateString = "";
long lastProgressReport;
StringBuffer diagnosticInfo = new StringBuffer();
TaskRunner runner;
@@ -355,7 +356,7 @@
/**
*/
public TaskStatus createStatus() {
- TaskStatus status = new TaskStatus(task.getTaskId(), progress, runstate, diagnosticInfo.toString());
+ TaskStatus status = new TaskStatus(task.getTaskId(), progress, runstate, diagnosticInfo.toString(), (stateString == null) ? "" : stateString);
if (diagnosticInfo.length() > 0) {
diagnosticInfo = new StringBuffer();
}
@@ -381,6 +382,7 @@
this.progress = p;
this.runstate = TaskStatus.RUNNING;
this.lastProgressReport = System.currentTimeMillis();
+ this.stateString = state;
}
/**
Modified: lucene/nutch/branches/mapred/src/webapps/jobtracker/jobdetails.jsp
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/webapps/jobtracker/jobdetails.jsp?rev=279603&r1=279602&r2=279603&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/webapps/jobtracker/jobdetails.jsp (original)
+++ lucene/nutch/branches/mapred/src/webapps/jobtracker/jobdetails.jsp Thu Sep 8 12:28:24 2005
@@ -38,16 +38,16 @@
<h2>Map Tasks</h2>
<center>
<table border=2 cellpadding="5" cellspacing="2">
- <tr><td align="center">Map Task Id</td><td>Pct Complete</td><td>Diagnostic Text</td></tr>
+ <tr><td align="center">Map Task Id</td><td>Pct Complete</td><td>State</td><td>Diagnostic Text</td></tr>
<%
for (int i = 0; i < mapTaskReports.length; i++) {
Vector v = mapTaskReports[i];
- out.print("<tr><td>" + v.elementAt(0) + "</td><td>" + v.elementAt(1) + "</td>");
- if (v.size() == 2) {
+ out.print("<tr><td>" + v.elementAt(0) + "</td><td>" + v.elementAt(1) + "</td><td>" + v.elementAt(2) + "</td>");
+ if (v.size() == 3) {
out.print("<td></td>");
} else {
- for (int j = 2; j < v.size(); j++) {
+ for (int j = 3; j < v.size(); j++) {
out.print("<td>" + v.elementAt(j) + "</td>");
}
}
@@ -62,16 +62,16 @@
<h2>Reduce Tasks</h2>
<center>
<table border=2 cellpadding="5" cellspacing="2">
- <tr><td align="center">Reduce Task Id</td><td>Pct Complete</td><td>Diagnostic Text</td></tr>
+ <tr><td align="center">Reduce Task Id</td><td>Pct Complete</td><td>State</td><td>Diagnostic Text</td></tr>
<%
for (int i = 0; i < reduceTaskReports.length; i++) {
Vector v = reduceTaskReports[i];
- out.print("<tr><td>" + v.elementAt(0) + "</td><td>" + v.elementAt(1) + "</td>");
- if (v.size() == 2) {
+ out.print("<tr><td>" + v.elementAt(0) + "</td><td>" + v.elementAt(1) + "</td><td>" + v.elementAt(2) + "</td>");
+ if (v.size() == 3) {
out.print("<td></td>");
} else {
- for (int j = 2; j < v.size(); j++) {
+ for (int j = 3; j < v.size(); j++) {
out.print("<td>" + v.elementAt(j) + "</td>");
}
}