You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/09/14 21:59:09 UTC
svn commit: r443452 - in /lucene/hadoop/trunk: ./
src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/util/
src/webapps/job/
Author: cutting
Date: Thu Sep 14 12:59:08 2006
New Revision: 443452
URL: http://svn.apache.org/viewvc?view=rev&rev=443452
Log:
HADOOP-243. Fix rounding in the display of task and job progress so that things are not shown as 100% complete until they are in fact finished. Contributed by Owen.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobStatus.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ToolBase.java
lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp
lucene/hadoop/trunk/src/webapps/job/jobtasks.jsp
lucene/hadoop/trunk/src/webapps/job/jobtracker.jsp
lucene/hadoop/trunk/src/webapps/job/machines.jsp
lucene/hadoop/trunk/src/webapps/job/taskdetails.jsp
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Sep 14 12:59:08 2006
@@ -1,6 +1,13 @@
Hadoop Change Log
+Trunk (unreleased changes)
+
+1. HADOOP-243. Fix rounding in the display of task and job progress
+ so that things are not shown to be 100% complete until they are in
+ fact finished. (omalley via cutting)
+
+
Release 0.6.1 - 2006-08-13
1. HADOOP-520. Fix a bug in libhdfs, where write failures were not
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java Thu Sep 14 12:59:08 2006
@@ -342,9 +342,10 @@
break;
}
running = jc.getJob(jobId);
- String report = (" map " + Math.round(running.mapProgress()*100)+
- "% reduce " +
- Math.round(running.reduceProgress()*100) + "%");
+ String report =
+ (" map " + StringUtils.formatPercent(running.mapProgress())+
+ " reduce " +
+ StringUtils.formatPercent(running.reduceProgress()));
if (!report.equals(lastReport)) {
LOG.info(report);
lastReport = report;
@@ -402,7 +403,6 @@
init();
// Process args
- String jobTrackerSpec = null;
String submitJobFile = null;
String jobid = null;
boolean getStatus = false;
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Thu Sep 14 12:59:08 2006
@@ -293,13 +293,16 @@
if (maps.length == 0) {
this.status.setMapProgress(1.0f);
} else {
- this.status.mapProgress += (progressDelta / maps.length);
+ this.status.setMapProgress((float) (this.status.mapProgress() +
+ progressDelta / maps.length));
}
} else {
if (reduces.length == 0) {
this.status.setReduceProgress(1.0f);
} else {
- this.status.reduceProgress += (progressDelta / reduces.length);
+ this.status.setReduceProgress
+ ((float) (this.status.reduceProgress() +
+ (progressDelta / reduces.length)));
}
}
}
@@ -477,6 +480,9 @@
}
}
if (allDone) {
+ if (tip.isMapTask()) {
+ this.status.setMapProgress(1.0f);
+ }
for (int i = 0; i < reduces.length; i++) {
if (! reduces[i].isComplete()) {
allDone = false;
@@ -489,7 +495,8 @@
// If all tasks are complete, then the job is done!
//
if (status.getRunState() == JobStatus.RUNNING && allDone) {
- this.status.runState = JobStatus.SUCCEEDED;
+ this.status.setRunState(JobStatus.SUCCEEDED);
+ this.status.setReduceProgress(1.0f);
this.finishTime = System.currentTimeMillis();
garbageCollect();
LOG.info("Job " + this.status.getJobId() +
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobStatus.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobStatus.java?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobStatus.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobStatus.java Thu Sep 14 12:59:08 2006
@@ -41,12 +41,12 @@
public static final int FAILED = 3;
public static final int PREP = 4;
- String jobid;
- float mapProgress;
- float reduceProgress;
- int runState;
- long startTime;
- String user;
+ private String jobid;
+ private float mapProgress;
+ private float reduceProgress;
+ private int runState;
+ private long startTime;
+ private String user;
/**
*/
public JobStatus() {
@@ -81,7 +81,10 @@
* Sets the map progress of this job
* @param p The value of map progress to set to
*/
- void setMapProgress(float p) { this.mapProgress = p; }
+ void setMapProgress(float p) {
+ this.mapProgress = (float) Math.min(1.0, Math.max(0.0, p));
+
+ }
/**
* @return Percentage of progress in reduce
@@ -92,12 +95,21 @@
* Sets the reduce progress of this Job
* @param p The value of reduce progress to set to
*/
- void setReduceProgress(float p) { this.reduceProgress = p; }
+ void setReduceProgress(float p) {
+ this.reduceProgress = (float) Math.min(1.0, Math.max(0.0, p));
+ }
/**
* @return running state of the job
*/
public int getRunState() { return runState; }
+
+ /**
+ * Change the current run state of the job.
+ */
+ public void setRunState(int state) {
+ this.runState = state;
+ }
/**
* Set the start time of the job
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Thu Sep 14 12:59:08 2006
@@ -49,7 +49,7 @@
private JobConf job;
private Random random = new Random();
- private JobStatus status = new JobStatus();
+ private JobStatus status;
private ArrayList mapIds = new ArrayList();
private MapOutputFile mapoutputFile;
private JobProfile profile;
@@ -73,8 +73,7 @@
this.job = new JobConf(localFile);
profile = new JobProfile(job.getUser(), id, file,
"http://localhost:8080/", job.getJobName());
- this.status.jobid = id;
- this.status.runState = JobStatus.RUNNING;
+ status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING);
jobs.put(id, this);
@@ -134,10 +133,10 @@
}
this.mapoutputFile.removeAll(reduceId);
- this.status.runState = JobStatus.SUCCEEDED;
+ this.status.setRunState(JobStatus.SUCCEEDED);
} catch (Throwable t) {
- this.status.runState = JobStatus.FAILED;
+ this.status.setRunState(JobStatus.FAILED);
LOG.warn(id, t);
} finally {
@@ -163,9 +162,9 @@
float taskIndex = mapIds.indexOf(taskId);
if (taskIndex >= 0) { // mapping
float numTasks = mapIds.size();
- status.mapProgress = (taskIndex/numTasks)+(progress/numTasks);
+ status.setMapProgress(taskIndex/numTasks + progress/numTasks);
} else {
- status.reduceProgress = progress;
+ status.setReduceProgress(progress);
}
}
@@ -180,9 +179,9 @@
public void done(String taskId) throws IOException {
int taskIndex = mapIds.indexOf(taskId);
if (taskIndex >= 0) { // mapping
- status.mapProgress = 1.0f;
+ status.setMapProgress(1.0f);
} else {
- status.reduceProgress = 1.0f;
+ status.setReduceProgress(1.0f);
}
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java Thu Sep 14 12:59:08 2006
@@ -52,7 +52,7 @@
return fullHostname;
}
- private static DecimalFormat numFormat = new DecimalFormat("0.0");
+ private static DecimalFormat oneDecimal = new DecimalFormat("0.0");
/**
* Given an integer, return a string that is in an approximate, but human
@@ -77,7 +77,15 @@
result = number / (1024.0 * 1024 * 1024);
suffix = "g";
}
- return numFormat.format(result) + suffix;
+ return oneDecimal.format(result) + suffix;
+ }
+
+ private static DecimalFormat percentFormat = new DecimalFormat("0.00%");
+
+ public static String formatPercent(double done) {
+ final int scale = 10000;
+ double rounded = Math.floor(done * scale);
+ return percentFormat.format(rounded / scale);
}
/**
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ToolBase.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ToolBase.java?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ToolBase.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ToolBase.java Thu Sep 14 12:59:08 2006
@@ -80,7 +80,7 @@
*
*/
public abstract class ToolBase implements Tool {
- public static final Log LOG = LogFactory.getLog(
+ private static final Log LOG = LogFactory.getLog(
"org.apache.hadoop.util.ToolBase");
public Configuration conf;
Modified: lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/jobdetails.jsp Thu Sep 14 12:59:08 2006
@@ -36,7 +36,7 @@
}
out.print("<tr><th><a href=\"/jobtasks.jsp?jobid=" + jobId +
"&type="+ kind + "&pagenum=1\">" + kind + "</a></th><td>" +
- percentFormat.format(100.0 * completePercent) +
+ StringUtils.formatPercent(completePercent) +
"</td><td>" + totalTasks + "</td><td>" +
(totalTasks - runningTasks - finishedTasks) + "</td><td>" +
runningTasks + "</td><td>" +
Modified: lucene/hadoop/trunk/src/webapps/job/jobtasks.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/jobtasks.jsp?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/jobtasks.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/jobtasks.jsp Thu Sep 14 12:59:08 2006
@@ -63,7 +63,8 @@
out.print("<tr><td><a href=\"taskdetails.jsp?jobid=" + jobid +
"&taskid=" + report.getTaskId() + "\">" +
report.getTaskId() + "</a></td>");
- out.print("<td>" + report.getProgress() + "</td>");
+ out.print("<td>" + StringUtils.formatPercent(report.getProgress()) +
+ "</td>");
out.print("<td>" + report.getState() + "</td>");
String[] diagnostics = report.getDiagnostics();
for (int j = 0; j < diagnostics.length ; j++) {
Modified: lucene/hadoop/trunk/src/webapps/job/jobtracker.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/jobtracker.jsp?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/jobtracker.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/jobtracker.jsp Thu Sep 14 12:59:08 2006
@@ -44,11 +44,11 @@
"<td>" + profile.getUser() + "</td>"
+ "<td>" + ("".equals(name) ? " " : name) + "</td>" +
"<td>" +
- percentFormat.format(100.0 * status.mapProgress()) +
- "%</td><td>" +
+ StringUtils.formatPercent(status.mapProgress()) +
+ "</td><td>" +
desiredMaps + "</td><td>" + completedMaps + "</td><td>" +
- percentFormat.format(100.0 * status.reduceProgress()) +
- "%</td><td>" +
+ StringUtils.formatPercent(status.reduceProgress()) +
+ "</td><td>" +
desiredReduces + "</td><td> " + completedReduces +
"</td></tr>\n");
}
Modified: lucene/hadoop/trunk/src/webapps/job/machines.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/machines.jsp?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/machines.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/machines.jsp Thu Sep 14 12:59:08 2006
@@ -12,7 +12,6 @@
JobTracker tracker = JobTracker.getTracker();
String trackerLabel =
StringUtils.simpleHostname(tracker.getJobTrackerMachine());
- private static DecimalFormat percentFormat = new DecimalFormat("##0.00");
public void generateTaskTrackerTable(JspWriter out) throws IOException {
Collection c = tracker.taskTrackers();
Modified: lucene/hadoop/trunk/src/webapps/job/taskdetails.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/taskdetails.jsp?view=diff&rev=443452&r1=443451&r2=443452
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/taskdetails.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/taskdetails.jsp Thu Sep 14 12:59:08 2006
@@ -6,6 +6,7 @@
import="java.lang.String"
import="java.util.*"
import="org.apache.hadoop.mapred.*"
+ import="org.apache.hadoop.util.*"
%>
<%
String jobid = request.getParameter("jobid");
@@ -65,7 +66,8 @@
out.print("<td>");
writeString(out, status.getRunState());
out.print("</td>");
- out.print("<td>"+status.getProgress()+"</td>");
+ out.print("<td>"+ StringUtils.formatPercent(status.getProgress()) +
+ "</td>");
out.print("<td><pre>" + status.getDiagnosticInfo() + "</pre></td>");
out.print("</tr>\n");
}