You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2011/12/15 00:34:10 UTC
svn commit: r1214546 [2/5] - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-map...
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java Wed Dec 14 23:34:04 2011
@@ -18,8 +18,9 @@
package org.apache.hadoop.mapreduce.v2.app.webapp;
-import static org.apache.hadoop.yarn.util.StringHelper.*;
+import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
/**
@@ -29,6 +30,9 @@ public class AMWebApp extends WebApp imp
@Override
public void setup() {
+ bind(JAXBContextResolver.class);
+ bind(GenericExceptionHandler.class);
+ bind(AMWebServices.class);
route("/", AppController.class);
route("/app", AppController.class);
route(pajoin("/job", JOB_ID), AppController.class, "job");
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java Wed Dec 14 23:34:04 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.JobAC
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AppInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -72,13 +73,14 @@ public class AppController extends Contr
* Render the /info page with an overview of current application.
*/
public void info() {
+ AppInfo info = new AppInfo(app, app.context);
info("Application Master Overview").
- _("Application ID:", $(APP_ID)).
- _("Application Name:", app.context.getApplicationName()).
- _("User:", app.context.getUser()).
- _("Started on:", Times.format(app.context.getStartTime())).
+ _("Application ID:", info.getId()).
+ _("Application Name:", info.getName()).
+ _("User:", info.getUser()).
+ _("Started on:", Times.format(info.getStartTime())).
_("Elasped: ", org.apache.hadoop.util.StringUtils.formatTime(
- Times.elapsed(app.context.getStartTime(), 0)));
+ info.getElapsedTime() ));
render(InfoPage.class);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java Wed Dec 14 23:34:04 2011
@@ -22,14 +22,14 @@ import static org.apache.hadoop.mapreduc
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
import java.io.IOException;
-import java.util.Map;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.ConfEntryInfo;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.ConfInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
@@ -71,11 +71,8 @@ public class ConfBlock extends HtmlBlock
}
Path confPath = job.getConfFile();
try {
- //Read in the configuration file and put it in a key/value table.
- FileContext fc = FileContext.getFileContext(confPath.toUri(), conf);
- Configuration jobConf = new Configuration(false);
- jobConf.addResource(fc.open(confPath));
-
+ ConfInfo info = new ConfInfo(job, this.conf);
+
html.div().h3(confPath.toString())._();
TBODY<TABLE<Hamlet>> tbody = html.
// Tasks table
@@ -87,10 +84,10 @@ public class ConfBlock extends HtmlBlock
_().
_().
tbody();
- for(Map.Entry<String, String> entry : jobConf) {
+ for (ConfEntryInfo entry : info.getProperties()) {
tbody.
tr().
- td(entry.getKey()).
+ td(entry.getName()).
td(entry.getValue()).
_();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java Wed Dec 14 23:34:04 2011
@@ -18,47 +18,32 @@
package org.apache.hadoop.mapreduce.v2.app.webapp;
-import com.google.inject.Inject;
+import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.JOB_ID;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._EVEN;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
+
import java.util.Date;
-import java.util.Map;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.mapreduce.v2.util.MRApps.TaskAttemptStateUI;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
-import static org.apache.hadoop.mapreduce.v2.app.webapp.AMWebApp.*;
-import static org.apache.hadoop.yarn.util.StringHelper.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+
+import com.google.inject.Inject;
public class JobBlock extends HtmlBlock {
final AppContext appContext;
- int runningMapTasks = 0;
- int pendingMapTasks = 0;
- int runningReduceTasks = 0;
- int pendingReduceTasks = 0;
-
- int newMapAttempts = 0;
- int runningMapAttempts = 0;
- int killedMapAttempts = 0;
- int failedMapAttempts = 0;
- int successfulMapAttempts = 0;
- int newReduceAttempts = 0;
- int runningReduceAttempts = 0;
- int killedReduceAttempts = 0;
- int failedReduceAttempts = 0;
- int successfulReduceAttempts = 0;
-
@Inject JobBlock(AppContext appctx) {
appContext = appctx;
}
@@ -77,23 +62,13 @@ public class JobBlock extends HtmlBlock
p()._("Sorry, ", jid, " not found.")._();
return;
}
- JobReport jobReport = job.getReport();
- String mapPct = percent(jobReport.getMapProgress());
- String reducePct = percent(jobReport.getReduceProgress());
- int mapTasks = job.getTotalMaps();
- int mapTasksComplete = job.getCompletedMaps();
- int reduceTasks = job.getTotalReduces();
- int reducesTasksComplete = job.getCompletedReduces();
- long startTime = jobReport.getStartTime();
- long finishTime = jobReport.getFinishTime();
- countTasksAndAttempts(job);
+ JobInfo jinfo = new JobInfo(job, true);
info("Job Overview").
- _("Job Name:", job.getName()).
- _("State:", job.getState()).
- _("Uberized:", job.isUber()).
- _("Started:", new Date(startTime)).
- _("Elapsed:", StringUtils.formatTime(
- Times.elapsed(startTime, finishTime)));
+ _("Job Name:", jinfo.getName()).
+ _("State:", jinfo.getState()).
+ _("Uberized:", jinfo.isUberized()).
+ _("Started:", new Date(jinfo.getStartTime())).
+ _("Elapsed:", StringUtils.formatTime(jinfo.getElapsedTime()));
html.
_(InfoBlock.class).
div(_INFO_WRAP).
@@ -112,25 +87,25 @@ public class JobBlock extends HtmlBlock
a(url("tasks", jid, "m"), "Map")._().
td().
div(_PROGRESSBAR).
- $title(join(mapPct, '%')). // tooltip
+ $title(join(jinfo.getMapProgressPercent(), '%')). // tooltip
div(_PROGRESSBAR_VALUE).
- $style(join("width:", mapPct, '%'))._()._()._().
- td(String.valueOf(mapTasks)).
- td(String.valueOf(pendingMapTasks)).
- td(String.valueOf(runningMapTasks)).
- td(String.valueOf(mapTasksComplete))._().
+ $style(join("width:", jinfo.getMapProgressPercent(), '%'))._()._()._().
+ td(String.valueOf(jinfo.getMapsTotal())).
+ td(String.valueOf(jinfo.getMapsPending())).
+ td(String.valueOf(jinfo.getMapsRunning())).
+ td(String.valueOf(jinfo.getMapsCompleted()))._().
tr(_EVEN).
th().
a(url("tasks", jid, "r"), "Reduce")._().
td().
div(_PROGRESSBAR).
- $title(join(reducePct, '%')). // tooltip
+ $title(join(jinfo.getReduceProgressPercent(), '%')). // tooltip
div(_PROGRESSBAR_VALUE).
- $style(join("width:", reducePct, '%'))._()._()._().
- td(String.valueOf(reduceTasks)).
- td(String.valueOf(pendingReduceTasks)).
- td(String.valueOf(runningReduceTasks)).
- td(String.valueOf(reducesTasksComplete))._()
+ $style(join("width:", jinfo.getReduceProgressPercent(), '%'))._()._()._().
+ td(String.valueOf(jinfo.getReducesTotal())).
+ td(String.valueOf(jinfo.getReducesPending())).
+ td(String.valueOf(jinfo.getReducesRunning())).
+ td(String.valueOf(jinfo.getReducesCompleted()))._()
._().
// Attempts table
@@ -145,110 +120,41 @@ public class JobBlock extends HtmlBlock
tr(_ODD).
th("Maps").
td().a(url("attempts", jid, "m",
- TaskAttemptStateUI.NEW.toString()),
- String.valueOf(newMapAttempts))._().
+ TaskAttemptStateUI.NEW.toString()),
+ String.valueOf(jinfo.getNewMapAttempts()))._().
td().a(url("attempts", jid, "m",
- TaskAttemptStateUI.RUNNING.toString()),
- String.valueOf(runningMapAttempts))._().
+ TaskAttemptStateUI.RUNNING.toString()),
+ String.valueOf(jinfo.getRunningMapAttempts()))._().
td().a(url("attempts", jid, "m",
- TaskAttemptStateUI.FAILED.toString()),
- String.valueOf(failedMapAttempts))._().
+ TaskAttemptStateUI.FAILED.toString()),
+ String.valueOf(jinfo.getFailedMapAttempts()))._().
td().a(url("attempts", jid, "m",
- TaskAttemptStateUI.KILLED.toString()),
- String.valueOf(killedMapAttempts))._().
+ TaskAttemptStateUI.KILLED.toString()),
+ String.valueOf(jinfo.getKilledMapAttempts()))._().
td().a(url("attempts", jid, "m",
- TaskAttemptStateUI.SUCCESSFUL.toString()),
- String.valueOf(successfulMapAttempts))._().
+ TaskAttemptStateUI.SUCCESSFUL.toString()),
+ String.valueOf(jinfo.getSuccessfulMapAttempts()))._().
_().
tr(_EVEN).
th("Reduces").
td().a(url("attempts", jid, "r",
- TaskAttemptStateUI.NEW.toString()),
- String.valueOf(newReduceAttempts))._().
+ TaskAttemptStateUI.NEW.toString()),
+ String.valueOf(jinfo.getNewReduceAttempts()))._().
td().a(url("attempts", jid, "r",
- TaskAttemptStateUI.RUNNING.toString()),
- String.valueOf(runningReduceAttempts))._().
+ TaskAttemptStateUI.RUNNING.toString()),
+ String.valueOf(jinfo.getRunningReduceAttempts()))._().
td().a(url("attempts", jid, "r",
- TaskAttemptStateUI.FAILED.toString()),
- String.valueOf(failedReduceAttempts))._().
+ TaskAttemptStateUI.FAILED.toString()),
+ String.valueOf(jinfo.getFailedReduceAttempts()))._().
td().a(url("attempts", jid, "r",
- TaskAttemptStateUI.KILLED.toString()),
- String.valueOf(killedReduceAttempts))._().
+ TaskAttemptStateUI.KILLED.toString()),
+ String.valueOf(jinfo.getKilledReduceAttempts()))._().
td().a(url("attempts", jid, "r",
- TaskAttemptStateUI.SUCCESSFUL.toString()),
- String.valueOf(successfulReduceAttempts))._().
+ TaskAttemptStateUI.SUCCESSFUL.toString()),
+ String.valueOf(jinfo.getSuccessfulReduceAttempts()))._().
_().
_().
_();
}
- private void countTasksAndAttempts(Job job) {
- Map<TaskId, Task> tasks = job.getTasks();
- for (Task task : tasks.values()) {
- switch (task.getType()) {
- case MAP:
- // Task counts
- switch (task.getState()) {
- case RUNNING:
- ++runningMapTasks;
- break;
- case SCHEDULED:
- ++pendingMapTasks;
- break;
- }
- break;
- case REDUCE:
- // Task counts
- switch (task.getState()) {
- case RUNNING:
- ++runningReduceTasks;
- break;
- case SCHEDULED:
- ++pendingReduceTasks;
- break;
- }
- break;
- }
-
- // Attempts counts
- Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
- for (TaskAttempt attempt : attempts.values()) {
-
- int newAttempts = 0, running = 0, successful = 0, failed = 0, killed =0;
-
- if (TaskAttemptStateUI.NEW.correspondsTo(attempt.getState())) {
- ++newAttempts;
- } else if (TaskAttemptStateUI.RUNNING.correspondsTo(attempt
- .getState())) {
- ++running;
- } else if (TaskAttemptStateUI.SUCCESSFUL.correspondsTo(attempt
- .getState())) {
- ++successful;
- } else if (TaskAttemptStateUI.FAILED
- .correspondsTo(attempt.getState())) {
- ++failed;
- } else if (TaskAttemptStateUI.KILLED
- .correspondsTo(attempt.getState())) {
- ++killed;
- }
-
- switch (task.getType()) {
- case MAP:
- newMapAttempts += newAttempts;
- runningMapAttempts += running;
- successfulMapAttempts += successful;
- failedMapAttempts += failed;
- killedMapAttempts += killed;
- break;
- case REDUCE:
- newReduceAttempts += newAttempts;
- runningReduceAttempts += running;
- successfulReduceAttempts += successful;
- failedReduceAttempts += failed;
- killedReduceAttempts += killed;
- break;
- }
- }
- }
- }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobsBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobsBlock.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobsBlock.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobsBlock.java Wed Dec 14 23:34:04 2011
@@ -18,18 +18,19 @@
package org.apache.hadoop.mapreduce.v2.app.webapp;
-import com.google.inject.Inject;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE;
-import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobInfo;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import static org.apache.hadoop.yarn.util.StringHelper.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+import com.google.inject.Inject;
public class JobsBlock extends HtmlBlock {
final AppContext appContext;
@@ -54,38 +55,31 @@ public class JobsBlock extends HtmlBlock
th("Reduces Total").
th("Reduces Completed")._()._().
tbody();
- for (Job job : appContext.getAllJobs().values()) {
- String jobID = MRApps.toString(job.getID());
- JobReport report = job.getReport();
- String mapPct = percent(report.getMapProgress());
- String mapsTotal = String.valueOf(job.getTotalMaps());
- String mapsCompleted = String.valueOf(job.getCompletedMaps());
- String reducePct = percent(report.getReduceProgress());
- String reduceTotal = String.valueOf(job.getTotalReduces());
- String reduceCompleted = String.valueOf(job.getCompletedReduces());
+ for (Job j : appContext.getAllJobs().values()) {
+ JobInfo job = new JobInfo(j, false);
tbody.
tr().
td().
- span().$title(String.valueOf(job.getID().getId()))._(). // for sorting
- a(url("job", jobID), jobID)._().
- td(job.getName().toString()).
- td(job.getState().toString()).
+ span().$title(String.valueOf(job.getId()))._(). // for sorting
+ a(url("job", job.getId()), job.getId())._().
+ td(job.getName()).
+ td(job.getState()).
td().
- span().$title(mapPct)._(). // for sorting
+ span().$title(job.getMapProgressPercent())._(). // for sorting
div(_PROGRESSBAR).
- $title(join(mapPct, '%')). // tooltip
+ $title(join(job.getMapProgressPercent(), '%')). // tooltip
div(_PROGRESSBAR_VALUE).
- $style(join("width:", mapPct, '%'))._()._()._().
- td(mapsTotal).
- td(mapsCompleted).
+ $style(join("width:", job.getMapProgressPercent(), '%'))._()._()._().
+ td(String.valueOf(job.getMapsTotal())).
+ td(String.valueOf(job.getMapsCompleted())).
td().
- span().$title(reducePct)._(). // for sorting
+ span().$title(job.getReduceProgressPercent())._(). // for sorting
div(_PROGRESSBAR).
- $title(join(reducePct, '%')). // tooltip
+ $title(join(job.getReduceProgressPercent(), '%')). // tooltip
div(_PROGRESSBAR_VALUE).
- $style(join("width:", reducePct, '%'))._()._()._().
- td(reduceTotal).
- td(reduceCompleted)._();
+ $style(join("width:", job.getReduceProgressPercent(), '%'))._()._()._().
+ td(String.valueOf(job.getReducesTotal())).
+ td(String.valueOf(job.getReducesCompleted()))._();
}
tbody._()._();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java Wed Dec 14 23:34:04 2011
@@ -18,23 +18,29 @@
package org.apache.hadoop.mapreduce.v2.app.webapp;
-import java.util.Collection;
+import static org.apache.hadoop.yarn.util.StringHelper.percent;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
-import com.google.common.base.Joiner;
-import com.google.inject.Inject;
+import java.util.Collection;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
-import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TD;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import static org.apache.hadoop.yarn.util.StringHelper.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+
+import com.google.inject.Inject;
public class TaskPage extends AppView {
@@ -66,24 +72,26 @@ public class TaskPage extends AppView {
th(".tsh", "Elapsed").
th(".note", "Note")._()._().
tbody();
- for (TaskAttempt ta : getTaskAttempts()) {
- String taid = MRApps.toString(ta.getID());
- String progress = percent(ta.getProgress());
- ContainerId containerId = ta.getAssignedContainerID();
+ for (TaskAttempt attempt : getTaskAttempts()) {
+ TaskAttemptInfo ta = new TaskAttemptInfo(attempt, true);
+ String taid = ta.getId();
+ String progress = percent(ta.getProgress() / 100);
+ ContainerId containerId = ta.getAssignedContainerId();
- String nodeHttpAddr = ta.getNodeHttpAddress();
- long startTime = ta.getLaunchTime();
+ String nodeHttpAddr = ta.getNode();
+ long startTime = ta.getStartTime();
long finishTime = ta.getFinishTime();
- long elapsed = Times.elapsed(startTime, finishTime);
+ long elapsed = ta.getElapsedTime();
+ String diag = ta.getNote() == null ? "" : ta.getNote();
TD<TR<TBODY<TABLE<Hamlet>>>> nodeTd = tbody.
tr().
td(".id", taid).
td(".progress", progress).
- td(".state", ta.getState().toString()).
+ td(".state", ta.getState()).
td().
a(".nodelink", url("http://", nodeHttpAddr), nodeHttpAddr);
if (containerId != null) {
- String containerIdStr = ConverterUtils.toString(containerId);
+ String containerIdStr = ta.getAssignedContainerIdStr();
nodeTd._(" ").
a(".logslink", url("http://", nodeHttpAddr, "node", "containerlogs",
containerIdStr, app.getJob().getUserName()), "logs");
@@ -92,7 +100,7 @@ public class TaskPage extends AppView {
td(".ts", Times.format(startTime)).
td(".ts", Times.format(finishTime)).
td(".dt", StringUtils.formatTime(elapsed)).
- td(".note", Joiner.on('\n').join(ta.getDiagnostics()))._();
+ td(".note", diag)._();
}
tbody._()._();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java Wed Dec 14 23:34:04 2011
@@ -18,21 +18,24 @@
package org.apache.hadoop.mapreduce.v2.app.webapp;
-import com.google.inject.Inject;
+import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_TYPE;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.util.StringHelper.percent;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import static org.apache.hadoop.mapreduce.v2.app.webapp.AMWebApp.*;
-import static org.apache.hadoop.yarn.util.StringHelper.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+import com.google.inject.Inject;
public class TasksBlock extends HtmlBlock {
final App app;
@@ -67,16 +70,16 @@ public class TasksBlock extends HtmlBloc
if (type != null && task.getType() != type) {
continue;
}
- String tid = MRApps.toString(task.getID());
- TaskReport report = task.getReport();
- String pct = percent(report.getProgress());
- long startTime = report.getStartTime();
- long finishTime = report.getFinishTime();
- long elapsed = Times.elapsed(startTime, finishTime);
+ TaskInfo info = new TaskInfo(task);
+ String tid = info.getId();
+ String pct = percent(info.getProgress() / 100);
+ long startTime = info.getStartTime();
+ long finishTime = info.getFinishTime();
+ long elapsed = info.getElapsedTime();
tbody.
tr().
td().
- br().$title(String.valueOf(task.getID().getId()))._(). // sorting
+ br().$title(String.valueOf(info.getTaskNum()))._(). // sorting
a(url("task", tid), tid)._().
td().
br().$title(pct)._().
@@ -84,7 +87,7 @@ public class TasksBlock extends HtmlBloc
$title(join(pct, '%')). // tooltip
div(_PROGRESSBAR_VALUE).
$style(join("width:", pct, '%'))._()._()._().
- td(report.getTaskState().toString()).
+ td(info.getState()).
td().
br().$title(String.valueOf(startTime))._().
_(Times.format(startTime))._().
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java Wed Dec 14 23:34:04 2011
@@ -183,6 +183,7 @@ public class TestMRClientService {
Assert.assertEquals(1, amInfo.getContainerId().getApplicationAttemptId()
.getAttemptId());
Assert.assertTrue(amInfo.getStartTime() > 0);
+ Assert.assertEquals(false, jr.isUber());
}
private void verifyTaskAttemptReport(TaskAttemptReport tar) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java Wed Dec 14 23:34:04 2011
@@ -118,7 +118,7 @@ public class TestRMContainerAllocator {
Job mockJob = mock(Job.class);
when(mockJob.getReport()).thenReturn(
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
- 0, 0, 0, 0, 0, 0, "jobfile", null));
+ 0, 0, 0, 0, 0, 0, "jobfile", null, false));
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
appAttemptId, mockJob);
@@ -195,7 +195,7 @@ public class TestRMContainerAllocator {
Job mockJob = mock(Job.class);
when(mockJob.getReport()).thenReturn(
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
- 0, 0, 0, 0, 0, 0, "jobfile", null));
+ 0, 0, 0, 0, 0, 0, "jobfile", null, false));
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
appAttemptId, mockJob);
@@ -261,7 +261,7 @@ public class TestRMContainerAllocator {
Job mockJob = mock(Job.class);
when(mockJob.getReport()).thenReturn(
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
- 0, 0, 0, 0, 0, 0, "jobfile", null));
+ 0, 0, 0, 0, 0, 0, "jobfile", null, false));
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
appAttemptId, mockJob);
@@ -375,7 +375,7 @@ public class TestRMContainerAllocator {
public JobReport getReport() {
return MRBuilderUtils.newJobReport(this.jobId, "job", "user",
JobState.RUNNING, 0, 0, 0, this.setupProgress, this.mapProgress,
- this.reduceProgress, this.cleanupProgress, "jobfile", null);
+ this.reduceProgress, this.cleanupProgress, "jobfile", null, false);
}
}
@@ -511,7 +511,7 @@ public class TestRMContainerAllocator {
Job mockJob = mock(Job.class);
when(mockJob.getReport()).thenReturn(
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
- 0, 0, 0, 0, 0, 0, "jobfile", null));
+ 0, 0, 0, 0, 0, 0, "jobfile", null, false));
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
appAttemptId, mockJob);
@@ -610,7 +610,7 @@ public class TestRMContainerAllocator {
Job mockJob = mock(Job.class);
when(mockJob.getReport()).thenReturn(
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
- 0, 0, 0, 0, 0, 0, "jobfile", null));
+ 0, 0, 0, 0, 0, 0, "jobfile", null, false));
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
appAttemptId, mockJob);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java Wed Dec 14 23:34:04 2011
@@ -754,8 +754,6 @@ public class TestRuntimeEstimators {
}
class MyAppContext implements AppContext {
- // I'll be making Avro objects by hand. Please don't do that very often.
-
private final ApplicationAttemptId myAppAttemptID;
private final ApplicationId myApplicationID;
private final JobId myJobID;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java Wed Dec 14 23:34:04 2011
@@ -288,7 +288,7 @@ public class TypeConverter {
.getMapProgress(), jobreport.getReduceProgress(), jobreport
.getCleanupProgress(), fromYarn(jobreport.getJobState()),
jobPriority, jobreport.getUser(), jobreport.getJobName(), jobreport
- .getJobFile(), trackingUrl);
+ .getJobFile(), trackingUrl, jobreport.isUber());
jobStatus.setFailureInfo(jobreport.getDiagnostics());
return jobStatus;
}
@@ -421,7 +421,7 @@ public class TypeConverter {
TypeConverter.fromYarn(application.getYarnApplicationState()),
org.apache.hadoop.mapreduce.JobPriority.NORMAL,
application.getUser(), application.getName(),
- application.getQueue(), jobFile, trackingUrl
+ application.getQueue(), jobFile, trackingUrl, false
);
jobStatus.setSchedulingInfo(trackingUrl); // Set AM tracking url
jobStatus.setStartTime(application.getStartTime());
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobReport.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobReport.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobReport.java Wed Dec 14 23:34:04 2011
@@ -36,6 +36,7 @@ public interface JobReport {
public abstract String getDiagnostics();
public abstract String getJobFile();
public abstract List<AMInfo> getAMInfos();
+ public abstract boolean isUber();
public abstract void setJobId(JobId jobId);
public abstract void setJobState(JobState jobState);
@@ -52,4 +53,5 @@ public interface JobReport {
public abstract void setDiagnostics(String diagnostics);
public abstract void setJobFile(String jobFile);
public abstract void setAMInfos(List<AMInfo> amInfos);
+ public abstract void setIsUber(boolean isUber);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/JobReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/JobReportPBImpl.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/JobReportPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/JobReportPBImpl.java Wed Dec 14 23:34:04 2011
@@ -332,4 +332,16 @@ public class JobReportPBImpl extends Pro
private JobState convertFromProtoFormat(JobStateProto e) {
return MRProtoUtils.convertFromProtoFormat(e);
}
+
+ @Override
+ public synchronized boolean isUber() {
+ JobReportProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getIsUber();
+ }
+
+ @Override
+ public synchronized void setIsUber(boolean isUber) {
+ maybeInitBuilder();
+ builder.setIsUber(isUber);
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java Wed Dec 14 23:34:04 2011
@@ -60,7 +60,8 @@ public class MRBuilderUtils {
public static JobReport newJobReport(JobId jobId, String jobName,
String userName, JobState state, long submitTime, long startTime, long finishTime,
float setupProgress, float mapProgress, float reduceProgress,
- float cleanupProgress, String jobFile, List<AMInfo> amInfos) {
+ float cleanupProgress, String jobFile, List<AMInfo> amInfos,
+ boolean isUber) {
JobReport report = Records.newRecord(JobReport.class);
report.setJobId(jobId);
report.setJobName(jobName);
@@ -75,6 +76,7 @@ public class MRBuilderUtils {
report.setReduceProgress(reduceProgress);
report.setJobFile(jobFile);
report.setAMInfos(amInfos);
+ report.setIsUber(isUber);
return report;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto Wed Dec 14 23:34:04 2011
@@ -152,6 +152,7 @@ message JobReportProto {
optional string jobFile = 13;
repeated AMInfoProto am_infos = 14;
optional int64 submit_time = 15;
+ optional bool is_uber = 16 [default = false];
}
message AMInfoProto {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java Wed Dec 14 23:34:04 2011
@@ -109,7 +109,14 @@ class JobQueueClient extends Configured
return exitcode;
}
+// format and print information about the passed in job queue.
+ void printJobQueueInfo(JobQueueInfo jobQueueInfo, Writer writer)
+ throws IOException {
+ printJobQueueInfo(jobQueueInfo, writer, "");
+ }
+
// format and print information about the passed in job queue.
+ @SuppressWarnings("deprecation")
void printJobQueueInfo(JobQueueInfo jobQueueInfo, Writer writer,
String prefix) throws IOException {
if (jobQueueInfo == null) {
@@ -136,7 +143,7 @@ class JobQueueClient extends Configured
private void displayQueueList() throws IOException {
JobQueueInfo[] rootQueues = jc.getRootQueues();
for (JobQueueInfo queue : rootQueues) {
- printJobQueueInfo(queue, new PrintWriter(System.out), "");
+ printJobQueueInfo(queue, new PrintWriter(System.out));
}
}
@@ -174,7 +181,7 @@ class JobQueueClient extends Configured
System.out.println("Queue \"" + queue + "\" does not exist.");
return;
}
- printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out), "");
+ printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out));
if (showJobs && (jobQueueInfo.getChildren() == null ||
jobQueueInfo.getChildren().size() == 0)) {
JobStatus[] jobs = jc.getJobsFromQueue(queue);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java Wed Dec 14 23:34:04 2011
@@ -97,7 +97,7 @@ public class JobStatus extends org.apach
String user, String jobName,
String jobFile, String trackingUrl) {
this(jobid, mapProgress, reduceProgress, cleanupProgress, runState,
- JobPriority.NORMAL, user, jobName, jobFile, trackingUrl);
+ JobPriority.NORMAL, user, jobName, jobFile, trackingUrl);
}
/**
@@ -135,7 +135,8 @@ public class JobStatus extends org.apach
String user, String jobName, String jobFile,
String trackingUrl) {
this(jobid, 0.0f, mapProgress, reduceProgress,
- cleanupProgress, runState, jp, user, jobName, jobFile, trackingUrl);
+ cleanupProgress, runState, jp, user, jobName, jobFile,
+ trackingUrl);
}
/**
@@ -157,9 +158,31 @@ public class JobStatus extends org.apach
int runState, JobPriority jp, String user, String jobName,
String jobFile, String trackingUrl) {
this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
- runState, jp,
- user, jobName, "default", jobFile, trackingUrl);
+ runState, jp, user, jobName, "default", jobFile, trackingUrl);
}
+
+ /**
+ * Create a job status object for a given jobid.
+ * @param jobid The jobid of the job
+ * @param setupProgress The progress made on the setup
+ * @param mapProgress The progress made on the maps
+ * @param reduceProgress The progress made on the reduces
+ * @param cleanupProgress The progress made on the cleanup
+ * @param runState The current state of the job
+ * @param jp Priority of the job.
+ * @param user userid of the person who submitted the job.
+ * @param jobName user-specified job name.
+ * @param jobFile job configuration file.
+ * @param trackingUrl link to the web-ui for details of the job.
+ * @param isUber Whether job running in uber mode
+ */
+ public JobStatus(JobID jobid, float setupProgress, float mapProgress,
+ float reduceProgress, float cleanupProgress,
+ int runState, JobPriority jp, String user, String jobName,
+ String jobFile, String trackingUrl, boolean isUber) {
+ this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+ runState, jp, user, jobName, "default", jobFile, trackingUrl, isUber);
+ }
/**
* Create a job status object for a given jobid.
@@ -173,27 +196,52 @@ public class JobStatus extends org.apach
* @param user userid of the person who submitted the job.
* @param jobName user-specified job name.
* @param queue job queue name.
+ * @param jobFile job configuration file.
+ * @param trackingUrl link to the web-ui for details of the job.
+ */
+ public JobStatus(JobID jobid, float setupProgress, float mapProgress,
+ float reduceProgress, float cleanupProgress,
+ int runState, JobPriority jp,
+ String user, String jobName, String queue,
+ String jobFile, String trackingUrl) {
+ this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+ runState, jp,
+ user, jobName, queue, jobFile, trackingUrl, false);
+ }
+
+ /**
+ * Create a job status object for a given jobid.
+ * @param jobid The jobid of the job
+ * @param setupProgress The progress made on the setup
+ * @param mapProgress The progress made on the maps
+ * @param reduceProgress The progress made on the reduces
+ * @param cleanupProgress The progress made on the cleanup
+ * @param runState The current state of the job
+ * @param jp Priority of the job.
+ * @param user userid of the person who submitted the job.
+ * @param jobName user-specified job name.
+ * @param queue job queue name.
* @param jobFile job configuration file.
* @param trackingUrl link to the web-ui for details of the job.
+ * @param isUber Whether job running in uber mode
*/
public JobStatus(JobID jobid, float setupProgress, float mapProgress,
float reduceProgress, float cleanupProgress,
int runState, JobPriority jp,
String user, String jobName, String queue,
- String jobFile, String trackingUrl) {
+ String jobFile, String trackingUrl, boolean isUber) {
super(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
getEnum(runState), org.apache.hadoop.mapreduce.JobPriority.valueOf(jp.name()),
- user, jobName, queue, jobFile, trackingUrl);
+ user, jobName, queue, jobFile, trackingUrl, isUber);
}
-
public static JobStatus downgrade(org.apache.hadoop.mapreduce.JobStatus stat){
JobStatus old = new JobStatus(JobID.downgrade(stat.getJobID()),
stat.getSetupProgress(), stat.getMapProgress(), stat.getReduceProgress(),
stat.getCleanupProgress(), stat.getState().getValue(),
JobPriority.valueOf(stat.getPriority().name()),
stat.getUsername(), stat.getJobName(), stat.getJobFile(),
- stat.getTrackingUrl());
+ stat.getTrackingUrl(), stat.isUber());
old.setStartTime(stat.getStartTime());
old.setFinishTime(stat.getFinishTime());
old.setSchedulingInfo(stat.getSchedulingInfo());
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java Wed Dec 14 23:34:04 2011
@@ -467,6 +467,7 @@ public class Job extends JobContextImpl
sb.append("Job File: ").append(status.getJobFile()).append("\n");
sb.append("Job Tracking URL : ").append(status.getTrackingUrl());
sb.append("\n");
+ sb.append("Uber job : ").append(status.isUber()).append("\n");
sb.append("map() completion: ");
sb.append(status.getMapProgress()).append("\n");
sb.append("reduce() completion: ");
@@ -1268,12 +1269,20 @@ public class Job extends JobContextImpl
Job.getProgressPollInterval(clientConf);
/* make sure to report full progress after the job is done */
boolean reportedAfterCompletion = false;
+ boolean reportedUberMode = false;
while (!isComplete() || !reportedAfterCompletion) {
if (isComplete()) {
reportedAfterCompletion = true;
} else {
Thread.sleep(progMonitorPollIntervalMillis);
}
+ if (status.getState() == JobStatus.State.PREP) {
+ continue;
+ }
+ if (!reportedUberMode) {
+ reportedUberMode = true;
+ LOG.info("Job " + jobId + " running in uber mode : " + isUber());
+ }
String report =
(" map " + StringUtils.formatPercent(mapProgress(), 0)+
" reduce " +
@@ -1497,4 +1506,10 @@ public class Job extends JobContextImpl
conf.set(Job.OUTPUT_FILTER, newValue.toString());
}
+ public boolean isUber() throws IOException, InterruptedException {
+ ensureState(JobState.RUNNING);
+ updateStatus();
+ return status.isUber();
+ }
+
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java Wed Dec 14 23:34:04 2011
@@ -97,7 +97,7 @@ public class JobStatus implements Writab
private int usedMem;
private int reservedMem;
private int neededMem;
-
+ private boolean isUber;
/**
*/
@@ -115,17 +115,17 @@ public class JobStatus implements Writab
* @param jp Priority of the job.
* @param user userid of the person who submitted the job.
* @param jobName user-specified job name.
- * @param jobFile job configuration file.
+ * @param jobFile job configuration file.
* @param trackingUrl link to the web-ui for details of the job.
*/
public JobStatus(JobID jobid, float setupProgress, float mapProgress,
- float reduceProgress, float cleanupProgress,
+ float reduceProgress, float cleanupProgress,
State runState, JobPriority jp, String user, String jobName,
String jobFile, String trackingUrl) {
this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
- runState, jp, user, jobName, "default", jobFile, trackingUrl);
+ runState, jp, user, jobName, "default", jobFile, trackingUrl, false);
}
-
+
/**
* Create a job status object for a given jobid.
* @param jobid The jobid of the job
@@ -138,14 +138,39 @@ public class JobStatus implements Writab
* @param user userid of the person who submitted the job.
* @param jobName user-specified job name.
* @param queue queue name
- * @param jobFile job configuration file.
+ * @param jobFile job configuration file.
* @param trackingUrl link to the web-ui for details of the job.
*/
public JobStatus(JobID jobid, float setupProgress, float mapProgress,
- float reduceProgress, float cleanupProgress,
- State runState, JobPriority jp,
- String user, String jobName, String queue,
+ float reduceProgress, float cleanupProgress,
+ State runState, JobPriority jp,
+ String user, String jobName, String queue,
String jobFile, String trackingUrl) {
+ this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+ runState, jp, user, jobName, queue, jobFile, trackingUrl, false);
+ }
+
+ /**
+ * Create a job status object for a given jobid.
+ * @param jobid The jobid of the job
+ * @param setupProgress The progress made on the setup
+ * @param mapProgress The progress made on the maps
+ * @param reduceProgress The progress made on the reduces
+ * @param cleanupProgress The progress made on the cleanup
+ * @param runState The current state of the job
+ * @param jp Priority of the job.
+ * @param user userid of the person who submitted the job.
+ * @param jobName user-specified job name.
+ * @param queue queue name
+ * @param jobFile job configuration file.
+ * @param trackingUrl link to the web-ui for details of the job.
+ * @param isUber Whether job running in uber mode
+ */
+ public JobStatus(JobID jobid, float setupProgress, float mapProgress,
+ float reduceProgress, float cleanupProgress,
+ State runState, JobPriority jp,
+ String user, String jobName, String queue,
+ String jobFile, String trackingUrl, boolean isUber) {
this.jobid = jobid;
this.setupProgress = setupProgress;
this.mapProgress = mapProgress;
@@ -161,8 +186,9 @@ public class JobStatus implements Writab
this.jobName = jobName;
this.jobFile = jobFile;
this.trackingUrl = trackingUrl;
+ this.isUber = isUber;
}
-
+
/**
* Sets the map progress of this job
@@ -411,6 +437,7 @@ public class JobStatus implements Writab
Text.writeString(out, jobName);
Text.writeString(out, trackingUrl);
Text.writeString(out, jobFile);
+ out.writeBoolean(isUber);
// Serialize the job's ACLs
out.writeInt(jobACLs.size());
@@ -438,6 +465,7 @@ public class JobStatus implements Writab
this.jobName = Text.readString(in);
this.trackingUrl = Text.readString(in);
this.jobFile = Text.readString(in);
+ this.isUber = in.readBoolean();
// De-serialize the job's ACLs
int numACLs = in.readInt();
@@ -562,9 +590,26 @@ public class JobStatus implements Writab
this.neededMem = n;
}
+ /**
+ * Whether job running in uber mode
+ * @return job in uber-mode
+ */
+ public synchronized boolean isUber() {
+ return isUber;
+ }
+
+ /**
+ * Set uber-mode flag
+ * @param isUber Whether job running in uber-mode
+ */
+ public synchronized void setUber(boolean isUber) {
+ this.isUber = isUber;
+ }
+
public String toString() {
StringBuffer buffer = new StringBuffer();
buffer.append("job-id : " + jobid);
+ buffer.append("uber-mode : " + isUber);
buffer.append("map-progress : " + mapProgress);
buffer.append("reduce-progress : " + reduceProgress);
buffer.append("cleanup-progress : " + cleanupProgress);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Wed Dec 14 23:34:04 2011
@@ -150,6 +150,10 @@ public interface MRJobConfig {
public static final String NUM_REDUCE_PROFILES = "mapreduce.task.profile.reduces";
+ public static final String TASK_MAP_PROFILE_PARAMS = "mapreduce.task.profile.map.params";
+
+ public static final String TASK_REDUCE_PROFILE_PARAMS = "mapreduce.task.profile.reduce.params";
+
public static final String TASK_TIMEOUT = "mapreduce.task.timeout";
public static final String TASK_ID = "mapreduce.task.id";
@@ -298,12 +302,6 @@ public interface MRJobConfig {
"mapreduce.job.ubertask.maxreduces";
public static final String JOB_UBERTASK_MAXBYTES =
"mapreduce.job.ubertask.maxbytes";
- public static final String UBERTASK_JAVA_OPTS =
- "mapreduce.ubertask.child.java.opts"; // or mapreduce.uber.java.opts?
- public static final String UBERTASK_ULIMIT =
- "mapreduce.ubertask.child.ulimit"; // or mapreduce.uber.ulimit?
- public static final String UBERTASK_ENV =
- "mapreduce.ubertask.child.env"; // or mapreduce.uber.env?
public static final String MR_PREFIX = "yarn.app.mapreduce.";
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java Wed Dec 14 23:34:04 2011
@@ -76,6 +76,7 @@ public abstract class AbstractCounters<C
TaskCounter.class.getName());
legacyMap.put("org.apache.hadoop.mapred.JobInProgress$Counter",
JobCounter.class.getName());
+ legacyMap.put("FileSystemCounter", FileSystemCounter.class.getName());
}
private final Limits limits = new Limits();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java Wed Dec 14 23:34:04 2011
@@ -353,7 +353,7 @@ public class JobHistoryParser {
* The class where job information is aggregated into after parsing
*/
public static class JobInfo {
- String errorInfo = "None";
+ String errorInfo = "";
long submitTime;
long finishTime;
JobID jobid;
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 14 23:34:04 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1166973-1213862
+/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1166973-1214510
/hadoop/core/branches/branch-0.19/mapred/src/java/mapred-default.xml:713112
/hadoop/core/trunk/src/mapred/mapred-default.xml:776175-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java Wed Dec 14 23:34:04 2011
@@ -22,6 +22,8 @@ import static org.junit.Assert.assertEqu
import java.io.IOException;
import java.text.ParseException;
+import org.apache.hadoop.mapred.Counters.Counter;
+import org.apache.hadoop.mapreduce.FileSystemCounter;
import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.TaskCounter;
import org.junit.Test;
@@ -102,6 +104,7 @@ public class TestCounters {
Counters counters = new Counters();
counters.incrCounter(TaskCounter.MAP_INPUT_RECORDS, 1);
counters.incrCounter(JobCounter.DATA_LOCAL_MAPS, 1);
+ counters.findCounter("file", FileSystemCounter.BYTES_READ).increment(1);
assertEquals("New name", 1, counters.findCounter(
TaskCounter.class.getName(), "MAP_INPUT_RECORDS").getValue());
@@ -114,6 +117,14 @@ public class TestCounters {
assertEquals("Legacy name", 1, counters.findCounter(
"org.apache.hadoop.mapred.JobInProgress$Counter",
"DATA_LOCAL_MAPS").getValue());
+
+ assertEquals("New name", 1, counters.findCounter(
+ FileSystemCounter.class.getName(), "FILE_BYTES_READ").getValue());
+ assertEquals("New name and method", 1, counters.findCounter("file",
+ FileSystemCounter.BYTES_READ).getValue());
+ assertEquals("Legacy name", 1, counters.findCounter(
+ "FileSystemCounter",
+ "FILE_BYTES_READ").getValue());
}
public static void main(String[] args) throws IOException {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobQueueClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobQueueClient.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobQueueClient.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobQueueClient.java Wed Dec 14 23:34:04 2011
@@ -45,7 +45,7 @@ public class TestJobQueueClient {
ByteArrayOutputStream bbos = new ByteArrayOutputStream();
PrintWriter writer = new PrintWriter(bbos);
- queueClient.printJobQueueInfo(parent, writer, "");
+ queueClient.printJobQueueInfo(parent, writer);
Assert.assertTrue("printJobQueueInfo did not print grandchild's name",
bbos.toString().contains("GrandChildQueue"));
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobMonitorAndPrint.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobMonitorAndPrint.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobMonitorAndPrint.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobMonitorAndPrint.java Wed Dec 14 23:34:04 2011
@@ -63,17 +63,20 @@ public class TestJobMonitorAndPrint exte
when(cluster.getConf()).thenReturn(conf);
when(cluster.getClient()).thenReturn(clientProtocol);
JobStatus jobStatus = new JobStatus(new JobID("job_000", 1), 0f, 0f, 0f, 0f,
- State.RUNNING, JobPriority.HIGH, "tmp-user", "tmp-jobname", "tmp-jobfile", "tmp-url");
+ State.RUNNING, JobPriority.HIGH, "tmp-user", "tmp-jobname",
+ "tmp-jobfile", "tmp-url");
job = Job.getInstance(cluster, jobStatus, conf);
job = spy(job);
}
@Test
public void testJobMonitorAndPrint() throws Exception {
- JobStatus jobStatus_1 = new JobStatus(new JobID("job_000", 1), 1f, 0.1f, 0.1f, 0f,
- State.RUNNING, JobPriority.HIGH, "tmp-user", "tmp-jobname", "tmp-jobfile", "tmp-url");
- JobStatus jobStatus_2 = new JobStatus(new JobID("job_000", 1), 1f, 1f, 1f, 1f,
- State.SUCCEEDED, JobPriority.HIGH, "tmp-user", "tmp-jobname", "tmp-jobfile", "tmp-url");
+ JobStatus jobStatus_1 = new JobStatus(new JobID("job_000", 1), 1f, 0.1f,
+ 0.1f, 0f, State.RUNNING, JobPriority.HIGH, "tmp-user", "tmp-jobname",
+ "tmp-queue", "tmp-jobfile", "tmp-url", true);
+ JobStatus jobStatus_2 = new JobStatus(new JobID("job_000", 1), 1f, 1f,
+ 1f, 1f, State.SUCCEEDED, JobPriority.HIGH, "tmp-user", "tmp-jobname",
+ "tmp-queue", "tmp-jobfile", "tmp-url", true);
doAnswer(
new Answer<TaskCompletionEvent[]>() {
@@ -102,15 +105,21 @@ public class TestJobMonitorAndPrint exte
String line;
boolean foundHundred = false;
boolean foundComplete = false;
- String match_1 = "map 100% reduce 100%";
- String match_2 = "completed successfully";
+ boolean foundUber = false;
+ String match_1 = "uber mode : true";
+ String match_2 = "map 100% reduce 100%";
+ String match_3 = "completed successfully";
while ((line = r.readLine()) != null) {
- foundHundred = line.contains(match_1);
+ if (line.contains(match_1)) {
+ foundUber = true;
+ }
+ foundHundred = line.contains(match_2);
if (foundHundred)
break;
}
line = r.readLine();
- foundComplete = line.contains(match_2);
+ foundComplete = line.contains(match_3);
+ assertTrue(foundUber);
assertTrue(foundHundred);
assertTrue(foundComplete);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java Wed Dec 14 23:34:04 2011
@@ -107,6 +107,7 @@ public class CompletedJob implements org
report.setTrackingUrl(JobHistoryUtils.getHistoryUrl(conf, TypeConverter
.toYarn(TypeConverter.fromYarn(jobId)).getAppId()));
report.setAMInfos(getAMInfos());
+ report.setIsUber(isUber());
}
@Override
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java Wed Dec 14 23:34:04 2011
@@ -27,12 +27,11 @@ import java.security.PrivilegedException
import java.util.Arrays;
import java.util.Collection;
-import org.apache.hadoop.ipc.Server;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
@@ -79,14 +78,14 @@ import org.apache.hadoop.yarn.webapp.Web
import org.apache.hadoop.yarn.webapp.WebApps;
/**
- * This module is responsible for talking to the
+ * This module is responsible for talking to the
* JobClient (user facing).
*
*/
public class HistoryClientService extends AbstractService {
private static final Log LOG = LogFactory.getLog(HistoryClientService.class);
-
+
private MRClientProtocol protocolHandler;
private Server server;
private WebApp webApp;
@@ -118,22 +117,22 @@ public class HistoryClientService extend
server =
rpc.getServer(MRClientProtocol.class, protocolHandler, address,
conf, null,
- conf.getInt(JHAdminConfig.MR_HISTORY_CLIENT_THREAD_COUNT,
+ conf.getInt(JHAdminConfig.MR_HISTORY_CLIENT_THREAD_COUNT,
JHAdminConfig.DEFAULT_MR_HISTORY_CLIENT_THREAD_COUNT));
-
+
// Enable service authorization?
if (conf.getBoolean(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
server.refreshServiceAcl(conf, new MRAMPolicyProvider());
}
-
+
server.start();
this.bindAddress =
NetUtils.createSocketAddr(hostNameResolved.getHostAddress()
+ ":" + server.getPort());
LOG.info("Instantiated MRClientService at " + this.bindAddress);
-
+
super.start();
}
@@ -141,7 +140,7 @@ public class HistoryClientService extend
webApp = new HsWebApp(history);
String bindAddress = conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS);
- WebApps.$for("jobhistory", this).with(conf).at(bindAddress).start(webApp);
+ WebApps.$for("jobhistory", HistoryClientService.class, this, "ws").with(conf).at(bindAddress).start(webApp);
}
@Override
@@ -158,7 +157,7 @@ public class HistoryClientService extend
private class MRClientProtocolHandler implements MRClientProtocol {
private RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-
+
private Job verifyAndGetJob(final JobId jobID) throws YarnRemoteException {
UserGroupInformation loginUgi = null;
Job job = null;
@@ -194,7 +193,7 @@ public class HistoryClientService extend
response.setCounters(job.getCounters());
return response;
}
-
+
@Override
public GetJobReportResponse getJobReport(GetJobReportRequest request) throws YarnRemoteException {
JobId jobId = request.getJobId();
@@ -227,23 +226,23 @@ public class HistoryClientService extend
JobId jobId = request.getJobId();
int fromEventId = request.getFromEventId();
int maxEvents = request.getMaxEvents();
-
+
Job job = verifyAndGetJob(jobId);
GetTaskAttemptCompletionEventsResponse response = recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class);
response.addAllCompletionEvents(Arrays.asList(job.getTaskAttemptCompletionEvents(fromEventId, maxEvents)));
return response;
}
-
+
@Override
public KillJobResponse killJob(KillJobRequest request) throws YarnRemoteException {
throw RPCUtil.getRemoteException("Invalid operation on completed job");
}
-
+
@Override
public KillTaskResponse killTask(KillTaskRequest request) throws YarnRemoteException {
throw RPCUtil.getRemoteException("Invalid operation on completed job");
}
-
+
@Override
public KillTaskAttemptResponse killTaskAttempt(KillTaskAttemptRequest request) throws YarnRemoteException {
throw RPCUtil.getRemoteException("Invalid operation on completed job");
@@ -252,15 +251,15 @@ public class HistoryClientService extend
@Override
public GetDiagnosticsResponse getDiagnostics(GetDiagnosticsRequest request) throws YarnRemoteException {
TaskAttemptId taskAttemptId = request.getTaskAttemptId();
-
+
Job job = verifyAndGetJob(taskAttemptId.getTaskId().getJobId());
-
+
GetDiagnosticsResponse response = recordFactory.newRecordInstance(GetDiagnosticsResponse.class);
response.addAllDiagnostics(job.getTask(taskAttemptId.getTaskId()).getAttempt(taskAttemptId).getDiagnostics());
return response;
}
- @Override
+ @Override
public FailTaskAttemptResponse failTaskAttempt(FailTaskAttemptRequest request) throws YarnRemoteException {
throw RPCUtil.getRemoteException("Invalid operation on completed job");
}
@@ -269,7 +268,7 @@ public class HistoryClientService extend
public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request) throws YarnRemoteException {
JobId jobId = request.getJobId();
TaskType taskType = request.getTaskType();
-
+
GetTaskReportsResponse response = recordFactory.newRecordInstance(GetTaskReportsResponse.class);
Job job = verifyAndGetJob(jobId);
Collection<Task> tasks = job.getTasks(taskType).values();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java?rev=1214546&r1=1214545&r2=1214546&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java Wed Dec 14 23:34:04 2011
@@ -21,7 +21,7 @@ package org.apache.hadoop.mapreduce.v2.h
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
-import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.HistoryInfo;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
@@ -45,8 +45,9 @@ public class HsAboutPage extends HsView
* @return AttemptsBlock.class
*/
@Override protected Class<? extends SubView> content() {
+ HistoryInfo info = new HistoryInfo();
info("History Server").
- _("BuildVersion", VersionInfo.getBuildVersion());
+ _("BuildVersion", info.getHadoopBuildVersion() + " on " + info.getHadoopVersionBuiltOn());
return InfoBlock.class;
}
}