You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2014/03/12 23:05:59 UTC
svn commit: r1576948 - in
/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/
hadoop-mapreduce-client/hadoop-mapreduce-client-commo...
Author: acmurthy
Date: Wed Mar 12 22:05:58 2014
New Revision: 1576948
URL: http://svn.apache.org/r1576948
Log:
Merge -c 1576946 from trunk to branch-2.4 to fix MAPREDUCE-5553. Allow users to easily access completed/pending/successful/failed tasks on MR AM web-ui. Contributed by Paul Han.
Modified:
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt?rev=1576948&r1=1576947&r2=1576948&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt Wed Mar 12 22:05:58 2014
@@ -25,6 +25,10 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5773. Provide dedicated MRAppMaster syslog length limit (Gera
Shegalov via jlowe)
+ MAPREDUCE-5553. Allow users to easily access
+ completed/pending/successful/failed tasks on MR AM web-ui. (Paul Han via
+ acmurthy)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java?rev=1576948&r1=1576947&r2=1576948&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java Wed Mar 12 22:05:58 2014
@@ -27,6 +27,7 @@ public interface AMParams {
static final String JOB_ID = "job.id";
static final String TASK_ID = "task.id";
static final String TASK_TYPE = "task.type";
+ static final String TASK_STATE = "task.state";
static final String ATTEMPT_STATE = "attempt.state";
static final String COUNTER_GROUP = "counter.group";
static final String COUNTER_NAME = "counter.name";
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java?rev=1576948&r1=1576947&r2=1576948&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java Wed Mar 12 22:05:58 2014
@@ -40,7 +40,7 @@ public class AMWebApp extends WebApp imp
route(pajoin("/jobcounters", JOB_ID), AppController.class, "jobCounters");
route(pajoin("/singlejobcounter",JOB_ID, COUNTER_GROUP, COUNTER_NAME),
AppController.class, "singleJobCounter");
- route(pajoin("/tasks", JOB_ID, TASK_TYPE), AppController.class, "tasks");
+ route(pajoin("/tasks", JOB_ID, TASK_TYPE, TASK_STATE), AppController.class, "tasks");
route(pajoin("/attempts", JOB_ID, TASK_TYPE, ATTEMPT_STATE),
AppController.class, "attempts");
route(pajoin("/task", TASK_ID), AppController.class, "task");
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java?rev=1576948&r1=1576947&r2=1576948&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java Wed Mar 12 22:05:58 2014
@@ -127,31 +127,28 @@ public class JobBlock extends HtmlBlock
th(_TH, "Running").
th(_TH, "Complete")._().
tr(_ODD).
- th().
- a(url("tasks", jid, "m"), "Map")._().
+ th("Map").
td().
div(_PROGRESSBAR).
$title(join(jinfo.getMapProgressPercent(), '%')). // tooltip
div(_PROGRESSBAR_VALUE).
$style(join("width:", jinfo.getMapProgressPercent(), '%'))._()._()._().
- td(String.valueOf(jinfo.getMapsTotal())).
- td(String.valueOf(jinfo.getMapsPending())).
- td(String.valueOf(jinfo.getMapsRunning())).
- td(String.valueOf(jinfo.getMapsCompleted()))._().
+ td().a(url("tasks", jid, "m", "ALL"),String.valueOf(jinfo.getMapsTotal()))._().
+ td().a(url("tasks", jid, "m", "PENDING"),String.valueOf(jinfo.getMapsPending()))._().
+ td().a(url("tasks", jid, "m", "RUNNING"),String.valueOf(jinfo.getMapsRunning()))._().
+ td().a(url("tasks", jid, "m", "COMPLETED"),String.valueOf(jinfo.getMapsCompleted()))._()._().
tr(_EVEN).
- th().
- a(url("tasks", jid, "r"), "Reduce")._().
+ th("Reduce").
td().
div(_PROGRESSBAR).
$title(join(jinfo.getReduceProgressPercent(), '%')). // tooltip
div(_PROGRESSBAR_VALUE).
$style(join("width:", jinfo.getReduceProgressPercent(), '%'))._()._()._().
- td(String.valueOf(jinfo.getReducesTotal())).
- td(String.valueOf(jinfo.getReducesPending())).
- td(String.valueOf(jinfo.getReducesRunning())).
- td(String.valueOf(jinfo.getReducesCompleted()))._()
+ td().a(url("tasks", jid, "r", "ALL"),String.valueOf(jinfo.getReducesTotal()))._().
+ td().a(url("tasks", jid, "r", "PENDING"),String.valueOf(jinfo.getReducesPending()))._().
+ td().a(url("tasks", jid, "r", "RUNNING"),String.valueOf(jinfo.getReducesRunning()))._().
+ td().a(url("tasks", jid, "r", "COMPLETED"),String.valueOf(jinfo.getReducesCompleted()))._()._()
._().
-
// Attempts table
table("#job").
tr().
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java?rev=1576948&r1=1576947&r2=1576948&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java Wed Mar 12 22:05:58 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapreduce.v2.app.webapp;
+import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_STATE;
import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_TYPE;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.percent;
@@ -71,6 +72,25 @@ public class TasksBlock extends HtmlBloc
if (type != null && task.getType() != type) {
continue;
}
+ String taskStateStr = $(TASK_STATE);
+ if (taskStateStr == null || taskStateStr.trim().equals("")) {
+ taskStateStr = "ALL";
+ }
+
+ if (!taskStateStr.equalsIgnoreCase("ALL"))
+ {
+ try {
+ // get stateUI enum
+ MRApps.TaskStateUI stateUI = MRApps.taskState(taskStateStr);
+ if (!stateUI.correspondsTo(task.getState()))
+ {
+ continue;
+ }
+ } catch (IllegalArgumentException e) {
+ continue; // not supported state, ignore
+ }
+ }
+
TaskInfo info = new TaskInfo(task);
String tid = info.getId();
String pct = percent(info.getProgress() / 100);
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1576948&r1=1576947&r2=1576948&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Wed Mar 12 22:05:58 2014
@@ -48,6 +48,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.ContainerLogAppender;
@@ -125,6 +126,23 @@ public class MRApps extends Apps {
}
}
+ public static enum TaskStateUI {
+ RUNNING(
+ new TaskState[]{TaskState.RUNNING}),
+ PENDING(new TaskState[]{TaskState.SCHEDULED}),
+ COMPLETED(new TaskState[]{TaskState.SUCCEEDED, TaskState.FAILED, TaskState.KILLED});
+
+ private final List<TaskState> correspondingStates;
+
+ private TaskStateUI(TaskState[] correspondingStates) {
+ this.correspondingStates = Arrays.asList(correspondingStates);
+ }
+
+ public boolean correspondsTo(TaskState state) {
+ return this.correspondingStates.contains(state);
+ }
+ }
+
public static TaskType taskType(String symbol) {
// JDK 7 supports switch on strings
if (symbol.equals("m")) return TaskType.MAP;
@@ -136,6 +154,10 @@ public class MRApps extends Apps {
return TaskAttemptStateUI.valueOf(attemptStateStr);
}
+ public static TaskStateUI taskState(String taskStateStr) {
+ return TaskStateUI.valueOf(taskStateStr);
+ }
+
// gets the base name of the MapReduce framework or null if no
// framework was configured
private static String getMRFrameworkName(Configuration conf) {
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1576948&r1=1576947&r2=1576948&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Wed Mar 12 22:05:58 2014
@@ -37,6 +37,7 @@ import org.apache.hadoop.mapreduce.filec
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -460,4 +461,13 @@ public class TestMRApps {
public void initialize(URI name, Configuration conf) throws IOException {}
}
+
+ @Test
+ public void testTaskStateUI() {
+ assertTrue(MRApps.TaskStateUI.PENDING.correspondsTo(TaskState.SCHEDULED));
+ assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.SUCCEEDED));
+ assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.FAILED));
+ assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.KILLED));
+ assertTrue(MRApps.TaskStateUI.RUNNING.correspondsTo(TaskState.RUNNING));
+ }
}