You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sa...@apache.org on 2013/12/20 22:00:26 UTC
svn commit: r1552808 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java...
Author: sandy
Date: Fri Dec 20 21:00:25 2013
New Revision: 1552808
URL: http://svn.apache.org/r1552808
Log:
MAPREDUCE-5550. Task Status message (reporter.setStatus) not shown in UI with Hadoop 2.0 (Gera Shegalov via Sandy Ryza)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptInfo.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Dec 20 21:00:25 2013
@@ -190,6 +190,9 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5692. Add explicit diagnostics when a task attempt is killed due
to speculative execution (Gera Shegalov via Sandy Ryza)
+ MAPREDUCE-5550. Task Status message (reporter.setStatus) not shown in UI
+ with Hadoop 2.0 (Gera Shegalov via Sandy Ryza)
+
OPTIMIZATIONS
MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java Fri Dec 20 21:00:25 2013
@@ -376,11 +376,15 @@ public abstract class TaskImpl implement
TaskReport report = recordFactory.newRecordInstance(TaskReport.class);
readLock.lock();
try {
+ TaskAttempt bestAttempt = selectBestAttempt();
report.setTaskId(taskId);
report.setStartTime(getLaunchTime());
report.setFinishTime(getFinishTime());
report.setTaskState(getState());
- report.setProgress(getProgress());
+ report.setProgress(bestAttempt == null ? 0f : bestAttempt.getProgress());
+ report.setStatus(bestAttempt == null
+ ? ""
+ : bestAttempt.getReport().getStateString());
for (TaskAttempt attempt : attempts.values()) {
if (TaskAttemptState.RUNNING.equals(attempt.getState())) {
@@ -400,7 +404,9 @@ public abstract class TaskImpl implement
// Add a copy of counters as the last step so that their lifetime on heap
// is as small as possible.
- report.setCounters(TypeConverter.toYarn(getCounters()));
+ report.setCounters(TypeConverter.toYarn(bestAttempt == null
+ ? TaskAttemptImpl.EMPTY_COUNTERS
+ : bestAttempt.getCounters()));
return report;
} finally {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java Fri Dec 20 21:00:25 2013
@@ -63,6 +63,7 @@ public class TaskPage extends AppView {
th(".id", "Attempt").
th(".progress", "Progress").
th(".state", "State").
+ th(".status", "Status").
th(".node", "Node").
th(".logs", "Logs").
th(".tsh", "Started").
@@ -84,6 +85,7 @@ public class TaskPage extends AppView {
.append(ta.getId()).append("\",\"")
.append(progress).append("\",\"")
.append(ta.getState().toString()).append("\",\"")
+ .append(ta.getStatus()).append("\",\"")
.append(nodeHttpAddr == null ? "N/A" :
"<a class='nodelink' href='" + MRWebAppUtil.getYARNWebappScheme() + nodeHttpAddr + "'>"
@@ -144,13 +146,13 @@ public class TaskPage extends AppView {
.append("\n,aoColumnDefs:[\n")
//logs column should not filterable (it includes container ID which may pollute searches)
- .append("\n{'aTargets': [ 4 ]")
+ .append("\n{'aTargets': [ 5 ]")
.append(", 'bSearchable': false }")
- .append("\n, {'sType':'numeric', 'aTargets': [ 5, 6")
+ .append("\n, {'sType':'numeric', 'aTargets': [ 6, 7")
.append(" ], 'mRender': renderHadoopDate }")
- .append("\n, {'sType':'numeric', 'aTargets': [ 7")
+ .append("\n, {'sType':'numeric', 'aTargets': [ 8")
.append(" ], 'mRender': renderHadoopElapsedTime }]")
// Sort by id upon page load
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java Fri Dec 20 21:00:25 2013
@@ -59,6 +59,7 @@ public class TasksBlock extends HtmlBloc
tr().
th("Task").
th("Progress").
+ th("Status").
th("State").
th("Start Time").
th("Finish Time").
@@ -81,6 +82,7 @@ public class TasksBlock extends HtmlBloc
.append(join(pct, '%')).append("'> ").append("<div class='")
.append(C_PROGRESSBAR_VALUE).append("' style='")
.append(join("width:", pct, '%')).append("'> </div> </div>\",\"")
+ .append(info.getStatus()).append("\",\"")
.append(info.getState()).append("\",\"")
.append(info.getStartTime()).append("\",\"")
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java Fri Dec 20 21:00:25 2013
@@ -50,10 +50,10 @@ public class TasksPage extends AppView {
.append(", 'mRender': parseHadoopProgress }")
- .append("\n, {'sType':'numeric', 'aTargets': [3, 4]")
+ .append("\n, {'sType':'numeric', 'aTargets': [4, 5]")
.append(", 'mRender': renderHadoopDate }")
- .append("\n, {'sType':'numeric', 'aTargets': [5]")
+ .append("\n, {'sType':'numeric', 'aTargets': [6]")
.append(", 'mRender': renderHadoopElapsedTime }]")
// Sort by id upon page load
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptInfo.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptInfo.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptInfo.java Fri Dec 20 21:00:25 2013
@@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlRoot
import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
@@ -45,6 +46,7 @@ public class TaskAttemptInfo {
protected String id;
protected String rack;
protected TaskAttemptState state;
+ protected String status;
protected String nodeHttpAddress;
protected String diagnostics;
protected String type;
@@ -61,29 +63,23 @@ public class TaskAttemptInfo {
}
public TaskAttemptInfo(TaskAttempt ta, TaskType type, Boolean isRunning) {
+ final TaskAttemptReport report = ta.getReport();
this.type = type.toString();
this.id = MRApps.toString(ta.getID());
this.nodeHttpAddress = ta.getNodeHttpAddress();
- this.startTime = ta.getLaunchTime();
- this.finishTime = ta.getFinishTime();
- this.assignedContainerId = ConverterUtils.toString(ta
- .getAssignedContainerID());
- this.assignedContainer = ta.getAssignedContainerID();
- this.progress = ta.getProgress() * 100;
- this.state = ta.getState();
+ this.startTime = report.getStartTime();
+ this.finishTime = report.getFinishTime();
+ this.assignedContainerId = ConverterUtils.toString(report.getContainerId());
+ this.assignedContainer = report.getContainerId();
+ this.progress = report.getProgress() * 100;
+ this.status = report.getStateString();
+ this.state = report.getTaskAttemptState();
this.elapsedTime = Times
.elapsed(this.startTime, this.finishTime, isRunning);
if (this.elapsedTime == -1) {
this.elapsedTime = 0;
}
- List<String> diagnostics = ta.getDiagnostics();
- if (diagnostics != null && !diagnostics.isEmpty()) {
- StringBuffer b = new StringBuffer();
- for (String diag : diagnostics) {
- b.append(diag);
- }
- this.diagnostics = b.toString();
- }
+ this.diagnostics = report.getDiagnosticInfo();
this.rack = ta.getNodeRackName();
}
@@ -99,6 +95,10 @@ public class TaskAttemptInfo {
return this.state.toString();
}
+ public String getStatus() {
+ return status;
+ }
+
public String getId() {
return this.id;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java Fri Dec 20 21:00:25 2013
@@ -43,6 +43,7 @@ public class TaskInfo {
protected TaskState state;
protected String type;
protected String successfulAttempt;
+ protected String status;
@XmlTransient
int taskNum;
@@ -66,6 +67,7 @@ public class TaskInfo {
this.elapsedTime = 0;
}
this.progress = report.getProgress() * 100;
+ this.status = report.getStatus();
this.id = MRApps.toString(task.getID());
this.taskNum = task.getID().getId();
this.successful = getSuccessfulAttempt(task);
@@ -121,4 +123,7 @@ public class TaskInfo {
return null;
}
+ public String getStatus() {
+ return status;
+ }
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java Fri Dec 20 21:00:25 2013
@@ -174,22 +174,37 @@ public class MockJobs extends MockApps {
report.setFinishTime(System.currentTimeMillis()
+ (int) (Math.random() * DT) + 1);
report.setProgress((float) Math.random());
+ report.setStatus("Moving average: " + Math.random());
report.setCounters(TypeConverter.toYarn(newCounters()));
report.setTaskState(TASK_STATES.next());
return report;
}
public static TaskAttemptReport newTaskAttemptReport(TaskAttemptId id) {
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
+ id.getTaskId().getJobId().getAppId(), 0);
+ ContainerId containerId = ContainerId.newInstance(appAttemptId, 0);
TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
report.setTaskAttemptId(id);
report
.setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT));
report.setFinishTime(System.currentTimeMillis()
+ (int) (Math.random() * DT) + 1);
+
+ if (id.getTaskId().getTaskType() == TaskType.REDUCE) {
+ report.setShuffleFinishTime(
+ (report.getFinishTime() + report.getStartTime()) / 2);
+ report.setSortFinishTime(
+ (report.getFinishTime() + report.getShuffleFinishTime()) / 2);
+ }
+
report.setPhase(PHASES.next());
report.setTaskAttemptState(TASK_ATTEMPT_STATES.next());
report.setProgress((float) Math.random());
report.setCounters(TypeConverter.toYarn(newCounters()));
+ report.setContainerId(containerId);
+ report.setDiagnosticInfo(DIAGS.next());
+ report.setStateString("Moving average " + Math.random());
return report;
}
@@ -230,8 +245,6 @@ public class MockJobs extends MockApps {
taid.setTaskId(tid);
taid.setId(i);
final TaskAttemptReport report = newTaskAttemptReport(taid);
- final List<String> diags = Lists.newArrayList();
- diags.add(DIAGS.next());
return new TaskAttempt() {
@Override
public NodeId getNodeId() throws UnsupportedOperationException{
@@ -250,12 +263,12 @@ public class MockJobs extends MockApps {
@Override
public long getLaunchTime() {
- return 0;
+ return report.getStartTime();
}
@Override
public long getFinishTime() {
- return 0;
+ return report.getFinishTime();
}
@Override
@@ -313,7 +326,7 @@ public class MockJobs extends MockApps {
@Override
public List<String> getDiagnostics() {
- return diags;
+ return Lists.newArrayList(report.getDiagnosticInfo());
}
@Override
@@ -323,12 +336,12 @@ public class MockJobs extends MockApps {
@Override
public long getShuffleFinishTime() {
- return 0;
+ return report.getShuffleFinishTime();
}
@Override
public long getSortFinishTime() {
- return 0;
+ return report.getSortFinishTime();
}
@Override
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java Fri Dec 20 21:00:25 2013
@@ -1,4 +1,5 @@
/**
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -425,9 +426,9 @@ public class TestAMWebServicesAttempts e
public void verifyAMTaskAttempt(JSONObject info, TaskAttempt att,
TaskType ttype) throws JSONException {
if (ttype == TaskType.REDUCE) {
- assertEquals("incorrect number of elements", 16, info.length());
+ assertEquals("incorrect number of elements", 17, info.length());
} else {
- assertEquals("incorrect number of elements", 11, info.length());
+ assertEquals("incorrect number of elements", 12, info.length());
}
verifyTaskAttemptGeneric(att, ttype, info.getString("id"),
@@ -532,11 +533,11 @@ public class TestAMWebServicesAttempts e
assertEquals("mergeFinishTime wrong", ta.getSortFinishTime(),
mergeFinishTime);
assertEquals("elapsedShuffleTime wrong",
- ta.getLaunchTime() - ta.getShuffleFinishTime(), elapsedShuffleTime);
+ ta.getShuffleFinishTime() - ta.getLaunchTime(), elapsedShuffleTime);
assertEquals("elapsedMergeTime wrong",
- ta.getShuffleFinishTime() - ta.getSortFinishTime(), elapsedMergeTime);
+ ta.getSortFinishTime() - ta.getShuffleFinishTime(), elapsedMergeTime);
assertEquals("elapsedReduceTime wrong",
- ta.getSortFinishTime() - ta.getFinishTime(), elapsedReduceTime);
+ ta.getFinishTime() - ta.getSortFinishTime(), elapsedReduceTime);
}
@Test
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java Fri Dec 20 21:00:25 2013
@@ -525,12 +525,13 @@ public class TestAMWebServicesTasks exte
public void verifyAMSingleTask(JSONObject info, Task task)
throws JSONException {
- assertEquals("incorrect number of elements", 8, info.length());
+ assertEquals("incorrect number of elements", 9, info.length());
verifyTaskGeneric(task, info.getString("id"), info.getString("state"),
info.getString("type"), info.getString("successfulAttempt"),
info.getLong("startTime"), info.getLong("finishTime"),
- info.getLong("elapsedTime"), (float) info.getDouble("progress"));
+ info.getLong("elapsedTime"), (float) info.getDouble("progress"),
+ info.getString("status"));
}
public void verifyAMTask(JSONArray arr, Job job, String type)
@@ -555,7 +556,7 @@ public class TestAMWebServicesTasks exte
public void verifyTaskGeneric(Task task, String id, String state,
String type, String successfulAttempt, long startTime, long finishTime,
- long elapsedTime, float progress) {
+ long elapsedTime, float progress, String status) {
TaskId taskid = task.getID();
String tid = MRApps.toString(taskid);
@@ -572,6 +573,7 @@ public class TestAMWebServicesTasks exte
assertEquals("finishTime wrong", report.getFinishTime(), finishTime);
assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f);
+ assertEquals("status wrong", report.getStatus(), status);
}
public void verifyAMSingleTaskXML(Element element, Task task) {
@@ -582,7 +584,8 @@ public class TestAMWebServicesTasks exte
WebServicesTestUtils.getXmlLong(element, "startTime"),
WebServicesTestUtils.getXmlLong(element, "finishTime"),
WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
- WebServicesTestUtils.getXmlFloat(element, "progress"));
+ WebServicesTestUtils.getXmlFloat(element, "progress"),
+ WebServicesTestUtils.getXmlString(element, "status"));
}
public void verifyAMTaskXML(NodeList nodes, Job job) {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java Fri Dec 20 21:00:25 2013
@@ -24,10 +24,10 @@ public interface TaskReport {
public abstract TaskId getTaskId();
public abstract TaskState getTaskState();
public abstract float getProgress();
+ public abstract String getStatus();
public abstract long getStartTime();
public abstract long getFinishTime();
public abstract Counters getCounters();
-
public abstract List<TaskAttemptId> getRunningAttemptsList();
public abstract TaskAttemptId getRunningAttempt(int index);
public abstract int getRunningAttemptsCount();
@@ -42,6 +42,7 @@ public interface TaskReport {
public abstract void setTaskId(TaskId taskId);
public abstract void setTaskState(TaskState taskState);
public abstract void setProgress(float progress);
+ public abstract void setStatus(String status);
public abstract void setStartTime(long startTime);
public abstract void setFinishTime(long finishTime);
public abstract void setCounters(Counters counters);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java Fri Dec 20 21:00:25 2013
@@ -49,6 +49,7 @@ public class TaskReportPBImpl extends Pr
private List<TaskAttemptId> runningAttempts = null;
private TaskAttemptId successfulAttemptId = null;
private List<String> diagnostics = null;
+ private String status;
public TaskReportPBImpl() {
@@ -172,10 +173,21 @@ public class TaskReportPBImpl extends Pr
}
@Override
+ public String getStatus() {
+ return status;
+ }
+
+ @Override
public void setProgress(float progress) {
maybeInitBuilder();
builder.setProgress((progress));
}
+
+ @Override
+ public void setStatus(String status) {
+ this.status = status;
+ }
+
@Override
public TaskState getTaskState() {
TaskReportProtoOrBuilder p = viaProto ? proto : builder;
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java Fri Dec 20 21:00:25 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.webapp.App;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil;
import org.apache.hadoop.yarn.util.Times;
@@ -89,6 +90,7 @@ public class HsTaskPage extends HsView {
headRow.
th(".id", "Attempt").
th(".state", "State").
+ th(".status", "Status").
th(".node", "Node").
th(".logs", "Logs").
th(".tsh", "Start Time");
@@ -113,15 +115,16 @@ public class HsTaskPage extends HsView {
// DataTables to display
StringBuilder attemptsTableData = new StringBuilder("[\n");
- for (TaskAttempt ta : getTaskAttempts()) {
- String taid = MRApps.toString(ta.getID());
+ for (TaskAttempt attempt : getTaskAttempts()) {
+ final TaskAttemptInfo ta = new TaskAttemptInfo(attempt, false);
+ String taid = ta.getId();
+
+ String nodeHttpAddr = ta.getNode();
+ String containerIdString = ta.getAssignedContainerIdStr();
+ String nodeIdString = attempt.getAssignedContainerMgrAddress();
+ String nodeRackName = ta.getRack();
- String nodeHttpAddr = ta.getNodeHttpAddress();
- String containerIdString = ta.getAssignedContainerID().toString();
- String nodeIdString = ta.getAssignedContainerMgrAddress();
- String nodeRackName = ta.getNodeRackName();
-
- long attemptStartTime = ta.getLaunchTime();
+ long attemptStartTime = ta.getStartTime();
long shuffleFinishTime = -1;
long sortFinishTime = -1;
long attemptFinishTime = ta.getFinishTime();
@@ -129,8 +132,8 @@ public class HsTaskPage extends HsView {
long elapsedSortTime = -1;
long elapsedReduceTime = -1;
if(type == TaskType.REDUCE) {
- shuffleFinishTime = ta.getShuffleFinishTime();
- sortFinishTime = ta.getSortFinishTime();
+ shuffleFinishTime = attempt.getShuffleFinishTime();
+ sortFinishTime = attempt.getSortFinishTime();
elapsedShuffleTime =
Times.elapsed(attemptStartTime, shuffleFinishTime, false);
elapsedSortTime =
@@ -140,11 +143,13 @@ public class HsTaskPage extends HsView {
}
long attemptElapsed =
Times.elapsed(attemptStartTime, attemptFinishTime, false);
- int sortId = ta.getID().getId() + (ta.getID().getTaskId().getId() * 10000);
+ int sortId = attempt.getID().getId()
+ + (attempt.getID().getTaskId().getId() * 10000);
attemptsTableData.append("[\"")
.append(sortId + " ").append(taid).append("\",\"")
- .append(ta.getState().toString()).append("\",\"")
+ .append(ta.getState()).append("\",\"")
+ .append(ta.getStatus()).append("\",\"")
.append("<a class='nodelink' href='" + MRWebAppUtil.getYARNWebappScheme() + nodeHttpAddr + "'>")
.append(nodeRackName + "/" + nodeHttpAddr + "</a>\",\"")
@@ -167,8 +172,9 @@ public class HsTaskPage extends HsView {
.append(elapsedReduceTime).append("\",\"");
}
attemptsTableData.append(attemptElapsed).append("\",\"")
- .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(
- Joiner.on('\n').join(ta.getDiagnostics())))).append("\"],\n");
+ .append(StringEscapeUtils.escapeJavaScript(
+ StringEscapeUtils.escapeHtml(ta.getNote())))
+ .append("\"],\n");
}
//Remove the last comma and close off the array of arrays
if(attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
@@ -185,6 +191,8 @@ public class HsTaskPage extends HsView {
th().input("search_init").$type(InputType.text).
$name("attempt_state").$value("State")._()._().
th().input("search_init").$type(InputType.text).
+ $name("attempt_status").$value("Status")._()._().
+ th().input("search_init").$type(InputType.text).
$name("attempt_node").$value("Node")._()._().
th().input("search_init").$type(InputType.text).
$name("attempt_node").$value("Logs")._()._().
@@ -283,19 +291,19 @@ public class HsTaskPage extends HsView {
.append("\n,aoColumnDefs:[\n")
//logs column should not filterable (it includes container ID which may pollute searches)
- .append("\n{'aTargets': [ 3 ]")
+ .append("\n{'aTargets': [ 4 ]")
.append(", 'bSearchable': false }")
.append("\n, {'sType':'numeric', 'aTargets': [ 0 ]")
.append(", 'mRender': parseHadoopAttemptID }")
- .append("\n, {'sType':'numeric', 'aTargets': [ 4, 5")
+ .append("\n, {'sType':'numeric', 'aTargets': [ 5, 6")
//Column numbers are different for maps and reduces
- .append(type == TaskType.REDUCE ? ", 6, 7" : "")
+ .append(type == TaskType.REDUCE ? ", 7, 8" : "")
.append(" ], 'mRender': renderHadoopDate }")
.append("\n, {'sType':'numeric', 'aTargets': [")
- .append(type == TaskType.REDUCE ? "8, 9, 10, 11" : "6")
+ .append(type == TaskType.REDUCE ? "9, 10, 11, 12" : "7")
.append(" ], 'mRender': renderHadoopElapsedTime }]")
// Sort by id upon page load
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java Fri Dec 20 21:00:25 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
@@ -138,11 +139,31 @@ public class TestBlocks {
when(attempt.getAssignedContainerMgrAddress()).thenReturn(
"assignedContainerMgrAddress");
when(attempt.getNodeRackName()).thenReturn("nodeRackName");
- when(attempt.getLaunchTime()).thenReturn(100002L);
- when(attempt.getFinishTime()).thenReturn(100012L);
- when(attempt.getShuffleFinishTime()).thenReturn(100010L);
- when(attempt.getSortFinishTime()).thenReturn(100011L);
- when(attempt.getState()).thenReturn(TaskAttemptState.SUCCEEDED);
+
+ final long taStartTime = 100002L;
+ final long taFinishTime = 100012L;
+ final long taShuffleFinishTime = 100010L;
+ final long taSortFinishTime = 100011L;
+ final TaskAttemptState taState = TaskAttemptState.SUCCEEDED;
+
+ when(attempt.getLaunchTime()).thenReturn(taStartTime);
+ when(attempt.getFinishTime()).thenReturn(taFinishTime);
+ when(attempt.getShuffleFinishTime()).thenReturn(taShuffleFinishTime);
+ when(attempt.getSortFinishTime()).thenReturn(taSortFinishTime);
+ when(attempt.getState()).thenReturn(taState);
+
+ TaskAttemptReport taReport = mock(TaskAttemptReport.class);
+ when(taReport.getStartTime()).thenReturn(taStartTime);
+ when(taReport.getFinishTime()).thenReturn(taFinishTime);
+ when(taReport.getShuffleFinishTime()).thenReturn(taShuffleFinishTime);
+ when(taReport.getSortFinishTime()).thenReturn(taSortFinishTime);
+ when(taReport.getContainerId()).thenReturn(containerId);
+ when(taReport.getProgress()).thenReturn(1.0f);
+ when(taReport.getStateString()).thenReturn("Processed 128/128 records");
+ when(taReport.getTaskAttemptState()).thenReturn(taState);
+ when(taReport.getDiagnosticInfo()).thenReturn("");
+
+ when(attempt.getReport()).thenReturn(taReport);
attempts.put(taId, attempt);
when(task.getAttempts()).thenReturn(attempts);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java Fri Dec 20 21:00:25 2013
@@ -444,9 +444,9 @@ public class TestHsWebServicesAttempts e
public void verifyHsTaskAttempt(JSONObject info, TaskAttempt att,
TaskType ttype) throws JSONException {
if (ttype == TaskType.REDUCE) {
- assertEquals("incorrect number of elements", 16, info.length());
+ assertEquals("incorrect number of elements", 17, info.length());
} else {
- assertEquals("incorrect number of elements", 11, info.length());
+ assertEquals("incorrect number of elements", 12, info.length());
}
verifyTaskAttemptGeneric(att, ttype, info.getString("id"),
@@ -551,11 +551,11 @@ public class TestHsWebServicesAttempts e
assertEquals("mergeFinishTime wrong", ta.getSortFinishTime(),
mergeFinishTime);
assertEquals("elapsedShuffleTime wrong",
- ta.getLaunchTime() - ta.getShuffleFinishTime(), elapsedShuffleTime);
+ ta.getShuffleFinishTime() - ta.getLaunchTime(), elapsedShuffleTime);
assertEquals("elapsedMergeTime wrong",
- ta.getShuffleFinishTime() - ta.getSortFinishTime(), elapsedMergeTime);
+ ta.getSortFinishTime() - ta.getShuffleFinishTime(), elapsedMergeTime);
assertEquals("elapsedReduceTime wrong",
- ta.getSortFinishTime() - ta.getFinishTime(), elapsedReduceTime);
+ ta.getFinishTime() - ta.getSortFinishTime(), elapsedReduceTime);
}
@Test
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java?rev=1552808&r1=1552807&r2=1552808&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java Fri Dec 20 21:00:25 2013
@@ -538,7 +538,7 @@ public class TestHsWebServicesTasks exte
public void verifyHsSingleTask(JSONObject info, Task task)
throws JSONException {
- assertEquals("incorrect number of elements", 8, info.length());
+ assertEquals("incorrect number of elements", 9, info.length());
verifyTaskGeneric(task, info.getString("id"), info.getString("state"),
info.getString("type"), info.getString("successfulAttempt"),