You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by jl...@apache.org on 2013/02/13 02:22:01 UTC
svn commit: r1445451 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/s...
Author: jlowe
Date: Wed Feb 13 01:22:01 2013
New Revision: 1445451
URL: http://svn.apache.org/r1445451
Log:
MAPREDUCE-4989. JSONify DataTables input data for Attempts page. Contributed by Ravi Prakash
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1445451&r1=1445450&r2=1445451&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Wed Feb 13 01:22:01 2013
@@ -22,6 +22,9 @@ Release 0.23.7 - UNRELEASED
MAPREDUCE-4905. test org.apache.hadoop.mapred.pipes
(Aleksey Gorshkov via bobby)
+ MAPREDUCE-4989. JSONify DataTables input data for Attempts page (Ravi
+ Prakash via jlowe)
+
OPTIMIZATIONS
MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java?rev=1445451&r1=1445450&r2=1445451&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java Wed Feb 13 01:22:01 2013
@@ -27,17 +27,11 @@ import static org.apache.hadoop.yarn.web
import java.util.Collection;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.SubView;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TD;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
@@ -59,7 +53,7 @@ public class TaskPage extends AppView {
h2($(TITLE));
return;
}
- TBODY<TABLE<Hamlet>> tbody = html.
+ html.
table("#attempts").
thead().
tr().
@@ -71,47 +65,46 @@ public class TaskPage extends AppView {
th(".tsh", "Started").
th(".tsh", "Finished").
th(".tsh", "Elapsed").
- th(".note", "Note")._()._().
- tbody();
+ th(".note", "Note")._()._();
+ // Write all the data into a JavaScript array of arrays for JQuery
+ // DataTables to display
+ StringBuilder attemptsTableData = new StringBuilder("[\n");
+
for (TaskAttempt attempt : getTaskAttempts()) {
TaskAttemptInfo ta = new TaskAttemptInfo(attempt, true);
- String taid = ta.getId();
String progress = percent(ta.getProgress() / 100);
- ContainerId containerId = ta.getAssignedContainerId();
String nodeHttpAddr = ta.getNode();
- long startTime = ta.getStartTime();
- long finishTime = ta.getFinishTime();
- long elapsed = ta.getElapsedTime();
String diag = ta.getNote() == null ? "" : ta.getNote();
- TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
- TD<TR<TBODY<TABLE<Hamlet>>>> nodeTd = row.
- td(".id", taid).
- td(".progress", progress).
- td(".state", ta.getState()).td();
- if (nodeHttpAddr == null) {
- nodeTd._("N/A");
- } else {
- nodeTd.
- a(".nodelink", url("http://", nodeHttpAddr), nodeHttpAddr);
- }
- nodeTd._();
- if (containerId != null) {
- String containerIdStr = ta.getAssignedContainerIdStr();
- row.td().
- a(".logslink", url("http://", nodeHttpAddr, "node", "containerlogs",
- containerIdStr, app.getJob().getUserName()), "logs")._();
- } else {
- row.td()._("N/A")._();
- }
-
- row.
- td(".ts", Times.format(startTime)).
- td(".ts", Times.format(finishTime)).
- td(".dt", StringUtils.formatTime(elapsed)).
- td(".note", diag)._();
+ attemptsTableData.append("[\"")
+ .append(ta.getId()).append("\",\"")
+ .append(progress).append("\",\"")
+ .append(ta.getState().toString()).append("\",\"")
+
+ .append(nodeHttpAddr == null ? "N/A" :
+ "<a class='nodelink' href='http://" + nodeHttpAddr + "'>"
+ + nodeHttpAddr + "</a>")
+ .append("\",\"")
+
+ .append(ta.getAssignedContainerId() == null ? "N/A" :
+ "<a class='logslink' href='" + url("http://", nodeHttpAddr, "node"
+ , "containerlogs", ta.getAssignedContainerIdStr(), app.getJob()
+ .getUserName()) + "'>logs</a>")
+ .append("\",\"")
+
+ .append(ta.getStartTime()).append("\",\"")
+ .append(ta.getFinishTime()).append("\",\"")
+ .append(ta.getElapsedTime()).append("\",\"")
+ .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(
+ diag))).append("\"],\n");
+ }
+ //Remove the last comma and close off the array of arrays
+ if(attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
+ attemptsTableData.delete(attemptsTableData.length()-2, attemptsTableData.length()-1);
}
- tbody._()._();
+ attemptsTableData.append("]");
+ html.script().$type("text/javascript").
+ _("var attemptsTableData=" + attemptsTableData)._();
}
protected boolean isValidRequest() {
@@ -137,9 +130,24 @@ public class TaskPage extends AppView {
}
private String attemptsTableInit() {
- return tableInit().
- // Sort by id upon page load
- append(", aaSorting: [[0, 'asc']]").
- append("}").toString();
+ return tableInit()
+ .append(", 'aaData': attemptsTableData")
+ .append(", bDeferRender: true")
+ .append(", bProcessing: true")
+ .append("\n,aoColumnDefs:[\n")
+
+ //logs column should not filterable (it includes container ID which may pollute searches)
+ .append("\n{'aTargets': [ 4 ]")
+ .append(", 'bSearchable': false }")
+
+ .append("\n, {'sType':'numeric', 'aTargets': [ 5, 6")
+ .append(" ], 'mRender': renderHadoopDate }")
+
+ .append("\n, {'sType':'numeric', 'aTargets': [ 7")
+ .append(" ], 'mRender': renderHadoopElapsedTime }]")
+
+ // Sort by id upon page load
+ .append("\n, aaSorting: [[0, 'asc']]")
+ .append("}").toString();
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java?rev=1445451&r1=1445450&r2=1445451&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java Wed Feb 13 01:22:01 2013
@@ -29,6 +29,7 @@ import static org.apache.hadoop.yarn.web
import java.util.Collection;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
@@ -109,13 +110,17 @@ public class HsTaskPage extends HsView {
th(".note", "Note");
TBODY<TABLE<Hamlet>> tbody = headRow._()._().tbody();
- for (TaskAttempt ta : getTaskAttempts()) {
+ // Write all the data into a JavaScript array of arrays for JQuery
+ // DataTables to display
+ StringBuilder attemptsTableData = new StringBuilder("[\n");
+
+ for (TaskAttempt ta : getTaskAttempts()) {
String taid = MRApps.toString(ta.getID());
String nodeHttpAddr = ta.getNodeHttpAddress();
String containerIdString = ta.getAssignedContainerID().toString();
String nodeIdString = ta.getAssignedContainerMgrAddress();
- String nodeRackName = ta.getNodeRackName();
+ String nodeRackName = ta.getNodeRackName();
long attemptStartTime = ta.getLaunchTime();
long shuffleFinishTime = -1;
@@ -137,58 +142,43 @@ public class HsTaskPage extends HsView {
long attemptElapsed =
Times.elapsed(attemptStartTime, attemptFinishTime, false);
int sortId = ta.getID().getId() + (ta.getID().getTaskId().getId() * 10000);
-
- TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
- TD<TR<TBODY<TABLE<Hamlet>>>> td = row.td();
-
- td.br().$title(String.valueOf(sortId))._(). // sorting
- _(taid)._().td(ta.getState().toString()).td().a(".nodelink",
- "http://"+ nodeHttpAddr,
- nodeRackName + "/" + nodeHttpAddr);
- td._();
- row.td().
- a(".logslink",
- url("logs", nodeIdString, containerIdString, taid, app.getJob()
- .getUserName()), "logs")._();
-
- row.td().
- br().$title(String.valueOf(attemptStartTime))._().
- _(Times.format(attemptStartTime))._();
+
+ attemptsTableData.append("[\"")
+ .append(sortId + " ").append(taid).append("\",\"")
+ .append(ta.getState().toString()).append("\",\"")
+
+ .append("<a class='nodelink' href='http://" + nodeHttpAddr + "'>")
+ .append(nodeRackName + "/" + nodeHttpAddr + "</a>\",\"")
+
+ .append("<a class='logslink' href='").append(url("logs", nodeIdString
+ , containerIdString, taid, app.getJob().getUserName()))
+ .append("'>logs</a>\",\"")
+
+ .append(attemptStartTime).append("\",\"");
if(type == TaskType.REDUCE) {
- row.td().
- br().$title(String.valueOf(shuffleFinishTime))._().
- _(Times.format(shuffleFinishTime))._();
- row.td().
- br().$title(String.valueOf(sortFinishTime))._().
- _(Times.format(sortFinishTime))._();
+ attemptsTableData.append(shuffleFinishTime).append("\",\"")
+ .append(sortFinishTime).append("\",\"");
}
- row.
- td().
- br().$title(String.valueOf(attemptFinishTime))._().
- _(Times.format(attemptFinishTime))._();
-
+ attemptsTableData.append(attemptFinishTime).append("\",\"");
+
if(type == TaskType.REDUCE) {
- row.td().
- br().$title(String.valueOf(elapsedShuffleTime))._().
- _(formatTime(elapsedShuffleTime))._();
- row.td().
- br().$title(String.valueOf(elapsedSortTime))._().
- _(formatTime(elapsedSortTime))._();
- row.td().
- br().$title(String.valueOf(elapsedReduceTime))._().
- _(formatTime(elapsedReduceTime))._();
+ attemptsTableData.append(elapsedShuffleTime).append("\",\"")
+ .append(elapsedSortTime).append("\",\"")
+ .append(elapsedReduceTime).append("\",\"");
}
-
- row.
- td().
- br().$title(String.valueOf(attemptElapsed))._().
- _(formatTime(attemptElapsed))._().
- td(".note", Joiner.on('\n').join(ta.getDiagnostics()));
- row._();
+ attemptsTableData.append(attemptElapsed).append("\",\"")
+ .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(
+ Joiner.on('\n').join(ta.getDiagnostics())))).append("\"],\n");
}
-
-
+ //Remove the last comma and close off the array of arrays
+ if(attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
+ attemptsTableData.delete(attemptsTableData.length()-2, attemptsTableData.length()-1);
+ }
+ attemptsTableData.append("]");
+ html.script().$type("text/javascript").
+ _("var attemptsTableData=" + attemptsTableData)._();
+
TR<TFOOT<TABLE<Hamlet>>> footRow = tbody._().tfoot().tr();
footRow.
th().input("search_init").$type(InputType.text).
@@ -236,10 +226,6 @@ public class HsTaskPage extends HsView {
footRow._()._()._();
}
- private String formatTime(long elapsed) {
- return elapsed < 0 ? "N/A" : StringUtils.formatTime(elapsed);
- }
-
/**
* @return true if this is a valid request else false.
*/
@@ -291,24 +277,34 @@ public class HsTaskPage extends HsView {
TaskId taskID = MRApps.toTaskID($(TASK_ID));
type = taskID.getTaskType();
}
- StringBuilder b = tableInit().
- append(",aoColumnDefs:[");
-
- b.append("{'sType':'title-numeric', 'aTargets': [ 0");
- if(type == TaskType.REDUCE) {
- b.append(", 7, 8, 9, 10");
- } else { //MAP
- b.append(", 5");
- }
- b.append(" ] }]");
-
- // Sort by id upon page load
- b.append(", aaSorting: [[0, 'asc']]");
-
- b.append("}");
- return b.toString();
+ StringBuilder b = tableInit()
+ .append(", 'aaData': attemptsTableData")
+ .append(", bDeferRender: true")
+ .append(", bProcessing: true")
+ .append("\n,aoColumnDefs:[\n")
+
+ //logs column should not filterable (it includes container ID which may pollute searches)
+ .append("\n{'aTargets': [ 3 ]")
+ .append(", 'bSearchable': false }")
+
+ .append("\n, {'sType':'numeric', 'aTargets': [ 0 ]")
+ .append(", 'mRender': parseHadoopAttemptID }")
+
+ .append("\n, {'sType':'numeric', 'aTargets': [ 4, 5")
+ //Column numbers are different for maps and reduces
+ .append(type == TaskType.REDUCE ? ", 6, 7" : "")
+ .append(" ], 'mRender': renderHadoopDate }")
+
+ .append("\n, {'sType':'numeric', 'aTargets': [")
+ .append(type == TaskType.REDUCE ? "8, 9, 10, 11" : "6")
+ .append(" ], 'mRender': renderHadoopElapsedTime }]")
+
+ // Sort by id upon page load
+ .append("\n, aaSorting: [[0, 'asc']]")
+ .append("}");
+ return b.toString();
}
-
+
private String attemptsPostTableInit() {
return "var asInitVals = new Array();\n" +
"$('tfoot input').keyup( function () \n{"+
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java?rev=1445451&r1=1445450&r2=1445451&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java Wed Feb 13 01:22:01 2013
@@ -28,7 +28,6 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
-import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
@@ -204,9 +203,4 @@ public class HsTasksBlock extends HtmlBl
footRow._()._()._();
}
-
- private String formatTime(long elapsed) {
- return elapsed < 0 ? "N/A" : StringUtils.formatTime(elapsed);
- }
-
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java?rev=1445451&r1=1445450&r2=1445451&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java Wed Feb 13 01:22:01 2013
@@ -67,33 +67,25 @@ public class HsTasksPage extends HsView
type = MRApps.taskType(symbol);
}
StringBuilder b = tableInit().
- append(", 'aaData': tasksTableData");
- b.append(", bDeferRender: true");
- b.append(", bProcessing: true");
+ append(", 'aaData': tasksTableData")
+ .append(", bDeferRender: true")
+ .append(", bProcessing: true")
- b.append("\n, aoColumnDefs: [\n");
- b.append("{'sType':'numeric', 'aTargets': [ 0 ]");
- b.append(", 'mRender': parseHadoopID }");
+ .append("\n, aoColumnDefs: [\n")
+ .append("{'sType':'numeric', 'aTargets': [ 0 ]")
+ .append(", 'mRender': parseHadoopID }")
- b.append(", {'sType':'numeric', 'aTargets': [ 4");
- if(type == TaskType.REDUCE) {
- b.append(", 9, 10, 11, 12");
- } else { //MAP
- b.append(", 7");
- }
- b.append(" ], 'mRender': renderHadoopElapsedTime }");
+ .append(", {'sType':'numeric', 'aTargets': [ 4")
+ .append(type == TaskType.REDUCE ? ", 9, 10, 11, 12" : ", 7")
+ .append(" ], 'mRender': renderHadoopElapsedTime }")
- b.append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5");
- if(type == TaskType.REDUCE) {
- b.append(", 6, 7, 8");
- } else { //MAP
- b.append(", 6");
- }
- b.append(" ], 'mRender': renderHadoopDate }]");
+ .append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5")
+ .append(type == TaskType.REDUCE ? ", 6, 7, 8" : ", 6")
+ .append(" ], 'mRender': renderHadoopDate }]")
// Sort by id upon page load
- b.append("\n, aaSorting: [[0, 'asc']]");
- b.append("}");
+ .append("\n, aaSorting: [[0, 'asc']]")
+ .append("}");
return b.toString();
}