You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ma...@apache.org on 2011/11/30 09:44:54 UTC
svn commit: r1208328 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
hadoop-mapreduce-client/hadoop-mapreduce-client-co...
Author: mahadev
Date: Wed Nov 30 08:44:52 2011
New Revision: 1208328
URL: http://svn.apache.org/viewvc?rev=1208328&view=rev
Log:
MAPREDUCE-3450. NM port info no longer available in JobHistory. (Siddharth Seth via mahadev) - Merging r1208327 from trunk
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Wed Nov 30 08:44:52 2011
@@ -151,6 +151,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3433. Finding counters by legacy group name returns empty
counters. (tomwhite)
+ MAPREDUCE-3450. NM port info no longer available in JobHistory.
+ (Siddharth Seth via mahadev)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Nov 30 08:44:52 2011
@@ -922,8 +922,11 @@ public abstract class TaskAttemptImpl im
TypeConverter.fromYarn(taskAttempt.attemptId.getTaskId()
.getTaskType()), attemptState.toString(),
taskAttempt.finishTime,
- taskAttempt.containerMgrAddress == null ? "UNKNOWN"
- : taskAttempt.containerMgrAddress, StringUtils.join(
+ taskAttempt.containerNodeId == null ? "UNKNOWN"
+ : taskAttempt.containerNodeId.getHost(),
+ taskAttempt.containerNodeId == null ? -1
+ : taskAttempt.containerNodeId.getPort(),
+ StringUtils.join(
LINE_SEPARATOR, taskAttempt.getDiagnostics()), taskAttempt
.getProgressSplitBlock().burst());
return tauce;
@@ -1273,6 +1276,7 @@ public abstract class TaskAttemptImpl im
finishTime,
this.containerNodeId == null ? "UNKNOWN"
: this.containerNodeId.getHost(),
+ this.containerNodeId == null ? -1 : this.containerNodeId.getPort(),
this.nodeRackName == null ? "UNKNOWN" : this.nodeRackName,
this.reportedStatus.stateString,
TypeConverter.fromYarn(getCounters()),
@@ -1288,7 +1292,8 @@ public abstract class TaskAttemptImpl im
this.reportedStatus.sortFinishTime,
finishTime,
this.containerNodeId == null ? "UNKNOWN"
- : this.containerNodeId.getHost(),
+ : this.containerNodeId.getHost(),
+ this.containerNodeId == null ? -1 : this.containerNodeId.getPort(),
this.nodeRackName == null ? "UNKNOWN" : this.nodeRackName,
this.reportedStatus.stateString,
TypeConverter.fromYarn(getCounters()),
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr Wed Nov 30 08:44:52 2011
@@ -136,6 +136,7 @@
{"name": "mapFinishTime", "type": "long"},
{"name": "finishTime", "type": "long"},
{"name": "hostname", "type": "string"},
+ {"name": "port", "type": "int"},
{"name": "rackname", "type": "string"},
{"name": "state", "type": "string"},
{"name": "counters", "type": "JhCounters"},
@@ -156,6 +157,7 @@
{"name": "sortFinishTime", "type": "long"},
{"name": "finishTime", "type": "long"},
{"name": "hostname", "type": "string"},
+ {"name": "port", "type": "int"},
{"name": "rackname", "type": "string"},
{"name": "state", "type": "string"},
{"name": "counters", "type": "JhCounters"},
@@ -199,6 +201,7 @@
{"name": "attemptId", "type": "string"},
{"name": "finishTime", "type": "long"},
{"name": "hostname", "type": "string"},
+ {"name": "port", "type": "int"},
{"name": "status", "type": "string"},
{"name": "error", "type": "string"},
{"name": "clockSplits", "type": { "type": "array", "items": "int"}},
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java Wed Nov 30 08:44:52 2011
@@ -209,6 +209,7 @@ public class JobHistoryParser {
attemptInfo.sortFinishTime = event.getSortFinishTime();
attemptInfo.counters = event.getCounters();
attemptInfo.hostname = event.getHostname();
+ attemptInfo.port = event.getPort();
attemptInfo.rackname = event.getRackName();
}
@@ -222,6 +223,7 @@ public class JobHistoryParser {
attemptInfo.mapFinishTime = event.getMapFinishTime();
attemptInfo.counters = event.getCounters();
attemptInfo.hostname = event.getHostname();
+ attemptInfo.port = event.getPort();
attemptInfo.rackname = event.getRackname();
}
@@ -234,6 +236,7 @@ public class JobHistoryParser {
attemptInfo.error = event.getError();
attemptInfo.status = event.getTaskStatus();
attemptInfo.hostname = event.getHostname();
+ attemptInfo.port = event.getPort();
attemptInfo.shuffleFinishTime = event.getFinishTime();
attemptInfo.sortFinishTime = event.getFinishTime();
attemptInfo.mapFinishTime = event.getFinishTime();
@@ -542,6 +545,7 @@ public class JobHistoryParser {
int httpPort;
int shufflePort;
String hostname;
+ int port;
String rackname;
ContainerId containerId;
@@ -552,6 +556,7 @@ public class JobHistoryParser {
startTime = finishTime = shuffleFinishTime = sortFinishTime =
mapFinishTime = -1;
error = state = trackerName = hostname = rackname = "";
+ port = -1;
httpPort = -1;
shufflePort = -1;
}
@@ -599,6 +604,8 @@ public class JobHistoryParser {
public String getTrackerName() { return trackerName; }
/** @return the host name */
public String getHostname() { return hostname; }
+ /** @return the port */
+ public int getPort() { return port; }
/** @return the rack name */
public String getRackname() { return rackname; }
/** @return the counters for the attempt */
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java Wed Nov 30 08:44:52 2011
@@ -44,6 +44,7 @@ public class MapAttemptFinishedEvent im
* @param mapFinishTime Finish time of the map phase
* @param finishTime Finish time of the attempt
* @param hostname Name of the host where the map executed
+ * @param port RPC port for the tracker host.
* @param rackName Name of the rack where the map executed
* @param state State string for the attempt
* @param counters Counters for the attempt
@@ -57,9 +58,8 @@ public class MapAttemptFinishedEvent im
*/
public MapAttemptFinishedEvent
(TaskAttemptID id, TaskType taskType, String taskStatus,
- long mapFinishTime, long finishTime, String hostname, String rackName,
- String state, Counters counters,
- int[][] allSplits) {
+ long mapFinishTime, long finishTime, String hostname, int port,
+ String rackName, String state, Counters counters, int[][] allSplits) {
datum.taskid = new Utf8(id.getTaskID().toString());
datum.attemptId = new Utf8(id.toString());
datum.taskType = new Utf8(taskType.name());
@@ -67,6 +67,7 @@ public class MapAttemptFinishedEvent im
datum.mapFinishTime = mapFinishTime;
datum.finishTime = finishTime;
datum.hostname = new Utf8(hostname);
+ datum.port = port;
datum.rackname = new Utf8(rackName);
datum.state = new Utf8(state);
datum.counters = EventWriter.toAvro(counters);
@@ -106,7 +107,7 @@ public class MapAttemptFinishedEvent im
(TaskAttemptID id, TaskType taskType, String taskStatus,
long mapFinishTime, long finishTime, String hostname,
String state, Counters counters) {
- this(id, taskType, taskStatus, mapFinishTime, finishTime, hostname, "",
+ this(id, taskType, taskStatus, mapFinishTime, finishTime, hostname, -1, "",
state, counters, null);
}
@@ -136,6 +137,8 @@ public class MapAttemptFinishedEvent im
public long getFinishTime() { return datum.finishTime; }
/** Get the host name */
public String getHostname() { return datum.hostname.toString(); }
+ /** Get the tracker rpc port */
+ public int getPort() { return datum.port; }
/** Get the rack name */
public String getRackname() { return datum.rackname.toString(); }
/** Get the state string */
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java Wed Nov 30 08:44:52 2011
@@ -46,6 +46,7 @@ public class ReduceAttemptFinishedEvent
* @param sortFinishTime Finish time of the sort phase
* @param finishTime Finish time of the attempt
* @param hostname Name of the host where the attempt executed
+ * @param port RPC port for the tracker host.
* @param rackName Name of the rack where the attempt executed
* @param state State of the attempt
* @param counters Counters for the attempt
@@ -57,8 +58,8 @@ public class ReduceAttemptFinishedEvent
public ReduceAttemptFinishedEvent
(TaskAttemptID id, TaskType taskType, String taskStatus,
long shuffleFinishTime, long sortFinishTime, long finishTime,
- String hostname, String rackName, String state, Counters counters,
- int[][] allSplits) {
+ String hostname, int port, String rackName, String state,
+ Counters counters, int[][] allSplits) {
datum.taskid = new Utf8(id.getTaskID().toString());
datum.attemptId = new Utf8(id.toString());
datum.taskType = new Utf8(taskType.name());
@@ -67,6 +68,7 @@ public class ReduceAttemptFinishedEvent
datum.sortFinishTime = sortFinishTime;
datum.finishTime = finishTime;
datum.hostname = new Utf8(hostname);
+ datum.port = port;
datum.rackname = new Utf8(rackName);
datum.state = new Utf8(state);
datum.counters = EventWriter.toAvro(counters);
@@ -108,7 +110,7 @@ public class ReduceAttemptFinishedEvent
String hostname, String state, Counters counters) {
this(id, taskType, taskStatus,
shuffleFinishTime, sortFinishTime, finishTime,
- hostname, "", state, counters, null);
+ hostname, -1, "", state, counters, null);
}
ReduceAttemptFinishedEvent() {}
@@ -138,6 +140,8 @@ public class ReduceAttemptFinishedEvent
public long getFinishTime() { return datum.finishTime; }
/** Get the name of the host where the attempt ran */
public String getHostname() { return datum.hostname.toString(); }
+ /** Get the tracker rpc port */
+ public int getPort() { return datum.port; }
/** Get the rack name of the node where the attempt ran */
public String getRackName() { return datum.rackname.toString(); }
/** Get the state string */
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java Wed Nov 30 08:44:52 2011
@@ -46,6 +46,7 @@ public class TaskAttemptUnsuccessfulComp
* @param status Status of the attempt
* @param finishTime Finish time of the attempt
* @param hostname Name of the host where the attempt executed
+ * @param port rpc port for for the tracker
* @param error Error string
* @param allSplits the "splits", or a pixelated graph of various
* measurable worker node state variables against progress.
@@ -55,13 +56,14 @@ public class TaskAttemptUnsuccessfulComp
public TaskAttemptUnsuccessfulCompletionEvent
(TaskAttemptID id, TaskType taskType,
String status, long finishTime,
- String hostname, String error,
+ String hostname, int port, String error,
int[][] allSplits) {
datum.taskid = new Utf8(id.getTaskID().toString());
datum.taskType = new Utf8(taskType.name());
datum.attemptId = new Utf8(id.toString());
datum.finishTime = finishTime;
datum.hostname = new Utf8(hostname);
+ datum.port = port;
datum.error = new Utf8(error);
datum.status = new Utf8(status);
@@ -97,7 +99,7 @@ public class TaskAttemptUnsuccessfulComp
(TaskAttemptID id, TaskType taskType,
String status, long finishTime,
String hostname, String error) {
- this(id, taskType, status, finishTime, hostname, error, null);
+ this(id, taskType, status, finishTime, hostname, -1, error, null);
}
TaskAttemptUnsuccessfulCompletionEvent() {}
@@ -121,6 +123,8 @@ public class TaskAttemptUnsuccessfulComp
public long getFinishTime() { return datum.finishTime; }
/** Get the name of the host where the attempt executed */
public String getHostname() { return datum.hostname.toString(); }
+ /** Get the rpc port for the host where the attempt executed */
+ public int getPort() { return datum.port; }
/** Get the error string */
public String getError() { return datum.error.toString(); }
/** Get the task status */
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java Wed Nov 30 08:44:52 2011
@@ -80,12 +80,11 @@ public class CompletedTaskAttempt implem
report.setStateString(attemptInfo.getState());
report.setCounters(getCounters());
report.setContainerId(attemptInfo.getContainerId());
- String []hostSplits = attemptInfo.getHostname().split(":");
- if (hostSplits.length != 2) {
+ if (attemptInfo.getHostname() == null) {
report.setNodeManagerHost("UNKNOWN");
} else {
- report.setNodeManagerHost(hostSplits[0]);
- report.setNodeManagerPort(Integer.parseInt(hostSplits[1]));
+ report.setNodeManagerHost(attemptInfo.getHostname());
+ report.setNodeManagerPort(attemptInfo.getPort());
}
report.setNodeManagerHttpPort(attemptInfo.getHttpPort());
}
@@ -97,7 +96,7 @@ public class CompletedTaskAttempt implem
@Override
public String getAssignedContainerMgrAddress() {
- return attemptInfo.getHostname();
+ return attemptInfo.getHostname() + ":" + attemptInfo.getPort();
}
@Override
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java Wed Nov 30 08:44:52 2011
@@ -165,6 +165,9 @@ public class TestJobHistoryEvents {
//Verify the wrong ctor is not being used. Remove after mrv1 is removed.
ContainerId fakeCid = BuilderUtils.newContainerId(-1, -1, -1, -1);
Assert.assertFalse(attempt.getAssignedContainerID().equals(fakeCid));
+ //Verify complete contianerManagerAddress
+ Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT,
+ attempt.getAssignedContainerMgrAddress());
}
static class MRAppWithHistory extends MRApp {
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java Wed Nov 30 08:44:52 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
@@ -64,13 +65,14 @@ public class TestJobHistoryParsing {
public static class MyResolver implements DNSToSwitchMapping {
@Override
public List<String> resolve(List<String> names) {
- return Arrays.asList(new String[]{"MyRackName"});
+ return Arrays.asList(new String[]{"/MyRackName"});
}
}
@Test
public void testHistoryParsing() throws Exception {
Configuration conf = new Configuration();
+ conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
long amStartTimeEst = System.currentTimeMillis();
conf.setClass(
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
@@ -165,10 +167,12 @@ public class TestJobHistoryParsing {
Assert.assertNotNull("TaskAttemptInfo not found", taskAttemptInfo);
Assert.assertEquals("Incorrect shuffle port for task attempt",
taskAttempt.getShufflePort(), taskAttemptInfo.getShufflePort());
+ Assert.assertEquals(MRApp.NM_HOST, taskAttemptInfo.getHostname());
+ Assert.assertEquals(MRApp.NM_PORT, taskAttemptInfo.getPort());
// Verify rack-name
Assert.assertEquals("rack-name is incorrect", taskAttemptInfo
- .getRackname(), "MyRackName");
+ .getRackname(), "/MyRackName");
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java Wed Nov 30 08:44:52 2011
@@ -2685,7 +2685,7 @@ public class JobInProgress {
MapAttemptFinishedEvent mfe = new MapAttemptFinishedEvent(
statusAttemptID, taskType, TaskStatus.State.SUCCEEDED.toString(),
status.getMapFinishTime(),
- status.getFinishTime(), trackerHostname, "",
+ status.getFinishTime(), trackerHostname, -1, "",
status.getStateString(),
new org.apache.hadoop.mapreduce.Counters(status.getCounters()),
tip.getSplits(statusAttemptID).burst()
@@ -2698,7 +2698,7 @@ public class JobInProgress {
statusAttemptID, taskType, TaskStatus.State.SUCCEEDED.toString(),
status.getShuffleFinishTime(),
status.getSortFinishTime(), status.getFinishTime(),
- trackerHostname, "", status.getStateString(),
+ trackerHostname, -1, "", status.getStateString(),
new org.apache.hadoop.mapreduce.Counters(status.getCounters()),
tip.getSplits(statusAttemptID).burst()
);
@@ -3208,7 +3208,7 @@ public class JobInProgress {
(taskid,
taskType, taskStatus.getRunState().toString(),
finishTime,
- taskTrackerHostName, diagInfo,
+ taskTrackerHostName, -1, diagInfo,
splits.burst());
jobHistory.logEvent(tue, taskid.getJobID());
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java Wed Nov 30 08:44:52 2011
@@ -83,7 +83,7 @@ public class TestJobHistoryEvents extend
for (TaskType t : types) {
TaskAttemptUnsuccessfulCompletionEvent tauce =
new TaskAttemptUnsuccessfulCompletionEvent
- (id, t, state, 0L, "", "", NULL_SPLITS_ARRAY);
+ (id, t, state, 0L, "", -1, "", NULL_SPLITS_ARRAY);
assertEquals(expected, tauce.getEventType());
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java Wed Nov 30 08:44:52 2011
@@ -938,12 +938,12 @@ public class TestRumenJobTraces {
(TaskAttemptID.forName("attempt_200904211745_0003_m_000004_1"),
TaskType.valueOf("MAP"), "STATUS", 1234567890L,
"/194\\.6\\.134\\.80/cluster50262\\.secondleveldomain\\.com",
- "MACHINE_EXPLODED", splits));
+ -1, "MACHINE_EXPLODED", splits));
subject.process(new TaskAttemptUnsuccessfulCompletionEvent
(TaskAttemptID.forName("attempt_200904211745_0003_m_000004_2"),
TaskType.valueOf("MAP"), "STATUS", 1234567890L,
"/194\\.6\\.134\\.80/cluster50263\\.secondleveldomain\\.com",
- "MACHINE_EXPLODED", splits));
+ -1, "MACHINE_EXPLODED", splits));
subject.process(new TaskStartedEvent(TaskID
.forName("task_200904211745_0003_m_000004"), 1234567890L, TaskType
.valueOf("MAP"),
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java Wed Nov 30 08:44:52 2011
@@ -73,7 +73,7 @@ public class MapAttempt20LineHistoryEven
that.originalTaskType, status,
Long.parseLong(finishTime),
Long.parseLong(finishTime),
- hostName, null, state, maybeParseCounters(counters),
+ hostName, -1, null, state, maybeParseCounters(counters),
null);
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java Wed Nov 30 08:44:52 2011
@@ -77,7 +77,7 @@ public class ReduceAttempt20LineHistoryE
Long.parseLong(shuffleFinish),
Long.parseLong(sortFinish),
Long.parseLong(finishTime),
- hostName, null,
+ hostName, -1, null,
state, maybeParseCounters(counters),
null);
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java?rev=1208328&r1=1208327&r2=1208328&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java Wed Nov 30 08:44:52 2011
@@ -141,7 +141,7 @@ public abstract class TaskAttempt20LineE
return new TaskAttemptUnsuccessfulCompletionEvent
(taskAttemptID,
that.originalTaskType, status, Long.parseLong(finishTime),
- hostName, error, null);
+ hostName, -1, error, null);
}
return null;