You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by at...@apache.org on 2011/12/27 19:18:41 UTC
svn commit: r1224965 [3/3] - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/mai...
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=1224965&r1=1224964&r2=1224965&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java Tue Dec 27 18:18:36 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.TaskA
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.jobhistory.AMStartedEvent;
import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobFinished;
import org.apache.hadoop.mapreduce.jobhistory.JobFinishedEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobInfoChangeEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobInitedEvent;
@@ -45,14 +46,15 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptFinished;
import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptFinishedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStartedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletion;
import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletionEvent;
+import org.apache.hadoop.mapreduce.jobhistory.TaskFailed;
import org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskFinished;
import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskUpdatedEvent;
import org.apache.hadoop.tools.rumen.Pre21JobHistoryConstants.Values;
-import org.apache.hadoop.tools.rumen.datatypes.JobProperties;
import org.apache.hadoop.util.StringUtils;
/**
@@ -67,16 +69,16 @@ public class JobBuilder {
private boolean finalized = false;
- private LoggedJob result = new LoggedJob();
+ private ParsedJob result = new ParsedJob();
- private Map<String, LoggedTask> mapTasks = new HashMap<String, LoggedTask>();
- private Map<String, LoggedTask> reduceTasks =
- new HashMap<String, LoggedTask>();
- private Map<String, LoggedTask> otherTasks =
- new HashMap<String, LoggedTask>();
+ private Map<String, ParsedTask> mapTasks = new HashMap<String, ParsedTask>();
+ private Map<String, ParsedTask> reduceTasks =
+ new HashMap<String, ParsedTask>();
+ private Map<String, ParsedTask> otherTasks =
+ new HashMap<String, ParsedTask>();
- private Map<String, LoggedTaskAttempt> attempts =
- new HashMap<String, LoggedTaskAttempt>();
+ private Map<String, ParsedTaskAttempt> attempts =
+ new HashMap<String, ParsedTaskAttempt>();
private Map<ParsedHost, ParsedHost> allHosts =
new HashMap<ParsedHost, ParsedHost>();
@@ -123,7 +125,7 @@ public class JobBuilder {
public void process(HistoryEvent event) {
if (finalized) {
throw new IllegalStateException(
- "JobBuilder.process(HistoryEvent event) called after LoggedJob built");
+ "JobBuilder.process(HistoryEvent event) called after ParsedJob built");
}
// these are in lexicographical order by class name.
@@ -229,12 +231,16 @@ public class JobBuilder {
public void process(Properties conf) {
if (finalized) {
throw new IllegalStateException(
- "JobBuilder.process(Properties conf) called after LoggedJob built");
+ "JobBuilder.process(Properties conf) called after ParsedJob built");
}
//TODO remove this once the deprecate APIs in LoggedJob are removed
- result.setQueue(extract(conf, JobConfPropertyNames.QUEUE_NAMES
- .getCandidates(), "default"));
+ String queue = extract(conf, JobConfPropertyNames.QUEUE_NAMES
+ .getCandidates(), null);
+ // set the queue name if existing
+ if (queue != null) {
+ result.setQueue(queue);
+ }
result.setJobName(extract(conf, JobConfPropertyNames.JOB_NAMES
.getCandidates(), null));
@@ -252,9 +258,9 @@ public class JobBuilder {
* Request the builder to build the final object. Once called, the
* {@link JobBuilder} would accept no more events or job-conf properties.
*
- * @return Parsed {@link LoggedJob} object.
+ * @return Parsed {@link ParsedJob} object.
*/
- public LoggedJob build() {
+ public ParsedJob build() {
// The main job here is to build CDFs and manage the conf
finalized = true;
@@ -416,7 +422,7 @@ public class JobBuilder {
}
private void processTaskUpdatedEvent(TaskUpdatedEvent event) {
- LoggedTask task = getTask(event.getTaskId().toString());
+ ParsedTask task = getTask(event.getTaskId().toString());
if (task == null) {
return;
}
@@ -424,7 +430,7 @@ public class JobBuilder {
}
private void processTaskStartedEvent(TaskStartedEvent event) {
- LoggedTask task =
+ ParsedTask task =
getOrMakeTask(event.getTaskType(), event.getTaskId().toString(), true);
task.setStartTime(event.getStartTime());
task.setPreferredLocations(preferredLocationForSplits(event
@@ -432,7 +438,7 @@ public class JobBuilder {
}
private void processTaskFinishedEvent(TaskFinishedEvent event) {
- LoggedTask task =
+ ParsedTask task =
getOrMakeTask(event.getTaskType(), event.getTaskId().toString(), false);
if (task == null) {
return;
@@ -443,18 +449,22 @@ public class JobBuilder {
}
private void processTaskFailedEvent(TaskFailedEvent event) {
- LoggedTask task =
+ ParsedTask task =
getOrMakeTask(event.getTaskType(), event.getTaskId().toString(), false);
if (task == null) {
return;
}
task.setFinishTime(event.getFinishTime());
task.setTaskStatus(getPre21Value(event.getTaskStatus()));
+ TaskFailed t = (TaskFailed)(event.getDatum());
+ task.putDiagnosticInfo(t.error.toString());
+ task.putFailedDueToAttemptId(t.failedDueToAttempt.toString());
+ // No counters in TaskFailedEvent
}
private void processTaskAttemptUnsuccessfulCompletionEvent(
TaskAttemptUnsuccessfulCompletionEvent event) {
- LoggedTaskAttempt attempt =
+ ParsedTaskAttempt attempt =
getOrMakeTaskAttempt(event.getTaskType(), event.getTaskId().toString(),
event.getTaskAttemptId().toString());
@@ -463,10 +473,11 @@ public class JobBuilder {
}
attempt.setResult(getPre21Value(event.getTaskStatus()));
- ParsedHost parsedHost = getAndRecordParsedHost(event.getHostname());
-
- if (parsedHost != null) {
- attempt.setLocation(parsedHost.makeLoggedLocation());
+ attempt.setHostName(event.getHostname(), event.getRackName());
+ ParsedHost pHost =
+ getAndRecordParsedHost(event.getRackName(), event.getHostname());
+ if (pHost != null) {
+ attempt.setLocation(pHost.makeLoggedLocation());
}
attempt.setFinishTime(event.getFinishTime());
@@ -475,28 +486,37 @@ public class JobBuilder {
attempt.arraySetCpuUsages(event.getCpuUsages());
attempt.arraySetVMemKbytes(event.getVMemKbytes());
attempt.arraySetPhysMemKbytes(event.getPhysMemKbytes());
+ TaskAttemptUnsuccessfulCompletion t =
+ (TaskAttemptUnsuccessfulCompletion) (event.getDatum());
+ attempt.putDiagnosticInfo(t.error.toString());
+ // No counters in TaskAttemptUnsuccessfulCompletionEvent
}
private void processTaskAttemptStartedEvent(TaskAttemptStartedEvent event) {
- LoggedTaskAttempt attempt =
+ ParsedTaskAttempt attempt =
getOrMakeTaskAttempt(event.getTaskType(), event.getTaskId().toString(),
event.getTaskAttemptId().toString());
if (attempt == null) {
return;
}
attempt.setStartTime(event.getStartTime());
+ attempt.putTrackerName(event.getTrackerName());
+ attempt.putHttpPort(event.getHttpPort());
+ attempt.putShufflePort(event.getShufflePort());
}
private void processTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) {
- LoggedTaskAttempt attempt =
+ ParsedTaskAttempt attempt =
getOrMakeTaskAttempt(event.getTaskType(), event.getTaskId().toString(),
event.getAttemptId().toString());
if (attempt == null) {
return;
}
attempt.setResult(getPre21Value(event.getTaskStatus()));
- attempt.setLocation(getAndRecordParsedHost(event.getHostname())
- .makeLoggedLocation());
+ ParsedHost pHost = getAndRecordParsedHost(event.getRackName(), event.getHostname());
+ if (pHost != null) {
+ attempt.setLocation(pHost.makeLoggedLocation());
+ }
attempt.setFinishTime(event.getFinishTime());
attempt
.incorporateCounters(((TaskAttemptFinished) event.getDatum()).counters);
@@ -504,7 +524,7 @@ public class JobBuilder {
private void processReduceAttemptFinishedEvent(
ReduceAttemptFinishedEvent event) {
- LoggedTaskAttempt attempt =
+ ParsedTaskAttempt attempt =
getOrMakeTaskAttempt(event.getTaskType(), event.getTaskId().toString(),
event.getAttemptId().toString());
if (attempt == null) {
@@ -512,6 +532,11 @@ public class JobBuilder {
}
attempt.setResult(getPre21Value(event.getTaskStatus()));
attempt.setHostName(event.getHostname(), event.getRackName());
+ ParsedHost pHost =
+ getAndRecordParsedHost(event.getRackName(), event.getHostname());
+ if (pHost != null) {
+ attempt.setLocation(pHost.makeLoggedLocation());
+ }
// XXX There may be redundant location info available in the event.
// We might consider extracting it from this event. Currently this
@@ -528,15 +553,21 @@ public class JobBuilder {
}
private void processMapAttemptFinishedEvent(MapAttemptFinishedEvent event) {
- LoggedTaskAttempt attempt =
+ ParsedTaskAttempt attempt =
getOrMakeTaskAttempt(event.getTaskType(), event.getTaskId().toString(),
event.getAttemptId().toString());
if (attempt == null) {
return;
}
attempt.setResult(getPre21Value(event.getTaskStatus()));
- attempt.setHostName(event.getHostname(), event.getRackname());
+ attempt.setHostName(event.getHostname(), event.getRackName());
+ ParsedHost pHost =
+ getAndRecordParsedHost(event.getRackName(), event.getHostname());
+ if (pHost != null) {
+ attempt.setLocation(pHost.makeLoggedLocation());
+ }
+
// XXX There may be redundant location info available in the event.
// We might consider extracting it from this event. Currently this
// is redundant, but making this will add future-proofing.
@@ -554,6 +585,7 @@ public class JobBuilder {
result.setOutcome(Pre21JobHistoryConstants.Values
.valueOf(event.getStatus()));
result.setFinishTime(event.getFinishTime());
+ // No counters in JobUnsuccessfulCompletionEvent
}
private void processJobSubmittedEvent(JobSubmittedEvent event) {
@@ -561,8 +593,14 @@ public class JobBuilder {
result.setJobName(event.getJobName());
result.setUser(event.getUserName());
result.setSubmitTime(event.getSubmitTime());
- // job queue name is set when conf file is processed.
- // See JobBuilder.process(Properties) method for details.
+ result.putJobConfPath(event.getJobConfPath());
+ result.putJobAcls(event.getJobAcls());
+
+ // set the queue name if existing
+ String queue = event.getJobQueueName();
+ if (queue != null) {
+ result.setQueue(queue);
+ }
}
private void processJobStatusChangedEvent(JobStatusChangedEvent event) {
@@ -589,10 +627,19 @@ public class JobBuilder {
result.setFinishTime(event.getFinishTime());
result.setJobID(jobID);
result.setOutcome(Values.SUCCESS);
+
+ JobFinished job = (JobFinished)event.getDatum();
+ Map<String, Long> countersMap =
+ JobHistoryUtils.extractCounters(job.totalCounters);
+ result.putTotalCounters(countersMap);
+ countersMap = JobHistoryUtils.extractCounters(job.mapCounters);
+ result.putMapCounters(countersMap);
+ countersMap = JobHistoryUtils.extractCounters(job.reduceCounters);
+ result.putReduceCounters(countersMap);
}
- private LoggedTask getTask(String taskIDname) {
- LoggedTask result = mapTasks.get(taskIDname);
+ private ParsedTask getTask(String taskIDname) {
+ ParsedTask result = mapTasks.get(taskIDname);
if (result != null) {
return result;
@@ -616,9 +663,9 @@ public class JobBuilder {
* if true, we can create a task.
* @return
*/
- private LoggedTask getOrMakeTask(TaskType type, String taskIDname,
+ private ParsedTask getOrMakeTask(TaskType type, String taskIDname,
boolean allowCreate) {
- Map<String, LoggedTask> taskMap = otherTasks;
+ Map<String, ParsedTask> taskMap = otherTasks;
List<LoggedTask> tasks = this.result.getOtherTasks();
switch (type) {
@@ -636,10 +683,10 @@ public class JobBuilder {
// no code
}
- LoggedTask result = taskMap.get(taskIDname);
+ ParsedTask result = taskMap.get(taskIDname);
if (result == null && allowCreate) {
- result = new LoggedTask();
+ result = new ParsedTask();
result.setTaskType(getPre21Value(type.toString()));
result.setTaskID(taskIDname);
taskMap.put(taskIDname, result);
@@ -649,13 +696,13 @@ public class JobBuilder {
return result;
}
- private LoggedTaskAttempt getOrMakeTaskAttempt(TaskType type,
+ private ParsedTaskAttempt getOrMakeTaskAttempt(TaskType type,
String taskIDName, String taskAttemptName) {
- LoggedTask task = getOrMakeTask(type, taskIDName, false);
- LoggedTaskAttempt result = attempts.get(taskAttemptName);
+ ParsedTask task = getOrMakeTask(type, taskIDName, false);
+ ParsedTaskAttempt result = attempts.get(taskAttemptName);
if (result == null && task != null) {
- result = new LoggedTaskAttempt();
+ result = new ParsedTaskAttempt();
result.setAttemptID(taskAttemptName);
attempts.put(taskAttemptName, result);
task.getAttempts().add(result);
@@ -665,7 +712,19 @@ public class JobBuilder {
}
private ParsedHost getAndRecordParsedHost(String hostName) {
- ParsedHost result = ParsedHost.parse(hostName);
+ return getAndRecordParsedHost(null, hostName);
+ }
+
+ private ParsedHost getAndRecordParsedHost(String rackName, String hostName) {
+ ParsedHost result = null;
+ if (rackName == null) {
+ // for old (pre-23) job history files where hostname was represented as
+ // /rackname/hostname
+ result = ParsedHost.parse(hostName);
+ } else {
+ // for new (post-23) job history files
+ result = new ParsedHost(rackName, hostName);
+ }
if (result != null) {
ParsedHost canonicalResult = allHosts.get(result);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobHistoryUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobHistoryUtils.java?rev=1224965&r1=1224964&r2=1224965&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobHistoryUtils.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobHistoryUtils.java Tue Dec 27 18:18:36 2011
@@ -18,10 +18,15 @@
package org.apache.hadoop.tools.rumen;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.jobhistory.JhCounter;
+import org.apache.hadoop.mapreduce.jobhistory.JhCounterGroup;
+import org.apache.hadoop.mapreduce.jobhistory.JhCounters;
import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
/**
@@ -143,4 +148,21 @@ public class JobHistoryUtils {
String jobId = extractJobIDFromConfFileName(fileName);
return jobId != null;
}
+
+ /**
+ * Extract/Add counters into the Map from the given JhCounters object.
+ * @param counters the counters to be extracted from
+ * @return the map of counters
+ */
+ static Map<String, Long> extractCounters(JhCounters counters) {
+ Map<String, Long> countersMap = new HashMap<String, Long>();
+ if (counters != null) {
+ for (JhCounterGroup group : counters.groups) {
+ for (JhCounter counter : group.counts) {
+ countersMap.put(counter.name.toString(), counter.value);
+ }
+ }
+ }
+ return countersMap;
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java?rev=1224965&r1=1224964&r2=1224965&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java Tue Dec 27 18:18:36 2011
@@ -360,6 +360,10 @@ public class LoggedJob implements DeepCo
this.relativeTime = relativeTime;
}
+ /**
+ * @return job queue name if it is available in job history file or
+ * job history conf file. Returns null otherwise.
+ */
public QueueName getQueue() {
return queue;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ParsedHost.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ParsedHost.java?rev=1224965&r1=1224964&r2=1224965&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ParsedHost.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/ParsedHost.java Tue Dec 27 18:18:36 2011
@@ -71,11 +71,17 @@ public class ParsedHost {
return new ParsedHost(matcher.group(1), matcher.group(2));
}
+ private String process(String name) {
+ return name == null
+ ? null
+ : name.startsWith("/") ? name.substring(1) : name;
+ }
+
public ParsedHost(LoggedLocation loc) {
List<NodeName> coordinates = loc.getLayers();
- rackName = coordinates.get(0).getRackName();
- nodeName = coordinates.get(1).getHostName();
+ rackName = process(coordinates.get(0).getRackName());
+ nodeName = process(coordinates.get(1).getHostName());
}
LoggedLocation makeLoggedLocation() {
@@ -101,8 +107,8 @@ public class ParsedHost {
// expects the broadest name first
ParsedHost(String rackName, String nodeName) {
- this.rackName = rackName;
- this.nodeName = nodeName;
+ this.rackName = process(rackName);
+ this.nodeName = process(nodeName);
}
@Override
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java?rev=1224965&r1=1224964&r2=1224965&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TaskAttempt20LineEventEmitter.java Tue Dec 27 18:18:36 2011
@@ -108,9 +108,12 @@ public abstract class TaskAttempt20LineE
TaskAttempt20LineEventEmitter that =
(TaskAttempt20LineEventEmitter) thatg;
+ ParsedHost pHost = ParsedHost.parse(hostName);
+
return new TaskAttemptFinishedEvent(taskAttemptID,
that.originalTaskType, status, Long.parseLong(finishTime),
- hostName, state, maybeParseCounters(counters));
+ pHost.getRackName(), pHost.getNodeName(), state,
+ maybeParseCounters(counters));
}
return null;
@@ -138,10 +141,19 @@ public abstract class TaskAttempt20LineE
TaskAttempt20LineEventEmitter that =
(TaskAttempt20LineEventEmitter) thatg;
+ ParsedHost pHost = ParsedHost.parse(hostName);
+ String rackName = null;
+
+ // Earlier versions of MR logged on hostnames (without rackname) for
+ // unsuccessful attempts
+ if (pHost != null) {
+ rackName = pHost.getRackName();
+ hostName = pHost.getNodeName();
+ }
return new TaskAttemptUnsuccessfulCompletionEvent
(taskAttemptID,
that.originalTaskType, status, Long.parseLong(finishTime),
- hostName, -1, error, null);
+ hostName, -1, rackName, error, null);
}
return null;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TopologyBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TopologyBuilder.java?rev=1224965&r1=1224964&r2=1224965&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TopologyBuilder.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/TopologyBuilder.java Tue Dec 27 18:18:36 2011
@@ -25,6 +25,8 @@ import java.util.StringTokenizer;
import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptFinishedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletionEvent;
+import org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinishedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.ReduceAttemptFinishedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent;
/**
@@ -46,6 +48,10 @@ public class TopologyBuilder {
processTaskAttemptUnsuccessfulCompletionEvent((TaskAttemptUnsuccessfulCompletionEvent) event);
} else if (event instanceof TaskStartedEvent) {
processTaskStartedEvent((TaskStartedEvent) event);
+ } else if (event instanceof MapAttemptFinishedEvent) {
+ processMapAttemptFinishedEvent((MapAttemptFinishedEvent) event);
+ } else if (event instanceof ReduceAttemptFinishedEvent) {
+ processReduceAttemptFinishedEvent((ReduceAttemptFinishedEvent) event);
}
// I do NOT expect these if statements to be exhaustive.
@@ -78,15 +84,40 @@ public class TopologyBuilder {
private void processTaskAttemptUnsuccessfulCompletionEvent(
TaskAttemptUnsuccessfulCompletionEvent event) {
- recordParsedHost(event.getHostname());
+ recordParsedHost(event.getHostname(), event.getRackName());
}
private void processTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) {
- recordParsedHost(event.getHostname());
+ recordParsedHost(event.getHostname(), event.getRackName());
}
- private void recordParsedHost(String hostName) {
- ParsedHost result = ParsedHost.parse(hostName);
+ private void processMapAttemptFinishedEvent(MapAttemptFinishedEvent event) {
+ recordParsedHost(event.getHostname(), event.getRackName());
+ }
+
+ private void processReduceAttemptFinishedEvent(ReduceAttemptFinishedEvent event) {
+ recordParsedHost(event.getHostname(), event.getRackName());
+ }
+
+ private void recordParsedHost(String hostName, String rackName) {
+ if (hostName == null) {
+ return;
+ }
+ ParsedHost result = null;
+ if (rackName == null) {
+ result = ParsedHost.parse(hostName);
+ } else {
+ result = new ParsedHost(rackName, hostName);
+ }
+
+
+ if (result != null && !allHosts.contains(result)) {
+ allHosts.add(result);
+ }
+ }
+
+ private void recordParsedHost(String nodeName) {
+ ParsedHost result = ParsedHost.parse(nodeName);
if (result != null && !allHosts.contains(result)) {
allHosts.add(result);
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 27 18:18:36 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1152502-1220940
+/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1152502-1224959
/hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
/hadoop/core/trunk/src/webapps/job:776175-785643