You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by cd...@apache.org on 2009/12/04 09:13:36 UTC
svn commit: r887096 - in /hadoop/mapreduce/trunk: ./
src/java/org/apache/hadoop/mapred/ src/test/mapred/org/apache/hadoop/mapred/
Author: cdouglas
Date: Fri Dec 4 08:13:25 2009
New Revision: 887096
URL: http://svn.apache.org/viewvc?rev=887096&view=rev
Log:
MAPREDUCE-1152. Distinguish between failed and killed tasks in
JobTrackerInstrumentation. Contributed by Sharad Agarwal
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=887096&r1=887095&r2=887096&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Dec 4 08:13:25 2009
@@ -113,6 +113,9 @@
MAPREDUCE-1260. Update Eclipse configuration to match changes to Ivy
configuration. (Edwin Chan via cos)
+ MAPREDUCE-1152. Distinguish between failed and killed tasks in
+ JobTrackerInstrumentation. (Sharad Agarwal via cdouglas)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=887096&r1=887095&r2=887096&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Fri Dec 4 08:13:25 2009
@@ -2954,7 +2954,6 @@
TaskStatus status,
TaskTracker taskTracker,
boolean wasRunning, boolean wasComplete) {
- final JobTrackerInstrumentation metrics = jobtracker.getInstrumentation();
// check if the TIP is already failed
boolean wasFailed = tip.isFailed();
boolean wasSpeculating = tip.isSpeculating();
@@ -2974,7 +2973,6 @@
launchedSetup = false;
} else if (tip.isMapTask()) {
runningMapTasks -= 1;
- metrics.failedMap(taskid);
// remove from the running queue and put it in the non-running cache
// if the tip is not complete i.e if the tip still needs to be run
if (!isComplete) {
@@ -2983,7 +2981,6 @@
}
} else {
runningReduceTasks -= 1;
- metrics.failedReduce(taskid);
// remove from the running queue and put in the failed queue if the tip
// is not complete
if (!isComplete) {
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java?rev=887096&r1=887095&r2=887096&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskInProgress.java Fri Dec 4 08:13:25 2009
@@ -697,6 +697,12 @@
if (tasks.contains(taskid)) {
if (taskState == TaskStatus.State.FAILED) {
numTaskFailures++;
+ if (isMapTask()) {
+ jobtracker.getInstrumentation().failedMap(taskid);
+ } else {
+ jobtracker.getInstrumentation().failedReduce(taskid);
+ }
+
machinesWhereFailed.add(trackerHostName);
if(maxSkipRecords>0) {
//skipping feature enabled
@@ -707,6 +713,11 @@
} else if (taskState == TaskStatus.State.KILLED) {
numKilledTasks++;
+ if (isMapTask()) {
+ jobtracker.getInstrumentation().killedMap(taskid);
+ } else {
+ jobtracker.getInstrumentation().killedReduce(taskid);
+ }
}
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java?rev=887096&r1=887095&r2=887096&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java Fri Dec 4 08:13:25 2009
@@ -214,6 +214,15 @@
updateTaskStatus(tip, status);
}
+ public void killTask(TaskAttemptID taskId) {
+ TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId);
+ TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId,
+ 1.0f, 1, TaskStatus.State.KILLED, "", "", tip
+ .machineWhereTaskRan(taskId), tip.isMapTask() ? Phase.MAP
+ : Phase.REDUCE, new Counters());
+ updateTaskStatus(tip, status);
+ }
+
public void cleanUpMetrics() {
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java?rev=887096&r1=887095&r2=887096&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java Fri Dec 4 08:13:25 2009
@@ -186,8 +186,13 @@
job1.finishTask(taskAttemptID[2]);
jobTracker.finalizeJob(job1);
+ assertEquals("Mismatch in number of failed map tasks",
+ 1, mi.numMapTasksFailed);
+ assertEquals("Mismatch in number of failed reduce tasks",
+ 1, mi.numReduceTasksFailed);
+
assertEquals("Mismatch in number of blacklisted trackers",
- mi.numTrackersBlackListed, 1);
+ 1, mi.numTrackersBlackListed);
assertEquals("Mismatch in blacklisted map slots",
mi.numBlackListedMapSlots,
@@ -321,6 +326,41 @@
trackers.length - 1, mi.numTrackers);
}
+ public void testKillTasks() throws IOException {
+ int numMaps, numReds;
+ JobConf conf = new JobConf();
+ conf.setSpeculativeExecution(false);
+ conf.setMaxTaskFailuresPerTracker(1);
+ conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false);
+ TaskAttemptID[] taskAttemptID = new TaskAttemptID[2];
+
+ numMaps = 1;
+ numReds = 1;
+ conf.setNumMapTasks(numMaps);
+ conf.setNumReduceTasks(numReds);
+ conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false);
+
+ assertEquals("Mismatch in number of killed map tasks",
+ 0, mi.numMapTasksKilled);
+ assertEquals("Mismatch in number of killed reduce tasks",
+ 0, mi.numReduceTasksKilled);
+
+ FakeJobInProgress job1 = new FakeJobInProgress(conf, jobTracker);
+ job1.setClusterSize(trackers.length);
+ job1.initTasks();
+ jobTracker.addJob(job1.getJobID(), job1);
+ taskAttemptID[0] = job1.findMapTask(trackers[0]);
+ job1.killTask(taskAttemptID[0]);
+ taskAttemptID[1] = job1.findReduceTask(trackers[0]);
+ job1.killTask(taskAttemptID[1]);
+ jobTracker.finalizeJob(job1);
+
+ assertEquals("Mismatch in number of killed map tasks",
+ 1, mi.numMapTasksKilled);
+ assertEquals("Mismatch in number of killed reduce tasks",
+ 1, mi.numReduceTasksKilled);
+ }
+
static class FakeTaskScheduler extends JobQueueTaskScheduler {
public FakeTaskScheduler() {
super();