You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/10/26 22:50:14 UTC
svn commit: r468127 - in /lucene/hadoop/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/JobInProgress.java
Author: cutting
Date: Thu Oct 26 13:50:13 2006
New Revision: 468127
URL: http://svn.apache.org/viewvc?view=rev&rev=468127
Log:
HADOOP-578. Failed tasks are no longer placed at the end of the task queue. Contributed by Sanjay.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=468127&r1=468126&r2=468127
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Oct 26 13:50:13 2006
@@ -75,6 +75,11 @@
20. HADOOP-624. Fix servlet path to stop a Jetty warning on startup.
(omalley via cutting)
+21. HADOOP-578. Failed tasks are no longer placed at the end of the
+ task queue. This was originally done to work around other
+ problems that have now been fixed. Re-executing failed tasks
+ sooner causes buggy jobs to fail faster. (Sanjay Dahiya via cutting)
+
Release 0.7.2 - 2006-10-18
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=468127&r1=468126&r2=468127
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Thu Oct 26 13:50:13 2006
@@ -56,8 +56,6 @@
long finishTime;
private JobConf conf;
- private int firstMapToTry = 0;
- private int firstReduceToTry = 0;
boolean tasksInited = false;
private LocalFileSystem localFs;
@@ -330,7 +328,7 @@
ArrayList mapCache = (ArrayList)hostToMaps.get(tts.getHost());
double avgProgress = status.mapProgress() / maps.length;
int target = findNewTask(tts, clusterSize, avgProgress,
- maps, firstMapToTry, mapCache);
+ maps, mapCache);
if (target == -1) {
return null;
}
@@ -359,7 +357,7 @@
double avgProgress = status.reduceProgress() / reduces.length;
int target = findNewTask(tts, clusterSize, avgProgress,
- reduces, firstReduceToTry, null);
+ reduces, null);
if (target == -1) {
return null;
}
@@ -388,7 +386,6 @@
int clusterSize,
double avgProgress,
TaskInProgress[] tasks,
- int firstTaskToTry,
List cachedTasks) {
String taskTracker = tts.getTrackerName();
//
@@ -419,8 +416,7 @@
int failedTarget = -1;
int specTarget = -1;
for (int i = 0; i < tasks.length; i++) {
- int realIdx = (i + firstTaskToTry) % tasks.length;
- TaskInProgress task = tasks[realIdx];
+ TaskInProgress task = tasks[i];
if (task.isRunnable()) {
// if it failed here and we haven't tried every machine, we
// don't schedule it here.
@@ -433,15 +429,15 @@
// failed tasks that aren't running can be scheduled as a last
// resort
if (!isRunning && failedTarget == -1) {
- failedTarget = realIdx;
+ failedTarget = i;
}
} else {
if (!isRunning) {
- LOG.info("Choosing normal task " + tasks[realIdx].getTIPId());
- return realIdx;
+ LOG.info("Choosing normal task " + tasks[i].getTIPId());
+ return i;
} else if (specTarget == -1 &&
task.hasSpeculativeTask(avgProgress)) {
- specTarget = realIdx;
+ specTarget = i;
}
}
}
@@ -625,10 +621,8 @@
// After this, try to assign tasks with the one after this, so that
// the failed task goes to the end of the list.
if (tip.isMapTask()) {
- firstMapToTry = (tip.getIdWithinJob() + 1) % maps.length;
failedMapTasks++;
} else {
- firstReduceToTry = (tip.getIdWithinJob() + 1) % reduces.length;
failedReduceTasks++;
}