You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by cd...@apache.org on 2011/05/10 22:18:18 UTC
svn commit: r1101628 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/JobInProgress.java
src/java/org/apache/hadoop/mapred/JobTracker.java
Author: cdouglas
Date: Tue May 10 20:18:17 2011
New Revision: 1101628
URL: http://svn.apache.org/viewvc?rev=1101628&view=rev
Log:
MAPREDUCE-2456. Log the reduce taskID and associated TaskTrackers with
failed fetch notifications in the JobTracker log.
Contributed by Jeffrey Naisbitt
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1101628&r1=1101627&r2=1101628&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue May 10 20:18:17 2011
@@ -82,7 +82,11 @@ Trunk (unreleased changes)
MAPREDUCE-2422. Removed unused internal methods from DistributedCache.
(tomwhite)
-
+
+ MAPREDUCE-2456. Log the reduce taskID and associated TaskTrackers with
+ failed fetch notifications in the JobTracker log.
+ (Jeffrey Naisbitt via cdouglas)
+
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1101628&r1=1101627&r2=1101628&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Tue May 10 20:18:17 2011
@@ -3429,13 +3429,17 @@ public class JobInProgress {
synchronized void fetchFailureNotification(TaskInProgress tip,
TaskAttemptID mapTaskId,
- String trackerName) {
+ String mapTrackerName,
+ TaskAttemptID reduceTaskId,
+ String reduceTrackerName) {
Integer fetchFailures = mapTaskIdToFetchFailuresMap.get(mapTaskId);
fetchFailures = (fetchFailures == null) ? 1 : (fetchFailures+1);
mapTaskIdToFetchFailuresMap.put(mapTaskId, fetchFailures);
- LOG.info("Failed fetch notification #" + fetchFailures + " for task " +
- mapTaskId);
-
+ LOG.info("Failed fetch notification #" + fetchFailures + " for map task: "
+ + mapTaskId + " running on tracker: " + mapTrackerName
+ + " and reduce task: " + reduceTaskId + " running on tracker: "
+ + reduceTrackerName);
+
float failureRate = (float)fetchFailures / runningReduceTasks;
// declare faulty if fetch-failures >= max-allowed-failures
boolean isMapFaulty = (failureRate >= MAX_ALLOWED_FETCH_FAILURES_PERCENT)
@@ -3449,7 +3453,7 @@ public class JobInProgress {
failedTask(tip, mapTaskId, "Too many fetch-failures",
(tip.isMapTask() ? TaskStatus.Phase.MAP :
TaskStatus.Phase.REDUCE),
- TaskStatus.State.FAILED, trackerName);
+ TaskStatus.State.FAILED, mapTrackerName);
mapTaskIdToFetchFailuresMap.remove(mapTaskId);
}
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1101628&r1=1101627&r2=1101628&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Tue May 10 20:18:17 2011
@@ -4095,9 +4095,11 @@ public class JobTracker implements MRCon
if (failedFetchTrackerName == null) {
failedFetchTrackerName = "Lost task tracker";
}
- failedFetchMap.getJob().fetchFailureNotification(failedFetchMap,
- mapTaskId,
- failedFetchTrackerName);
+ failedFetchMap.getJob().fetchFailureNotification(failedFetchMap,
+ mapTaskId,
+ failedFetchTrackerName,
+ taskId,
+ trackerName);
}
}
}