You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/06/17 00:31:18 UTC
svn commit: r1136713 - in /hadoop/common/branches/branch-0.20-security-204:
CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobTracker.java
Author: omalley
Date: Thu Jun 16 22:31:17 2011
New Revision: 1136713
URL: http://svn.apache.org/viewvc?rev=1136713&view=rev
Log:
MAPREDUCE-2490. Add logging to graylist and blacklist activity to aid
diagnosis of related issues. (Jonathan Eagles via cdouglas)
Modified:
hadoop/common/branches/branch-0.20-security-204/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Modified: hadoop/common/branches/branch-0.20-security-204/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/CHANGES.txt?rev=1136713&r1=1136712&r2=1136713&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security-204/CHANGES.txt Thu Jun 16 22:31:17 2011
@@ -36,6 +36,9 @@ Release 0.20.204.0 - unreleased
HADOOP-7356. Fix bin/hadoop scripts (eyang via omalley)
+ MAPREDUCE-2490. Add logging to graylist and blacklist activity to aid
+ diagnosis of related issues. (Jonathan Eagles via cdouglas)
+
MAPREDUCE-2447. Fix Child.java to set Task.jvmContext sooner to avoid
corner cases in error handling. (Siddharth Seth via acmurthy)
Propchange: hadoop/common/branches/branch-0.20-security-204/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 16 22:31:17 2011
@@ -1,5 +1,5 @@
/hadoop/common/branches/branch-0.20/CHANGES.txt:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,1044225
-/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1097202,1098837,1100336,1127362,1131277,1131286,1131290,1131299,1131737,1134140
+/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1097202,1098837,1100336,1125589,1127362,1131277,1131286,1131290,1131299,1131737,1134140
/hadoop/common/branches/branch-0.20-security-203/CHANGES.txt:1096071,1097012-1099333,1102071,1128115
/hadoop/common/branches/branch-0.20-security-205/CHANGES.txt:1132788,1133133,1133274,1133282
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
Modified: hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1136713&r1=1136712&r2=1136713&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/JobTracker.java Thu Jun 16 22:31:17 2011
@@ -887,21 +887,25 @@ public class JobTracker implements MRCon
}
private void incrBlacklistedTrackers(int count) {
+ LOG.info("Incrementing blacklisted trackers by " + count);
numBlacklistedTrackers += count;
getInstrumentation().addBlackListedTrackers(count);
}
private void decrBlacklistedTrackers(int count) {
+ LOG.info("Decrementing blacklisted trackers by " + count);
numBlacklistedTrackers -= count;
getInstrumentation().decBlackListedTrackers(count);
}
private void incrGraylistedTrackers(int count) {
+ LOG.info("Incrementing graylisted trackers by " + count);
numGraylistedTrackers += count;
getInstrumentation().addGrayListedTrackers(count);
}
private void decrGraylistedTrackers(int count) {
+ LOG.info("Decrementing graylisted trackers by " + count);
numGraylistedTrackers -= count;
getInstrumentation().decGrayListedTrackers(count);
}
@@ -986,13 +990,13 @@ public class JobTracker implements MRCon
if (listed && rfbSet.contains(rfb)) {
if (fi.removeBlacklistedReason(rfb, gray)) {
if (fi.getReasonForBlacklisting(gray).isEmpty()) {
+ LOG.info("Un" + (gray? "gray" : "black") + "listing tracker : " +
+ hostName);
if (gray) {
decrGraylistedTrackers(getNumTaskTrackersOnHost(hostName));
} else {
addHostCapacity(hostName);
}
- LOG.info("Un" + (gray? "gray" : "black") + "listing tracker : " +
- hostName);
fi.unBlacklist(gray);
// We have unblack/graylisted tracker, so tracker should definitely
// be healthy. Check fault count; if zero, don't keep it in memory.
@@ -1032,11 +1036,11 @@ public class JobTracker implements MRCon
if (fi != null) {
// a tracker can be both blacklisted and graylisted, so check both
if (fi.isGraylisted()) {
- LOG.info("Removing " + hostName + " from graylist");
+ LOG.info("Marking " + hostName + " healthy from graylist");
decrGraylistedTrackers(getNumTaskTrackersOnHost(hostName));
}
if (fi.isBlacklisted()) {
- LOG.info("Removing " + hostName + " from blacklist");
+ LOG.info("Marking " + hostName + " healthy from blacklist");
addHostCapacity(hostName);
}
// no need for fi.unBlacklist() for either one: fi is already gone
@@ -4880,15 +4884,18 @@ public class JobTracker implements MRCon
// Remove a tracker from the system
private void removeTracker(TaskTracker tracker) {
String trackerName = tracker.getTrackerName();
+ String hostName = JobInProgress.convertTrackerNameToHostName(trackerName);
// Remove completely after marking the tasks as 'KILLED'
lostTaskTracker(tracker);
// tracker is lost; if it is blacklisted and/or graylisted, remove
// it from the relevant count(s) of trackers in the cluster
if (isBlacklisted(trackerName)) {
- faultyTrackers.decrBlacklistedTrackers(1);
+ LOG.info("Removing " + hostName + " from blacklist");
+ faultyTrackers.decrBlacklistedTrackers(1);
}
if (isGraylisted(trackerName)) {
- faultyTrackers.decrGraylistedTrackers(1);
+ LOG.info("Removing " + hostName + " from graylist");
+ faultyTrackers.decrGraylistedTrackers(1);
}
updateTaskTrackerStatus(trackerName, null);
statistics.taskTrackerRemoved(trackerName);