You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2009/11/02 08:54:43 UTC
svn commit: r831813 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/JobTracker.java
src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java
Author: sharad
Date: Mon Nov 2 07:54:43 2009
New Revision: 831813
URL: http://svn.apache.org/viewvc?rev=831813&view=rev
Log:
MAPREDUCE-1153. Fix tasktracker metrics when trackers are decommissioned. Contributed by Sharad Agarwal.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=831813&r1=831812&r2=831813&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Nov 2 07:54:43 2009
@@ -61,6 +61,9 @@
to avoid filling up jobtracker logs on a busy cluster.
(Ravi Gummadi and Hong Tang via yhemanth)
+ MAPREDUCE-1153. Fix tasktracker metrics when trackers are decommissioned.
+ (sharad)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=831813&r1=831812&r2=831813&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Mon Nov 2 07:54:43 2009
@@ -420,15 +420,7 @@
if ((now - newProfile.getLastSeen()) >
tasktrackerExpiryInterval) {
// Remove completely after marking the tasks as 'KILLED'
- lostTaskTracker(current);
- // tracker is lost, and if it is blacklisted, remove
- // it from the count of blacklisted trackers in the cluster
- if (isBlacklisted(trackerName)) {
- faultyTrackers.decrBlackListedTrackers(1);
- }
- updateTaskTrackerStatus(trackerName, null);
- statistics.taskTrackerRemoved(trackerName);
- getInstrumentation().decTrackers(1);
+ removeTracker(current);
// remove the mapping from the hosts list
String hostname = newProfile.getHost();
hostnameToTaskTracker.get(hostname).remove(trackerName);
@@ -443,6 +435,19 @@
}
}
+ private void removeTracker(TaskTracker tracker) {
+ lostTaskTracker(tracker);
+ String trackerName = tracker.getStatus().getTrackerName();
+ // tracker is lost, and if it is blacklisted, remove
+ // it from the count of blacklisted trackers in the cluster
+ if (isBlacklisted(trackerName)) {
+ faultyTrackers.decrBlackListedTrackers(1);
+ }
+ updateTaskTrackerStatus(trackerName, null);
+ statistics.taskTrackerRemoved(trackerName);
+ getInstrumentation().decTrackers(1);
+ }
+
public synchronized void retireJob(JobID jobid, String historyFile) {
synchronized (jobs) {
JobInProgress job = jobs.get(jobid);
@@ -3828,9 +3833,7 @@
for (TaskTracker tracker : trackers) {
LOG.info("Decommission: Losing tracker " + tracker +
" on host " + host);
- lostTaskTracker(tracker); // lose the tracker
- updateTaskTrackerStatus(
- tracker.getStatus().getTrackerName(), null);
+ removeTracker(tracker);
}
trackersDecommissioned += trackers.size();
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java?rev=831813&r1=831812&r2=831813&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java Mon Nov 2 07:54:43 2009
@@ -298,6 +298,18 @@
}
public void testDecomissionedTrackers() throws IOException {
+ // create TaskTrackerStatus and send heartbeats
+ TaskTrackerStatus[] status = new TaskTrackerStatus[trackers.length];
+ status[0] = getTTStatus(trackers[0], new ArrayList<TaskStatus>());
+ status[1] = getTTStatus(trackers[1], new ArrayList<TaskStatus>());
+ status[2] = getTTStatus(trackers[2], new ArrayList<TaskStatus>());
+ for (int i = 0; i< trackers.length; i++) {
+ FakeObjectUtilities.sendHeartBeat(jobTracker, status[i], false,
+ false, trackers[i], responseId);
+ }
+
+ assertEquals("Mismatch in number of trackers",
+ trackers.length, mi.numTrackers);
Set<String> dHosts = new HashSet<String>();
dHosts.add(hosts[1]);
assertEquals("Mismatch in number of decommissioned trackers",
@@ -305,6 +317,8 @@
jobTracker.decommissionNodes(dHosts);
assertEquals("Mismatch in number of decommissioned trackers",
1, mi.numTrackersDecommissioned);
+ assertEquals("Mismatch in number of trackers",
+ trackers.length - 1, mi.numTrackers);
}
static class FakeTaskScheduler extends JobQueueTaskScheduler {