You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2009/05/05 00:12:46 UTC
svn commit: r771468 - in /hadoop/core/trunk: CHANGES.txt
src/mapred/org/apache/hadoop/mapred/JobTracker.java
src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java
src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java
Author: cdouglas
Date: Mon May 4 22:12:45 2009
New Revision: 771468
URL: http://svn.apache.org/viewvc?rev=771468&view=rev
Log:
HADOOP-5733. Add map/reduce slot capacity and blacklisted capacity to JobTracker metrics. Contributed by Sreekanth Ramakrishnan
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=771468&r1=771467&r2=771468&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon May 4 22:12:45 2009
@@ -295,6 +295,9 @@
HADOOP-5364. Add certificate expiration warning to HsftpFileSystem and HDFS
proxy. (Zhiyong Zhang via cdouglas)
+ HADOOP-5733. Add map/reduce slot capacity and blacklisted capacity to
+ JobTracker metrics. (Sreekanth Ramakrishnan via cdouglas)
+
OPTIMIZATIONS
HADOOP-5595. NameNode does not need to run a replicator to choose a
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=771468&r1=771467&r2=771468&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java Mon May 4 22:12:45 2009
@@ -606,11 +606,16 @@
synchronized (taskTrackers) {
// remove the capacity of trackers on this host
for (TaskTrackerStatus status : getStatusesOnHost(hostName)) {
- totalMapTaskCapacity -= status.getMaxMapTasks();
- totalReduceTaskCapacity -= status.getMaxReduceTasks();
+ int mapSlots = status.getMaxMapTasks();
+ totalMapTaskCapacity -= mapSlots;
+ int reduceSlots = status.getMaxReduceTasks();
+ totalReduceTaskCapacity -= reduceSlots;
+ getInstrumentation().addBlackListedMapSlots(
+ mapSlots);
+ getInstrumentation().addBlackListedReduceSlots(
+ reduceSlots);
}
- numBlacklistedTrackers +=
- uniqueHostsMap.remove(hostName);
+ numBlacklistedTrackers += uniqueHostsMap.remove(hostName);
}
}
@@ -620,9 +625,13 @@
int numTrackersOnHost = 0;
// add the capacity of trackers on the host
for (TaskTrackerStatus status : getStatusesOnHost(hostName)) {
- totalMapTaskCapacity += status.getMaxMapTasks();
- totalReduceTaskCapacity += status.getMaxReduceTasks();
+ int mapSlots = status.getMaxMapTasks();
+ totalMapTaskCapacity += mapSlots;
+ int reduceSlots = status.getMaxReduceTasks();
+ totalReduceTaskCapacity += reduceSlots;
numTrackersOnHost++;
+ getInstrumentation().decBlackListedMapSlots(mapSlots);
+ getInstrumentation().decBlackListedReduceSlots(reduceSlots);
}
uniqueHostsMap.put(hostName,
numTrackersOnHost);
@@ -2627,8 +2636,10 @@
totalMaps -= oldStatus.countMapTasks();
totalReduces -= oldStatus.countReduceTasks();
if (!faultyTrackers.isBlacklisted(oldStatus.getHost())) {
- totalMapTaskCapacity -= oldStatus.getMaxMapTasks();
- totalReduceTaskCapacity -= oldStatus.getMaxReduceTasks();
+ int mapSlots = oldStatus.getMaxMapTasks();
+ totalMapTaskCapacity -= mapSlots;
+ int reduceSlots = oldStatus.getMaxReduceTasks();
+ totalReduceTaskCapacity -= reduceSlots;
}
if (status == null) {
taskTrackers.remove(trackerName);
@@ -2647,8 +2658,10 @@
totalMaps += status.countMapTasks();
totalReduces += status.countReduceTasks();
if (!faultyTrackers.isBlacklisted(status.getHost())) {
- totalMapTaskCapacity += status.getMaxMapTasks();
- totalReduceTaskCapacity += status.getMaxReduceTasks();
+ int mapSlots = status.getMaxMapTasks();
+ totalMapTaskCapacity += mapSlots;
+ int reduceSlots = status.getMaxReduceTasks();
+ totalReduceTaskCapacity += reduceSlots;
}
boolean alreadyPresent = false;
if (taskTrackers.containsKey(trackerName)) {
@@ -2666,6 +2679,8 @@
uniqueHostsMap.put(status.getHost(), numTaskTrackersInHost);
}
}
+ getInstrumentation().setMapSlots(totalMapTaskCapacity);
+ getInstrumentation().setReduceSlots(totalReduceTaskCapacity);
return oldStatus != null;
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java?rev=771468&r1=771467&r2=771468&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java Mon May 4 22:12:45 2009
@@ -66,4 +66,22 @@
public void decWaitingReduces(JobID id, int task)
{ }
+
+ public void setMapSlots(int slots)
+ { }
+
+ public void setReduceSlots(int slots)
+ { }
+
+ public void addBlackListedMapSlots(int slots)
+ { }
+
+ public void decBlackListedMapSlots(int slots)
+ { }
+
+ public void addBlackListedReduceSlots(int slots)
+ { }
+
+ public void decBlackListedReduceSlots(int slots)
+ { }
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java?rev=771468&r1=771467&r2=771468&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java Mon May 4 22:12:45 2009
@@ -38,7 +38,13 @@
private int numJobsCompleted = 0;
private int numWaitingMaps = 0;
private int numWaitingReduces = 0;
-
+
+ //Cluster status fields.
+ private volatile int numMapSlots = 0;
+ private volatile int numReduceSlots = 0;
+ private int numBlackListedMapSlots = 0;
+ private int numBlackListedReduceSlots = 0;
+
public JobTrackerMetricsInst(JobTracker tracker, JobConf conf) {
super(tracker, conf);
String sessionId = conf.getSessionId();
@@ -57,6 +63,11 @@
*/
public void doUpdates(MetricsContext unused) {
synchronized (this) {
+ metricsRecord.setMetric("map_slots", numMapSlots);
+ metricsRecord.setMetric("reduce_slots", numReduceSlots);
+ metricsRecord.incrMetric("blacklisted_maps", numBlackListedMapSlots);
+ metricsRecord.incrMetric("blacklisted_reduces",
+ numBlackListedReduceSlots);
metricsRecord.incrMetric("maps_launched", numMapTasksLaunched);
metricsRecord.incrMetric("maps_completed", numMapTasksCompleted);
metricsRecord.incrMetric("maps_failed", numMapTasksFailed);
@@ -78,6 +89,8 @@
numJobsCompleted = 0;
numWaitingMaps = 0;
numWaitingReduces = 0;
+ numBlackListedMapSlots = 0;
+ numBlackListedReduceSlots = 0;
}
metricsRecord.update();
@@ -151,4 +164,34 @@
public synchronized void decWaitingReduces(JobID id, int task){
numWaitingReduces -= task;
}
+
+ @Override
+ public void setMapSlots(int slots) {
+ numMapSlots = slots;
+ }
+
+ @Override
+ public void setReduceSlots(int slots) {
+ numReduceSlots = slots;
+ }
+
+ @Override
+ public synchronized void addBlackListedMapSlots(int slots){
+ numBlackListedMapSlots += slots;
+ }
+
+ @Override
+ public synchronized void decBlackListedMapSlots(int slots){
+ numBlackListedMapSlots -= slots;
+ }
+
+ @Override
+ public synchronized void addBlackListedReduceSlots(int slots){
+ numBlackListedReduceSlots += slots;
+ }
+
+ @Override
+ public synchronized void decBlackListedReduceSlots(int slots){
+ numBlackListedReduceSlots -= slots;
+ }
}