svn commit: r547649 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JobInProgress.java src/java/org/apache/hadoop/mapred/JobInProgress_Counter.properties

[to...@apache.org]

Author: tomwhite
Date: Fri Jun 15 05:27:01 2007
New Revision: 547649

URL: http://svn.apache.org/viewvc?view=rev&rev=547649
Log:
HADOOP-1457.  Add counters for monitoring task assignments.  Contributed by Arun C Murthy.

Added:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress_Counter.properties
Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=547649&r1=547648&r2=547649
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Jun 15 05:27:01 2007
@@ -127,6 +127,9 @@
      making images appear in PDF.  Also update copyright date for all
      docs.  (Luke Nezda via cutting)
 
+ 41. HADOOP-1457.  Add counters for monitoring task assignments.
+     (Arun C Murthy via tomwhite)
+
 
 Release 0.13.0 - 2007-06-08
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=547649&r1=547648&r2=547649
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Fri Jun 15 05:27:01 2007
@@ -101,7 +101,10 @@
   // Per-job counters
   public static enum Counter { 
     NUM_FAILED_MAPS, 
-    NUM_FAILED_REDUCES
+    NUM_FAILED_REDUCES,
+    TOTAL_LAUNCHED_MAPS,
+    TOTAL_LAUNCHED_REDUCES,
+    DATA_LOCAL_MAPS
   }
   private Counters jobCounters = new Counters();
   
@@ -508,12 +511,14 @@
       LOG.info("Cannot create task split for " + profile.getJobId());
       return null;
     }
+    
     ArrayList mapCache = (ArrayList)hostToMaps.get(tts.getHost());
     int target = findNewTask(tts, clusterSize, status.mapProgress(), 
                              maps, mapCache);
     if (target == -1) {
       return null;
     }
+    
     boolean wasRunning = maps[target].isRunning();
     Task result = maps[target].getTaskToRun(tts.getTrackerName());
     if (!wasRunning) {
@@ -522,6 +527,9 @@
                                  maps[target].getTIPId(), Values.MAP.name(),
                                  System.currentTimeMillis());
     }
+
+    jobCounters.incrCounter(Counter.TOTAL_LAUNCHED_MAPS, 1);
+
     return result;
   }    
 
@@ -543,6 +551,7 @@
     if (target == -1) {
       return null;
     }
+    
     boolean wasRunning = reduces[target].isRunning();
     Task result = reduces[target].getTaskToRun(tts.getTrackerName());
     if (!wasRunning) {
@@ -551,6 +560,9 @@
                                  reduces[target].getTIPId(), Values.REDUCE.name(),
                                  System.currentTimeMillis());
     }
+    
+    jobCounters.incrCounter(Counter.TOTAL_LAUNCHED_REDUCES, 1);
+
     return result;
   }
     
@@ -670,6 +682,7 @@
             !tip.hasFailedOnMachine(taskTracker)) {
           LOG.info("Choosing cached task " + tip.getTIPId());
           int cacheTarget = tip.getIdWithinJob();
+          jobCounters.incrCounter(Counter.DATA_LOCAL_MAPS, 1);
           return cacheTarget;
         }
       }
@@ -717,6 +730,7 @@
       LOG.info("Choosing failed task " + 
                tasks[failedTarget].getTIPId());          
     }
+    
     return specTarget != -1 ? specTarget : failedTarget;
   }
 

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress_Counter.properties
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress_Counter.properties?view=auto&rev=547649
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress_Counter.properties (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress_Counter.properties Fri Jun 15 05:27:01 2007
@@ -0,0 +1,10 @@
+# ResourceBundle properties file for job-level counters
+
+CounterGroupName=              Job Counters 
+
+NUM_FAILED_MAPS.name=          Failed map tasks
+NUM_FAILED_REDUCES.name=       Failed reduce tasks
+TOTAL_LAUNCHED_MAPS.name=      Launched map tasks
+TOTAL_LAUNCHED_REDUCES.name=   Launched reduce tasks
+DATA_LOCAL_MAPS.name=          Data-local map tasks
+