svn commit: r1079255 - in /hadoop/mapreduce/branches/yahoo-merge/src: java/org/apache/hadoop/mapred/JobInProgress.java test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java

[om...@apache.org]

Author: omalley
Date: Tue Mar  8 06:00:53 2011
New Revision: 1079255

URL: http://svn.apache.org/viewvc?rev=1079255&view=rev
Log:
commit 6c69dabd04c260f528b0860ca6b9856a1da30356
Author: Greg Roelofs <ro...@yahoo-inc.com>
Date:   Tue Jan 25 17:18:36 2011 -0800

    Fixes addressing Dick's review comments.  Skipped (optional) TestMiniMRWithDFS/
    TestServiceLevelAuthorization enum fix for now--dislike tying two unit tests
    together more tightly, even though they're already tied.
    
    This appears to complete the main y-trunk uberization/MR-1220 changes.  (There
    are still a dozen or so "FUTURE UBER TODO" items on the list, however.)

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1079255&r1=1079254&r2=1079255&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java Tue Mar  8 06:00:53 2011
@@ -68,6 +68,8 @@ import org.apache.hadoop.mapreduce.jobhi
 import org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent;
 import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent;
 import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent;
+import org.apache.hadoop.mapreduce.lib.chain.ChainMapper;
+import org.apache.hadoop.mapreduce.lib.chain.ChainReducer;
 import org.apache.hadoop.mapreduce.security.TokenCache;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.mapreduce.security.token.DelegationTokenRenewal;
@@ -691,12 +693,7 @@ public class JobInProgress {
         // ignoring overhead due to UberTask and statics as negligible here:
         && (Math.max(memoryPerMap, memoryPerReduce) <= sysMemSizeForUberSlot
             || sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT)
-        && (conf.get(MRJobConfig.MAP_CLASS_ATTR) == null ||
-            conf.get(MRJobConfig.MAP_CLASS_ATTR).lastIndexOf("ChainMapper")
-            == -1)
-        && (conf.get(MRJobConfig.REDUCE_CLASS_ATTR) == null ||
-            conf.get(MRJobConfig.REDUCE_CLASS_ATTR).lastIndexOf("ChainReducer")
-            == -1);
+        && !isChainJob();
 
     if (isUber()) {
       // save internal details for UI and abort-cleanup
@@ -780,6 +777,34 @@ public class JobInProgress {
     }
   }
 
+  boolean isChainJob() {
+    boolean isChainJob = false;
+
+    try {
+      String mapClassName = conf.get(MRJobConfig.MAP_CLASS_ATTR);
+      if (mapClassName != null) {
+        Class<?> mapClass = Class.forName(mapClassName);
+        if (ChainMapper.class.isAssignableFrom(mapClass))
+          isChainJob = true;
+      }
+    } catch (ClassNotFoundException cnfe) {
+      // don't care; assume it's not derived from ChainMapper
+    }
+
+    try {
+      String reduceClassName = conf.get(MRJobConfig.REDUCE_CLASS_ATTR);
+      if (reduceClassName != null) {
+        Class<?> reduceClass = Class.forName(reduceClassName);
+        if (ChainReducer.class.isAssignableFrom(reduceClass))
+          isChainJob = true;
+      }
+    } catch (ClassNotFoundException cnfe) {
+      // don't care; assume it's not derived from ChainReducer
+    }
+
+    return isChainJob;
+  }
+
   // Returns true if the job is empty (0 maps, 0 reduces and no setup-cleanup)
   // else return false.
   synchronized boolean isJobEmpty() {

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java?rev=1079255&r1=1079254&r2=1079255&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java Tue Mar  8 06:00:53 2011
@@ -211,19 +211,16 @@ public class TestWebUIAuthorization exte
         cluster.getJobTrackerRunner().getJobTracker().getJob(jobId);
     TaskReport[] taskReports = null;
 
-    TaskID tipId = null;
-    do { // make sure that the map task is running
+    do { // make sure that the map task (or ubertask) is running
       Thread.sleep(200);
       taskReports = jip.isUber()
           ? client.getReduceTaskReports(jobId)
           : client.getMapTaskReports(jobId);
     } while (taskReports.length == 0);
 
-    for (TaskReport r : taskReports) {
-      tipId = r.getTaskID();
-      break;// because we have only one map (or one reduce if uberized)
-    }
-    return tipId;
+    assertEquals("unexpected number of task reports", 1, taskReports.length);
+
+    return taskReports[0].getTaskID();
   }
 
   /**