svn commit: r1077429 - in /hadoop/common/branches/branch-0.20-security-patches/src: mapred/org/apache/hadoop/mapred/JobClient.java test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java

[om...@apache.org]

Author: omalley
Date: Fri Mar  4 04:13:59 2011
New Revision: 1077429

URL: http://svn.apache.org/viewvc?rev=1077429&view=rev
Log:
commit d3166d2216c20177b5423d66323ec73c986b6b97
Author: Arun C Murthy <ac...@apache.org>
Date:   Mon Apr 26 11:42:39 2010 -0700

    MAPREDUCE-1641. Bug-fix to ensure command line options such as -files/-archives are checked for duplicate artifacts in the DistributedCache. Contributed by Amareshwari Sreeramadasu.
    
    +++ b/YAHOO-CHANGES.txt
    +    MAPREDUCE-1641. Bug-fix to ensure command line options such as
    +    -files/-archives are checked for duplicate artifacts in the
    +    DistributedCache. (Amareshwari Sreeramadasu via acmurthy)
    +

Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=1077429&r1=1077428&r2=1077429&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java Fri Mar  4 04:13:59 2011
@@ -40,7 +40,6 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 
-import javax.security.auth.login.LoginException;
 import java.security.PrivilegedExceptionAction;
 
 import org.apache.commons.logging.Log;
@@ -657,6 +656,8 @@ public class JobClient extends Configure
      }
     }
     
+    // First we check whether the cached archives and files are legal.
+    TrackerDistributedCacheManager.validate(job);
     //  set the timestamps of the archives and files
     TrackerDistributedCacheManager.determineTimestamps(job);
     //  set the public/private visibility of the archives and files
@@ -766,9 +767,6 @@ public class JobClient extends Configure
                (new Path("file:///" +  binaryTokenFilename), jobCopy);
           }
 
-          // First we check whether the cached archives and files are legal.
-          TrackerDistributedCacheManager.validate(jobCopy);
-
           copyAndConfigureFiles(jobCopy, submitJobDir);
 
           // get delegation token for the dir

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java?rev=1077429&r1=1077428&r2=1077429&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java Fri Mar  4 04:13:59 2011
@@ -22,6 +22,7 @@ import java.io.FileOutputStream;
 import junit.framework.TestCase;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -80,12 +81,46 @@ public class TestCommandLineJobSubmissio
       assertTrue("not failed ", ret != -1);
       f.delete();
       thisbuildDir.delete();
+      
+      // test duplicate uris for options -files and -archives
+      testDuplicateURI(mr, dfs);
     } finally {
       if (dfs != null) {dfs.shutdown();};
       if (mr != null) {mr.shutdown();};
     }
   }
   
+  
+  private void testDuplicateURI(MiniMRCluster mr, MiniDFSCluster dfs)
+      throws Exception {
+    Configuration jobConf = mr.createJobConf();
+    FileSystem fs = dfs.getFileSystem();
+    Path dfsPath = new Path("/test/testjob.jar");
+    fs.copyFromLocalFile(new Path("build/test/testjar/testjob.jar"), dfsPath);
+    String url = fs.getDefaultUri(jobConf).toString() + dfsPath.toString();
+    String[] args = new String[6];
+    args[0] = "-files";
+    args[1] = url;
+    args[2] = "-archives";
+    args[3] = url;
+    args[4] = input.toString();
+    args[5] = output.toString();
+
+    Exception ex = null;
+    try {
+      int ret = ToolRunner
+          .run(jobConf, new testshell.ExternalMapReduce(), args);
+    } catch (Exception e) {
+      ex = e;
+    }
+    assertNotNull("No exception thrown", ex);
+    assertTrue("Exception is not InvalidJobConfException.",
+        ex instanceof InvalidJobConfException);
+    assertEquals("Wrong message for the exception", "The core URI, \"" + url
+        + "\" is listed both in " + DistributedCache.CACHE_FILES + " and in "
+        + DistributedCache.CACHE_ARCHIVES + " .", ex.getMessage());
+  }
+
   @SuppressWarnings("unchecked")
   private Class loadLibJar(JobConf jobConf) {
     try {