You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2013/02/07 05:05:34 UTC
svn commit: r1443317 - in
/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/
hadoop-mapreduce-client/hadoop-mapreduce-client-jo...
Author: vinodkv
Date: Thu Feb 7 04:05:34 2013
New Revision: 1443317
URL: http://svn.apache.org/viewvc?rev=1443317&view=rev
Log:
MAPREDUCE-4983. Fixed various platform specific assumptions in various tests, so that they can pass on Windows too. Contributed by Chris Nauroth.
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt?rev=1443317&r1=1443316&r2=1443317&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt Thu Feb 7 04:05:34 2013
@@ -13,3 +13,5 @@ branch-trunk-win changes - unreleased
MAPREDUCE-4870. Fix TestMRJobsWithHistoryService. (Chris Nauroth via acmurthy)
+ MAPREDUCE-4983. Fixed various platform specific assumptions in various tests,
+ so that they can pass on Windows too. (Chris Nauroth via vinodkv)
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1443317&r1=1443316&r2=1443317&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Thu Feb 7 04:05:34 2013
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -39,6 +40,8 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -69,9 +72,9 @@ public class TestMRApps {
}
private static void delete(File dir) throws IOException {
- Path p = new Path("file://"+dir.getAbsolutePath());
Configuration conf = new Configuration();
- FileSystem fs = p.getFileSystem(conf);
+ FileSystem fs = FileSystem.getLocal(conf);
+ Path p = fs.makeQualified(new Path(dir.getAbsolutePath()));
fs.delete(p, true);
}
@@ -165,18 +168,21 @@ public class TestMRApps {
Job job = Job.getInstance();
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, job.getConfiguration());
- assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
+ assertTrue(environment.get("CLASSPATH").startsWith(
+ ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
String yarnAppClasspath =
job.getConfiguration().get(
YarnConfiguration.YARN_APPLICATION_CLASSPATH);
if (yarnAppClasspath != null) {
- yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", ":").trim();
+ yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", File.pathSeparator)
+ .trim();
}
assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
String mrAppClasspath =
job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH);
if (mrAppClasspath != null) {
- mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", ":").trim();
+ mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", File.pathSeparator)
+ .trim();
}
assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
}
@@ -188,16 +194,18 @@ public class TestMRApps {
out.close();
Job job = Job.getInstance();
Configuration conf = job.getConfiguration();
- conf.set(MRJobConfig.CLASSPATH_ARCHIVES, "file://"
- + testTGZ.getAbsolutePath());
- conf.set(MRJobConfig.CACHE_ARCHIVES, "file://"
- + testTGZ.getAbsolutePath() + "#testTGZ");
+ String testTGZQualifiedPath = FileSystem.getLocal(conf).makeQualified(new Path(
+ testTGZ.getAbsolutePath())).toString();
+ conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath);
+ conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ");
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, conf);
- assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
+ assertTrue(environment.get("CLASSPATH").startsWith(
+ ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
String confClasspath = job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
if (confClasspath != null) {
- confClasspath = confClasspath.replaceAll(",\\s*", ":").trim();
+ confClasspath = confClasspath.replaceAll(",\\s*", File.pathSeparator)
+ .trim();
}
assertTrue(environment.get("CLASSPATH").contains(confClasspath));
assertTrue(environment.get("CLASSPATH").contains("testTGZ"));
@@ -213,8 +221,12 @@ public class TestMRApps {
fail("Got exception while setting classpath");
}
String env_str = env.get("CLASSPATH");
- assertSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
- env_str.indexOf("$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*"), 0);
+ String expectedClasspath = StringUtils.join(File.pathSeparator,
+ Arrays.asList(ApplicationConstants.Environment.PWD.$(), "job.jar/job.jar",
+ "job.jar/classes/", "job.jar/lib/*",
+ ApplicationConstants.Environment.PWD.$() + "/*"));
+ assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
+ env_str.startsWith(expectedClasspath));
}
@Test public void testSetClasspathWithNoUserPrecendence() {
@@ -227,12 +239,13 @@ public class TestMRApps {
fail("Got exception while setting classpath");
}
String env_str = env.get("CLASSPATH");
- int index =
- env_str.indexOf("job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*");
- assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not"
- + " in the classpath!", index, -1);
- assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
- index, 0);
+ String expectedClasspath = StringUtils.join(File.pathSeparator,
+ Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
+ ApplicationConstants.Environment.PWD.$() + "/*"));
+ assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
+ + " the classpath!", env_str.contains(expectedClasspath));
+ assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
+ env_str.startsWith(expectedClasspath));
}
@Test public void testSetClasspathWithJobClassloader() throws IOException {
@@ -242,13 +255,16 @@ public class TestMRApps {
MRApps.setClasspath(env, conf);
String cp = env.get("CLASSPATH");
String appCp = env.get("APP_CLASSPATH");
- assertSame("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is"
- + " in the classpath!", cp.indexOf("jar:job"), -1);
- assertSame("MAPREDUCE_JOB_CLASSLOADER true, but PWD is"
- + " in the classpath!", cp.indexOf("PWD"), -1);
- assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not"
- + " in the app classpath!",
- "$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*", appCp);
+ assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
+ + " classpath!", cp.contains("jar" + File.pathSeparator + "job"));
+ assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
+ cp.contains("PWD"));
+ String expectedAppClasspath = StringUtils.join(File.pathSeparator,
+ Arrays.asList(ApplicationConstants.Environment.PWD.$(), "job.jar/job.jar",
+ "job.jar/classes/", "job.jar/lib/*",
+ ApplicationConstants.Environment.PWD.$() + "/*"));
+ assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
+ + " classpath!", expectedAppClasspath, appCp);
}
@Test
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java?rev=1443317&r1=1443316&r2=1443317&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java Thu Feb 7 04:05:34 2013
@@ -61,8 +61,12 @@ import org.apache.hadoop.util.Reflection
*/
public class TestMapProgress extends TestCase {
public static final Log LOG = LogFactory.getLog(TestMapProgress.class);
- private static String TEST_ROOT_DIR = new File(System.getProperty(
- "test.build.data", "/tmp")).getAbsolutePath() + "/mapPahseprogress";
+ private static String TEST_ROOT_DIR;
+ static {
+ String root = new File(System.getProperty("test.build.data", "/tmp"))
+ .getAbsolutePath();
+ TEST_ROOT_DIR = new Path(root, "mapPhaseprogress").toString();
+ }
static class FakeUmbilical implements TaskUmbilicalProtocol {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1443317&r1=1443316&r2=1443317&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java Thu Feb 7 04:05:34 2013
@@ -25,6 +25,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.HashMap;
+import java.util.Map;
import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry;
import org.apache.commons.io.FileUtils;
@@ -103,6 +105,8 @@ public class TestMRJobs {
private static Path TEST_ROOT_DIR = new Path("target",
TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs);
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
+ private static final String OUTPUT_ROOT_DIR = "/tmp/" +
+ TestMRJobs.class.getSimpleName();
@BeforeClass
public static void setup() throws IOException {
@@ -226,8 +230,7 @@ public class TestMRJobs {
mrCluster.getConfig().set(RandomTextWriterJob.TOTAL_BYTES, "3072");
mrCluster.getConfig().set(RandomTextWriterJob.BYTES_PER_MAP, "1024");
Job job = randomWriterJob.createJob(mrCluster.getConfig());
- Path outputDir =
- new Path(mrCluster.getTestWorkDir().getAbsolutePath(), "random-output");
+ Path outputDir = new Path(OUTPUT_ROOT_DIR, "random-output");
FileOutputFormat.setOutputPath(job, outputDir);
job.setSpeculativeExecution(false);
job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
@@ -342,9 +345,8 @@ public class TestMRJobs {
job.setMapperClass(FailingMapper.class);
job.setNumReduceTasks(0);
- FileOutputFormat.setOutputPath(job,
- new Path(mrCluster.getTestWorkDir().getAbsolutePath(),
- "failmapper-output"));
+ FileOutputFormat.setOutputPath(job, new Path(OUTPUT_ROOT_DIR,
+ "failmapper-output"));
job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
job.submit();
String trackingUrl = job.getTrackingURL();
@@ -425,14 +427,22 @@ public class TestMRJobs {
Assert.assertEquals(2, archives.length);
// Check lengths of the files
- Assert.assertEquals(1, localFs.getFileStatus(files[1]).getLen());
- Assert.assertTrue(localFs.getFileStatus(files[2]).getLen() > 1);
+ Map<String, Path> filesMap = pathsToMap(files);
+ Assert.assertTrue(filesMap.containsKey("distributed.first.symlink"));
+ Assert.assertEquals(1, localFs.getFileStatus(
+ filesMap.get("distributed.first.symlink")).getLen());
+ Assert.assertTrue(filesMap.containsKey("distributed.second.jar"));
+ Assert.assertTrue(localFs.getFileStatus(
+ filesMap.get("distributed.second.jar")).getLen() > 1);
// Check extraction of the archive
- Assert.assertTrue(localFs.exists(new Path(archives[0],
- "distributed.jar.inside3")));
- Assert.assertTrue(localFs.exists(new Path(archives[1],
- "distributed.jar.inside4")));
+ Map<String, Path> archivesMap = pathsToMap(archives);
+ Assert.assertTrue(archivesMap.containsKey("distributed.third.jar"));
+ Assert.assertTrue(localFs.exists(new Path(
+ archivesMap.get("distributed.third.jar"), "distributed.jar.inside3")));
+ Assert.assertTrue(archivesMap.containsKey("distributed.fourth.jar"));
+ Assert.assertTrue(localFs.exists(new Path(
+ archivesMap.get("distributed.fourth.jar"), "distributed.jar.inside4")));
// Check the class loaders
LOG.info("Java Classpath: " + System.getProperty("java.class.path"));
@@ -460,6 +470,23 @@ public class TestMRJobs {
Assert.assertTrue(FileUtils.isSymlink(jobJarDir));
Assert.assertTrue(jobJarDir.isDirectory());
}
+
+ /**
+ * Returns a mapping of the final component of each path to the corresponding
+ * Path instance. This assumes that every given Path has a unique string in
+ * the final path component, which is true for these tests.
+ *
+ * @param paths Path[] to map
+ * @return Map<String, Path> mapping the final component of each path to the
+ * corresponding Path instance
+ */
+ private static Map<String, Path> pathsToMap(Path[] paths) {
+ Map<String, Path> map = new HashMap<String, Path>();
+ for (Path path: paths) {
+ map.put(path.getName(), path);
+ }
+ return map;
+ }
}
public void _testDistributedCache(String jobJarPath) throws Exception {