You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by su...@apache.org on 2013/05/23 22:42:01 UTC
svn commit: r1485845 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ bin/
hadoop-mapreduce-client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-c...
Author: suresh
Date: Thu May 23 20:41:53 2013
New Revision: 1485845
URL: http://svn.apache.org/r1485845
Log:
HADOOP-8562. Merge r1453486 from trunk
Added:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt
- copied unchanged from r1453486, hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/bin/mapred
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testshell/ExternalMapReduce.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
hadoop/common/branches/branch-2/hadoop-mapreduce-project/pom.xml
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/bin/mapred
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/bin/mapred?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/bin/mapred (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/bin/mapred Thu May 23 20:41:53 2013
@@ -135,10 +135,6 @@ for f in $HADOOP_MAPRED_HOME/modules/*.j
CLASSPATH=${CLASSPATH}:$f;
done
-if $cygwin; then
- CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-fi
-
if [ "$COMMAND" = "classpath" ] ; then
echo $CLASSPATH
exit
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java Thu May 23 20:41:53 2013
@@ -162,7 +162,6 @@ public class MapReduceChildJVM {
Vector<String> vargs = new Vector<String>(8);
- vargs.add("exec");
vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
// Add child (task) java-vm options.
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java Thu May 23 20:41:53 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.junit.Test;
@@ -37,7 +38,7 @@ public class TestMapReduceChildJVM {
private static final Log LOG = LogFactory.getLog(TestMapReduceChildJVM.class);
- @Test
+ @Test (timeout = 30000)
public void testCommandLine() throws Exception {
MyMRApp app = new MyMRApp(1, 0, true, this.getClass().getName(), true);
@@ -46,10 +47,10 @@ public class TestMapReduceChildJVM {
app.verifyCompleted();
Assert.assertEquals(
- "[exec $JAVA_HOME/bin/java" +
+ "[" + envVar("JAVA_HOME") + "/bin/java" +
" -Djava.net.preferIPv4Stack=true" +
" -Dhadoop.metrics.log.level=WARN" +
- " -Xmx200m -Djava.io.tmpdir=$PWD/tmp" +
+ " -Xmx200m -Djava.io.tmpdir=" + envVar("PWD") + "/tmp" +
" -Dlog4j.configuration=container-log4j.properties" +
" -Dyarn.app.mapreduce.container.log.dir=<LOG_DIR>" +
" -Dyarn.app.mapreduce.container.log.filesize=0" +
@@ -89,4 +90,16 @@ public class TestMapReduceChildJVM {
};
}
}
+
+ /**
+ * Returns platform-specific string for retrieving the value of an environment
+ * variable with the given name. On Unix, this returns $name. On Windows,
+ * this returns %name%.
+ *
+ * @param name String environment variable name
+ * @return String for retrieving value of environment variable
+ */
+ private static String envVar(String name) {
+ return Shell.WINDOWS ? '%' + name + '%' : '$' + name;
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Thu May 23 20:41:53 2013
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -37,6 +38,10 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -68,31 +73,35 @@ public class TestMRApps {
}
private static void delete(File dir) throws IOException {
- Path p = new Path("file://"+dir.getAbsolutePath());
Configuration conf = new Configuration();
- FileSystem fs = p.getFileSystem(conf);
+ FileSystem fs = FileSystem.getLocal(conf);
+ Path p = fs.makeQualified(new Path(dir.getAbsolutePath()));
fs.delete(p, true);
}
- @Test public void testJobIDtoString() {
+ @Test (timeout = 120000)
+ public void testJobIDtoString() {
JobId jid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class);
jid.setAppId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class));
assertEquals("job_0_0000", MRApps.toString(jid));
}
- @Test public void testToJobID() {
+ @Test (timeout = 120000)
+ public void testToJobID() {
JobId jid = MRApps.toJobID("job_1_1");
assertEquals(1, jid.getAppId().getClusterTimestamp());
assertEquals(1, jid.getAppId().getId());
assertEquals(1, jid.getId()); // tests against some proto.id and not a job.id field
}
- @Test(expected=IllegalArgumentException.class) public void testJobIDShort() {
+ @Test (timeout = 120000, expected=IllegalArgumentException.class)
+ public void testJobIDShort() {
MRApps.toJobID("job_0_0_0");
}
//TODO_get.set
- @Test public void testTaskIDtoString() {
+ @Test (timeout = 120000)
+ public void testTaskIDtoString() {
TaskId tid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskId.class);
tid.setJobId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class));
tid.getJobId().setAppId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class));
@@ -107,7 +116,8 @@ public class TestMRApps {
assertEquals("task_0_0000_r_000000", MRApps.toString(tid));
}
- @Test public void testToTaskID() {
+ @Test (timeout = 120000)
+ public void testToTaskID() {
TaskId tid = MRApps.toTaskID("task_1_2_r_3");
assertEquals(1, tid.getJobId().getAppId().getClusterTimestamp());
assertEquals(2, tid.getJobId().getAppId().getId());
@@ -119,16 +129,19 @@ public class TestMRApps {
assertEquals(TaskType.MAP, tid.getTaskType());
}
- @Test(expected=IllegalArgumentException.class) public void testTaskIDShort() {
+ @Test(timeout = 120000, expected=IllegalArgumentException.class)
+ public void testTaskIDShort() {
MRApps.toTaskID("task_0_0000_m");
}
- @Test(expected=IllegalArgumentException.class) public void testTaskIDBadType() {
+ @Test(timeout = 120000, expected=IllegalArgumentException.class)
+ public void testTaskIDBadType() {
MRApps.toTaskID("task_0_0000_x_000000");
}
//TODO_get.set
- @Test public void testTaskAttemptIDtoString() {
+ @Test (timeout = 120000)
+ public void testTaskAttemptIDtoString() {
TaskAttemptId taid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskAttemptId.class);
taid.setTaskId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskId.class));
taid.getTaskId().setTaskType(TaskType.MAP);
@@ -137,7 +150,8 @@ public class TestMRApps {
assertEquals("attempt_0_0000_m_000000_0", MRApps.toString(taid));
}
- @Test public void testToTaskAttemptID() {
+ @Test (timeout = 120000)
+ public void testToTaskAttemptID() {
TaskAttemptId taid = MRApps.toTaskAttemptID("attempt_0_1_m_2_3");
assertEquals(0, taid.getTaskId().getJobId().getAppId().getClusterTimestamp());
assertEquals(1, taid.getTaskId().getJobId().getAppId().getId());
@@ -146,11 +160,13 @@ public class TestMRApps {
assertEquals(3, taid.getId());
}
- @Test(expected=IllegalArgumentException.class) public void testTaskAttemptIDShort() {
+ @Test(timeout = 120000, expected=IllegalArgumentException.class)
+ public void testTaskAttemptIDShort() {
MRApps.toTaskAttemptID("attempt_0_0_0_m_0");
}
- @Test public void testGetJobFileWithUser() {
+ @Test (timeout = 120000)
+ public void testGetJobFileWithUser() {
Configuration conf = new Configuration();
conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/my/path/to/staging");
String jobFile = MRApps.getJobFile(conf, "dummy-user",
@@ -160,49 +176,57 @@ public class TestMRApps {
"/my/path/to/staging/dummy-user/.staging/job_dummy-job_12345/job.xml", jobFile);
}
- @Test public void testSetClasspath() throws IOException {
+ @Test (timeout = 120000)
+ public void testSetClasspath() throws IOException {
Job job = Job.getInstance();
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, job.getConfiguration());
- assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
+ assertTrue(environment.get("CLASSPATH").startsWith(
+ ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
String yarnAppClasspath =
job.getConfiguration().get(
YarnConfiguration.YARN_APPLICATION_CLASSPATH);
if (yarnAppClasspath != null) {
- yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", ":").trim();
+ yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", File.pathSeparator)
+ .trim();
}
assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
String mrAppClasspath =
job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH);
if (mrAppClasspath != null) {
- mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", ":").trim();
+ mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", File.pathSeparator)
+ .trim();
}
assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
}
- @Test public void testSetClasspathWithArchives () throws IOException {
+ @Test (timeout = 120000)
+ public void testSetClasspathWithArchives () throws IOException {
File testTGZ = new File(testWorkDir, "test.tgz");
FileOutputStream out = new FileOutputStream(testTGZ);
out.write(0);
out.close();
Job job = Job.getInstance();
Configuration conf = job.getConfiguration();
- conf.set(MRJobConfig.CLASSPATH_ARCHIVES, "file://"
- + testTGZ.getAbsolutePath());
- conf.set(MRJobConfig.CACHE_ARCHIVES, "file://"
- + testTGZ.getAbsolutePath() + "#testTGZ");
+ String testTGZQualifiedPath = FileSystem.getLocal(conf).makeQualified(new Path(
+ testTGZ.getAbsolutePath())).toString();
+ conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath);
+ conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ");
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, conf);
- assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
+ assertTrue(environment.get("CLASSPATH").startsWith(
+ ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
String confClasspath = job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
if (confClasspath != null) {
- confClasspath = confClasspath.replaceAll(",\\s*", ":").trim();
+ confClasspath = confClasspath.replaceAll(",\\s*", File.pathSeparator)
+ .trim();
}
assertTrue(environment.get("CLASSPATH").contains(confClasspath));
assertTrue(environment.get("CLASSPATH").contains("testTGZ"));
}
- @Test public void testSetClasspathWithUserPrecendence() {
+ @Test (timeout = 120000)
+ public void testSetClasspathWithUserPrecendence() {
Configuration conf = new Configuration();
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
Map<String, String> env = new HashMap<String, String>();
@@ -212,11 +236,16 @@ public class TestMRApps {
fail("Got exception while setting classpath");
}
String env_str = env.get("CLASSPATH");
- assertSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
- env_str.indexOf("$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*"), 0);
+ String expectedClasspath = StringUtils.join(File.pathSeparator,
+ Arrays.asList(ApplicationConstants.Environment.PWD.$(), "job.jar/job.jar",
+ "job.jar/classes/", "job.jar/lib/*",
+ ApplicationConstants.Environment.PWD.$() + "/*"));
+ assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
+ env_str.startsWith(expectedClasspath));
}
- @Test public void testSetClasspathWithNoUserPrecendence() {
+ @Test (timeout = 120000)
+ public void testSetClasspathWithNoUserPrecendence() {
Configuration conf = new Configuration();
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
Map<String, String> env = new HashMap<String, String>();
@@ -226,31 +255,36 @@ public class TestMRApps {
fail("Got exception while setting classpath");
}
String env_str = env.get("CLASSPATH");
- int index =
- env_str.indexOf("job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*");
- assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not"
- + " in the classpath!", index, -1);
- assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
- index, 0);
+ String expectedClasspath = StringUtils.join(File.pathSeparator,
+ Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
+ ApplicationConstants.Environment.PWD.$() + "/*"));
+ assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
+ + " the classpath!", env_str.contains(expectedClasspath));
+ assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
+ env_str.startsWith(expectedClasspath));
}
- @Test public void testSetClasspathWithJobClassloader() throws IOException {
+ @Test (timeout = 120000)
+ public void testSetClasspathWithJobClassloader() throws IOException {
Configuration conf = new Configuration();
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
Map<String, String> env = new HashMap<String, String>();
MRApps.setClasspath(env, conf);
String cp = env.get("CLASSPATH");
String appCp = env.get("APP_CLASSPATH");
- assertSame("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is"
- + " in the classpath!", cp.indexOf("jar:job"), -1);
- assertSame("MAPREDUCE_JOB_CLASSLOADER true, but PWD is"
- + " in the classpath!", cp.indexOf("PWD"), -1);
- assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not"
- + " in the app classpath!",
- "$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*", appCp);
+ assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
+ + " classpath!", cp.contains("jar" + File.pathSeparator + "job"));
+ assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
+ cp.contains("PWD"));
+ String expectedAppClasspath = StringUtils.join(File.pathSeparator,
+ Arrays.asList(ApplicationConstants.Environment.PWD.$(), "job.jar/job.jar",
+ "job.jar/classes/", "job.jar/lib/*",
+ ApplicationConstants.Environment.PWD.$() + "/*"));
+ assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
+ + " classpath!", expectedAppClasspath, appCp);
}
- @Test
+ @Test (timeout = 30000)
public void testSetupDistributedCacheEmpty() throws IOException {
Configuration conf = new Configuration();
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
@@ -260,7 +294,6 @@ public class TestMRApps {
}
@SuppressWarnings("deprecation")
- @Test
public void testSetupDistributedCacheConflicts() throws Exception {
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
@@ -299,7 +332,6 @@ public class TestMRApps {
}
@SuppressWarnings("deprecation")
- @Test
public void testSetupDistributedCacheConflictsFiles() throws Exception {
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
@@ -335,7 +367,7 @@ public class TestMRApps {
}
@SuppressWarnings("deprecation")
- @Test
+ @Test (timeout = 30000)
public void testSetupDistributedCache() throws Exception {
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java Thu May 23 20:41:53 2013
@@ -61,8 +61,12 @@ import org.apache.hadoop.util.Reflection
*/
public class TestMapProgress extends TestCase {
public static final Log LOG = LogFactory.getLog(TestMapProgress.class);
- private static String TEST_ROOT_DIR = new File(System.getProperty(
- "test.build.data", "/tmp")).getAbsolutePath() + "/mapPahseprogress";
+ private static String TEST_ROOT_DIR;
+ static {
+ String root = new File(System.getProperty("test.build.data", "/tmp"))
+ .getAbsolutePath();
+ TEST_ROOT_DIR = new Path(root, "mapPhaseprogress").toString();
+ }
static class FakeUmbilical implements TaskUmbilicalProtocol {
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java Thu May 23 20:41:53 2013
@@ -25,6 +25,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.HashMap;
+import java.util.Map;
import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry;
import org.apache.commons.io.FileUtils;
@@ -103,6 +105,8 @@ public class TestMRJobs {
private static Path TEST_ROOT_DIR = new Path("target",
TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs);
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
+ private static final String OUTPUT_ROOT_DIR = "/tmp/" +
+ TestMRJobs.class.getSimpleName();
@BeforeClass
public static void setup() throws IOException {
@@ -140,7 +144,7 @@ public class TestMRJobs {
}
}
- @Test
+ @Test (timeout = 300000)
public void testSleepJob() throws IOException, InterruptedException,
ClassNotFoundException {
LOG.info("\n\n\nStarting testSleepJob().");
@@ -211,7 +215,7 @@ public class TestMRJobs {
}
}
- @Test
+ @Test (timeout = 30000)
public void testRandomWriter() throws IOException, InterruptedException,
ClassNotFoundException {
@@ -226,8 +230,7 @@ public class TestMRJobs {
mrCluster.getConfig().set(RandomTextWriterJob.TOTAL_BYTES, "3072");
mrCluster.getConfig().set(RandomTextWriterJob.BYTES_PER_MAP, "1024");
Job job = randomWriterJob.createJob(mrCluster.getConfig());
- Path outputDir =
- new Path(mrCluster.getTestWorkDir().getAbsolutePath(), "random-output");
+ Path outputDir = new Path(OUTPUT_ROOT_DIR, "random-output");
FileOutputFormat.setOutputPath(job, outputDir);
job.setSpeculativeExecution(false);
job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
@@ -274,7 +277,7 @@ public class TestMRJobs {
&& counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue() != 0);
}
- @Test
+ @Test (timeout = 30000)
public void testFailingMapper() throws IOException, InterruptedException,
ClassNotFoundException {
@@ -342,9 +345,8 @@ public class TestMRJobs {
job.setMapperClass(FailingMapper.class);
job.setNumReduceTasks(0);
- FileOutputFormat.setOutputPath(job,
- new Path(mrCluster.getTestWorkDir().getAbsolutePath(),
- "failmapper-output"));
+ FileOutputFormat.setOutputPath(job, new Path(OUTPUT_ROOT_DIR,
+ "failmapper-output"));
job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
job.submit();
String trackingUrl = job.getTrackingURL();
@@ -357,7 +359,7 @@ public class TestMRJobs {
return job;
}
- //@Test
+ //@Test (timeout = 30000)
public void testSleepJobWithSecurityOn() throws IOException,
InterruptedException, ClassNotFoundException {
@@ -425,14 +427,22 @@ public class TestMRJobs {
Assert.assertEquals(2, archives.length);
// Check lengths of the files
- Assert.assertEquals(1, localFs.getFileStatus(files[1]).getLen());
- Assert.assertTrue(localFs.getFileStatus(files[2]).getLen() > 1);
+ Map<String, Path> filesMap = pathsToMap(files);
+ Assert.assertTrue(filesMap.containsKey("distributed.first.symlink"));
+ Assert.assertEquals(1, localFs.getFileStatus(
+ filesMap.get("distributed.first.symlink")).getLen());
+ Assert.assertTrue(filesMap.containsKey("distributed.second.jar"));
+ Assert.assertTrue(localFs.getFileStatus(
+ filesMap.get("distributed.second.jar")).getLen() > 1);
// Check extraction of the archive
- Assert.assertTrue(localFs.exists(new Path(archives[0],
- "distributed.jar.inside3")));
- Assert.assertTrue(localFs.exists(new Path(archives[1],
- "distributed.jar.inside4")));
+ Map<String, Path> archivesMap = pathsToMap(archives);
+ Assert.assertTrue(archivesMap.containsKey("distributed.third.jar"));
+ Assert.assertTrue(localFs.exists(new Path(
+ archivesMap.get("distributed.third.jar"), "distributed.jar.inside3")));
+ Assert.assertTrue(archivesMap.containsKey("distributed.fourth.jar"));
+ Assert.assertTrue(localFs.exists(new Path(
+ archivesMap.get("distributed.fourth.jar"), "distributed.jar.inside4")));
// Check the class loaders
LOG.info("Java Classpath: " + System.getProperty("java.class.path"));
@@ -460,6 +470,23 @@ public class TestMRJobs {
Assert.assertTrue(FileUtils.isSymlink(jobJarDir));
Assert.assertTrue(jobJarDir.isDirectory());
}
+
+ /**
+ * Returns a mapping of the final component of each path to the corresponding
+ * Path instance. This assumes that every given Path has a unique string in
+ * the final path component, which is true for these tests.
+ *
+ * @param paths Path[] to map
+ * @return Map<String, Path> mapping the final component of each path to the
+ * corresponding Path instance
+ */
+ private static Map<String, Path> pathsToMap(Path[] paths) {
+ Map<String, Path> map = new HashMap<String, Path>();
+ for (Path path: paths) {
+ map.put(path.getName(), path);
+ }
+ return map;
+ }
}
public void _testDistributedCache(String jobJarPath) throws Exception {
@@ -515,7 +542,7 @@ public class TestMRJobs {
trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
}
- @Test
+ @Test (timeout = 300000)
public void testDistributedCache() throws Exception {
// Test with a local (file:///) Job Jar
Path localJobJarPath = makeJobJarWithLib(TEST_ROOT_DIR.toUri().toString());
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java Thu May 23 20:41:53 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.v2;
import java.io.File;
import java.io.IOException;
+import java.util.EnumSet;
import java.util.List;
import junit.framework.Assert;
@@ -58,6 +59,9 @@ public class TestMRJobsWithHistoryServic
private static final Log LOG =
LogFactory.getLog(TestMRJobsWithHistoryService.class);
+ private static final EnumSet<RMAppState> TERMINAL_RM_APP_STATES =
+ EnumSet.of(RMAppState.FINISHED, RMAppState.FAILED, RMAppState.KILLED);
+
private static MiniMRYarnCluster mrCluster;
private static Configuration conf = new Configuration();
@@ -108,7 +112,7 @@ public class TestMRJobsWithHistoryServic
}
}
- @Test
+ @Test (timeout = 30000)
public void testJobHistoryData() throws IOException, InterruptedException,
AvroRemoteException, ClassNotFoundException {
if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
@@ -129,12 +133,24 @@ public class TestMRJobsWithHistoryServic
Counters counterMR = job.getCounters();
JobId jobId = TypeConverter.toYarn(job.getJobID());
ApplicationId appID = jobId.getAppId();
+ int pollElapsed = 0;
while (true) {
Thread.sleep(1000);
- if (mrCluster.getResourceManager().getRMContext().getRMApps()
- .get(appID).getState().equals(RMAppState.FINISHED))
+ pollElapsed += 1000;
+
+ if (TERMINAL_RM_APP_STATES.contains(
+ mrCluster.getResourceManager().getRMContext().getRMApps().get(appID)
+ .getState())) {
+ break;
+ }
+
+ if (pollElapsed >= 60000) {
+ LOG.warn("application did not reach terminal state within 60 seconds");
break;
+ }
}
+ Assert.assertEquals(RMAppState.FINISHED, mrCluster.getResourceManager()
+ .getRMContext().getRMApps().get(appID).getState());
Counters counterHS = job.getCounters();
//TODO the Assert below worked. need to check
//Should we compare each field or convert to V2 counter and compare
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testshell/ExternalMapReduce.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testshell/ExternalMapReduce.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testshell/ExternalMapReduce.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testshell/ExternalMapReduce.java Thu May 23 20:41:53 2013
@@ -72,7 +72,7 @@ public class ExternalMapReduce extends C
}
//fork off ls to see if the file exists.
// java file.exists() will not work on
- // cygwin since it is a symlink
+ // Windows since it is a symlink
String[] argv = new String[7];
argv[0] = "ls";
argv[1] = "files_tmp";
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java Thu May 23 20:41:53 2013
@@ -69,8 +69,10 @@ public class FadvisedChunkedFile extends
}
if (manageOsCache && getEndOffset() - getStartOffset() > 0) {
try {
- NativeIO.posixFadviseIfPossible(fd, getStartOffset(), getEndOffset()
- - getStartOffset(), NativeIO.POSIX_FADV_DONTNEED);
+ NativeIO.POSIX.posixFadviseIfPossible(
+ fd,
+ getStartOffset(), getEndOffset() - getStartOffset(),
+ NativeIO.POSIX.POSIX_FADV_DONTNEED);
} catch (Throwable t) {
LOG.warn("Failed to manage OS cache for " + identifier, t);
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java Thu May 23 20:41:53 2013
@@ -71,8 +71,9 @@ public class FadvisedFileRegion extends
}
if (manageOsCache && getCount() > 0) {
try {
- NativeIO.posixFadviseIfPossible(fd, getPosition(), getCount(),
- NativeIO.POSIX_FADV_DONTNEED);
+ NativeIO.POSIX.posixFadviseIfPossible(
+ fd, getPosition(), getCount(),
+ NativeIO.POSIX.POSIX_FADV_DONTNEED);
} catch (Throwable t) {
LOG.warn("Failed to manage OS cache for " + identifier, t);
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java Thu May 23 20:41:53 2013
@@ -338,7 +338,7 @@ public class TestShuffleHandler {
}
// Retrieve file owner name
FileInputStream is = new FileInputStream(fileMap.get(0));
- String owner = NativeIO.getFstat(is.getFD()).getOwner();
+ String owner = NativeIO.POSIX.getFstat(is.getFD()).getOwner();
is.close();
String message =
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml Thu May 23 20:41:53 2013
@@ -176,6 +176,10 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
+ <environmentVariables>
+ <!-- HADOOP_HOME required for tests on Windows to find winutils -->
+ <HADOOP_HOME>${basedir}/../../../hadoop-common-project/hadoop-common/target</HADOOP_HOME>
+ </environmentVariables>
<properties>
<property>
<name>listener</name>
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/pom.xml?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/pom.xml Thu May 23 20:41:53 2013
@@ -182,15 +182,8 @@
<target if="tar">
<!-- Using Unix script to preserve symlinks -->
<echo file="${project.build.directory}/dist-maketar.sh">
-
- which cygpath 2> /dev/null
- if [ $? = 1 ]; then
- BUILD_DIR="${project.build.directory}"
- else
- BUILD_DIR=`cygpath --unix '${project.build.directory}'`
- fi
- cd $BUILD_DIR
- tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version}
+ cd "${project.build.directory}"
+ tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
</echo>
<exec executable="sh" dir="${project.build.directory}" failonerror="true">
<arg line="./dist-maketar.sh"/>
@@ -217,6 +210,7 @@
<excludes>
<exclude>.eclipse.templates/</exclude>
<exclude>CHANGES.txt</exclude>
+ <exclude>CHANGES.branch-trunk-win.txt</exclude>
<exclude>lib/jdiff/**</exclude>
</excludes>
</configuration>