You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2014/03/16 19:32:06 UTC
svn commit: r1578135 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/...
Author: vinodkv
Date: Sun Mar 16 18:32:05 2014
New Revision: 1578135
URL: http://svn.apache.org/r1578135
Log:
YARN-1824. Improved NodeManager and clients to be able to handle cross platform application submissions. Contributed by Jian He.
MAPREDUCE-4052. Improved MapReduce clients to use NodeManagers' ability to handle cross platform application submissions. Contributed by Jian He.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Sun Mar 16 18:32:05 2014
@@ -196,6 +196,9 @@ Release 2.4.0 - UNRELEASED
completed/pending/successful/failed tasks on MR AM web-ui. (Paul Han via
acmurthy)
+ MAPREDUCE-4052. Improved MapReduce clients to use NodeManagers' ability to
+ handle cross platform application submissions. (Jian He via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java Sun Mar 16 18:32:05 2014
@@ -32,7 +32,6 @@ import org.apache.hadoop.mapreduce.v2.ut
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.util.Apps;
@SuppressWarnings("deprecation")
public class MapReduceChildJVM {
@@ -69,10 +68,9 @@ public class MapReduceChildJVM {
Task task) {
JobConf conf = task.conf;
-
// Add the env variables passed by the user
String mapredChildEnv = getChildEnv(conf, task.isMapTask());
- Apps.setEnvFromInputString(environment, mapredChildEnv);
+ MRApps.setEnvFromInputString(environment, mapredChildEnv, conf);
// Set logging level in the environment.
// This is so that, if the child forks another "bin/hadoop" (common in
@@ -164,7 +162,8 @@ public class MapReduceChildJVM {
Vector<String> vargs = new Vector<String>(8);
- vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
+ vargs.add(MRApps.crossPlatformifyMREnv(task.conf, Environment.JAVA_HOME)
+ + "/bin/java");
// Add child (task) java-vm options.
//
@@ -201,7 +200,7 @@ public class MapReduceChildJVM {
vargs.add(javaOptsSplit[i]);
}
- Path childTmpDir = new Path(Environment.PWD.$(),
+ Path childTmpDir = new Path(MRApps.crossPlatformifyMREnv(conf, Environment.PWD),
YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
vargs.add("-Djava.io.tmpdir=" + childTmpDir);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Sun Mar 16 18:32:05 2014
@@ -24,6 +24,7 @@ import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -31,7 +32,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.Collection;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
@@ -132,7 +132,6 @@ import org.apache.hadoop.yarn.state.Mult
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -631,7 +630,6 @@ public abstract class TaskAttemptImpl im
Token<JobTokenIdentifier> jobToken,
final org.apache.hadoop.mapred.JobID oldJobId,
Credentials credentials) {
-
// Application resources
Map<String, LocalResource> localResources =
new HashMap<String, LocalResource>();
@@ -747,16 +745,16 @@ public abstract class TaskAttemptImpl im
}
}
- Apps.addToEnvironment(
+ MRApps.addToEnvironment(
environment,
Environment.CLASSPATH.name(),
- getInitialClasspath(conf));
+ getInitialClasspath(conf), conf);
if (initialAppClasspath != null) {
- Apps.addToEnvironment(
+ MRApps.addToEnvironment(
environment,
Environment.APP_CLASSPATH.name(),
- initialAppClasspath);
+ initialAppClasspath, conf);
}
} catch (IOException e) {
throw new YarnRuntimeException(e);
@@ -771,17 +769,17 @@ public abstract class TaskAttemptImpl im
);
// Add pwd to LD_LIBRARY_PATH, add this before adding anything else
- Apps.addToEnvironment(
+ MRApps.addToEnvironment(
environment,
Environment.LD_LIBRARY_PATH.name(),
- Environment.PWD.$());
+ MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
// Add the env variables passed by the admin
- Apps.setEnvFromInputString(
+ MRApps.setEnvFromInputString(
environment,
conf.get(
MRJobConfig.MAPRED_ADMIN_USER_ENV,
- MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV)
+ MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf
);
// Construct the actual Container
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java Sun Mar 16 18:32:05 2014
@@ -23,6 +23,8 @@ import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.MRApp;
@@ -30,7 +32,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
-import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.junit.Test;
@@ -42,15 +44,17 @@ public class TestMapReduceChildJVM {
public void testCommandLine() throws Exception {
MyMRApp app = new MyMRApp(1, 0, true, this.getClass().getName(), true);
- Job job = app.submit(new Configuration());
+ Configuration conf = new Configuration();
+ conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
+ Job job = app.submit(conf);
app.waitForState(job, JobState.SUCCEEDED);
app.verifyCompleted();
Assert.assertEquals(
- "[" + envVar("JAVA_HOME") + "/bin/java" +
+ "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
" -Djava.net.preferIPv4Stack=true" +
" -Dhadoop.metrics.log.level=WARN" +
- " -Xmx200m -Djava.io.tmpdir=" + envVar("PWD") + "/tmp" +
+ " -Xmx200m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
" -Dlog4j.configuration=container-log4j.properties" +
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
" -Dyarn.app.container.log.filesize=0" +
@@ -90,16 +94,4 @@ public class TestMapReduceChildJVM {
};
}
}
-
- /**
- * Returns platform-specific string for retrieving the value of an environment
- * variable with the given name. On Unix, this returns $name. On Windows,
- * this returns %name%.
- *
- * @param name String environment variable name
- * @return String for retrieving value of environment variable
- */
- private static String envVar(String name) {
- return Shell.WINDOWS ? '%' + name + '%' : '$' + name;
- }
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Sun Mar 16 18:32:05 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapreduce.v2.util;
+import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
@@ -29,16 +30,20 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.InvalidJobConfException;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
@@ -50,6 +55,8 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.ContainerLogAppender;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -186,29 +193,33 @@ public class MRApps extends Apps {
Map<String, String> environment, Configuration conf) throws IOException {
// Propagate the system classpath when using the mini cluster
if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
- System.getProperty("java.class.path"));
+ MRApps.addToEnvironment(environment, Environment.CLASSPATH.name(),
+ System.getProperty("java.class.path"), conf);
}
+ boolean crossPlatform =
+ conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
+ MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM);
// if the framework is specified then only use the MR classpath
String frameworkName = getMRFrameworkName(conf);
if (frameworkName == null) {
// Add standard Hadoop classes
- for (String c : conf.getStrings(
- YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
- .trim());
+ for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ crossPlatform
+ ? YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH
+ : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
+ MRApps.addToEnvironment(environment, Environment.CLASSPATH.name(),
+ c.trim(), conf);
}
}
boolean foundFrameworkInClasspath = (frameworkName == null);
- for (String c : conf.getStrings(
- MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
- StringUtils.getStrings(
- MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))){
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
- .trim());
+ for (String c : conf.getStrings(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
+ crossPlatform ?
+ StringUtils.getStrings(MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH)
+ : StringUtils.getStrings(MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))) {
+ MRApps.addToEnvironment(environment, Environment.CLASSPATH.name(),
+ c.trim(), conf);
if (!foundFrameworkInClasspath) {
foundFrameworkInClasspath = c.contains(frameworkName);
}
@@ -232,28 +243,27 @@ public class MRApps extends Apps {
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();
- Apps.addToEnvironment(environment,
- classpathEnvVar,
- Environment.PWD.$());
+ MRApps.addToEnvironment(environment,
+ classpathEnvVar, crossPlatformifyMREnv(conf, Environment.PWD), conf);
if (!userClassesTakesPrecedence) {
MRApps.setMRFrameworkClasspath(environment, conf);
}
- Apps.addToEnvironment(
+ MRApps.addToEnvironment(
environment,
classpathEnvVar,
- MRJobConfig.JOB_JAR + Path.SEPARATOR + MRJobConfig.JOB_JAR);
- Apps.addToEnvironment(
+ MRJobConfig.JOB_JAR + Path.SEPARATOR + MRJobConfig.JOB_JAR, conf);
+ MRApps.addToEnvironment(
environment,
classpathEnvVar,
- MRJobConfig.JOB_JAR + Path.SEPARATOR + "classes" + Path.SEPARATOR);
- Apps.addToEnvironment(
+ MRJobConfig.JOB_JAR + Path.SEPARATOR + "classes" + Path.SEPARATOR, conf);
+ MRApps.addToEnvironment(
environment,
classpathEnvVar,
- MRJobConfig.JOB_JAR + Path.SEPARATOR + "lib" + Path.SEPARATOR + "*");
- Apps.addToEnvironment(
+ MRJobConfig.JOB_JAR + Path.SEPARATOR + "lib" + Path.SEPARATOR + "*", conf);
+ MRApps.addToEnvironment(
environment,
classpathEnvVar,
- Environment.PWD.$() + Path.SEPARATOR + "*");
+ crossPlatformifyMREnv(conf, Environment.PWD) + Path.SEPARATOR + "*", conf);
// a * in the classpath will only find a .jar, so we need to filter out
// all .jars and add everything else
addToClasspathIfNotJar(DistributedCache.getFileClassPaths(conf),
@@ -306,10 +316,10 @@ public class MRApps extends Apps {
name = p.getName();
}
if(!name.toLowerCase().endsWith(".jar")) {
- Apps.addToEnvironment(
+ MRApps.addToEnvironment(
environment,
classpathEnvVar,
- Environment.PWD.$() + Path.SEPARATOR + name);
+ crossPlatformifyMREnv(conf, Environment.PWD) + Path.SEPARATOR + name, conf);
}
}
}
@@ -549,4 +559,31 @@ public class MRApps extends Apps {
}
return null;
}
+
+ public static void setEnvFromInputString(Map<String, String> env,
+ String envString, Configuration conf) {
+ String classPathSeparator =
+ conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
+ MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
+ ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
+ Apps.setEnvFromInputString(env, envString, classPathSeparator);
+ }
+
+ @Public
+ @Unstable
+ public static void addToEnvironment(Map<String, String> environment,
+ String variable, String value, Configuration conf) {
+ String classPathSeparator =
+ conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
+ MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
+ ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
+ Apps.addToEnvironment(environment, variable, value, classPathSeparator);
+ }
+
+ public static String crossPlatformifyMREnv(Configuration conf, Environment env) {
+ boolean crossPlatform =
+ conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
+ MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM);
+ return crossPlatform ? env.$$() : env.$();
+ }
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Sun Mar 16 18:32:05 2014
@@ -18,6 +18,15 @@
package org.apache.hadoop.mapreduce.v2.util;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -33,6 +42,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.InvalidJobConfException;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -40,7 +50,6 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -51,8 +60,6 @@ import org.apache.hadoop.yarn.factory.pr
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
public class TestMRApps {
private static File testWorkDir = null;
@@ -178,27 +185,32 @@ public class TestMRApps {
@Test (timeout = 120000)
public void testSetClasspath() throws IOException {
- Job job = Job.getInstance();
+ Configuration conf = new Configuration();
+ conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
+ Job job = Job.getInstance(conf);
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, job.getConfiguration());
assertTrue(environment.get("CLASSPATH").startsWith(
- ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
+ ApplicationConstants.Environment.PWD.$$()
+ + ApplicationConstants.CLASS_PATH_SEPARATOR));
String yarnAppClasspath = job.getConfiguration().get(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
StringUtils.join(",",
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
+ YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
if (yarnAppClasspath != null) {
- yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", File.pathSeparator)
- .trim();
+ yarnAppClasspath =
+ yarnAppClasspath.replaceAll(",\\s*",
+ ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
}
assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
String mrAppClasspath =
job.getConfiguration().get(
MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
- MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH);
+ MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH);
if (mrAppClasspath != null) {
- mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", File.pathSeparator)
- .trim();
+ mrAppClasspath =
+ mrAppClasspath.replaceAll(",\\s*",
+ ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
}
assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
}
@@ -209,8 +221,10 @@ public class TestMRApps {
FileOutputStream out = new FileOutputStream(testTGZ);
out.write(0);
out.close();
- Job job = Job.getInstance();
- Configuration conf = job.getConfiguration();
+ Configuration conf = new Configuration();
+ conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
+ Job job = Job.getInstance(conf);
+ conf = job.getConfiguration();
String testTGZQualifiedPath = FileSystem.getLocal(conf).makeQualified(new Path(
testTGZ.getAbsolutePath())).toString();
conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath);
@@ -218,13 +232,13 @@ public class TestMRApps {
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, conf);
assertTrue(environment.get("CLASSPATH").startsWith(
- ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
+ ApplicationConstants.Environment.PWD.$$() + ApplicationConstants.CLASS_PATH_SEPARATOR));
String confClasspath = job.getConfiguration().get(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
StringUtils.join(",",
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
+ YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
if (confClasspath != null) {
- confClasspath = confClasspath.replaceAll(",\\s*", File.pathSeparator)
+ confClasspath = confClasspath.replaceAll(",\\s*", ApplicationConstants.CLASS_PATH_SEPARATOR)
.trim();
}
assertTrue(environment.get("CLASSPATH").contains(confClasspath));
@@ -234,6 +248,7 @@ public class TestMRApps {
@Test (timeout = 120000)
public void testSetClasspathWithUserPrecendence() {
Configuration conf = new Configuration();
+ conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
Map<String, String> env = new HashMap<String, String>();
try {
@@ -242,10 +257,10 @@ public class TestMRApps {
fail("Got exception while setting classpath");
}
String env_str = env.get("CLASSPATH");
- String expectedClasspath = StringUtils.join(File.pathSeparator,
- Arrays.asList(ApplicationConstants.Environment.PWD.$(), "job.jar/job.jar",
+ String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
+ Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
"job.jar/classes/", "job.jar/lib/*",
- ApplicationConstants.Environment.PWD.$() + "/*"));
+ ApplicationConstants.Environment.PWD.$$() + "/*"));
assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
env_str.startsWith(expectedClasspath));
}
@@ -253,6 +268,7 @@ public class TestMRApps {
@Test (timeout = 120000)
public void testSetClasspathWithNoUserPrecendence() {
Configuration conf = new Configuration();
+ conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
Map<String, String> env = new HashMap<String, String>();
try {
@@ -261,9 +277,9 @@ public class TestMRApps {
fail("Got exception while setting classpath");
}
String env_str = env.get("CLASSPATH");
- String expectedClasspath = StringUtils.join(File.pathSeparator,
+ String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
- ApplicationConstants.Environment.PWD.$() + "/*"));
+ ApplicationConstants.Environment.PWD.$$() + "/*"));
assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
+ " the classpath!", env_str.contains(expectedClasspath));
assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
@@ -273,19 +289,20 @@ public class TestMRApps {
@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
Configuration conf = new Configuration();
+ conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
Map<String, String> env = new HashMap<String, String>();
MRApps.setClasspath(env, conf);
String cp = env.get("CLASSPATH");
String appCp = env.get("APP_CLASSPATH");
assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
- + " classpath!", cp.contains("jar" + File.pathSeparator + "job"));
+ + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
cp.contains("PWD"));
- String expectedAppClasspath = StringUtils.join(File.pathSeparator,
- Arrays.asList(ApplicationConstants.Environment.PWD.$(), "job.jar/job.jar",
+ String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
+ Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
"job.jar/classes/", "job.jar/lib/*",
- ApplicationConstants.Environment.PWD.$() + "/*"));
+ ApplicationConstants.Environment.PWD.$$() + "/*"));
assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
+ " classpath!", expectedAppClasspath, appCp);
}
@@ -295,6 +312,7 @@ public class TestMRApps {
final String FRAMEWORK_NAME = "some-framework-name";
final String FRAMEWORK_PATH = "some-framework-path#" + FRAMEWORK_NAME;
Configuration conf = new Configuration();
+ conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
conf.set(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH, FRAMEWORK_PATH);
Map<String, String> env = new HashMap<String, String>();
try {
@@ -310,11 +328,11 @@ public class TestMRApps {
final String FRAMEWORK_CLASSPATH = FRAMEWORK_NAME + "/*.jar";
conf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, FRAMEWORK_CLASSPATH);
MRApps.setClasspath(env, conf);
- final String stdClasspath = StringUtils.join(File.pathSeparator,
+ final String stdClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
- ApplicationConstants.Environment.PWD.$() + "/*"));
- String expectedClasspath = StringUtils.join(File.pathSeparator,
- Arrays.asList(ApplicationConstants.Environment.PWD.$(),
+ ApplicationConstants.Environment.PWD.$$() + "/*"));
+ String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
+ Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
FRAMEWORK_CLASSPATH, stdClasspath));
assertEquals("Incorrect classpath with framework and no user precedence",
expectedClasspath, env.get("CLASSPATH"));
@@ -322,8 +340,8 @@ public class TestMRApps {
env.clear();
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
MRApps.setClasspath(env, conf);
- expectedClasspath = StringUtils.join(File.pathSeparator,
- Arrays.asList(ApplicationConstants.Environment.PWD.$(),
+ expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
+ Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
stdClasspath, FRAMEWORK_CLASSPATH));
assertEquals("Incorrect classpath with framework and user precedence",
expectedClasspath, env.get("CLASSPATH"));
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java Sun Mar 16 18:32:05 2014
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapreduce;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
* Place holder for cluster level configuration keys.
@@ -113,5 +115,14 @@ public interface MRConfig {
= "mapreduce.minicluster.control-resource-monitoring";
public static final boolean
DEFAULT_MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false;
+
+ @Public
+ @Unstable
+ public static final String MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM =
+ "mapreduce.app-submission.cross-platform";
+ @Public
+ @Unstable
+ public static final boolean DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM =
+ false;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Sun Mar 16 18:32:05 2014
@@ -18,8 +18,11 @@
package org.apache.hadoop.mapreduce;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.util.Apps;
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -692,14 +695,31 @@ public interface MRJobConfig {
"mapreduce.application.framework.path";
/**
- * Default CLASSPATH for all YARN MapReduce applications.
+ * Default CLASSPATH for all YARN MapReduce applications constructed with
+ * platform-agnostic syntax.
*/
- public final String
- DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = Shell.WINDOWS ?
- "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\*,"
- + "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\lib\\*" :
- "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,"
- + "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*";
+ @Public
+ @Unstable
+ public final String DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH = Apps
+ .crossPlatformify("HADOOP_MAPRED_HOME")
+ + "/share/hadoop/mapreduce/*,"
+ + Apps.crossPlatformify("HADOOP_MAPRED_HOME")
+ + "/share/hadoop/mapreduce/lib/*";
+
+ /**
+ * Default platform-specific CLASSPATH for all YARN MapReduce applications
+ * constructed based on client OS syntax.
+ * <p>
+ * Note: Use {@link DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH}
+ * for cross-platform practice i.e. submit an application from a Windows
+ * client to a Linux/Unix server or vice versa.
+ * </p>
+ */
+ public final String DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH =
+ Shell.WINDOWS ? "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\*,"
+ + "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\lib\\*"
+ : "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,"
+ + "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*";
public static final String WORKFLOW_ID = "mapreduce.workflow.id";
@@ -732,5 +752,4 @@ public interface MRJobConfig {
public static final String TASK_PREEMPTION =
"mapreduce.job.preemption";
-
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Sun Mar 16 18:32:05 2014
@@ -1108,21 +1108,38 @@
<description>CLASSPATH for MR applications. A comma-separated list
of CLASSPATH entries. If mapreduce.application.framework is set then this
must specify the appropriate classpath for that archive, and the name of
- the archive must be present in the classpath.
- When this value is empty, the following default CLASSPATH for MR
- applications would be used.
+ the archive must be present in the classpath.
+ If mapreduce.app-submission.cross-platform is false, platform-specific
+ environment vairable expansion syntax would be used to construct the default
+ CLASSPATH entries.
For Linux:
$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,
$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*.
For Windows:
%HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/*,
%HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/lib/*.
+
+ If mapreduce.app-submission.cross-platform is true, platform-agnostic default
+ CLASSPATH for MR applications would be used:
+ {{HADOOP_MAPRED_HOME}}/share/hadoop/mapreduce/*,
+ {{HADOOP_MAPRED_HOME}}/share/hadoop/mapreduce/lib/*
+ Parameter expansion marker will be replaced by NodeManager on container
+ launch based on the underlying OS accordingly.
</description>
<name>mapreduce.application.classpath</name>
<value></value>
</property>
<property>
+ <description>If enabled, user can submit an application cross-platform
+ i.e. submit an application from a Windows client to a Linux/Unix server or
+ vice versa.
+ </description>
+ <name>mapreduce.app-submission.cross-platform</name>
+ <value>false</value>
+</property>
+
+<property>
<description>Path to the MapReduce framework archive. If set, the framework
archive will automatically be distributed along with the job, and this
path would normally reside in a public location in an HDFS filesystem. As
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Sun Mar 16 18:32:05 2014
@@ -387,7 +387,8 @@ public class YARNRunner implements Clien
// Setup the command to run the AM
List<String> vargs = new ArrayList<String>(8);
- vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
+ vargs.add(MRApps.crossPlatformifyMREnv(jobConf, Environment.JAVA_HOME)
+ + "/bin/java");
// TODO: why do we use 'conf' some places and 'jobConf' others?
long logSize = jobConf.getLong(MRJobConfig.MR_AM_LOG_KB,
@@ -445,13 +446,13 @@ public class YARNRunner implements Clien
// i.e. add { Hadoop jars, job jar, CWD } to classpath.
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, conf);
-
+
// Setup the environment variables for Admin first
MRApps.setEnvFromInputString(environment,
- conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV));
+ conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), conf);
// Setup the environment variables (LD_LIBRARY_PATH, etc)
MRApps.setEnvFromInputString(environment,
- conf.get(MRJobConfig.MR_AM_ENV));
+ conf.get(MRJobConfig.MR_AM_ENV), conf);
// Parse distributed cache
MRApps.setupDistributedCache(jobConf, localResources);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java Sun Mar 16 18:32:05 2014
@@ -22,33 +22,34 @@ import static org.junit.Assert.assertNot
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.*;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.lib.IdentityReducer;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.util.Apps;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
/**
* Class to test mapred task's
@@ -410,12 +411,12 @@ public class TestMiniMRChildTask {
String setupHadoopHomeCommand = Shell.WINDOWS ?
"HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home" :
"HADOOP_COMMON_HOME=/fake/path/to/hadoop/common/home";
- Apps.setEnvFromInputString(environment, setupHadoopHomeCommand);
-
+ MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
+
// Add the env variables passed by the admin
- Apps.setEnvFromInputString(environment, conf.get(
+ MRApps.setEnvFromInputString(environment, conf.get(
MRJobConfig.MAPRED_ADMIN_USER_ENV,
- MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV));
+ MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
String executionPaths = environment.get(
Shell.WINDOWS ? "PATH" : "LD_LIBRARY_PATH");
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java?rev=1578135&r1=1578134&r2=1578135&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java Sun Mar 16 18:32:05 2014
@@ -89,7 +89,7 @@ public class TestEncryptedShuffle {
conf.set("hadoop.security.authentication", "simple");
String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
StringUtils.join(",",
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH))
+ YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH))
+ File.pathSeparator + classpathDir;
conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp);
dfsCluster = new MiniDFSCluster(conf, 1, true, null);