You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2015/05/27 23:31:58 UTC
hadoop git commit: YARN-3626. On Windows localized resources are not
moved to the front of the classpath when they should be. Contributed by Craig
Welch.
Repository: hadoop
Updated Branches:
refs/heads/trunk 3164e7d83 -> 4102e5882
YARN-3626. On Windows localized resources are not moved to the front of the classpath when they should be. Contributed by Craig Welch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4102e588
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4102e588
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4102e588
Branch: refs/heads/trunk
Commit: 4102e5882e17b75507ae5cf8b8979485b3e24cbc
Parents: 3164e7d
Author: cnauroth <cn...@apache.org>
Authored: Wed May 27 14:21:05 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Wed May 27 14:31:49 2015 -0700
----------------------------------------------------------------------
.../mapreduce/v2/app/job/impl/TaskAttemptImpl.java | 6 ++++++
.../java/org/apache/hadoop/mapreduce/v2/util/MRApps.java | 5 -----
.../org/apache/hadoop/yarn/api/ApplicationConstants.java | 8 ++++++++
.../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 11 +----------
.../containermanager/launcher/ContainerLaunch.java | 9 +++++----
.../containermanager/launcher/TestContainerLaunch.java | 8 +++-----
6 files changed, 23 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4102e588/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 7e82df2..3055a25 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -931,10 +931,16 @@ public abstract class TaskAttemptImpl implements
// Fill in the fields needed per-container that are missing in the common
// spec.
+ boolean userClassesTakesPrecedence =
+ conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
+
// Setup environment by cloning from common env.
Map<String, String> env = commonContainerSpec.getEnvironment();
Map<String, String> myEnv = new HashMap<String, String>(env.size());
myEnv.putAll(env);
+ if (userClassesTakesPrecedence) {
+ myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
+ }
MapReduceChildJVM.setVMEnv(myEnv, remoteTask);
// Set up the launch command
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4102e588/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
index 6d82104..c8d8a44 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
@@ -241,11 +241,6 @@ public class MRApps extends Apps {
boolean userClassesTakesPrecedence =
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
- if (userClassesTakesPrecedence) {
- conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
- "true");
- }
-
String classpathEnvVar =
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4102e588/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index d42cd67..42464da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.api;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
@@ -191,6 +192,13 @@ public interface ApplicationConstants {
HADOOP_YARN_HOME("HADOOP_YARN_HOME"),
/**
+ * $CLASSPATH_PREPEND_DISTCACHE
+ * Private, Windows specific
+ */
+ @Private
+ CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"),
+
+ /**
* $CONTAINER_ID
* Final, exported by NodeManager and non-modifiable by users.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4102e588/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index cc9b44a..1b6d9c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -611,6 +611,7 @@ public class YarnConfiguration extends Configuration {
ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
+ ApplicationConstants.Environment.CLASSPATH_PREPEND_DISTCACHE.key(),
ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
/** address of node manager IPC.*/
@@ -1280,16 +1281,6 @@ public class YarnConfiguration extends Configuration {
+ "application.classpath";
/**
- * Whether or not entries from the distributed cache should be preferred over
- * the rest of the YARN CLASSPATH
- */
- public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE =
- YARN_PREFIX + "application.classpath.prepend.distcache";
-
- public static final boolean
- DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false;
-
- /**
* Default platform-agnostic CLASSPATH for YARN applications. A
* comma-separated list of CLASSPATH entries. The parameter expansion marker
* will be replaced with real parameter expansion marker ('%' for Windows and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4102e588/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 0b7f762..af168c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -749,11 +749,12 @@ public class ContainerLaunch implements Callable<Integer> {
//jar is created and so they "are lost" and have to be explicitly
//added to the classpath instead. This also means that their position
//is lost relative to other non-distcache classpath entries which will
- //break things like mapreduce.job.user.classpath.first.
+ //break things like mapreduce.job.user.classpath.first. An environment
+ //variable can be set to indicate that distcache entries should come
+ //first
- boolean preferLocalizedJars = conf.getBoolean(
- YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE
+ boolean preferLocalizedJars = Boolean.valueOf(
+ environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name())
);
boolean needsSeparator = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4102e588/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index efeeb1d..2856357 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -417,7 +417,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
- userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH");
+ userSetEnv.put(Environment.CLASSPATH.name(), "APATH");
containerLaunchContext.setEnvironment(userSetEnv);
Container container = mock(Container.class);
when(container.getContainerId()).thenReturn(cId);
@@ -463,14 +463,12 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Assert.assertTrue(
result.get(result.size() - 1).endsWith("userjarlink.jar"));
- //Now move userjar to the front
+ //Then, with user classpath first
+ userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
cId = ContainerId.newContainerId(appAttemptId, 1);
when(container.getContainerId()).thenReturn(cId);
- conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
- "true");
-
launch = new ContainerLaunch(distContext, conf,
dispatcher, exec, null, container, dirsHandler, containerManager);