You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by bi...@apache.org on 2013/04/19 21:30:17 UTC
svn commit: r1469998 - in /hadoop/common/trunk/hadoop-yarn-project:
CHANGES.txt
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
Author: bikas
Date: Fri Apr 19 19:29:22 2013
New Revision: 1469998
URL: http://svn.apache.org/r1469998
Log:
YARN-593. container launch on Windows does not correctly populate classpath with new process's environment variables and localized resources (Chris Nauroth via bikas)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1469998&r1=1469997&r2=1469998&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Fri Apr 19 19:29:22 2013
@@ -37,6 +37,10 @@ Trunk - Unreleased
YARN-493. Fixed some shell related flaws in YARN on Windows. (Chris Nauroth
via vinodkv)
+ YARN-593. container launch on Windows does not correctly populate
+ classpath with new process's environment variables and localized resources
+ (Chris Nauroth via bikas)
+
BREAKDOWN OF HADOOP-8562 SUBTASKS
YARN-158. Yarn creating package-info.java must not depend on sh.
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1469998&r1=1469997&r2=1469998&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Fri Apr 19 19:29:22 2013
@@ -28,6 +28,7 @@ import java.io.OutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -211,7 +212,7 @@ public class ContainerLaunch implements
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
// Sanitize the container's environment
- sanitizeEnv(environment, containerWorkDir, appDirs);
+ sanitizeEnv(environment, containerWorkDir, appDirs, localResources);
// Write out the environment
writeLaunchEnv(containerScriptOutStream, environment, localResources,
@@ -506,9 +507,17 @@ public class ContainerLaunch implements
@Override
protected void link(Path src, Path dst) throws IOException {
- line(String.format("@%s symlink \"%s\" \"%s\"", Shell.WINUTILS,
- new File(dst.toString()).getPath(),
- new File(src.toUri().getPath()).getPath()));
+ File srcFile = new File(src.toUri().getPath());
+ String srcFileStr = srcFile.getPath();
+ String dstFileStr = new File(dst.toString()).getPath();
+ // If not on Java7+ on Windows, then copy file instead of symlinking.
+ // See also FileUtil#symLink for full explanation.
+ if (!Shell.isJava7OrAbove() && srcFile.isFile()) {
+ line(String.format("@copy \"%s\" \"%s\"", srcFileStr, dstFileStr));
+ } else {
+ line(String.format("@%s symlink \"%s\" \"%s\"", Shell.WINUTILS,
+ dstFileStr, srcFileStr));
+ }
}
@Override
@@ -532,7 +541,8 @@ public class ContainerLaunch implements
}
public void sanitizeEnv(Map<String, String> environment,
- Path pwd, List<Path> appDirs) throws IOException {
+ Path pwd, List<Path> appDirs, Map<Path, List<String>> resources)
+ throws IOException {
/**
* Non-modifiable environment variables
*/
@@ -566,16 +576,6 @@ public class ContainerLaunch implements
environment.put("JVM_PID", "$$");
}
- // TODO: Remove Windows check and use this approach on all platforms after
- // additional testing. See YARN-358.
- if (Shell.WINDOWS) {
- String inputClassPath = environment.get(Environment.CLASSPATH.name());
- if (inputClassPath != null && !inputClassPath.isEmpty()) {
- environment.put(Environment.CLASSPATH.name(),
- FileUtil.createJarWithClassPath(inputClassPath, pwd));
- }
- }
-
/**
* Modifiable environment variables
*/
@@ -594,6 +594,57 @@ public class ContainerLaunch implements
YarnConfiguration.NM_ADMIN_USER_ENV,
YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV)
);
+
+ // TODO: Remove Windows check and use this approach on all platforms after
+ // additional testing. See YARN-358.
+ if (Shell.WINDOWS) {
+ String inputClassPath = environment.get(Environment.CLASSPATH.name());
+ if (inputClassPath != null && !inputClassPath.isEmpty()) {
+ StringBuilder newClassPath = new StringBuilder(inputClassPath);
+
+ // Localized resources do not exist at the desired paths yet, because the
+ // container launch script has not run to create symlinks yet. This
+ // means that FileUtil.createJarWithClassPath can't automatically expand
+ // wildcards to separate classpath entries for each file in the manifest.
+ // To resolve this, append classpath entries explicitly for each
+ // resource.
+ for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
+ boolean targetIsDirectory = new File(entry.getKey().toUri().getPath())
+ .isDirectory();
+
+ for (String linkName : entry.getValue()) {
+ // Append resource.
+ newClassPath.append(File.pathSeparator).append(pwd.toString())
+ .append(Path.SEPARATOR).append(linkName);
+
+ // FileUtil.createJarWithClassPath must use File.toURI to convert
+ // each file to a URI to write into the manifest's classpath. For
+ // directories, the classpath must have a trailing '/', but
+ // File.toURI only appends the trailing '/' if it is a directory that
+ // already exists. To resolve this, add the classpath entries with
+ // explicit trailing '/' here for any localized resource that targets
+ // a directory. Then, FileUtil.createJarWithClassPath will guarantee
+ // that the resulting entry in the manifest's classpath will have a
+ // trailing '/', and thus refer to a directory instead of a file.
+ if (targetIsDirectory) {
+ newClassPath.append(Path.SEPARATOR);
+ }
+ }
+ }
+
+ // When the container launches, it takes the parent process's environment
+ // and then adds/overwrites with the entries from the container launch
+ // context. Do the same thing here for correct substitution of
+ // environment variables in the classpath jar manifest.
+ Map<String, String> mergedEnv = new HashMap<String, String>(
+ System.getenv());
+ mergedEnv.putAll(environment);
+
+ String classPathJar = FileUtil.createJarWithClassPath(
+ newClassPath.toString(), pwd, mergedEnv);
+ environment.put(Environment.CLASSPATH.name(), classPathJar);
+ }
+ }
}
static void writeLaunchEnv(OutputStream out,