You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by su...@apache.org on 2013/01/29 22:40:42 UTC
svn commit: r1440159 - in
/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn...
Author: suresh
Date: Tue Jan 29 21:40:41 2013
New Revision: 1440159
URL: http://svn.apache.org/viewvc?rev=1440159&view=rev
Log:
YARN-316. YARN container launch may exceed maximum Windows command line length due to long classpath. Contributed by Chris Nauroth.
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/CHANGES.branch-trunk-win.txt
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/CHANGES.branch-trunk-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/CHANGES.branch-trunk-win.txt?rev=1440159&r1=1440158&r2=1440159&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/CHANGES.branch-trunk-win.txt (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/CHANGES.branch-trunk-win.txt Tue Jan 29 21:40:41 2013
@@ -22,3 +22,5 @@ branch-trunk-win changes - unreleased
YARN-259. Fix LocalDirsHandlerService to use Path rather than URIs. (Xuan
Gong via acmurthy)
+ YARN-316. YARN container launch may exceed maximum Windows command line
+ length due to long classpath. (Chris Nauroth via suresh)
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1440159&r1=1440158&r2=1440159&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Tue Jan 29 21:40:41 2013
@@ -55,6 +55,8 @@ public class DefaultContainerExecutor ex
private static final Log LOG = LogFactory
.getLog(DefaultContainerExecutor.class);
+ private static final int WIN_MAX_PATH = 260;
+
private final FileContext lfs;
public DefaultContainerExecutor() {
@@ -148,6 +150,17 @@ public class DefaultContainerExecutor ex
new WindowsLocalWrapperScriptBuilder(containerIdStr, containerWorkDir) :
new UnixLocalWrapperScriptBuilder(containerWorkDir);
+ // Fail fast if attempting to launch the wrapper script would fail due to
+ // Windows path length limitation.
+ if (Shell.WINDOWS &&
+ sb.getWrapperScriptPath().toString().length() > WIN_MAX_PATH) {
+ throw new IOException(String.format(
+ "Cannot launch container using script at path %s, because it exceeds " +
+ "the maximum supported path length of %d characters. Consider " +
+ "configuring shorter directories in %s.", sb.getWrapperScriptPath(),
+ WIN_MAX_PATH, YarnConfiguration.NM_LOCAL_DIRS));
+ }
+
Path pidFile = getPidFilePath(containerId);
if (pidFile != null) {
sb.writeLocalWrapperScript(launchDst, pidFile);
@@ -168,7 +181,7 @@ public class DefaultContainerExecutor ex
// Setup command to run
String[] command = Shell.getRunCommand(
- sb.getWrapperScriptPath().toUri().getPath().toString(), containerIdStr);
+ sb.getWrapperScriptPath().toString(), containerIdStr);
LOG.info("launchContainer: " + Arrays.toString(command));
shExec = new ShellCommandExecutor(
@@ -277,7 +290,7 @@ public class DefaultContainerExecutor ex
".tmp");
pout.println("@move /Y " + normalizedPidFile + ".tmp " +
normalizedPidFile);
- pout.println("@call " + launchDst.toUri().getPath().toString());
+ pout.println("@call " + launchDst.toString());
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1440159&r1=1440158&r2=1440159&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Tue Jan 29 21:40:41 2013
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
@@ -132,7 +133,7 @@ public class ContainerLaunch implements
for (String str : command) {
// TODO: Should we instead work via symlinks without this grammar?
newCmds.add(str.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
- containerLogDir.toUri().getPath()));
+ containerLogDir.toString()));
}
launchContext.setCommands(newCmds);
@@ -143,7 +144,7 @@ public class ContainerLaunch implements
entry.setValue(
value.replace(
ApplicationConstants.LOG_DIR_EXPANSION_VAR,
- containerLogDir.toUri().getPath())
+ containerLogDir.toString())
);
}
// /////////////////////////// End of variable expansion
@@ -531,7 +532,7 @@ public class ContainerLaunch implements
}
public void sanitizeEnv(Map<String, String> environment,
- Path pwd, List<Path> appDirs) {
+ Path pwd, List<Path> appDirs) throws IOException {
/**
* Non-modifiable environment variables
*/
@@ -565,6 +566,14 @@ public class ContainerLaunch implements
environment.put("JVM_PID", "$$");
}
+ // TODO: Remove Windows check and use this approach on all platforms after
+ // additional testing. See YARN-358.
+ if (Shell.WINDOWS) {
+ String inputClassPath = environment.get(Environment.CLASSPATH.name());
+ environment.put(Environment.CLASSPATH.name(),
+ FileUtil.createJarWithClassPath(inputClassPath, pwd));
+ }
+
/**
* Modifiable environment variables
*/
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1440159&r1=1440158&r2=1440159&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Tue Jan 29 21:40:41 2013
@@ -29,6 +29,8 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -83,15 +85,51 @@ public class MiniYARNCluster extends Com
super(testName.replace("$", ""));
this.numLocalDirs = numLocalDirs;
this.numLogDirs = numLogDirs;
- this.testWorkDir = new File("target",
- testName.replace("$", ""));
+ String testSubDir = testName.replace("$", "");
+ File targetWorkDir = new File("target", testSubDir);
try {
FileContext.getLocalFSFileContext().delete(
- new Path(testWorkDir.getAbsolutePath()), true);
+ new Path(targetWorkDir.getAbsolutePath()), true);
} catch (Exception e) {
LOG.warn("COULD NOT CLEANUP", e);
throw new YarnException("could not cleanup test dir", e);
}
+
+ if (Shell.WINDOWS) {
+ // The test working directory can exceed the maximum path length supported
+ // by some Windows APIs and cmd.exe (260 characters). To work around this,
+ // create a symlink in temporary storage with a much shorter path,
+ // targeting the full path to the test working directory. Then, use the
+ // symlink as the test working directory.
+ String targetPath = targetWorkDir.getAbsolutePath();
+ File link = new File(System.getProperty("java.io.tmpdir"),
+ String.valueOf(System.currentTimeMillis()));
+ String linkPath = link.getAbsolutePath();
+
+ try {
+ FileContext.getLocalFSFileContext().delete(new Path(linkPath), true);
+ } catch (IOException e) {
+ throw new YarnException("could not cleanup symlink: " + linkPath, e);
+ }
+
+ // Guarantee target exists before creating symlink.
+ targetWorkDir.mkdirs();
+
+ ShellCommandExecutor shexec = new ShellCommandExecutor(
+ Shell.getSymlinkCommand(targetPath, linkPath));
+ try {
+ shexec.execute();
+ } catch (IOException e) {
+ throw new YarnException(String.format(
+ "failed to create symlink from %s to %s, shell output: %s", linkPath,
+ targetPath, shexec.getOutput()), e);
+ }
+
+ this.testWorkDir = link;
+ } else {
+ this.testWorkDir = targetWorkDir;
+ }
+
resourceManagerWrapper = new ResourceManagerWrapper();
addService(resourceManagerWrapper);
nodeManagers = new CustomNodeManager[noOfNodeManagers];
@@ -192,6 +230,19 @@ public class MiniYARNCluster extends Com
resourceManager.stop();
}
super.stop();
+
+ if (Shell.WINDOWS) {
+ // On Windows, clean up the short temporary symlink that was created to
+ // work around path length limitation.
+ String testWorkDirPath = testWorkDir.getAbsolutePath();
+ try {
+ FileContext.getLocalFSFileContext().delete(new Path(testWorkDirPath),
+ true);
+ } catch (IOException e) {
+ LOG.warn("could not cleanup symlink: " +
+ testWorkDir.getAbsolutePath());
+ }
+ }
}
}
@@ -220,7 +271,7 @@ public class MiniYARNCluster extends Com
for (int i = 0; i < numDirs; i++) {
dirs[i]= new File(testWorkDir, MiniYARNCluster.this.getName()
+ "-" + dirType + "Dir-nm-" + index + "_" + i);
- dirs[i].mkdir();
+ dirs[i].mkdirs();
LOG.info("Created " + dirType + "Dir in " + dirs[i].getAbsolutePath());
String delimiter = (i > 0) ? "," : "";
dirsString = dirsString.concat(delimiter + dirs[i].getAbsolutePath());