You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2014/10/21 21:34:34 UTC
git commit: YARN-2720. Windows: Wildcard classpath variables not
expanded against resources contained in archives. Contributed by Craig Welch.
Repository: hadoop
Updated Branches:
refs/heads/trunk 4e134a02a -> 6637e3cf9
YARN-2720. Windows: Wildcard classpath variables not expanded against resources contained in archives. Contributed by Craig Welch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6637e3cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6637e3cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6637e3cf
Branch: refs/heads/trunk
Commit: 6637e3cf95b3a9be8d6b9cd66bc849a0607e8ed5
Parents: 4e134a0
Author: cnauroth <cn...@apache.org>
Authored: Tue Oct 21 12:33:21 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Tue Oct 21 12:33:21 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/FileUtil.java | 17 +++++++++++++----
.../java/org/apache/hadoop/util/Classpath.java | 2 +-
.../java/org/apache/hadoop/fs/TestFileUtil.java | 4 +++-
hadoop-yarn-project/CHANGES.txt | 3 +++
.../WindowsSecureContainerExecutor.java | 6 ++++--
.../containermanager/launcher/ContainerLaunch.java | 6 ++++--
6 files changed, 28 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6637e3cf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 2b05293..0848cb1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -1212,10 +1212,11 @@ public class FileUtil {
* @param pwd Path to working directory to save jar
* @param callerEnv Map<String, String> caller's environment variables to use
* for expansion
- * @return String absolute path to new jar
+ * @return String[] with absolute path to new jar in position 0 and
+ * unexpanded wild card entry path in position 1
* @throws IOException if there is an I/O error while writing the jar file
*/
- public static String createJarWithClassPath(String inputClassPath, Path pwd,
+ public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
Map<String, String> callerEnv) throws IOException {
// Replace environment variables, case-insensitive on Windows
@SuppressWarnings("unchecked")
@@ -1235,6 +1236,7 @@ public class FileUtil {
LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
}
+ StringBuilder unexpandedWildcardClasspath = new StringBuilder();
// Append all entries
List<String> classPathEntryList = new ArrayList<String>(
classPathEntries.length);
@@ -1243,16 +1245,22 @@ public class FileUtil {
continue;
}
if (classPathEntry.endsWith("*")) {
+ boolean foundWildCardJar = false;
// Append all jars that match the wildcard
Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
.globStatus(globPath);
if (wildcardJars != null) {
for (FileStatus wildcardJar: wildcardJars) {
+ foundWildCardJar = true;
classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
.toExternalForm());
}
}
+ if (!foundWildCardJar) {
+ unexpandedWildcardClasspath.append(File.pathSeparator);
+ unexpandedWildcardClasspath.append(classPathEntry);
+ }
} else {
// Append just this entry
File fileCpEntry = null;
@@ -1300,7 +1308,8 @@ public class FileUtil {
} finally {
IOUtils.cleanup(LOG, jos, bos, fos);
}
-
- return classPathJar.getCanonicalPath();
+ String[] jarCp = {classPathJar.getCanonicalPath(),
+ unexpandedWildcardClasspath.toString()};
+ return jarCp;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6637e3cf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
index 0d3df2d..29ffbe1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
@@ -94,7 +94,7 @@ public final class Classpath {
final String tmpJarPath;
try {
tmpJarPath = FileUtil.createJarWithClassPath(classPath, workingDir,
- System.getenv());
+ System.getenv())[0];
} catch (IOException e) {
terminate(1, "I/O error creating jar: " + e.getMessage());
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6637e3cf/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 2a03988..41794b8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -1036,8 +1036,10 @@ public class TestFileUtil {
List<String> classPaths = Arrays.asList("", "cp1.jar", "cp2.jar", wildcardPath,
"cp3.jar", nonExistentSubdir);
String inputClassPath = StringUtils.join(File.pathSeparator, classPaths);
- String classPathJar = FileUtil.createJarWithClassPath(inputClassPath,
+ String[] jarCp = FileUtil.createJarWithClassPath(inputClassPath + File.pathSeparator + "unexpandedwildcard/*",
new Path(tmp.getCanonicalPath()), System.getenv());
+ String classPathJar = jarCp[0];
+ assertNotEquals("Unexpanded wildcard was not placed in extra classpath", jarCp[1].indexOf("unexpanded"), -1);
// verify classpath by reading manifest from jar file
JarFile jarFile = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6637e3cf/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index af056b3..06c0eab 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -708,6 +708,9 @@ Release 2.6.0 - UNRELEASED
YARN-2717. Avoided duplicate logging when container logs are not found. (Xuan
Gong via zjshen)
+ YARN-2720. Windows: Wildcard classpath variables not expanded against
+ resources contained in archives. (Craig Welch via cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6637e3cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
index fc6c64f..578a9a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
@@ -148,10 +148,12 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
// Passing CLASSPATH explicitly is *way* too long for command line.
String classPath = System.getProperty("java.class.path");
Map<String, String> env = new HashMap<String, String>(System.getenv());
- String classPathJar = FileUtil.createJarWithClassPath(classPath, appStorageDir, env);
+ String[] jarCp = FileUtil.createJarWithClassPath(classPath, appStorageDir, env);
+ String classPathJar = jarCp[0];
localizeClasspathJar(new Path(classPathJar), user);
+ String replacementClassPath = classPathJar + jarCp[1];
command.add("-classpath");
- command.add(classPathJar);
+ command.add(replacementClassPath);
String javaLibPath = System.getProperty("java.library.path");
if (javaLibPath != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6637e3cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index f87ed6a..30abef5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -764,11 +764,13 @@ public class ContainerLaunch implements Callable<Integer> {
System.getenv());
mergedEnv.putAll(environment);
- String classPathJar = FileUtil.createJarWithClassPath(
+ String[] jarCp = FileUtil.createJarWithClassPath(
newClassPath.toString(), pwd, mergedEnv);
+ String classPathJar = jarCp[0];
// In a secure cluster the classpath jar must be localized to grant access
this.exec.localizeClasspathJar(new Path(classPathJar), container.getUser());
- environment.put(Environment.CLASSPATH.name(), classPathJar);
+ String replacementClassPath = classPathJar + jarCp[1];
+ environment.put(Environment.CLASSPATH.name(), replacementClassPath);
}
}
// put AuxiliaryService data to environment