You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2014/08/03 06:58:43 UTC
svn commit: r1615386 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/bin/
src/main/java/org/apache/hadoop/util/ src/site/apt/
src/test/java/org/apache/hadoop/util/
Author: cnauroth
Date: Sun Aug 3 04:58:42 2014
New Revision: 1615386
URL: http://svn.apache.org/r1615386
Log:
HADOOP-10903. Enhance hadoop classpath command to expand wildcards or write classpath into jar manifest. Contributed by Chris Nauroth.
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1615386&r1=1615385&r2=1615386&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Sun Aug 3 04:58:42 2014
@@ -479,6 +479,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10900. CredentialShell args should use single-dash style. (wang)
+ HADOOP-10903. Enhance hadoop classpath command to expand wildcards or write
+ classpath into jar manifest. (cnauroth)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1615386&r1=1615385&r2=1615386&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop Sun Aug 3 04:58:42 2014
@@ -90,11 +90,6 @@ case $COMMAND in
fi
;;
- classpath)
- echo $CLASSPATH
- exit
- ;;
-
#core commands
*)
# the core commands
@@ -118,6 +113,14 @@ case $COMMAND in
CLASSPATH=${CLASSPATH}:${TOOL_PATH}
elif [ "$COMMAND" = "credential" ] ; then
CLASS=org.apache.hadoop.security.alias.CredentialShell
+ elif [ "$COMMAND" = "classpath" ] ; then
+ if [ "$#" -eq 1 ]; then
+ # No need to bother starting up a JVM for this simple case.
+ echo $CLASSPATH
+ exit
+ else
+ CLASS=org.apache.hadoop.util.Classpath
+ fi
elif [[ "$COMMAND" = -* ]] ; then
# class and package names cannot begin with a -
echo "Error: No command named \`$COMMAND' was found. Perhaps you meant \`hadoop ${COMMAND#-}'"
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd?rev=1615386&r1=1615385&r2=1615386&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd Sun Aug 3 04:58:42 2014
@@ -115,11 +115,14 @@ call :updatepath %HADOOP_BIN_PATH%
)
if %hadoop-command% == classpath (
- @echo %CLASSPATH%
- goto :eof
+ if not defined hadoop-command-arguments (
+ @rem No need to bother starting up a JVM for this simple case.
+ @echo %CLASSPATH%
+ exit /b
+ )
)
- set corecommands=fs version jar checknative distcp daemonlog archive
+ set corecommands=fs version jar checknative distcp daemonlog archive classpath
for %%i in ( %corecommands% ) do (
if %hadoop-command% == %%i set corecommand=true
)
@@ -175,6 +178,10 @@ call :updatepath %HADOOP_BIN_PATH%
set CLASSPATH=%CLASSPATH%;%TOOL_PATH%
goto :eof
+:classpath
+ set CLASS=org.apache.hadoop.util.Classpath
+ goto :eof
+
:updatepath
set path_to_add=%*
set current_path_comparable=%path%
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java?rev=1615386&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java (added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java Sun Aug 3 04:58:42 2014
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.CommandFormat;
+import org.apache.hadoop.fs.shell.CommandFormat.UnknownOptionException;
+
+/**
+ * Command-line utility for getting the full classpath needed to launch a Hadoop
+ * client application. If the hadoop script is called with "classpath" as the
+ * command, then it simply prints the classpath and exits immediately without
+ * launching a JVM. The output likely will include wildcards in the classpath.
+ * If there are arguments passed to the classpath command, then this class gets
+ * called. With the --glob argument, it prints the full classpath with wildcards
+ * expanded. This is useful in situations where wildcard syntax isn't usable.
+ * With the --jar argument, it writes the classpath as a manifest in a jar file.
+ * This is useful in environments with short limitations on the maximum command
+ * line length, where it may not be possible to specify the full classpath in a
+ * command. For example, the maximum command line length on Windows is 8191
+ * characters.
+ */
+@InterfaceAudience.Private
+public final class Classpath {
+ private static final String usage =
+ "classpath [--glob|--jar <path>|-h|--help] :\n"
+ + " Prints the classpath needed to get the Hadoop jar and the required\n"
+ + " libraries.\n"
+ + " Options:\n"
+ + "\n"
+ + " --glob expand wildcards\n"
+ + " --jar <path> write classpath as manifest in jar named <path>\n"
+ + " -h, --help print help\n";
+
+ /**
+ * Main entry point.
+ *
+ * @param args command-line arguments
+ */
+ public static void main(String[] args) {
+ if (args.length < 1 || args[0].equals("-h") || args[0].equals("--help")) {
+ System.out.println(usage);
+ return;
+ }
+
+ // Copy args, because CommandFormat mutates the list.
+ List<String> argsList = new ArrayList<String>(Arrays.asList(args));
+ CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "-glob", "-jar");
+ try {
+ cf.parse(argsList);
+ } catch (UnknownOptionException e) {
+ terminate(1, "unrecognized option");
+ return;
+ }
+
+ String classPath = System.getProperty("java.class.path");
+
+ if (cf.getOpt("-glob")) {
+ // The classpath returned from the property has been globbed already.
+ System.out.println(classPath);
+ } else if (cf.getOpt("-jar")) {
+ if (argsList.isEmpty() || argsList.get(0) == null ||
+ argsList.get(0).isEmpty()) {
+ terminate(1, "-jar option requires path of jar file to write");
+ return;
+ }
+
+ // Write the classpath into the manifest of a temporary jar file.
+ Path workingDir = new Path(System.getProperty("user.dir"));
+ final String tmpJarPath;
+ try {
+ tmpJarPath = FileUtil.createJarWithClassPath(classPath, workingDir,
+ System.getenv());
+ } catch (IOException e) {
+ terminate(1, "I/O error creating jar: " + e.getMessage());
+ return;
+ }
+
+ // Rename the temporary file to its final location.
+ String jarPath = argsList.get(0);
+ try {
+ FileUtil.replaceFile(new File(tmpJarPath), new File(jarPath));
+ } catch (IOException e) {
+ terminate(1, "I/O error renaming jar temporary file to path: " +
+ e.getMessage());
+ return;
+ }
+ }
+ }
+
+ /**
+ * Prints a message to stderr and exits with a status code.
+ *
+ * @param status exit code
+ * @param msg message
+ */
+ private static void terminate(int status, String msg) {
+ System.err.println(msg);
+ ExitUtil.terminate(status, msg);
+ }
+}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm?rev=1615386&r1=1615385&r2=1615386&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm Sun Aug 3 04:58:42 2014
@@ -296,9 +296,24 @@ User Commands
* <<<classpath>>>
Prints the class path needed to get the Hadoop jar and the required
- libraries.
+ libraries. If called without arguments, then prints the classpath set up by
+ the command scripts, which is likely to contain wildcards in the classpath
+ entries. Additional options print the classpath after wildcard expansion or
+ write the classpath into the manifest of a jar file. The latter is useful in
+ environments where wildcards cannot be used and the expanded classpath exceeds
+ the maximum supported command line length.
- Usage: <<<hadoop classpath>>>
+ Usage: <<<hadoop classpath [--glob|--jar <path>|-h|--help]>>>
+
+*-----------------+-----------------------------------------------------------+
+|| COMMAND_OPTION || Description
+*-----------------+-----------------------------------------------------------+
+| --glob | expand wildcards
+*-----------------+-----------------------------------------------------------+
+| --jar <path> | write classpath as manifest in jar named <path>
+*-----------------+-----------------------------------------------------------+
+| -h, --help | print help
+*-----------------+-----------------------------------------------------------+
Administration Commands
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java?rev=1615386&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java (added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java Sun Aug 3 04:58:42 2014
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.charset.Charset;
+import java.util.jar.Attributes;
+import java.util.jar.JarFile;
+import java.util.jar.Manifest;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests covering the classpath command-line utility.
+ */
+public class TestClasspath {
+
+ private static final Log LOG = LogFactory.getLog(TestClasspath.class);
+ private static final File TEST_DIR = new File(
+ System.getProperty("test.build.data", "/tmp"), "TestClasspath");
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
+ static {
+ ExitUtil.disableSystemExit();
+ }
+
+ private PrintStream oldStdout, oldStderr;
+ private ByteArrayOutputStream stdout, stderr;
+ private PrintStream printStdout, printStderr;
+
+ @Before
+ public void setUp() {
+ assertTrue(FileUtil.fullyDelete(TEST_DIR));
+ assertTrue(TEST_DIR.mkdirs());
+ oldStdout = System.out;
+ oldStderr = System.err;
+
+ stdout = new ByteArrayOutputStream();
+ printStdout = new PrintStream(stdout);
+ System.setOut(printStdout);
+
+ stderr = new ByteArrayOutputStream();
+ printStderr = new PrintStream(stderr);
+ System.setErr(printStderr);
+ }
+
+ @After
+ public void tearDown() {
+ System.setOut(oldStdout);
+ System.setErr(oldStderr);
+ IOUtils.cleanup(LOG, printStdout, printStderr);
+ assertTrue(FileUtil.fullyDelete(TEST_DIR));
+ }
+
+ @Test
+ public void testGlob() {
+ Classpath.main(new String[] { "--glob" });
+ String strOut = new String(stdout.toByteArray(), UTF8);
+ assertEquals(System.getProperty("java.class.path"), strOut.trim());
+ assertTrue(stderr.toByteArray().length == 0);
+ }
+
+ @Test
+ public void testJar() throws IOException {
+ File file = new File(TEST_DIR, "classpath.jar");
+ Classpath.main(new String[] { "--jar", file.getAbsolutePath() });
+ assertTrue(stdout.toByteArray().length == 0);
+ assertTrue(stderr.toByteArray().length == 0);
+ assertTrue(file.exists());
+ assertJar(file);
+ }
+
+ @Test
+ public void testJarReplace() throws IOException {
+ // Run the command twice with the same output jar file, and expect success.
+ testJar();
+ testJar();
+ }
+
+ @Test
+ public void testJarFileMissing() throws IOException {
+ try {
+ Classpath.main(new String[] { "--jar" });
+ fail("expected exit");
+ } catch (ExitUtil.ExitException e) {
+ assertTrue(stdout.toByteArray().length == 0);
+ String strErr = new String(stderr.toByteArray(), UTF8);
+ assertTrue(strErr.contains("requires path of jar"));
+ }
+ }
+
+ @Test
+ public void testHelp() {
+ Classpath.main(new String[] { "--help" });
+ String strOut = new String(stdout.toByteArray(), UTF8);
+ assertTrue(strOut.contains("Prints the classpath"));
+ assertTrue(stderr.toByteArray().length == 0);
+ }
+
+ @Test
+ public void testHelpShort() {
+ Classpath.main(new String[] { "-h" });
+ String strOut = new String(stdout.toByteArray(), UTF8);
+ assertTrue(strOut.contains("Prints the classpath"));
+ assertTrue(stderr.toByteArray().length == 0);
+ }
+
+ @Test
+ public void testUnrecognized() {
+ try {
+ Classpath.main(new String[] { "--notarealoption" });
+ fail("expected exit");
+ } catch (ExitUtil.ExitException e) {
+ assertTrue(stdout.toByteArray().length == 0);
+ String strErr = new String(stderr.toByteArray(), UTF8);
+ assertTrue(strErr.contains("unrecognized option"));
+ }
+ }
+
+ /**
+ * Asserts that the specified file is a jar file with a manifest containing a
+ * non-empty classpath attribute.
+ *
+ * @param file File to check
+ * @throws IOException if there is an I/O error
+ */
+ private static void assertJar(File file) throws IOException {
+ JarFile jarFile = null;
+ try {
+ jarFile = new JarFile(file);
+ Manifest manifest = jarFile.getManifest();
+ assertNotNull(manifest);
+ Attributes mainAttributes = manifest.getMainAttributes();
+ assertNotNull(mainAttributes);
+ assertTrue(mainAttributes.containsKey(Attributes.Name.CLASS_PATH));
+ String classPathAttr = mainAttributes.getValue(Attributes.Name.CLASS_PATH);
+ assertNotNull(classPathAttr);
+ assertFalse(classPathAttr.isEmpty());
+ } finally {
+ // It's too bad JarFile doesn't implement Closeable.
+ if (jarFile != null) {
+ try {
+ jarFile.close();
+ } catch (IOException e) {
+ LOG.warn("exception closing jarFile: " + jarFile, e);
+ }
+ }
+ }
+ }
+}