You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2015/04/28 13:57:03 UTC
spark git commit: [SPARK-6435] spark-shell --jars option does not add
all jars to classpath
Repository: spark
Updated Branches:
refs/heads/master 75905c57c -> 268c419f1
[SPARK-6435] spark-shell --jars option does not add all jars to classpath
Modified to accept double-quotated args properly in spark-shell.cmd.
Author: Masayoshi TSUZUKI <ts...@oss.nttdata.co.jp>
Closes #5227 from tsudukim/feature/SPARK-6435-2 and squashes the following commits:
ac55787 [Masayoshi TSUZUKI] removed unnecessary argument.
60789a7 [Masayoshi TSUZUKI] Merge branch 'master' of https://github.com/apache/spark into feature/SPARK-6435-2
1fee420 [Masayoshi TSUZUKI] fixed test code for escaping '='.
0d4dc41 [Masayoshi TSUZUKI] - escaped comman and semicolon in CommandBuilderUtils.java - added random string to the temporary filename - double-quotation followed by `cmd /c` did not worked properly - no need to escape `=` by `^` - if double-quoted string ended with `\` like classpath, the last `\` is parsed as the escape charactor and the closing `"` didn't work properly
2a332e5 [Masayoshi TSUZUKI] Merge branch 'master' into feature/SPARK-6435-2
04f4291 [Masayoshi TSUZUKI] [SPARK-6435] spark-shell --jars option does not add all jars to classpath
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/268c419f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/268c419f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/268c419f
Branch: refs/heads/master
Commit: 268c419f1586110b90e68f98cd000a782d18828c
Parents: 75905c5
Author: Masayoshi TSUZUKI <ts...@oss.nttdata.co.jp>
Authored: Tue Apr 28 07:55:21 2015 -0400
Committer: Sean Owen <so...@cloudera.com>
Committed: Tue Apr 28 07:56:36 2015 -0400
----------------------------------------------------------------------
bin/spark-class2.cmd | 5 ++++-
.../java/org/apache/spark/launcher/CommandBuilderUtils.java | 9 ++++-----
launcher/src/main/java/org/apache/spark/launcher/Main.java | 6 +-----
.../org/apache/spark/launcher/CommandBuilderUtilsSuite.java | 5 ++++-
4 files changed, 13 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/268c419f/bin/spark-class2.cmd
----------------------------------------------------------------------
diff --git a/bin/spark-class2.cmd b/bin/spark-class2.cmd
index 3d068dd..db09fa2 100644
--- a/bin/spark-class2.cmd
+++ b/bin/spark-class2.cmd
@@ -61,7 +61,10 @@ if not "x%JAVA_HOME%"=="x" set RUNNER=%JAVA_HOME%\bin\java
rem The launcher library prints the command to be executed in a single line suitable for being
rem executed by the batch interpreter. So read all the output of the launcher into a variable.
-for /f "tokens=*" %%i in ('cmd /C ""%RUNNER%" -cp %LAUNCH_CLASSPATH% org.apache.spark.launcher.Main %*"') do (
+set LAUNCHER_OUTPUT=%temp%\spark-class-launcher-output-%RANDOM%.txt
+"%RUNNER%" -cp %LAUNCH_CLASSPATH% org.apache.spark.launcher.Main %* > %LAUNCHER_OUTPUT%
+for /f "tokens=*" %%i in (%LAUNCHER_OUTPUT%) do (
set SPARK_CMD=%%i
)
+del %LAUNCHER_OUTPUT%
%SPARK_CMD%
http://git-wip-us.apache.org/repos/asf/spark/blob/268c419f/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
index 8028e42..2614028 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
@@ -244,7 +244,7 @@ class CommandBuilderUtils {
boolean needsQuotes = false;
for (int i = 0; i < arg.length(); i++) {
int c = arg.codePointAt(i);
- if (Character.isWhitespace(c) || c == '"' || c == '=') {
+ if (Character.isWhitespace(c) || c == '"' || c == '=' || c == ',' || c == ';') {
needsQuotes = true;
break;
}
@@ -261,15 +261,14 @@ class CommandBuilderUtils {
quoted.append('"');
break;
- case '=':
- quoted.append('^');
- break;
-
default:
break;
}
quoted.appendCodePoint(cp);
}
+ if (arg.codePointAt(arg.length() - 1) == '\\') {
+ quoted.append("\\");
+ }
quoted.append("\"");
return quoted.toString();
}
http://git-wip-us.apache.org/repos/asf/spark/blob/268c419f/launcher/src/main/java/org/apache/spark/launcher/Main.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/Main.java b/launcher/src/main/java/org/apache/spark/launcher/Main.java
index 206acfb..929b29a 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/Main.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/Main.java
@@ -101,12 +101,9 @@ class Main {
* The method quotes all arguments so that spaces are handled as expected. Quotes within arguments
* are "double quoted" (which is batch for escaping a quote). This page has more details about
* quoting and other batch script fun stuff: http://ss64.com/nt/syntax-esc.html
- *
- * The command is executed using "cmd /c" and formatted in single line, since that's the
- * easiest way to consume this from a batch script (see spark-class2.cmd).
*/
private static String prepareWindowsCommand(List<String> cmd, Map<String, String> childEnv) {
- StringBuilder cmdline = new StringBuilder("cmd /c \"");
+ StringBuilder cmdline = new StringBuilder();
for (Map.Entry<String, String> e : childEnv.entrySet()) {
cmdline.append(String.format("set %s=%s", e.getKey(), e.getValue()));
cmdline.append(" && ");
@@ -115,7 +112,6 @@ class Main {
cmdline.append(quoteForBatchScript(arg));
cmdline.append(" ");
}
- cmdline.append("\"");
return cmdline.toString();
}
http://git-wip-us.apache.org/repos/asf/spark/blob/268c419f/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
----------------------------------------------------------------------
diff --git a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
index 1ae42ee..bc513ec 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
@@ -74,7 +74,10 @@ public class CommandBuilderUtilsSuite {
assertEquals("\"a b c\"", quoteForBatchScript("a b c"));
assertEquals("\"a \"\"b\"\" c\"", quoteForBatchScript("a \"b\" c"));
assertEquals("\"a\"\"b\"\"c\"", quoteForBatchScript("a\"b\"c"));
- assertEquals("\"ab^=\"\"cd\"\"\"", quoteForBatchScript("ab=\"cd\""));
+ assertEquals("\"ab=\"\"cd\"\"\"", quoteForBatchScript("ab=\"cd\""));
+ assertEquals("\"a,b,c\"", quoteForBatchScript("a,b,c"));
+ assertEquals("\"a;b;c\"", quoteForBatchScript("a;b;c"));
+ assertEquals("\"a,b,c\\\\\"", quoteForBatchScript("a,b,c\\"));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org