You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2015/03/18 14:07:11 UTC
spark git commit: [SPARK-6372] [core] Propagate --conf to child
processes.
Repository: spark
Updated Branches:
refs/heads/master 357900311 -> 6205a255a
[SPARK-6372] [core] Propagate --conf to child processes.
And add unit test.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #5057 from vanzin/SPARK-6372 and squashes the following commits:
b33728b [Marcelo Vanzin] [SPARK-6372] [core] Propagate --conf to child processes.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6205a255
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6205a255
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6205a255
Branch: refs/heads/master
Commit: 6205a255aea0652dddadf953771f5405065b5bec
Parents: 3579003
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Wed Mar 18 09:06:57 2015 -0400
Committer: Sean Owen <so...@cloudera.com>
Committed: Wed Mar 18 09:06:57 2015 -0400
----------------------------------------------------------------------
.../apache/spark/launcher/SparkSubmitCommandBuilder.java | 10 +---------
.../spark/launcher/SparkSubmitCommandBuilderSuite.java | 4 ++++
2 files changed, 5 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6205a255/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
index 6ffdff6..91dcf70 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
@@ -253,12 +253,6 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
private class OptionParser extends SparkSubmitOptionParser {
- private final List<String> driverJvmKeys = Arrays.asList(
- SparkLauncher.DRIVER_EXTRA_CLASSPATH,
- SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS,
- SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH,
- SparkLauncher.DRIVER_MEMORY);
-
@Override
protected boolean handle(String opt, String value) {
if (opt.equals(MASTER)) {
@@ -278,9 +272,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
} else if (opt.equals(CONF)) {
String[] setConf = value.split("=", 2);
checkArgument(setConf.length == 2, "Invalid argument to %s: %s", CONF, value);
- if (driverJvmKeys.contains(setConf[0])) {
- conf.put(setConf[0], setConf[1]);
- }
+ conf.put(setConf[0], setConf[1]);
} else if (opt.equals(CLASS)) {
// The special classes require some special command line handling, since they allow
// mixing spark-submit arguments with arguments that should be propagated to the shell
http://git-wip-us.apache.org/repos/asf/spark/blob/6205a255/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
----------------------------------------------------------------------
diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
index 815edc4..626116a 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
@@ -68,6 +68,8 @@ public class SparkSubmitCommandBuilderSuite {
parser.DRIVER_JAVA_OPTIONS,
"extraJavaOpt",
parser.CONF,
+ "spark.randomOption=foo",
+ parser.CONF,
SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH + "=/driverLibPath");
Map<String, String> env = new HashMap<String, String>();
List<String> cmd = buildCommand(sparkSubmitArgs, env);
@@ -77,6 +79,8 @@ public class SparkSubmitCommandBuilderSuite {
assertTrue(findInStringList(findArgValue(cmd, "-cp"), File.pathSeparator, "/driverCp"));
assertTrue("Driver -Xms should be configured.", cmd.contains("-Xms42g"));
assertTrue("Driver -Xmx should be configured.", cmd.contains("-Xmx42g"));
+ assertTrue("Command should contain user-defined conf.",
+ Collections.indexOfSubList(cmd, Arrays.asList(parser.CONF, "spark.randomOption=foo")) > 0);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org