You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2015/10/15 18:50:21 UTC
spark git commit: [SPARK-11099] [SPARK SHELL] [SPARK SUBMIT] Default conf property file i…
Repository: spark
Updated Branches:
refs/heads/master 0f62c2282 -> aec4400be
[SPARK-11099] [SPARK SHELL] [SPARK SUBMIT] Default conf property file i…
Please help review it. Thanks
Author: Jeff Zhang <zj...@apache.org>
Closes #9114 from zjffdu/SPARK-11099.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/aec4400b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/aec4400b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/aec4400b
Branch: refs/heads/master
Commit: aec4400beffc569c13cceea2d0c481dfa3f34175
Parents: 0f62c22
Author: Jeff Zhang <zj...@apache.org>
Authored: Thu Oct 15 09:49:19 2015 -0700
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Thu Oct 15 09:50:00 2015 -0700
----------------------------------------------------------------------
.../spark/launcher/AbstractCommandBuilder.java | 14 ++++------
.../SparkSubmitCommandBuilderSuite.java | 28 +++++++++++++-------
launcher/src/test/resources/spark-defaults.conf | 21 +++++++++++++++
3 files changed, 45 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/aec4400b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index cf3729b..3ee6bd9 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -272,15 +272,11 @@ abstract class AbstractCommandBuilder {
Map<String, String> getEffectiveConfig() throws IOException {
if (effectiveConfig == null) {
- if (propertiesFile == null) {
- effectiveConfig = conf;
- } else {
- effectiveConfig = new HashMap<>(conf);
- Properties p = loadPropertiesFile();
- for (String key : p.stringPropertyNames()) {
- if (!effectiveConfig.containsKey(key)) {
- effectiveConfig.put(key, p.getProperty(key));
- }
+ effectiveConfig = new HashMap<>(conf);
+ Properties p = loadPropertiesFile();
+ for (String key : p.stringPropertyNames()) {
+ if (!effectiveConfig.containsKey(key)) {
+ effectiveConfig.put(key, p.getProperty(key));
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/aec4400b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
----------------------------------------------------------------------
diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
index d5397b0..6aad47a 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
@@ -48,12 +48,14 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
@Test
public void testDriverCmdBuilder() throws Exception {
- testCmdBuilder(true);
+ testCmdBuilder(true, true);
+ testCmdBuilder(true, false);
}
@Test
public void testClusterCmdBuilder() throws Exception {
- testCmdBuilder(false);
+ testCmdBuilder(false, true);
+ testCmdBuilder(false, false);
}
@Test
@@ -149,7 +151,7 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
assertEquals("arg1", cmd.get(cmd.size() - 1));
}
- private void testCmdBuilder(boolean isDriver) throws Exception {
+ private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) throws Exception {
String deployMode = isDriver ? "client" : "cluster";
SparkSubmitCommandBuilder launcher =
@@ -161,14 +163,20 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
launcher.appResource = "/foo";
launcher.appName = "MyApp";
launcher.mainClass = "my.Class";
- launcher.setPropertiesFile(dummyPropsFile.getAbsolutePath());
launcher.appArgs.add("foo");
launcher.appArgs.add("bar");
- launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "1g");
- launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, "/driver");
- launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Ddriver -XX:MaxPermSize=256m");
- launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native");
launcher.conf.put("spark.foo", "foo");
+ // either set the property through "--conf" or through default property file
+ if (!useDefaultPropertyFile) {
+ launcher.setPropertiesFile(dummyPropsFile.getAbsolutePath());
+ launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "1g");
+ launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, "/driver");
+ launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Ddriver -XX:MaxPermSize=256m");
+ launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native");
+ } else {
+ launcher.childEnv.put("SPARK_CONF_DIR", System.getProperty("spark.test.home")
+ + "/launcher/src/test/resources");
+ }
Map<String, String> env = new HashMap<String, String>();
List<String> cmd = launcher.buildCommand(env);
@@ -216,7 +224,9 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
}
// Checks below are the same for both driver and non-driver mode.
- assertEquals(dummyPropsFile.getAbsolutePath(), findArgValue(cmd, parser.PROPERTIES_FILE));
+ if (!useDefaultPropertyFile) {
+ assertEquals(dummyPropsFile.getAbsolutePath(), findArgValue(cmd, parser.PROPERTIES_FILE));
+ }
assertEquals("yarn", findArgValue(cmd, parser.MASTER));
assertEquals(deployMode, findArgValue(cmd, parser.DEPLOY_MODE));
assertEquals("my.Class", findArgValue(cmd, parser.CLASS));
http://git-wip-us.apache.org/repos/asf/spark/blob/aec4400b/launcher/src/test/resources/spark-defaults.conf
----------------------------------------------------------------------
diff --git a/launcher/src/test/resources/spark-defaults.conf b/launcher/src/test/resources/spark-defaults.conf
new file mode 100644
index 0000000..239fc57
--- /dev/null
+++ b/launcher/src/test/resources/spark-defaults.conf
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spark.driver.memory=1g
+spark.driver.extraClassPath=/driver
+spark.driver.extraJavaOptions=-Ddriver -XX:MaxPermSize=256m
+spark.driver.extraLibraryPath=/native
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org