You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by bz...@apache.org on 2015/04/20 04:51:25 UTC
incubator-zeppelin git commit: ZEPPELIN-46: set only non-empty values
for for spark.* properties
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master f86a4e92e -> cdd343b1f
ZEPPELIN-46: set only non-empty values for for spark.* properties
https://issues.apache.org/jira/browse/ZEPPELIN-46
Author: Jongyoul Lee <jo...@gmail.com>
Author: Alexander Bezzubov <ab...@nflabs.com>
Author: Alexander Bezzubov <bz...@apache.org>
Author: Alexander <ab...@nflabs.com>
Closes #38 from bzz/fix-non-empty-spark-conf-paraments and squashes the following commits:
56a5ce1 [Alexander Bezzubov] ZEPPELIN-46: fix style convention
50632f1 [Alexander Bezzubov] Merge branch 'master' into fix-non-empty-spark-conf-paraments
922b523 [Alexander] Merge pull request #1 from jongyoul/ZEPPELIN-46
cd0b4d4 [Jongyoul Lee] [ZEPPELIN-46] Some spark env must have a valid value - Fixed unused imports
f259a5a [Jongyoul Lee] [ZEPPELIN-46] Some spark env must have a valid value - Fixed styles
aec31d3 [Jongyoul Lee] [ZEPPELIN-46] Some spark env must have a valid value - Fixed styles - Fixed some test cases
5e22509 [Jongyoul Lee] Resolve conflicts
9d94910 [Alexander Bezzubov] ZEPPELIN-46: make tests pass on local Spark
3ff8460 [Alexander Bezzubov] ZEPPELIN-46 adding tests, by @jongyoul
00e4676 [Alexander Bezzubov] ZEPPELIN-46 fixing a typo
0ae83f4 [Alexander Bezzubov] ZEPPELIN-46: set only non-empty properties for spark.*
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/cdd343b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/cdd343b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/cdd343b1
Branch: refs/heads/master
Commit: cdd343b1fcb64f90d96e4c9aedb35132514a8340
Parents: f86a4e9
Author: Jongyoul Lee <jo...@gmail.com>
Authored: Fri Apr 17 17:19:03 2015 +0900
Committer: Alexander Bezzubov <bz...@apache.org>
Committed: Mon Apr 20 11:51:16 2015 +0900
----------------------------------------------------------------------
.../apache/zeppelin/spark/SparkInterpreter.java | 17 +++++++++++------
.../zeppelin/spark/SparkInterpreterTest.java | 15 +++++++++++++++
2 files changed, 26 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/cdd343b1/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index b038dd6..c875e85 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -244,9 +244,12 @@ public class SparkInterpreter extends Interpreter {
new SparkConf()
.setMaster(getProperty("master"))
.setAppName(getProperty("spark.app.name"))
- .setJars(jars)
.set("spark.repl.class.uri", classServerUri);
+ if (jars.length > 0) {
+ conf.setJars(jars);
+ }
+
if (execUri != null) {
conf.set("spark.executor.uri", execUri);
}
@@ -259,17 +262,19 @@ public class SparkInterpreter extends Interpreter {
for (Object k : intpProperty.keySet()) {
String key = (String) k;
- Object value = intpProperty.get(key);
- logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, value));
- conf.set(key, (String) value);
+ String val = toString(intpProperty.get(key));
+ if (!key.startsWith("spark.") || !val.trim().isEmpty()) {
+ logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val));
+ conf.set(key, val);
+ }
}
SparkContext sparkContext = new SparkContext(conf);
return sparkContext;
}
- public static boolean isEmptyString(Object val) {
- return val instanceof String && ((String) val).trim().isEmpty();
+ static final String toString(Object o) {
+ return (o instanceof String) ? (String) o : "";
}
public static String getSystemDefault(
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/cdd343b1/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
----------------------------------------------------------------------
diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
index a5e0fe2..87df793 100644
--- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
+++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
+import org.apache.spark.SparkConf;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
@@ -138,4 +139,18 @@ public class SparkInterpreterTest {
repl.interpret("z.load(\"org.apache.commons:commons-csv:1.1\")", context);
assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("import org.apache.commons.csv.CSVFormat", context).code());
}
+
+ @Test
+ public void emptyConfigurationVariablesOnlyForNonSparkProperties() {
+ Properties intpProperty = repl.getProperty();
+ SparkConf sparkConf = repl.getSparkContext().getConf();
+ for (Object oKey : intpProperty.keySet()) {
+ String key = (String) oKey;
+ String value = (String) intpProperty.get(key);
+ repl.logger.debug(String.format("[%s]: [%s]", key, value));
+ if (key.startsWith("spark.") && value.isEmpty()) {
+ assertTrue(String.format("configuration starting from 'spark.' should not be empty. [%s]", key), !sparkConf.contains(key) || !sparkConf.get(key).isEmpty());
+ }
+ }
+ }
}