You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mi...@apache.org on 2016/08/04 08:32:10 UTC
zeppelin git commit: ZEPPELIN-1267. PySparkInterpreter doesn't work
in spark 2.0
Repository: zeppelin
Updated Branches:
refs/heads/master 1d0028bfe -> 161dd0efd
ZEPPELIN-1267. PySparkInterpreter doesn't work in spark 2.0
### What is this PR for?
PySparkInterpreter doesn't work in spark 2.0 because pyspark and py4j is not distributed to executors. This PR extract the setup staff for pyspark interpreter into method setupConfForPySpark and use it for both spark1 and spark2. But this is just a short term solution, as I think this should be handled by spark rather than zeppelin, here zeppelin duplicate part of spark's work. In the long term, I'd like to resolve it in `ZEPPELIN-1263`.
### What type of PR is it?
[Bug Fix]
### Todos
* https://issues.apache.org/jira/browse/ZEPPELIN-1263
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1267
### How should this be tested?
Verify it manually.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #1260 from zjffdu/ZEPPELIN-1267 and squashes the following commits:
81d1d56 [Jeff Zhang] ZEPPELIN-1267. PySparkInterpreter doesn't work in spark 2.0
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/161dd0ef
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/161dd0ef
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/161dd0ef
Branch: refs/heads/master
Commit: 161dd0efd21e233687d7472d97ccebe86a992582
Parents: 1d0028b
Author: Jeff Zhang <zj...@apache.org>
Authored: Tue Aug 2 17:41:21 2016 +0800
Committer: Mina Lee <mi...@apache.org>
Committed: Thu Aug 4 17:32:02 2016 +0900
----------------------------------------------------------------------
.../org/apache/zeppelin/spark/SparkInterpreter.java | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/161dd0ef/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index 879ed4a..89d24e6 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -329,6 +329,7 @@ public class SparkInterpreter extends Interpreter {
}
}
+ setupConfForPySpark(conf);
Class SparkSession = Utils.findClass("org.apache.spark.sql.SparkSession");
Object builder = Utils.invokeStaticMethod(SparkSession, "builder");
Utils.invokeMethod(builder, "config", new Class[]{ SparkConf.class }, new Object[]{ conf });
@@ -442,8 +443,12 @@ public class SparkInterpreter extends Interpreter {
conf.set(key, val);
}
}
+ setupConfForPySpark(conf);
+ SparkContext sparkContext = new SparkContext(conf);
+ return sparkContext;
+ }
- //TODO(jongyoul): Move these codes into PySparkInterpreter.java
+ private void setupConfForPySpark(SparkConf conf) {
String pysparkBasePath = getSystemDefault("SPARK_HOME", null, null);
File pysparkPath;
if (null == pysparkBasePath) {
@@ -456,7 +461,8 @@ public class SparkInterpreter extends Interpreter {
}
//Only one of py4j-0.9-src.zip and py4j-0.8.2.1-src.zip should exist
- String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip"};
+ String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip",
+ "py4j-0.10.1-src.zip"};
ArrayList<String> pythonLibUris = new ArrayList<>();
for (String lib : pythonLibs) {
File libFile = new File(pysparkPath, lib);
@@ -486,9 +492,6 @@ public class SparkInterpreter extends Interpreter {
if (getProperty("master").equals("yarn-client")) {
conf.set("spark.yarn.isPython", "true");
}
-
- SparkContext sparkContext = new SparkContext(conf);
- return sparkContext;
}
static final String toString(Object o) {