You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mi...@apache.org on 2016/10/06 01:47:36 UTC
zeppelin git commit: [ZEPPELIN-1466] Make %dep work for spark 2.0
when SPARK_HOME is not defined
Repository: zeppelin
Updated Branches:
refs/heads/master 578fdf3e0 -> 3f5a500cc
[ZEPPELIN-1466] Make %dep work for spark 2.0 when SPARK_HOME is not defined
### What is this PR for?
%dep does not work for spark 2.0 when SPARK_HOME is not defined.
Problem described in the discussion https://issues.apache.org/jira/browse/ZEPPELIN-1466
### What type of PR is it?
Bug Fix
### Todos
* [x] - Construct classpath arg correctly
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-1466
### How should this be tested?
make sure SPARK_HOME is NOT defined.
run
```
%dep
z.load("org.apache.commons:commons-csv:1.1")
```
and
```
%spark
import org.apache.commons.csv.CSVParser
```
sequentially and see import statement success
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no
Author: Lee moon soo <mo...@apache.org>
Closes #1481 from Leemoonsoo/ZEPPELIN-1466 and squashes the following commits:
5cdf99f [Lee moon soo] pass depInterpreter loaded jar in scala compiler's classpath arg
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/3f5a500c
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/3f5a500c
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/3f5a500c
Branch: refs/heads/master
Commit: 3f5a500ccf0f540fe2d9d488193a94f1eb737a3f
Parents: 578fdf3
Author: Lee moon soo <mo...@apache.org>
Authored: Tue Oct 4 18:28:30 2016 +0900
Committer: Mina Lee <mi...@apache.org>
Committed: Thu Oct 6 10:47:25 2016 +0900
----------------------------------------------------------------------
.../apache/zeppelin/spark/SparkInterpreter.java | 52 ++++++++++++++------
1 file changed, 36 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3f5a500c/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index 44c2a74..dccc12e 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -584,6 +584,24 @@ public class SparkInterpreter extends Interpreter {
argList.add(arg);
}
+ DepInterpreter depInterpreter = getDepInterpreter();
+ String depInterpreterClasspath = "";
+ if (depInterpreter != null) {
+ SparkDependencyContext depc = depInterpreter.getDependencyContext();
+ if (depc != null) {
+ List<File> files = depc.getFiles();
+ if (files != null) {
+ for (File f : files) {
+ if (depInterpreterClasspath.length() > 0) {
+ depInterpreterClasspath += File.pathSeparator;
+ }
+ depInterpreterClasspath += f.getAbsolutePath();
+ }
+ }
+ }
+ }
+
+
if (Utils.isScala2_10()) {
scala.collection.immutable.List<String> list =
JavaConversions.asScalaBuffer(argList).toList();
@@ -611,10 +629,22 @@ public class SparkInterpreter extends Interpreter {
argList.add("-Yrepl-class-based");
argList.add("-Yrepl-outdir");
argList.add(outputDir.getAbsolutePath());
+
+ String classpath = "";
if (conf.contains("spark.jars")) {
- String jars = StringUtils.join(conf.get("spark.jars").split(","), File.separator);
+ classpath = StringUtils.join(conf.get("spark.jars").split(","), File.separator);
+ }
+
+ if (!depInterpreterClasspath.isEmpty()) {
+ if (!classpath.isEmpty()) {
+ classpath += File.separator;
+ }
+ classpath += depInterpreterClasspath;
+ }
+
+ if (!classpath.isEmpty()) {
argList.add("-classpath");
- argList.add(jars);
+ argList.add(classpath);
}
scala.collection.immutable.List<String> list =
@@ -626,6 +656,7 @@ public class SparkInterpreter extends Interpreter {
// set classpath for scala compiler
PathSetting pathSettings = settings.classpath();
String classpath = "";
+
List<File> paths = currentClassPath();
for (File f : paths) {
if (classpath.length() > 0) {
@@ -644,21 +675,10 @@ public class SparkInterpreter extends Interpreter {
}
// add dependency from DepInterpreter
- DepInterpreter depInterpreter = getDepInterpreter();
- if (depInterpreter != null) {
- SparkDependencyContext depc = depInterpreter.getDependencyContext();
- if (depc != null) {
- List<File> files = depc.getFiles();
- if (files != null) {
- for (File f : files) {
- if (classpath.length() > 0) {
- classpath += File.pathSeparator;
- }
- classpath += f.getAbsolutePath();
- }
- }
- }
+ if (classpath.length() > 0) {
+ classpath += File.pathSeparator;
}
+ classpath += depInterpreterClasspath;
// add dependency from local repo
String localRepo = getProperty("zeppelin.interpreter.localRepo");