You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by js...@apache.org on 2017/06/27 06:39:16 UTC
[35/50] [abbrv] incubator-livy git commit: LIVY-323. Fixed livy-repl
doesn't start with a lot of spark packages. (#312)
LIVY-323. Fixed livy-repl doesn't start with a lot of spark packages. (#312)
Some Spark packages are depending on scala-reflect 2.11.0 and it conflicts with Spark's scala version 2.11.8.
This's not Livy's fault actually but doesn't hurt to make Livy more fault tolerant.
Since the scala-reflect jar with the correct version must already be in CLASSPATH, fixed livy-repl to not load user supplied scala-reflect jars.
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/07f6072f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/07f6072f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/07f6072f
Branch: refs/heads/master
Commit: 07f6072f9d90ec970a1f9fbaccf0452f3a783538
Parents: 221aa9c
Author: Alex Man <al...@users.noreply.github.com>
Authored: Sat Apr 8 17:12:37 2017 -0700
Committer: GitHub <no...@github.com>
Committed: Sat Apr 8 17:12:37 2017 -0700
----------------------------------------------------------------------
.../src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala | 4 ++++
.../src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala | 4 ++++
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/07f6072f/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala b/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
index ec12929..5ef5491 100644
--- a/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
+++ b/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
@@ -92,6 +92,10 @@ class SparkInterpreter(conf: SparkConf,
.filter { u => u.getProtocol == "file" && new File(u.getPath).isFile }
// Livy rsc and repl are also in the extra jars list. Filter them out.
.filterNot { u => Paths.get(u.toURI).getFileName.toString.startsWith("livy-") }
+ // Some bad spark packages depend on the wrong version of scala-reflect. Blacklist it.
+ .filterNot { u =>
+ Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect")
+ }
extraJarPath.foreach { p => debug(s"Adding $p to Scala interpreter's class path...") }
sparkIMain.addUrlsToClassPath(extraJarPath: _*)
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/07f6072f/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala b/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
index bf2f680..6735b3a 100644
--- a/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
+++ b/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
@@ -78,6 +78,10 @@ class SparkInterpreter(conf: SparkConf,
.filter { u => u.getProtocol == "file" && new File(u.getPath).isFile }
// Livy rsc and repl are also in the extra jars list. Filter them out.
.filterNot { u => Paths.get(u.toURI).getFileName.toString.startsWith("livy-") }
+ // Some bad spark packages depend on the wrong version of scala-reflect. Blacklist it.
+ .filterNot { u =>
+ Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect")
+ }
extraJarPath.foreach { p => debug(s"Adding $p to Scala interpreter's class path...") }
sparkILoop.addUrlsToClassPath(extraJarPath: _*)