You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lr...@apache.org on 2016/01/11 22:02:16 UTC
[34/50] [abbrv] incubator-toree git commit: Build interpreter
classpath based on Classloader hierarchy
Build interpreter classpath based on Classloader hierarchy
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/cac39e9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/cac39e9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/cac39e9c
Branch: refs/heads/master
Commit: cac39e9c8a05784dd7c1f9856818c8d3c50a1f05
Parents: 655a35b
Author: Gino Bustelo <pa...@us.ibm.com>
Authored: Wed Dec 2 17:24:24 2015 -0600
Committer: Gino Bustelo <pa...@us.ibm.com>
Committed: Thu Dec 3 09:21:09 2015 -0600
----------------------------------------------------------------------
etc/bin/spark-kernel | 1 -
.../interpreter/scala/ScalaInterpreter.scala | 34 ++++++++++++++------
.../scala/ScalaInterpreterSpec.scala | 33 ++++++++++++++++++-
3 files changed, 56 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/cac39e9c/etc/bin/spark-kernel
----------------------------------------------------------------------
diff --git a/etc/bin/spark-kernel b/etc/bin/spark-kernel
index 18cce1d..b827265 100755
--- a/etc/bin/spark-kernel
+++ b/etc/bin/spark-kernel
@@ -32,5 +32,4 @@ export PYTHONHASHSEED=0
exec "$SPARK_HOME"/bin/spark-submit \
${SPARK_OPTS} \
- --driver-class-path $PROG_HOME/lib/${KERNEL_ASSEMBLY} \
--class com.ibm.spark.SparkKernel $PROG_HOME/lib/${KERNEL_ASSEMBLY} "$@"
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/cac39e9c/scala-interpreter/src/main/scala/com/ibm/spark/kernel/interpreter/scala/ScalaInterpreter.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/main/scala/com/ibm/spark/kernel/interpreter/scala/ScalaInterpreter.scala b/scala-interpreter/src/main/scala/com/ibm/spark/kernel/interpreter/scala/ScalaInterpreter.scala
index 6fc0aa6..078054a 100644
--- a/scala-interpreter/src/main/scala/com/ibm/spark/kernel/interpreter/scala/ScalaInterpreter.scala
+++ b/scala-interpreter/src/main/scala/com/ibm/spark/kernel/interpreter/scala/ScalaInterpreter.scala
@@ -33,12 +33,13 @@ import org.apache.spark.SparkContext
import org.apache.spark.repl.{SparkCommandLine, SparkIMain, SparkJLineCompletion}
import org.slf4j.LoggerFactory
+import scala.annotation.tailrec
import scala.concurrent.{Await, Future}
import scala.language.reflectiveCalls
import scala.tools.nsc.backend.JavaPlatform
import scala.tools.nsc.interpreter.{OutputStream, IR, JPrintWriter, InputStream}
import scala.tools.nsc.io.AbstractFile
-import scala.tools.nsc.util.MergedClassPath
+import scala.tools.nsc.util.{ClassPath, MergedClassPath}
import scala.tools.nsc.{Global, Settings, io}
import scala.util.{Try => UtilTry}
@@ -193,15 +194,7 @@ class ScalaInterpreter() extends Interpreter {
val args = interpreterArgs(kernel)
this.settings = newSettings(args)
- val urls = _thisClassloader match {
- case cl: java.net.URLClassLoader => cl.getURLs.toList
- case a => // TODO: Should we really be using sys.error here?
- sys.error("[SparkInterpreter] Unexpected class loader: " + a.getClass)
- }
- val classpath = urls.map(_.toString)
-
- this.settings.classpath.value =
- classpath.distinct.mkString(java.io.File.pathSeparator)
+ this.settings.classpath.value = buildClasspath(_thisClassloader)
this.settings.embeddedDefaults(_runtimeClassloader)
maxInterpreterThreads = maxInterpreterThreads(kernel)
@@ -212,6 +205,27 @@ class ScalaInterpreter() extends Interpreter {
this
}
+ protected[scala] def buildClasspath(classLoader: ClassLoader): String = {
+
+ def toClassLoaderList( classLoader: ClassLoader ): Seq[ClassLoader] = {
+ @tailrec
+ def toClassLoaderListHelper( aClassLoader: ClassLoader, theList: Seq[ClassLoader]):Seq[ClassLoader] = {
+ if( aClassLoader == null )
+ return theList
+
+ toClassLoaderListHelper( aClassLoader.getParent, aClassLoader +: theList )
+ }
+ toClassLoaderListHelper(classLoader, Seq())
+ }
+
+ val urls = toClassLoaderList(classLoader).flatMap{
+ case cl: java.net.URLClassLoader => cl.getURLs.toList
+ case a => List()
+ }
+
+ urls.foldLeft("")((l, r) => ClassPath.join(l, r.toString))
+ }
+
protected def interpreterArgs(kernel: KernelLike): List[String] = {
import scala.collection.JavaConverters._
kernel.config.getStringList("interpreter_args").asScala.toList
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/cac39e9c/scala-interpreter/src/test/scala/com/ibm/spark/kernel/interpreter/scala/ScalaInterpreterSpec.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/test/scala/com/ibm/spark/kernel/interpreter/scala/ScalaInterpreterSpec.scala b/scala-interpreter/src/test/scala/com/ibm/spark/kernel/interpreter/scala/ScalaInterpreterSpec.scala
index bf8c5d8..1b89df3 100644
--- a/scala-interpreter/src/test/scala/com/ibm/spark/kernel/interpreter/scala/ScalaInterpreterSpec.scala
+++ b/scala-interpreter/src/test/scala/com/ibm/spark/kernel/interpreter/scala/ScalaInterpreterSpec.scala
@@ -17,7 +17,7 @@
package com.ibm.spark.kernel.interpreter.scala
import java.io.{File, InputStream, OutputStream}
-import java.net.URL
+import java.net.{URLClassLoader, URL}
import com.ibm.spark.interpreter.Results.Result
import com.ibm.spark.interpreter._
@@ -34,6 +34,7 @@ import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
import scala.concurrent.Future
import scala.tools.nsc.Settings
import scala.tools.nsc.interpreter.{JPrintWriter, IR}
+import scala.tools.nsc.util.ClassPath
class ScalaInterpreterSpec extends FunSpec
with Matchers with MockitoSugar with BeforeAndAfter
@@ -149,6 +150,36 @@ class ScalaInterpreterSpec extends FunSpec
}
}
+ describe("#buildClasspath") {
+ it("should return classpath based on classloader hierarchy") {
+ // Needed to access runtimeClassloader method
+ import scala.language.reflectiveCalls
+
+ // Create a new interpreter exposing the internal runtime classloader
+ val itInterpreter = new StubbedStartInterpreter
+
+ val parentUrls = Array(
+ new URL("file:/some/dir/a.jar"),
+ new URL("file:/some/dir/b.jar"),
+ new URL("file:/some/dir/c.jar")
+ )
+
+ val theParentClassloader = new URLClassLoader(parentUrls, null)
+
+ val urls = Array(
+ new URL("file:/some/dir/1.jar"),
+ new URL("file:/some/dir/2.jar"),
+ new URL("file:/some/dir/3.jar")
+ )
+
+ val theClassloader = new URLClassLoader(urls, theParentClassloader)
+
+ val expected = ClassPath.join((parentUrls ++ urls).map(_.toString) :_*)
+
+ itInterpreter.buildClasspath(theClassloader) should be(expected)
+ }
+ }
+
describe("#interrupt") {
it("should fail a require if the interpreter is not started") {
intercept[IllegalArgumentException] {