You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lb...@apache.org on 2016/03/29 20:03:08 UTC
[1/2] incubator-toree git commit: Converted magics to plugins
Repository: incubator-toree
Updated Branches:
refs/heads/master 429c74c67 -> 9c8824fff
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
index c6771be..be36c8f 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
@@ -23,15 +23,15 @@ import java.net.URL
import org.apache.toree.magic._
import org.apache.toree.magic.dependencies._
import org.apache.toree.utils.ArgumentParsingSupport
-
import scala.util.Try
+import org.apache.toree.plugins.annotations.Event
class AddDeps extends LineMagic with IncludeInterpreter
with IncludeOutputStream with IncludeSparkContext with ArgumentParsingSupport
with IncludeDependencyDownloader with IncludeKernel
{
- private lazy val printStream = new PrintStream(outputStream)
+ private def printStream = new PrintStream(outputStream)
private val _transitive = parser.accepts(
"transitive", "Retrieve dependencies recursively"
@@ -59,6 +59,7 @@ class AddDeps extends LineMagic with IncludeInterpreter
* @param code The single line of code
* @return The output of the magic
*/
+ @Event(name = "adddeps")
override def execute(code: String): Unit = {
val nonOptionArgs = parseArgs(code)
dependencyDownloader.setPrintStream(printStream)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/magic/builtin/AddJar.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddJar.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddJar.scala
index 65abccc..125ee0a 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddJar.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddJar.scala
@@ -26,6 +26,7 @@ import org.apache.toree.magic.builtin.AddJar._
import org.apache.toree.magic.dependencies._
import org.apache.toree.utils.{ArgumentParsingSupport, DownloadSupport, LogLike}
import com.typesafe.config.Config
+import org.apache.toree.plugins.annotations.Event
object AddJar {
@@ -47,7 +48,7 @@ object AddJar {
class AddJar
extends LineMagic with IncludeInterpreter with IncludeSparkContext
with IncludeOutputStream with DownloadSupport with ArgumentParsingSupport
- with IncludeKernel with IncludeMagicLoader with IncludeConfig with LogLike
+ with IncludeKernel with IncludePluginManager with IncludeConfig with LogLike
{
// Option to mark re-downloading of jars
private val _force =
@@ -58,7 +59,7 @@ class AddJar
parser.accepts("magic", "loads jar as a magic extension")
// Lazy because the outputStream is not provided at construction
- private lazy val printStream = new PrintStream(outputStream)
+ private def printStream = new PrintStream(outputStream)
/**
* Retrieves file name from URL.
@@ -82,6 +83,7 @@ class AddJar
*
* @param code The line containing the location of the jar
*/
+ @Event(name = "addjar")
override def execute(code: String): Unit = {
val nonOptionArgs = parseArgs(code.trim)
@@ -103,12 +105,14 @@ class AddJar
// Ensure the URL actually contains a jar or zip file
if (!jarName.endsWith(".jar") && !jarName.endsWith(".zip")) {
- throw new IllegalArgumentException(s"The jar file $jarName must end in .jar or .zip.")
+ throw new IllegalArgumentException(
+ s"The jar file $jarName must end in .jar or .zip."
+ )
}
val downloadLocation = getJarDir(config) + "/" + jarName
- logger.debug( "Downloading jar to %s".format(downloadLocation) )
+ logger.debug("Downloading jar to %s".format(downloadLocation))
val fileDownloadLocation = new File(downloadLocation)
@@ -128,18 +132,12 @@ class AddJar
printStream.println(s"Using cached version of $jarName")
}
-
- if (_magic)
- {
-
- magicLoader.addJar(fileDownloadLocation.toURI.toURL)
-
- }
- else
- {
+ if (_magic) {
+ val plugins = pluginManager.loadPlugins(fileDownloadLocation)
+ pluginManager.initializePlugins(plugins)
+ } else {
interpreter.addJars(fileDownloadLocation.toURI.toURL)
sparkContext.addJar(fileDownloadLocation.getCanonicalPath)
-
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/magic/builtin/Html.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/Html.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/Html.scala
index 8060a46..7975ce7 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/Html.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/Html.scala
@@ -24,13 +24,15 @@ import org.apache.toree.magic._
import org.apache.toree.magic.dependencies.IncludeOutputStream
import org.apache.toree.utils.ArgumentParsingSupport
import com.google.common.base.Strings
+import org.apache.toree.plugins.annotations.Event
class Html extends CellMagic with ArgumentParsingSupport
with IncludeOutputStream {
// Lazy because the outputStream is not provided at construction
- private lazy val printStream = new PrintStream(outputStream)
-
+ private def printStream = new PrintStream(outputStream)
+
+ @Event(name = "html")
override def execute(code: String): CellMagicOutput = {
def printHelpAndReturn: CellMagicOutput = {
printHelp(printStream, """%%Html <string_code>""")
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/magic/builtin/JavaScript.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/JavaScript.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/JavaScript.scala
index a3bd585..f6b7dbf 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/JavaScript.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/JavaScript.scala
@@ -25,13 +25,15 @@ import org.apache.toree.magic._
import org.apache.toree.magic.dependencies.IncludeOutputStream
import org.apache.toree.utils.ArgumentParsingSupport
import org.slf4j.LoggerFactory
+import org.apache.toree.plugins.annotations.Event
class JavaScript extends CellMagic with ArgumentParsingSupport
with IncludeOutputStream {
// Lazy because the outputStream is not provided at construction
- private lazy val printStream = new PrintStream(outputStream)
-
+ private def printStream = new PrintStream(outputStream)
+
+ @Event(name = "javascript")
override def execute(code: String): CellMagicOutput = {
def printHelpAndReturn: CellMagicOutput = {
printHelp(printStream, """%JavaScript <string_code>""")
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/magic/builtin/LSMagic.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/LSMagic.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/LSMagic.scala
index 260db25..48e5ad4 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/LSMagic.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/LSMagic.scala
@@ -21,16 +21,18 @@ import java.io.PrintStream
import org.apache.toree.magic._
import org.apache.toree.magic.dependencies.IncludeOutputStream
+import org.apache.toree.plugins.annotations.Event
class LSMagic extends LineMagic with IncludeOutputStream {
- private lazy val printStream = new PrintStream(outputStream)
+ private def printStream = new PrintStream(outputStream)
/**
* Lists all available magics.
* @param code The single line of code
* @return The output of the magic
*/
+ @Event(name = "lsmagic")
override def execute(code: String): Unit = {
val classes = new BuiltinLoader().loadClasses().toList
val lineMagics = magicNames("%", classOf[LineMagic], classes)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/magic/builtin/RDD.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/RDD.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/RDD.scala
index 32244d0..b165ac8 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/RDD.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/RDD.scala
@@ -24,6 +24,7 @@ import org.apache.toree.magic.dependencies.{IncludeKernelInterpreter, IncludeInt
import org.apache.toree.utils.LogLike
import org.apache.toree.utils.json.RddToJson
import org.apache.spark.sql.SchemaRDD
+import org.apache.toree.plugins.annotations.Event
/**
* Temporary magic to show an RDD as JSON
@@ -57,6 +58,7 @@ class RDD extends CellMagic with IncludeKernelInterpreter with LogLike {
}
}
+ @Event(name = "rdd")
override def execute(code: String): CellMagicOutput =
convertToJson(code)
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/magic/builtin/ShowTypes.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/ShowTypes.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/ShowTypes.scala
index 62cfb3a..fb9e4c9 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/ShowTypes.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/ShowTypes.scala
@@ -20,11 +20,13 @@ import org.apache.toree.magic.LineMagic
import org.apache.toree.magic.dependencies.IncludeOutputStream
import java.io.PrintStream
import org.apache.toree.kernel.api.KernelOptions
+import org.apache.toree.plugins.annotations.Event
class ShowTypes extends LineMagic with IncludeOutputStream {
- private lazy val printStream = new PrintStream(outputStream)
+ private def printStream = new PrintStream(outputStream)
+ @Event(name = "showtypes")
override def execute(code: String): Unit = {
code match {
case "on" =>
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/magic/builtin/Truncation.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/Truncation.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/Truncation.scala
index 0443199..1a07827 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/Truncation.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/Truncation.scala
@@ -20,11 +20,13 @@ import org.apache.toree.magic.LineMagic
import org.apache.toree.magic.dependencies.IncludeOutputStream
import java.io.PrintStream
import org.apache.toree.kernel.api.KernelOptions
+import org.apache.toree.plugins.annotations.Event
class Truncation extends LineMagic with IncludeOutputStream {
- private lazy val printStream = new PrintStream(outputStream)
+ private def printStream = new PrintStream(outputStream)
+ @Event(name = "truncation")
override def execute(code: String): Unit = {
code match {
case "on" =>
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala b/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
index 0118cec..f8ae1eb 100644
--- a/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
@@ -19,19 +19,18 @@ package org.apache.toree.kernel.api
import java.io.{InputStream, PrintStream}
+import com.typesafe.config.Config
+import org.apache.spark.{SparkConf, SparkContext}
import org.apache.toree.boot.layer.InterpreterManager
import org.apache.toree.comm.CommManager
+import org.apache.toree.global.ExecuteRequestState
import org.apache.toree.interpreter._
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
-import org.apache.toree.magic.MagicLoader
-import com.typesafe.config.Config
-import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.toree.plugins.PluginManager
import org.mockito.Mockito._
-import org.mockito.Matchers._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
-import org.apache.toree.global.ExecuteRequestState
class KernelSpec extends FunSpec with Matchers with MockitoSugar
with BeforeAndAfter
@@ -50,7 +49,7 @@ class KernelSpec extends FunSpec with Matchers with MockitoSugar
private var mockInterpreter: Interpreter = _
private var mockInterpreterManager: InterpreterManager = _
private var mockCommManager: CommManager = _
- private var mockMagicLoader: MagicLoader = _
+ private var mockPluginManager: PluginManager = _
private var kernel: Kernel = _
private var spyKernel: Kernel = _
@@ -60,6 +59,7 @@ class KernelSpec extends FunSpec with Matchers with MockitoSugar
mockInterpreterManager = mock[InterpreterManager]
mockSparkContext = mock[SparkContext]
mockSparkConf = mock[SparkConf]
+ mockPluginManager = mock[PluginManager]
when(mockInterpreterManager.defaultInterpreter)
.thenReturn(Some(mockInterpreter))
when(mockInterpreterManager.interpreters)
@@ -74,11 +74,10 @@ class KernelSpec extends FunSpec with Matchers with MockitoSugar
mockCommManager = mock[CommManager]
mockActorLoader = mock[ActorLoader]
- mockMagicLoader = mock[MagicLoader]
kernel = new Kernel(
mockConfig, mockActorLoader, mockInterpreterManager, mockCommManager,
- mockMagicLoader
+ mockPluginManager
)
spyKernel = spy(kernel)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParserSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParserSpec.scala b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParserSpec.scala
index 705161e..86d320c 100644
--- a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParserSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParserSpec.scala
@@ -17,7 +17,7 @@
package org.apache.toree.kernel.protocol.v5.magic
-import org.apache.toree.magic.MagicLoader
+import org.apache.toree.magic.{CellMagic, Magic, MagicManager}
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpec, Matchers}
import org.mockito.Mockito._
@@ -32,14 +32,14 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
|foo
|bean
""".stripMargin
- val parser = spy(new MagicParser(mock[MagicLoader]))
+ val parser = spy(new MagicParser(mock[MagicManager]))
parser.parse(codeBlob)
verify(parser).parseCell(codeBlob.trim)
}
it("should call parseLines if the code is not a cell magic") {
val codeBlob = """%magic foo bean"""
- val parser = spy(new MagicParser(mock[MagicLoader]))
+ val parser = spy(new MagicParser(mock[MagicManager]))
parser.parse(codeBlob)
verify(parser).parseLines(codeBlob.trim)
}
@@ -47,8 +47,10 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
describe("#parseCell") {
it("should substitute the magic code for kernel code when magic is valid") {
- val magicLoader = mock[MagicLoader]
- doReturn(true).when(magicLoader).hasCellMagic(anyString())
+ val magicManager = mock[MagicManager]
+ val mockMagic = mock[Magic]
+ doReturn(mockMagic).when(magicManager).findMagic(anyString())
+ doReturn(true).when(magicManager).isCellMagic(mockMagic)
val magicName = "magic"
val args = "foo\nbean\nbar"
@@ -56,7 +58,7 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
s"""%%$magicName
|$args
""".stripMargin
- val parser = spy(new MagicParser(magicLoader))
+ val parser = spy(new MagicParser(magicManager))
val result = parser.parseCell(codeBlob)
verify(parser).substitute(magicName, args)
@@ -64,8 +66,10 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
}
it("should return an error if the magic invocation is invalid") {
- val magicLoader = mock[MagicLoader]
- doReturn(false).when(magicLoader).hasCellMagic(anyString())
+ val magicManager = mock[MagicManager]
+ val mockMagic = mock[Magic]
+ doReturn(mockMagic).when(magicManager).findMagic(anyString())
+ doReturn(false).when(magicManager).isCellMagic(mockMagic)
val magicName = "magic"
val args = "foo\nbean\nbar"
@@ -73,7 +77,7 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
s"""%%$magicName
|$args
""".stripMargin
- val parser = spy(new MagicParser(magicLoader))
+ val parser = spy(new MagicParser(magicManager))
val result = parser.parseCell(codeBlob)
verify(parser, times(0)).substitute(anyString(), anyString())
@@ -81,14 +85,16 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
}
it("should return original code if code contains no magic invocations") {
- val magicLoader = mock[MagicLoader]
- doReturn(false).when(magicLoader).hasCellMagic(anyString())
+ val magicManager = mock[MagicManager]
+ val mockMagic = mock[Magic]
+ doReturn(mockMagic).when(magicManager).findMagic(anyString())
+ doReturn(false).when(magicManager).isCellMagic(mockMagic)
val codeBlob =
s"""val x = 3
|println(x + 2)
""".stripMargin
- val parser = spy(new MagicParser(magicLoader))
+ val parser = spy(new MagicParser(magicManager))
val result = parser.parseCell(codeBlob)
verify(parser, times(0)).substitute(anyString(), anyString())
@@ -100,14 +106,16 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
describe("#parseLines") {
it("should call substituteLine for each line of code " +
"when there are no invalid magic invocations") {
- val magicLoader = mock[MagicLoader]
- doReturn(true).when(magicLoader).hasLineMagic(anyString())
+ val magicManager = mock[MagicManager]
+ val mockMagic = mock[Magic]
+ doReturn(mockMagic).when(magicManager).findMagic(anyString())
+ doReturn(true).when(magicManager).isLineMagic(mockMagic)
val codeBlob =
s"""val x = 3
|%lineMagic
""".stripMargin
- val parser = spy(new MagicParser(magicLoader))
+ val parser = spy(new MagicParser(magicManager))
val result = parser.parseLines(codeBlob)
verify(parser, times(2)).substituteLine(anyString())
@@ -115,14 +123,16 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
}
it("should return an error when there are invalid magic invocations") {
- val magicLoader = mock[MagicLoader]
- doReturn(false).when(magicLoader).hasLineMagic(anyString())
+ val magicManager = mock[MagicManager]
+ val mockMagic = mock[Magic]
+ doReturn(mockMagic).when(magicManager).findMagic(anyString())
+ doReturn(false).when(magicManager).isLineMagic(mockMagic)
val codeBlob =
s"""val x = 3
|%lineMagic
""".stripMargin
- val parser = spy(new MagicParser(magicLoader))
+ val parser = spy(new MagicParser(magicManager))
val result = parser.parseLines(codeBlob)
verify(parser, times(0)).substituteLine(anyString())
@@ -130,14 +140,16 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
}
it("should return original code when there are no magic invocations") {
- val magicLoader = mock[MagicLoader]
- doReturn(false).when(magicLoader).hasLineMagic(anyString())
+ val magicManager = mock[MagicManager]
+ val mockMagic = mock[Magic]
+ doReturn(mockMagic).when(magicManager).findMagic(anyString())
+ doReturn(false).when(magicManager).isLineMagic(mockMagic)
val codeBlob =
s"""val x = 3
|val y = x + 2
""".stripMargin
- val parser = spy(new MagicParser(magicLoader))
+ val parser = spy(new MagicParser(magicManager))
val result = parser.parseLines(codeBlob.trim)
result.isLeft should be(true)
@@ -150,7 +162,7 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
val magicName = "foobar"
val magicArgs = "baz\nbean"
val codeBlob = s"""%%$magicName\n$magicArgs"""
- val parser = new MagicParser(mock[MagicLoader])
+ val parser = new MagicParser(mock[MagicManager])
parser.parseMagic(codeBlob) should be(Some((magicName, magicArgs)))
}
@@ -158,7 +170,7 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
val magicName = "foobar"
val magicArgs = "baz\nbean"
val codeBlob = s"""%$magicName $magicArgs"""
- val parser = new MagicParser(mock[MagicLoader])
+ val parser = new MagicParser(mock[MagicManager])
parser.parseMagic(codeBlob) should be(Some((magicName, magicArgs)))
}
@@ -166,7 +178,7 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
val magicName = "foobar"
val magicArgs = "baz\nbean"
val codeBlob = s"""$magicName\n$magicArgs"""
- val parser = new MagicParser(mock[MagicLoader])
+ val parser = new MagicParser(mock[MagicManager])
parser.parseMagic(codeBlob) should be(None)
}
}
@@ -176,7 +188,7 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
val magicName = "magic"
val args = "-v foo bar"
val codeLine = s"""%$magicName $args"""
- val parser = spy(new MagicParser(mock[MagicLoader]))
+ val parser = spy(new MagicParser(mock[MagicManager]))
doReturn(true).when(parser).isValidLineMagic(anyString())
parser.substituteLine(codeLine)
verify(parser).substitute(magicName, args)
@@ -185,7 +197,7 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
it("should return original line of code when it's not a valid +" +
"magic invocation") {
val codeLine = """val x = 3"""
- val parser = spy(new MagicParser(mock[MagicLoader]))
+ val parser = spy(new MagicParser(mock[MagicManager]))
doReturn(false).when(parser).isValidLineMagic(anyString())
parser.substituteLine(codeLine) should be(codeLine)
}
@@ -196,7 +208,7 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
it("should replace a magic invocation with an equivalent kernel call") {
val magicName = "magic"
val args = "foo bean"
- val parser = new MagicParser(mock[MagicLoader])
+ val parser = new MagicParser(mock[MagicManager])
val equivalent =
s"""${parser.kernelObjectName}.$magicName(\"\"\"$args\"\"\")"""
@@ -213,7 +225,7 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
|%$magicOne bar baz
|%$magicTwo quo bean
""".stripMargin
- val parser = spy(new MagicParser(mock[MagicLoader]))
+ val parser = spy(new MagicParser(mock[MagicManager]))
doReturn(false).when(parser).isValidLineMagic(anyString())
parser.parseOutInvalidMagics(codeBlob) should be(List(magicOne, magicTwo))
@@ -227,7 +239,7 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
|%$magicOne bar baz
|%$magicTwo quo bean
""".stripMargin
- val parser = spy(new MagicParser(mock[MagicLoader]))
+ val parser = spy(new MagicParser(mock[MagicManager]))
doReturn(true).when(parser).isValidLineMagic(anyString())
parser.parseOutInvalidMagics(codeBlob) should be(Nil)
@@ -236,18 +248,22 @@ class MagicParserSpec extends FunSpec with Matchers with MockitoSugar {
describe("#isValidLineMagic") {
it("should return true if the line magic invocation is valid") {
- val magicLoader = mock[MagicLoader]
- doReturn(true).when(magicLoader).hasLineMagic(anyString())
+ val magicManager = mock[MagicManager]
+ val mockMagic = mock[Magic]
+ doReturn(mockMagic).when(magicManager).findMagic(anyString())
+ doReturn(true).when(magicManager).isLineMagic(mockMagic)
- val parser = new MagicParser(magicLoader)
+ val parser = new MagicParser(magicManager)
parser.isValidLineMagic("%foobar baz") should be(true)
}
it("should return false if the line magic invocation is not valid") {
- val magicLoader = mock[MagicLoader]
- doReturn(false).when(magicLoader).hasLineMagic(anyString())
+ val magicManager = mock[MagicManager]
+ val mockMagic = mock[Magic]
+ doReturn(mockMagic).when(magicManager).findMagic(anyString())
+ doReturn(false).when(magicManager).isLineMagic(mockMagic)
- val parser = new MagicParser(magicLoader)
+ val parser = new MagicParser(magicManager)
parser.isValidLineMagic("%foobar baz") should be(false)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelaySpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelaySpec.scala b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelaySpec.scala
index eccc5fc..0661191 100644
--- a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelaySpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelaySpec.scala
@@ -26,9 +26,9 @@ import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content._
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.magic.{MagicParser, PostProcessor}
-import org.apache.toree.magic.MagicLoader
-import org.apache.toree.magic.dependencies.DependencyMap
import com.typesafe.config.ConfigFactory
+import org.apache.toree.plugins.PluginManager
+import org.apache.toree.plugins.dependencies.DependencyManager
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, FunSpecLike, Matchers}
@@ -66,10 +66,10 @@ class ExecuteRequestRelaySpec extends TestKit(
val executeRequest =
ExecuteRequest("%myMagic", false, true, UserExpressions(), true)
- val mockMagicLoader = mock[MagicLoader]
val mockPostProcessor = mock[PostProcessor]
- val mockDependencyMap = mock[DependencyMap]
- doReturn(mockDependencyMap).when(mockMagicLoader).dependencyMap
+ val mockPluginManager = mock[PluginManager]
+ val mockDependencyManager = mock[DependencyManager]
+ doReturn(mockDependencyManager).when(mockPluginManager).dependencyManager
val mockMagicParser = mock[MagicParser]
doReturn(Left(executeRequest.code))
@@ -77,7 +77,7 @@ class ExecuteRequestRelaySpec extends TestKit(
val executeRequestRelay = system.actorOf(Props(
classOf[ExecuteRequestRelay], mockActorLoader,
- mockMagicLoader, mockMagicParser, mockPostProcessor
+ mockPluginManager, mockMagicParser, mockPostProcessor
))
// Send the message to the ExecuteRequestRelay
@@ -103,10 +103,10 @@ class ExecuteRequestRelaySpec extends TestKit(
val executeRequest =
ExecuteRequest("%myMagic", false, true, UserExpressions(), true)
- val mockMagicLoader = mock[MagicLoader]
val mockPostProcessor = mock[PostProcessor]
- val mockDependencyMap = mock[DependencyMap]
- doReturn(mockDependencyMap).when(mockMagicLoader).dependencyMap
+ val mockPluginManager = mock[PluginManager]
+ val mockDependencyManager = mock[DependencyManager]
+ doReturn(mockDependencyManager).when(mockPluginManager).dependencyManager
val mockMagicParser = mock[MagicParser]
doReturn(Left(executeRequest.code))
@@ -114,7 +114,7 @@ class ExecuteRequestRelaySpec extends TestKit(
val executeRequestRelay = system.actorOf(Props(
classOf[ExecuteRequestRelay], mockActorLoader,
- mockMagicLoader, mockMagicParser, mockPostProcessor
+ mockPluginManager, mockMagicParser, mockPostProcessor
))
// Send the message to the ExecuteRequestRelay
@@ -144,13 +144,12 @@ class ExecuteRequestRelaySpec extends TestKit(
val executeRequest =
ExecuteRequest("notAMagic", false, true, UserExpressions(), true)
- val mockMagicLoader = mock[MagicLoader]
val mockPostProcessor = mock[PostProcessor]
doReturn(Data(MIMEType.PlainText -> expected))
.when(mockPostProcessor).process(expected)
-
- val mockDependencyMap = mock[DependencyMap]
- doReturn(mockDependencyMap).when(mockMagicLoader).dependencyMap
+ val mockPluginManager = mock[PluginManager]
+ val mockDependencyManager = mock[DependencyManager]
+ doReturn(mockDependencyManager).when(mockPluginManager).dependencyManager
val mockMagicParser = mock[MagicParser]
doReturn(Left(executeRequest.code))
@@ -158,7 +157,7 @@ class ExecuteRequestRelaySpec extends TestKit(
val executeRequestRelay = system.actorOf(Props(
classOf[ExecuteRequestRelay], mockActorLoader,
- mockMagicLoader, mockMagicParser, mockPostProcessor
+ mockPluginManager, mockMagicParser, mockPostProcessor
))
// Send the message to the ExecuteRequestRelay
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/test/scala/org/apache/toree/magic/builtin/AddJarSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/magic/builtin/AddJarSpec.scala b/kernel/src/test/scala/org/apache/toree/magic/builtin/AddJarSpec.scala
index 255853d..169fb60 100644
--- a/kernel/src/test/scala/org/apache/toree/magic/builtin/AddJarSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/magic/builtin/AddJarSpec.scala
@@ -25,12 +25,11 @@ import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.dependencies.{IncludeConfig, IncludeOutputStream, IncludeInterpreter, IncludeSparkContext}
import com.typesafe.config.ConfigFactory
import org.apache.spark.SparkContext
+import org.apache.toree.plugins.PluginManager
import org.scalatest.{Matchers, FunSpec}
import org.scalatest.mock.MockitoSugar
-
import org.mockito.Mockito._
import org.mockito.Matchers._
-import org.apache.toree.magic.MagicLoader
class AddJarSpec extends FunSpec with Matchers with MockitoSugar {
describe("AddJar"){
@@ -40,7 +39,7 @@ class AddJarSpec extends FunSpec with Matchers with MockitoSugar {
val mockSparkContext = mock[SparkContext]
val mockInterpreter = mock[Interpreter]
val mockOutputStream = mock[OutputStream]
- val mockMagicLoader = mock[MagicLoader]
+ val mockPluginManager = mock[PluginManager]
val testConfig = ConfigFactory.load()
val addJarMagic = new AddJar
@@ -52,7 +51,7 @@ class AddJarSpec extends FunSpec with Matchers with MockitoSugar {
override val sparkContext: SparkContext = mockSparkContext
override val interpreter: Interpreter = mockInterpreter
override val outputStream: OutputStream = mockOutputStream
- override lazy val magicLoader: MagicLoader = mockMagicLoader
+ override lazy val pluginManager: PluginManager = mockPluginManager
override val config = testConfig
override def downloadFile(fileUrl: URL, destinationUrl: URL): URL =
new URL("file://someFile") // Cannot mock URL
@@ -62,7 +61,7 @@ class AddJarSpec extends FunSpec with Matchers with MockitoSugar {
verify(mockSparkContext).addJar(anyString())
verify(mockInterpreter).addJars(any[URL])
- verify(mockMagicLoader, times(0)).addJar(any())
+ verify(mockPluginManager, times(0)).loadPlugins(any())
}
it("should raise exception if jar file does not end in .jar or .zip") {
@@ -191,7 +190,7 @@ class AddJarSpec extends FunSpec with Matchers with MockitoSugar {
val mockSparkContext = mock[SparkContext]
val mockInterpreter = mock[Interpreter]
val mockOutputStream = mock[OutputStream]
- val mockMagicLoader = mock[MagicLoader]
+ val mockPluginManager = mock[PluginManager]
val testConfig = ConfigFactory.load()
val addJarMagic = new AddJar
@@ -203,7 +202,7 @@ class AddJarSpec extends FunSpec with Matchers with MockitoSugar {
override val sparkContext: SparkContext = mockSparkContext
override val interpreter: Interpreter = mockInterpreter
override val outputStream: OutputStream = mockOutputStream
- override lazy val magicLoader: MagicLoader = mockMagicLoader
+ override lazy val pluginManager: PluginManager = mockPluginManager
override val config = testConfig
override def downloadFile(fileUrl: URL, destinationUrl: URL): URL =
new URL("file://someFile") // Cannot mock URL
@@ -212,7 +211,7 @@ class AddJarSpec extends FunSpec with Matchers with MockitoSugar {
addJarMagic.execute(
"""--magic http://www.example.com/someJar.jar""")
- verify(mockMagicLoader).addJar(any())
+ verify(mockPluginManager).loadPlugins(any())
verify(mockSparkContext, times(0)).addJar(anyString())
verify(mockInterpreter, times(0)).addJars(any[URL])
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/test/scala/system/KernelCommSpecForSystem.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/system/KernelCommSpecForSystem.scala b/kernel/src/test/scala/system/KernelCommSpecForSystem.scala
index 6cc50f5..9ebe146 100644
--- a/kernel/src/test/scala/system/KernelCommSpecForSystem.scala
+++ b/kernel/src/test/scala/system/KernelCommSpecForSystem.scala
@@ -17,17 +17,18 @@
package system
-import akka.testkit.{TestProbe}
+import akka.testkit.TestProbe
import org.apache.toree.communication.ZMQMessage
-import org.apache.toree.kernel.protocol.v5.kernel.{ActorLoader, Utilities}
-import Utilities._
import org.apache.toree.kernel.protocol.v5
-import org.apache.toree.kernel.protocol.v5.{KernelMessage, SocketType, KMBuilder}
import org.apache.toree.kernel.protocol.v5.content._
+import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
+import org.apache.toree.kernel.protocol.v5.kernel.Utilities._
+import org.apache.toree.kernel.protocol.v5.{KMBuilder, KernelMessage, SocketType}
import org.scalatest._
import play.api.libs.json.Json
+import test.utils.NoArgSparkKernelTestKit
+
import scala.concurrent.duration._
-import test.utils.{NoArgSparkKernelTestKit}
/**
@@ -44,7 +45,7 @@ class KernelCommSpecForSystem
{
private val MaxFishTime = 5.seconds
- import org.apache.toree.boot.layer.SparkKernelDeployer._
+ import test.utils.SparkKernelDeployer._
private def waitForExecuteReply(
shell: TestProbe, headerId: v5.UUID, maxTime: Duration = MaxFishTime
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/test/scala/system/SuiteForSystem.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/system/SuiteForSystem.scala b/kernel/src/test/scala/system/SuiteForSystem.scala
index 19f256a..4556e5a 100644
--- a/kernel/src/test/scala/system/SuiteForSystem.scala
+++ b/kernel/src/test/scala/system/SuiteForSystem.scala
@@ -17,8 +17,8 @@
package system
-import org.apache.toree.boot.layer.SparkKernelDeployer
import org.scalatest.{BeforeAndAfterAll, Suites}
+import test.utils.SparkKernelDeployer
class SuiteForSystem extends Suites(
new KernelCommSpecForSystem,
@@ -29,4 +29,10 @@ class SuiteForSystem extends Suites(
println("Initializing kernel for system tests")
SparkKernelDeployer.noArgKernelBootstrap
}
+
+ override protected def afterAll(): Unit = {
+ println("Shutting down kernel for system tests")
+ SparkKernelDeployer.noArgKernelBootstrap.shutdown()
+ println("Finished shutting down kernel!")
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/test/scala/system/TruncationTests.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/system/TruncationTests.scala b/kernel/src/test/scala/system/TruncationTests.scala
index 2fe107f..118a609 100644
--- a/kernel/src/test/scala/system/TruncationTests.scala
+++ b/kernel/src/test/scala/system/TruncationTests.scala
@@ -28,7 +28,6 @@ import org.scalatest._
import play.api.libs.json.Json
import scala.concurrent.duration._
import test.utils.NoArgSparkKernelTestKit
-import org.apache.toree.boot.layer.SparkKernelDeployer
/**
@@ -44,8 +43,7 @@ class TruncationTests
with FunSpecLike with Matchers
{
private val MaxFishTime = 30.seconds
-
- import SparkKernelDeployer._
+ import test.utils.SparkKernelDeployer._
private def waitForExecuteReply(
shell: TestProbe, headerId: v5.UUID, maxTime: Duration = MaxFishTime
@@ -194,10 +192,7 @@ class TruncationTests
str
}
-
-
-
- describe("Test Truncation") {
+ describe( "Test Truncation") {
it("should show or not show types based on %showtypes") {
withNoArgSparkKernel { (actorLoader, heartbeat, shell, ioPub) =>
@@ -216,7 +211,6 @@ class TruncationTests
}
it("should truncate or not truncate based on %truncate") {
withNoArgSparkKernel { (actorLoader, heartbeat, shell, ioPub) =>
-
executeCode("for ( a <- 1 to 300 ) yield a",actorLoader,ioPub) should endWith("...")
executeCode(
@@ -224,18 +218,11 @@ class TruncationTests
|for ( a <- 1 to 300 ) yield a
""".stripMargin,actorLoader,ioPub) should endWith("300)")
-
-
executeCode(
"""%Truncation on
|for ( a <- 1 to 300 ) yield a
""".stripMargin,actorLoader,ioPub) should endWith("...")
-
-
-
}
}
- }
-
-
}
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/test/scala/test/utils/NoArgSparkKernelTestKit.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/test/utils/NoArgSparkKernelTestKit.scala b/kernel/src/test/scala/test/utils/NoArgSparkKernelTestKit.scala
index 3052cfe..3a06a21 100644
--- a/kernel/src/test/scala/test/utils/NoArgSparkKernelTestKit.scala
+++ b/kernel/src/test/scala/test/utils/NoArgSparkKernelTestKit.scala
@@ -18,7 +18,6 @@
package test.utils
import akka.testkit.TestKit
-import org.apache.toree.boot.layer.SparkKernelDeployer
class NoArgSparkKernelTestKit
extends TestKit(SparkKernelDeployer.getNoArgSparkKernelActorSystem)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/test/scala/test/utils/SparkKernelDeployer.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/test/utils/SparkKernelDeployer.scala b/kernel/src/test/scala/test/utils/SparkKernelDeployer.scala
index b157eaa..62548c7 100644
--- a/kernel/src/test/scala/test/utils/SparkKernelDeployer.scala
+++ b/kernel/src/test/scala/test/utils/SparkKernelDeployer.scala
@@ -15,27 +15,19 @@
* limitations under the License
*/
-package org.apache.toree.boot.layer
+package test.utils
-import java.io.OutputStream
-
-import akka.actor.{Actor, Props, ActorRef, ActorSystem}
+import akka.actor.{Actor, ActorRef, ActorSystem, Props}
import akka.testkit.TestProbe
+import com.typesafe.config.Config
+import org.apache.toree.boot.layer.{StandardBareInitialization, StandardComponentInitialization, StandardHandlerInitialization, StandardHookInitialization}
import org.apache.toree.boot.{CommandLineOptions, KernelBootstrap}
-import org.apache.toree.kernel.api.KernelLike
-import org.apache.toree.kernel.interpreter.scala.{StandardTaskManagerProducer, StandardSparkIMainProducer, StandardSettingsProducer, ScalaInterpreter}
-import org.apache.toree.kernel.protocol.v5.{KMBuilder, SocketType}
+import org.apache.toree.kernel.protocol.v5.SocketType
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.kernel.socket._
import org.apache.toree.utils.LogLike
-import com.typesafe.config.Config
import org.scalatest.mock.MockitoSugar
import play.api.libs.json.Json
-import scala.collection.JavaConverters._
-import test.utils.SparkContextProvider
-import org.apache.spark.{SparkContext, SparkConf}
-import org.apache.toree.kernel.protocol.v5.stream.KernelOutputStream
-import org.apache.toree.global
/**
* Represents an object that can deploy a singleton Spark Kernel for tests,
@@ -172,7 +164,6 @@ object SparkKernelDeployer extends LogLike with MockitoSugar {
* KernelBootstrap through an actor loader and socket test probes.
*
* @param testCode The test code to execute
- *
* @return The results from the test code
*/
def withNoArgSparkKernel(
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/plugins/src/main/scala/org/apache/toree/plugins/Plugin.scala
----------------------------------------------------------------------
diff --git a/plugins/src/main/scala/org/apache/toree/plugins/Plugin.scala b/plugins/src/main/scala/org/apache/toree/plugins/Plugin.scala
index 66cd4a8..42f0eb8 100644
--- a/plugins/src/main/scala/org/apache/toree/plugins/Plugin.scala
+++ b/plugins/src/main/scala/org/apache/toree/plugins/Plugin.scala
@@ -36,11 +36,17 @@ object Plugin {
*/
trait Plugin {
/** Plugin manager containing the plugin */
- @Internal private var _pluginManager: PluginManager = null
+ @Internal private var _internalPluginManager: PluginManager = null
- /** Represents the name of the plugin. */
+ /** Represents the fully qualified name of the plugin. */
final val name: String = getClass.getName
+ /**
+ * Represents the simple name of the plugin. In the case of a anonymous class
+ * it will just be the name of the anonymous class.
+ */
+ final val simpleName: String = getClass.getSimpleName
+
/** Represents the priority of the plugin. */
final val priority: Long = {
Option(getClass.getAnnotation(classOf[annotations.Priority]))
@@ -49,13 +55,19 @@ trait Plugin {
}
/** Sets the plugin manager pointer for this plugin. */
- @Internal private[plugins] final def pluginManager_=(_pluginManager: PluginManager) = {
- require(this._pluginManager == null, "Plugin manager cannot be reassigned!")
- this._pluginManager = _pluginManager
+ @Internal private[plugins] final def internalPluginManager_=(
+ _pluginManager: PluginManager
+ ) = {
+ require(
+ this._internalPluginManager == null,
+ "Plugin manager cannot be reassigned!"
+ )
+ this._internalPluginManager = _pluginManager
}
/** Returns the plugin manager pointer for this plugin. */
- @Internal private[plugins] final def pluginManager = _pluginManager
+ @Internal private[plugins] final def internalPluginManager =
+ _internalPluginManager
/** Represents all @init methods in the plugin. */
@Internal private[plugins] final lazy val initMethods: Seq[PluginMethod] = {
@@ -109,7 +121,7 @@ trait Plugin {
* @tparam T The dependency's type
*/
protected def register[T <: AnyRef](name: String, value: T)(implicit typeTag: TypeTag[T]): Unit = {
- assert(_pluginManager != null, "Internal plugin manager reference invalid!")
- _pluginManager.dependencyManager.add(name, value)
+ assert(_internalPluginManager != null, "Internal plugin manager reference invalid!")
+ _internalPluginManager.dependencyManager.add(name, value)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/plugins/src/main/scala/org/apache/toree/plugins/PluginManager.scala
----------------------------------------------------------------------
diff --git a/plugins/src/main/scala/org/apache/toree/plugins/PluginManager.scala b/plugins/src/main/scala/org/apache/toree/plugins/PluginManager.scala
index 4b7338f..4b611a1 100644
--- a/plugins/src/main/scala/org/apache/toree/plugins/PluginManager.scala
+++ b/plugins/src/main/scala/org/apache/toree/plugins/PluginManager.scala
@@ -91,6 +91,8 @@ class PluginManager(
* @return The collection of loaded plugins
*/
def loadPlugins(paths: File*): Seq[Plugin] = {
+ require(paths.nonEmpty, "Plugin paths cannot be empty!")
+
// Search for plugins in our new paths, then add loaded plugins to list
// NOTE: Iterator returned from plugin searcher, so avoid building a
// large collection by performing all tasks together
@@ -136,7 +138,7 @@ class PluginManager(
// Attempt to cast as plugin type to add to active plugins
tryInstance.transform({
case p: Plugin =>
- p.pluginManager_=(this)
+ p.internalPluginManager_=(this)
activePlugins.put(p.name, p)
Success(p)
case x =>
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/plugins/src/main/scala/org/apache/toree/plugins/PluginMethod.scala
----------------------------------------------------------------------
diff --git a/plugins/src/main/scala/org/apache/toree/plugins/PluginMethod.scala b/plugins/src/main/scala/org/apache/toree/plugins/PluginMethod.scala
index cb87b65..50aafe3 100644
--- a/plugins/src/main/scala/org/apache/toree/plugins/PluginMethod.scala
+++ b/plugins/src/main/scala/org/apache/toree/plugins/PluginMethod.scala
@@ -81,9 +81,6 @@ case class PluginMethod(
* (as needed)
* @return The result from invoking the plugin
*/
- @throws[DepNameNotFoundException]
- @throws[DepClassNotFoundException]
- @throws[DepUnexpectedClassException]
def invoke(dependencyManager: DependencyManager): PluginMethodResult = Try({
// Get dependency info (if has specific name or just use class)
val depInfo = method.getParameterAnnotations
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala
----------------------------------------------------------------------
diff --git a/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala b/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala
index e6c9141..4686a50 100644
--- a/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala
+++ b/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala
@@ -48,7 +48,7 @@ class PluginSearcher {
*
* @return The new class finder
*/
- protected def newClassFinder(): ClassFinder = ClassFinder()
+ protected def newClassFinder(): ClassFinder = ClassFinder(classpath)
/**
* Creates a new class finder for the given paths.
@@ -83,24 +83,28 @@ class PluginSearcher {
classes: Map[String, ClassInfo]
): Iterator[ClassInfo] = {
@tailrec def classMatches(
- ancestorClassInfo: ClassInfo,
classesToCheck: Seq[ClassInfo]
): Boolean = {
if (classesToCheck.isEmpty) false
- else if (classesToCheck.exists(_.name == ancestorClassInfo.name)) true
- else if (classesToCheck.exists(_.superClassName == ancestorClassInfo.name)) true
- else if (classesToCheck.exists(_ implements ancestorClassInfo.name)) true
+ else if (classesToCheck.exists(_.name == ancestor)) true
+ else if (classesToCheck.exists(_.superClassName == ancestor)) true
+ else if (classesToCheck.exists(_ implements ancestor)) true
else {
val superClasses = classesToCheck.map(_.superClassName).flatMap(classes.get)
val interfaces = classesToCheck.flatMap(_.interfaces).flatMap(classes.get)
- classMatches(ancestorClassInfo, superClasses ++ interfaces)
+ classMatches(superClasses ++ interfaces)
}
}
- classes.get(ancestor).map(ci => {
- classes.values.toIterator
- .filter(_.isConcrete)
- .filter(c => classMatches(ci, Seq(c)))
- }).getOrElse(Iterator.empty)
+ classes.values.toIterator
+ .filter(_.isConcrete)
+ .filter(c => classMatches(Seq(c)))
}
+
+ private def classpath = System.getProperty("java.class.path")
+ .split(File.pathSeparator)
+ .map(s => if (s.trim.length == 0) "." else s)
+ .map(new File(_))
+ .filter(_.getAbsolutePath.toLowerCase.contains("toree"))
+ .toList
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/plugins/src/test/scala/integration/PluginManagerSpecForIntegration.scala
----------------------------------------------------------------------
diff --git a/plugins/src/test/scala/integration/PluginManagerSpecForIntegration.scala b/plugins/src/test/scala/integration/PluginManagerSpecForIntegration.scala
index b2e9d00..c97e61a 100644
--- a/plugins/src/test/scala/integration/PluginManagerSpecForIntegration.scala
+++ b/plugins/src/test/scala/integration/PluginManagerSpecForIntegration.scala
@@ -26,6 +26,15 @@ class PluginManagerSpecForIntegration extends FunSpec with Matchers
private val pluginManager = new PluginManager
describe("PluginManager") {
+ it("should be able to load and initialize internal plugins") {
+ val plugins = pluginManager.initialize()
+ plugins.map(_.name) should contain allOf (
+ classOf[NonCircularPlugin].getName,
+ classOf[RegisterPluginA].getName,
+ classOf[ConsumePluginA].getName
+ )
+ }
+
it("should be able to initialize plugins with dependencies provided by other plugins") {
val cpa = pluginManager.loadPlugin("", classOf[ConsumePluginA]).get
val rpa = pluginManager.loadPlugin("", classOf[RegisterPluginA]).get
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/plugins/src/test/scala/org/apache/toree/plugins/PluginManagerSpec.scala
----------------------------------------------------------------------
diff --git a/plugins/src/test/scala/org/apache/toree/plugins/PluginManagerSpec.scala b/plugins/src/test/scala/org/apache/toree/plugins/PluginManagerSpec.scala
index 09e0f10..86097c8 100644
--- a/plugins/src/test/scala/org/apache/toree/plugins/PluginManagerSpec.scala
+++ b/plugins/src/test/scala/org/apache/toree/plugins/PluginManagerSpec.scala
@@ -116,7 +116,7 @@ class PluginManagerSpec extends FunSpec with Matchers
it("should set the internal plugin manager of the new plugin") {
val p = pluginManager.loadPlugin("name", classOf[TestPlugin])
- p.get.pluginManager should be (pluginManager)
+ p.get.internalPluginManager should be (pluginManager)
}
it("should add the new plugin to the list of active plugins") {
@@ -146,6 +146,12 @@ class PluginManagerSpec extends FunSpec with Matchers
}
describe("#loadPlugins") {
+ it("should throw an IllegalArgumentException if provided no paths") {
+ intercept[IllegalArgumentException] {
+ pluginManager.loadPlugins()
+ }
+ }
+
it("should load nothing if the plugin searcher returns empty handed") {
val expected = Nil
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/plugins/src/test/scala/org/apache/toree/plugins/PluginSpec.scala
----------------------------------------------------------------------
diff --git a/plugins/src/test/scala/org/apache/toree/plugins/PluginSpec.scala b/plugins/src/test/scala/org/apache/toree/plugins/PluginSpec.scala
index fd5e1f0..5ea58a7 100644
--- a/plugins/src/test/scala/org/apache/toree/plugins/PluginSpec.scala
+++ b/plugins/src/test/scala/org/apache/toree/plugins/PluginSpec.scala
@@ -30,12 +30,12 @@ class PluginSpec extends FunSpec with Matchers with OneInstancePerTest with Mock
private val mockPluginManager = mock[PluginManager]
private val testPlugin = {
val plugin = new TestPlugin
- plugin.pluginManager_=(mockPluginManager)
+ plugin.internalPluginManager_=(mockPluginManager)
plugin
}
private val extendedTestPlugin = {
val extendedPlugin = new ExtendedTestPlugin
- extendedPlugin.pluginManager_=(mockPluginManager)
+ extendedPlugin.internalPluginManager_=(mockPluginManager)
extendedPlugin
}
private val registerPlugin = new RegisterPlugin
@@ -53,6 +53,26 @@ class PluginSpec extends FunSpec with Matchers with OneInstancePerTest with Mock
}
}
+ describe("#simpleName") {
+ it("should be the simple name of the class implementing the plugin") {
+ val expected = classOf[TestPlugin].getSimpleName
+
+ val actual = testPlugin.simpleName
+
+ actual should be (expected)
+ }
+
+ it("should be the simple name of an anonymous class implementing the plugin") {
+ val anonymousPlugin = new TestPlugin {}
+ val anonymousClass = anonymousPlugin.getClass
+ val expected = anonymousClass.getSimpleName
+
+ val actual = anonymousPlugin.simpleName
+
+ actual should be(expected)
+ }
+ }
+
describe("#priority") {
it("should return the priority set by the plugin's annotation") {
val expected = 999
@@ -236,7 +256,7 @@ class PluginSpec extends FunSpec with Matchers with OneInstancePerTest with Mock
}
it("should create a new name for the dependency if not specified") {
- registerPlugin.pluginManager_=(mockPluginManager)
+ registerPlugin.internalPluginManager_=(mockPluginManager)
val value = new AnyRef
val mockDependencyManager = mock[DependencyManager]
@@ -247,7 +267,7 @@ class PluginSpec extends FunSpec with Matchers with OneInstancePerTest with Mock
}
it("should add the dependency using the provided name") {
- registerPlugin.pluginManager_=(mockPluginManager)
+ registerPlugin.internalPluginManager_=(mockPluginManager)
val name = "some name"
val value = new AnyRef
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/project/Build.scala
----------------------------------------------------------------------
diff --git a/project/Build.scala b/project/Build.scala
index baf8baf..8984570 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -100,7 +100,8 @@ trait SubProjects extends Settings with TestTasks {
settings = fullSettings ++ Seq(
test in assembly := {}
)
- )) dependsOn(
+ // Enable forking to load correct classes with plugin loader during tests
+ ), doFork = true) dependsOn(
macros % "test->test;compile->compile",
protocol % "test->test;compile->compile",
communication % "test->test;compile->compile",
@@ -227,7 +228,8 @@ trait SubProjects extends Settings with TestTasks {
id = "toree-plugins",
base = file("plugins"),
settings = fullSettings
- )) dependsOn(
+ // Enable forking to load correct classes with plugin loader during tests
+ ), doFork = true) dependsOn(
macros % "test->test;compile->compile"
)
@@ -251,7 +253,7 @@ trait SubProjects extends Settings with TestTasks {
* scratch:test - runs temporary tests
*/
trait TestTasks {
- def addTestTasksToProject(project: Project): Project =
+ def addTestTasksToProject(project: Project, doFork: Boolean = false): Project =
project
.configs( UnitTest )
.configs( IntegrationTest )
@@ -266,6 +268,12 @@ trait TestTasks {
testOptions in IntegrationTest := Seq(Tests.Filter(intFilter)),
testOptions in SystemTest := Seq(Tests.Filter(sysFilter)),
testOptions in ScratchTest := Seq(Tests.Filter(scratchFilter))
+ ).settings(
+ fork in Test := doFork,
+ fork in UnitTest := doFork,
+ fork in IntegrationTest := doFork,
+ fork in SystemTest := doFork,
+ fork in ScratchTest := doFork
)
def scratchFilter(name: String): Boolean =
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/project/Common.scala
----------------------------------------------------------------------
diff --git a/project/Common.scala b/project/Common.scala
index dd1c8aa..872db2e 100644
--- a/project/Common.scala
+++ b/project/Common.scala
@@ -115,7 +115,8 @@ object Common {
//"-Xlint", // Scala 2.11.x only
"-Xfatal-warnings",
"-Ywarn-all",
- "-language:reflectiveCalls"
+ "-language:reflectiveCalls",
+ "-target:jvm-1.6"
),
// Java-based options for compilation (all tasks)
@@ -126,7 +127,19 @@ object Common {
javacOptions in (Compile, compile) ++= Seq(
"-Xlint:all", // Enable all Java-based warnings
"-Xlint:-path", // Suppress path warnings since we get tons of them
- "-Werror" // Treat warnings as errors
+ "-Xlint:-options",
+ "-Xlint:-processing",
+ "-Werror", // Treat warnings as errors
+ "-source", "1.6",
+ "-target", "1.6"
+ ),
+
+ // Options provided to forked JVMs through sbt, based on our .jvmopts file
+ javaOptions ++= Seq(
+ "-Xms1024M", "-Xmx4096M", "-Xss2m", "-XX:MaxPermSize=1024M",
+ "-XX:ReservedCodeCacheSize=256M", "-XX:+TieredCompilation",
+ "-XX:+CMSPermGenSweepingEnabled", "-XX:+CMSClassUnloadingEnabled",
+ "-XX:+UseConcMarkSweepGC", "-XX:+HeapDumpOnOutOfMemoryError"
),
// Add additional test option to show time taken per test
@@ -143,20 +156,10 @@ object Common {
credentials += Credentials("Sonatype Nexus Repository Manager", repoHost, repoUsername, repoPassword),
- // Change destination of local delivery (building ivy.xml) to have *-ivy.xml
- deliverLocalConfiguration := {
- val newDestinationPath = crossTarget.value / s"${name.value}-ivy.xml"
- val dlc = deliverLocalConfiguration.value
- new DeliverConfiguration(
- newDestinationPath.absolutePath, dlc.status,
- dlc.configurations, dlc.logging)
- },
-
// Add rebuild ivy xml to the following tasks
compile <<= (compile in Compile) dependsOn (rebuildIvyXml dependsOn deliverLocal)
) ++ rebuildIvyXmlSettings // Include our rebuild ivy xml settings
-
buildLibraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % sparkVersion % "provided" excludeAll( // Apache v2
// Exclude netty (org.jboss.netty is for 3.2.2.Final only)
@@ -187,7 +190,7 @@ object Common {
lazy val rebuildIvyXmlSettings = Seq(
rebuildIvyXml := {
val s: TaskStreams = streams.value
- val inputFile = (crossTarget.value / s"${name.value}-ivy.xml").getAbsoluteFile
+ val inputFile = (crossTarget.value / s"ivy-${version.value}.xml").getAbsoluteFile
val outputFile =
((resourceDirectory in Compile).value / s"${name.value}-ivy.xml").getAbsoluteFile
s.log.info(s"Copying ${inputFile.getPath} to ${outputFile.getPath}")
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index f0ef85b..081776e 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -50,4 +50,4 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "0.8.1")
addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.3")
// Provides alternative resolving/downloading over sbt
-addSbtPlugin("com.github.alexarchambault" % "coursier-sbt-plugin" % "1.0.0-M9")
+addSbtPlugin("com.github.alexarchambault" % "coursier-sbt-plugin" % "1.0.0-M10")
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/pyspark-interpreter/src/main/scala/org/apache/toree/magic/builtin/PySpark.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/magic/builtin/PySpark.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/magic/builtin/PySpark.scala
index 7510656..015320d 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/magic/builtin/PySpark.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/magic/builtin/PySpark.scala
@@ -21,11 +21,13 @@ import org.apache.toree.kernel.interpreter.pyspark.{PySparkInterpreter, PySparkE
import org.apache.toree.kernel.protocol.v5.MIMEType
import org.apache.toree.magic.{CellMagicOutput, CellMagic}
import org.apache.toree.magic.dependencies.IncludeKernel
+import org.apache.toree.plugins.annotations.Event
/**
* Represents the magic interface to use the PySpark interpreter.
*/
class PySpark extends CellMagic with IncludeKernel {
+ @Event(name = "pyspark")
override def execute(code: String): CellMagicOutput = {
val pySpark = kernel.interpreter("PySpark")
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/resources/compile/log4j.properties
----------------------------------------------------------------------
diff --git a/resources/compile/log4j.properties b/resources/compile/log4j.properties
index c760493..82fcf92 100644
--- a/resources/compile/log4j.properties
+++ b/resources/compile/log4j.properties
@@ -26,6 +26,7 @@ log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%p] %c{1.
# Override log levels of external packages that clutter our output
log4j.logger.org.eclipse.jetty=WARN
log4j.logger.org.apache=WARN
+log4j.logger.org.apache.toree=INFO
log4j.logger.org.spark-project=WARN
log4j.logger.io.netty=WARN
log4j.logger.akka=WARN
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/scala-interpreter/src/main/scala/org/apache/toree/magic/builtin/Scala.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/main/scala/org/apache/toree/magic/builtin/Scala.scala b/scala-interpreter/src/main/scala/org/apache/toree/magic/builtin/Scala.scala
index 7d18405..ceeda00 100644
--- a/scala-interpreter/src/main/scala/org/apache/toree/magic/builtin/Scala.scala
+++ b/scala-interpreter/src/main/scala/org/apache/toree/magic/builtin/Scala.scala
@@ -21,11 +21,13 @@ import org.apache.toree.kernel.interpreter.scala.{ScalaException, ScalaInterpret
import org.apache.toree.kernel.protocol.v5.MIMEType
import org.apache.toree.magic.dependencies.IncludeKernel
import org.apache.toree.magic.{CellMagic, CellMagicOutput}
-
+import org.apache.toree.plugins.annotations.Event
/**
* Represents the magic interface to use the Scala interpreter.
*/
class Scala extends CellMagic with IncludeKernel {
+
+ @Event(name = "scala")
override def execute(code: String): CellMagicOutput = {
val scala = kernel.interpreter("Scala")
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/sparkr-interpreter/src/main/scala/org/apache/toree/magic/builtin/SparkR.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/magic/builtin/SparkR.scala b/sparkr-interpreter/src/main/scala/org/apache/toree/magic/builtin/SparkR.scala
index 9f12973..facf3ac 100644
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/magic/builtin/SparkR.scala
+++ b/sparkr-interpreter/src/main/scala/org/apache/toree/magic/builtin/SparkR.scala
@@ -21,11 +21,14 @@ import org.apache.toree.kernel.interpreter.sparkr.{SparkRInterpreter, SparkRExce
import org.apache.toree.kernel.protocol.v5.MIMEType
import org.apache.toree.magic.{CellMagicOutput, CellMagic}
import org.apache.toree.magic.dependencies.IncludeKernel
+import org.apache.toree.plugins.annotations.Event
/**
* Represents the magic interface to use the SparkR interpreter.
*/
class SparkR extends CellMagic with IncludeKernel {
+
+ @Event(name = "sparkr")
override def execute(code: String): CellMagicOutput = {
val sparkR = kernel.interpreter("SparkR")
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/sql-interpreter/src/main/scala/org/apache/toree/magic/builtin/Sql.scala
----------------------------------------------------------------------
diff --git a/sql-interpreter/src/main/scala/org/apache/toree/magic/builtin/Sql.scala b/sql-interpreter/src/main/scala/org/apache/toree/magic/builtin/Sql.scala
index 955bdd5..4144cf4 100644
--- a/sql-interpreter/src/main/scala/org/apache/toree/magic/builtin/Sql.scala
+++ b/sql-interpreter/src/main/scala/org/apache/toree/magic/builtin/Sql.scala
@@ -21,11 +21,14 @@ import org.apache.toree.kernel.interpreter.sql.{SqlInterpreter, SqlException}
import org.apache.toree.kernel.protocol.v5.MIMEType
import org.apache.toree.magic.{CellMagicOutput, CellMagic}
import org.apache.toree.magic.dependencies.IncludeKernel
+import org.apache.toree.plugins.annotations.Event
/**
* Represents the magic interface to use the SQL interpreter.
*/
class Sql extends CellMagic with IncludeKernel {
+
+ @Event(name = "sql")
override def execute(code: String): CellMagicOutput = {
val sparkR = kernel.interpreter("SQL")
[2/2] incubator-toree git commit: Converted magics to plugins
Posted by lb...@apache.org.
Converted magics to plugins
Fixed plugin test classpath to run script
Updated internal plugin loading to use only Toree-related classes
Updated compilation to use Java 1.6 to avoid issues on JDK 8
Fixed kernel hanging on shutdown
Added JVM forking options for sbt
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/9c8824ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/9c8824ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/9c8824ff
Branch: refs/heads/master
Commit: 9c8824fff063ff6e84b2e4298088954ae610612b
Parents: 429c74c
Author: Chip Senkbeil and Corey Stubbs <chip.senkbeil@gmail.com & cas5542@gmail.com>
Authored: Fri Mar 25 15:03:22 2016 -0500
Committer: Chip Senkbeil and Corey Stubbs <chip.senkbeil@gmail.com & cas5542@gmail.com>
Committed: Tue Mar 29 12:20:04 2016 -0500
----------------------------------------------------------------------
.jvmopts | 11 +
Makefile | 1 +
etc/bin/run.sh | 5 +-
.../toree/interpreter/broker/BrokerState.scala | 4 +-
.../scala/org/apache/toree/magic/Magic.scala | 24 +-
.../org/apache/toree/magic/MagicExecutor.scala | 51 ---
.../org/apache/toree/magic/MagicLoader.scala | 138 --------
.../org/apache/toree/magic/MagicManager.scala | 111 +++++++
.../toree/magic/MagicNotFoundException.scala | 22 ++
.../magic/dependencies/DependencyMap.scala | 171 ----------
.../magic/dependencies/IncludeConfig.scala | 8 +-
.../IncludeDependencyDownloader.scala | 9 +-
.../magic/dependencies/IncludeInterpreter.scala | 10 +-
.../magic/dependencies/IncludeKernel.scala | 8 +-
.../dependencies/IncludeKernelInterpreter.scala | 11 +-
.../magic/dependencies/IncludeMagicLoader.scala | 31 --
.../dependencies/IncludeOutputStream.scala | 12 +-
.../dependencies/IncludePluginManager.scala | 30 ++
.../magic/dependencies/IncludeSQLContext.scala | 12 +-
.../dependencies/IncludeSparkContext.scala | 13 +-
.../apache/toree/magic/MagicLoaderSpec.scala | 184 -----------
.../apache/toree/magic/MagicManagerSpec.scala | 324 +++++++++++++++++++
.../src/test/scala/test/utils/SomeMagic.scala | 25 ++
.../org/apache/toree/boot/KernelBootstrap.scala | 5 +-
.../boot/layer/ComponentInitialization.scala | 89 ++---
.../boot/layer/HandlerInitialization.scala | 28 +-
.../toree/boot/layer/HookInitialization.scala | 4 -
.../org/apache/toree/kernel/api/Kernel.scala | 44 +--
.../kernel/protocol/v5/magic/MagicParser.scala | 23 +-
.../protocol/v5/relay/ExecuteRequestRelay.scala | 12 +-
.../apache/toree/magic/builtin/AddDeps.scala | 5 +-
.../org/apache/toree/magic/builtin/AddJar.scala | 26 +-
.../org/apache/toree/magic/builtin/Html.scala | 6 +-
.../apache/toree/magic/builtin/JavaScript.scala | 6 +-
.../apache/toree/magic/builtin/LSMagic.scala | 4 +-
.../org/apache/toree/magic/builtin/RDD.scala | 2 +
.../apache/toree/magic/builtin/ShowTypes.scala | 4 +-
.../apache/toree/magic/builtin/Truncation.scala | 4 +-
.../apache/toree/kernel/api/KernelSpec.scala | 15 +-
.../protocol/v5/magic/MagicParserSpec.scala | 86 +++--
.../v5/relay/ExecuteRequestRelaySpec.scala | 29 +-
.../apache/toree/magic/builtin/AddJarSpec.scala | 15 +-
.../scala/system/KernelCommSpecForSystem.scala | 13 +-
.../src/test/scala/system/SuiteForSystem.scala | 8 +-
.../src/test/scala/system/TruncationTests.scala | 19 +-
.../test/utils/NoArgSparkKernelTestKit.scala | 1 -
.../scala/test/utils/SparkKernelDeployer.scala | 19 +-
.../scala/org/apache/toree/plugins/Plugin.scala | 28 +-
.../apache/toree/plugins/PluginManager.scala | 4 +-
.../org/apache/toree/plugins/PluginMethod.scala | 3 -
.../apache/toree/plugins/PluginSearcher.scala | 26 +-
.../PluginManagerSpecForIntegration.scala | 9 +
.../toree/plugins/PluginManagerSpec.scala | 8 +-
.../org/apache/toree/plugins/PluginSpec.scala | 28 +-
project/Build.scala | 14 +-
project/Common.scala | 29 +-
project/plugins.sbt | 2 +-
.../apache/toree/magic/builtin/PySpark.scala | 2 +
resources/compile/log4j.properties | 1 +
.../org/apache/toree/magic/builtin/Scala.scala | 4 +-
.../org/apache/toree/magic/builtin/SparkR.scala | 3 +
.../org/apache/toree/magic/builtin/Sql.scala | 3 +
62 files changed, 964 insertions(+), 882 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/.jvmopts
----------------------------------------------------------------------
diff --git a/.jvmopts b/.jvmopts
new file mode 100644
index 0000000..f22a607
--- /dev/null
+++ b/.jvmopts
@@ -0,0 +1,11 @@
+-Xms1024M
+-Xmx4096M
+-Xss2m
+-XX:MaxPermSize=1024M
+-XX:ReservedCodeCacheSize=256M
+-XX:+TieredCompilation
+-XX:+CMSPermGenSweepingEnabled
+-XX:+CMSClassUnloadingEnabled
+-XX:+UseConcMarkSweepGC
+-XX:+HeapDumpOnOutOfMemoryError
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/Makefile
----------------------------------------------------------------------
diff --git a/Makefile b/Makefile
index 530b41c..cdf525a 100644
--- a/Makefile
+++ b/Makefile
@@ -72,6 +72,7 @@ clean-dist:
clean: VM_WORKDIR=/src/toree-kernel
clean: clean-dist
$(call RUN,$(ENV_OPTS) sbt clean)
+ rm -r `find . -name target -type d`
.example-image: EXTRA_CMD?=pip install jupyter_declarativewidgets==0.4.0; jupyter declarativewidgets install --user; jupyter declarativewidgets activate; pip install jupyter_dashboards; jupyter dashboards install --user; jupyter dashboards activate; apt-get update; apt-get install --yes curl; curl --silent --location https://deb.nodesource.com/setup_0.12 | sudo bash -; apt-get install --yes nodejs; npm install -g bower;
.example-image:
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/etc/bin/run.sh
----------------------------------------------------------------------
diff --git a/etc/bin/run.sh b/etc/bin/run.sh
index bdeb402..67b5a29 100755
--- a/etc/bin/run.sh
+++ b/etc/bin/run.sh
@@ -30,7 +30,8 @@ KERNEL_ASSEMBLY=`(cd ${PROG_HOME}/lib; ls -1 toree-kernel-assembly-*.jar;)`
# disable randomized hash for string in Python 3.3+
export PYTHONHASHSEED=0
-
+TOREE_ASSEMBLY=${PROG_HOME}/lib/${KERNEL_ASSEMBLY}
exec "$SPARK_HOME"/bin/spark-submit \
${SPARK_OPTS} \
- --class org.apache.toree.Main $PROG_HOME/lib/${KERNEL_ASSEMBLY} "$@"
+ --driver-class-path ${TOREE_ASSEMBLY} \
+ --class org.apache.toree.Main ${TOREE_ASSEMBLY} "$@"
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
index a07f724..f97152d 100644
--- a/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
import org.apache.toree.interpreter.broker.BrokerTypes._
import org.slf4j.LoggerFactory
-import scala.concurrent.{Future, promise}
+import scala.concurrent.{Promise, Future, promise}
/**
* Represents the state structure of broker.
@@ -57,7 +57,7 @@ class BrokerState(private val maxQueuedCode: Int) {
// Generate our promise that will be fulfilled when the code is executed
// and the results are sent back
- val codeExecutionPromise = promise[CodeResults]()
+ val codeExecutionPromise = Promise[CodeResults]()
// Build the code representation to send to Broker
val uniqueId = java.util.UUID.randomUUID().toString
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/Magic.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/Magic.scala b/kernel-api/src/main/scala/org/apache/toree/magic/Magic.scala
index 584202b..5f18a8f 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/Magic.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/Magic.scala
@@ -17,14 +17,18 @@
package org.apache.toree.magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.{Event, DepName}
+
/**
- * Represents the base structure for a magic that is loaded and executed.
- */
-trait Magic {
- /**
- * Execute a magic.
- * @param code The code
- * @return The output of the magic
- */
- def execute(code: String): Any
- }
+ * Represents the base structure for a magic that is loaded and executed.
+ */
+trait Magic extends Plugin {
+ /**
+ * Execute a magic.
+ *
+ * @param code The code
+ * @return The output of the magic
+ */
+ def execute(code: String): Any
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/MagicExecutor.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/MagicExecutor.scala b/kernel-api/src/main/scala/org/apache/toree/magic/MagicExecutor.scala
deleted file mode 100644
index e1e1372..0000000
--- a/kernel-api/src/main/scala/org/apache/toree/magic/MagicExecutor.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.toree.magic
-
-import org.apache.toree.utils.DynamicReflectionSupport
-
-import scala.language.dynamics
-
-class MagicExecutor(magicLoader: MagicLoader) extends Dynamic {
-
- val executeMethod = classOf[Magic].getDeclaredMethods.head.getName
-
- def applyDynamic(name: String)(args: Any*): Either[CellMagicOutput, LineMagicOutput] = {
- val className = magicLoader.magicClassName(name)
- val isCellMagic = magicLoader.hasCellMagic(className)
- val isLineMagic = magicLoader.hasLineMagic(className)
-
- (isCellMagic, isLineMagic) match {
- case (true, false) =>
- val result = executeMagic(className, args)
- Left(result.asInstanceOf[CellMagicOutput])
- case (false, true) =>
- executeMagic(className, args)
- Right(LineMagicOutput)
- case (_, _) =>
- Left(CellMagicOutput("text/plain" ->
- s"Magic ${className} could not be executed."))
- }
- }
-
- private def executeMagic(className: String, args: Seq[Any]) = {
- val inst = magicLoader.createMagicInstance(className)
- val dynamicSupport = new DynamicReflectionSupport(inst.getClass, inst)
- dynamicSupport.applyDynamic(executeMethod)(args)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/MagicLoader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/MagicLoader.scala b/kernel-api/src/main/scala/org/apache/toree/magic/MagicLoader.scala
deleted file mode 100644
index e8b706c..0000000
--- a/kernel-api/src/main/scala/org/apache/toree/magic/MagicLoader.scala
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.toree.magic
-
-import java.net.{URL, URLClassLoader}
-
-import com.google.common.reflect.ClassPath
-import org.apache.toree.magic.dependencies.DependencyMap
-
-import scala.reflect.runtime.{universe => runtimeUniverse}
-import scala.collection.JavaConversions._
-
-class MagicLoader(
- var dependencyMap: DependencyMap = new DependencyMap(),
- urls: Array[URL] = Array(),
- parentLoader: ClassLoader = null
-) extends URLClassLoader(urls, parentLoader) {
- private val magicPackage = "org.apache.toree.magic.builtin"
-
- /**
- * Checks whether a magic with a given name, implementing a given interface,
- * exists.
- * @param name case insensitive magic name
- * @param interface interface
- * @return true if a magic with the given name and interface exists
- */
- private def hasSpecificMagic(name: String, interface: Class[_]) : Boolean = {
- val className = magicClassName(name)
- try {
- val clazz = loadClass(className)
- clazz.getInterfaces.contains(interface)
- } catch {
- case _: Throwable => false
- }
- }
-
- /**
- * Checks whether a line magic exists.
- * @param name case insensitive line magic name
- * @return true if the line magic exists
- */
- def hasLineMagic(name: String): Boolean =
- hasSpecificMagic(name, classOf[LineMagic])
-
- /**
- * Checks whether a cell magic exists.
- * @param name case insensitive cell magic name
- * @return true if the cell magic exists
- */
- def hasCellMagic(name: String): Boolean =
- hasSpecificMagic(name, classOf[CellMagic])
-
- /**
- * Attempts to load a class with a given name from a package.
- * @param name the name of the class
- * @param resolve whether to resolve the class or not
- * @return the class if found
- */
- override def loadClass(name: String, resolve: Boolean): Class[_] =
- try {
- super.loadClass(magicPackage + "." + name, resolve)
- } catch {
- case ex: ClassNotFoundException =>
- super.loadClass(name, resolve)
- }
-
- /**
- * Returns the class name for a case insensitive magic name query.
- * If no match is found, returns the query.
- * @param query a magic name, e.g. jAvasCRipt
- * @return the queried magic name's corresponding class, e.g. JavaScript
- */
- def magicClassName(query: String): String = {
- lowercaseClassMap(magicClassNames).getOrElse(query.toLowerCase, query)
- }
-
- /**
- * @return list of magic class names in magicPackage.
- */
- protected def magicClassNames : List[String] = {
- val classPath: ClassPath = ClassPath.from(this)
- val classes = classPath.getTopLevelClasses(magicPackage)
- classes.asList.map(_.getSimpleName).toList
- }
-
- /**
- * @param names list of class names
- * @return map of lowercase class names to class names
- */
- private def lowercaseClassMap(names: List[String]): Map[String, String] = {
- names.map(n => (n.toLowerCase, n)).toMap
- }
-
- def addJar(jar: URL) = addURL(jar)
- /**
- * Creates a instance of the specified magic with dependencies added.
- * @param name name of magic class
- * @return instance of the Magic corresponding to the given name
- */
- protected[magic] def createMagicInstance(name: String): Any = {
- val magicClass = loadClass(name) // Checks parent loadClass first
-
- val runtimeMirror = runtimeUniverse.runtimeMirror(this)
- val classSymbol = runtimeMirror.staticClass(magicClass.getCanonicalName)
- val classMirror = runtimeMirror.reflectClass(classSymbol)
- val selfType = classSymbol.selfType
-
- val classConstructorSymbol =
- selfType.declaration(runtimeUniverse.nme.CONSTRUCTOR).asMethod
- val classConstructorMethod =
- classMirror.reflectConstructor(classConstructorSymbol)
-
- val magicInstance = classConstructorMethod()
-
-
- // Add all of our dependencies to the new instance
- dependencyMap.internalMap.filter(selfType <:< _._1).values.foreach(
- _(magicInstance.asInstanceOf[Magic])
- )
-
- magicInstance
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/MagicManager.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/MagicManager.scala b/kernel-api/src/main/scala/org/apache/toree/magic/MagicManager.scala
new file mode 100644
index 0000000..51a38c2
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/MagicManager.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.toree.magic
+
+import org.apache.toree.plugins.{Plugin, PluginMethodResult, PluginManager}
+import org.slf4j.LoggerFactory
+
+import scala.annotation.tailrec
+import scala.language.dynamics
+import scala.runtime.BoxedUnit
+import scala.util.{Try, Failure, Success}
+
+class MagicManager(private val pluginManager: PluginManager) extends Dynamic {
+ protected val logger = LoggerFactory.getLogger(this.getClass.getName)
+ /**
+ * Checks if the provided magic is a line magic.
+ *
+ * @param magic The magic instance
+ * @return True if the magic is an instance of a line magic
+ */
+ def isLineMagic(magic: Magic): Boolean =
+ magic.getClass.getInterfaces.contains(classOf[LineMagic])
+
+ /**
+ * Checks if the provided magic is a cell magic.
+ *
+ * @param magic The magic instance
+ * @return True if the magic is an instance of a cell magic
+ */
+ def isCellMagic(magic: Magic): Boolean =
+ magic.getClass.getInterfaces.contains(classOf[CellMagic])
+
+ /**
+ * Finds a magic whose class name ends with a case insensitive name.
+ *
+ * @param name The name to search for
+ * @return The magic
+ * @throws MagicNotFoundException when no magics match name
+ */
+ @throws[MagicNotFoundException]
+ def findMagic(name: String): Magic = {
+ @tailrec def inheritsMagic(klass: Class[_]): Boolean = {
+ if (klass == null) false
+ else if (klass.getInterfaces.exists(classOf[Magic].isAssignableFrom)) true
+ else inheritsMagic(klass.getSuperclass)
+ }
+
+ val magics = pluginManager.plugins
+ .filter(p => inheritsMagic(p.getClass))
+ .filter(_.simpleName.split("\\.").last.toLowerCase == name.toLowerCase)
+
+ if (magics.size <= 0){
+ logger.error(s"No magic found for $name!")
+ throw new MagicNotFoundException(name)
+ } else if (magics.size > 1) {
+ logger.warn(s"More than one magic found for $name!")
+ }
+
+ magics.head.asInstanceOf[Magic]
+ }
+
+ @throws[MagicNotFoundException]
+ def applyDynamic(name: String)(args: Any*): Either[CellMagicOutput, LineMagicOutput] = {
+ val arg = args.headOption.map(_.toString).getOrElse("")
+
+ import org.apache.toree.plugins.Implicits._
+ val result = pluginManager.fireEventFirstResult(
+ name.toLowerCase(),
+ "input" -> arg
+ )
+
+ result match {
+ case Some(r: PluginMethodResult) => handleMagicResult(name, r.toTry)
+ case None => throw new MagicNotFoundException(name)
+ }
+ }
+
+ private def handleMagicResult(name: String, result: Try[Any]) = result match {
+ case Success(magicOutput) => magicOutput match {
+ case null | _: BoxedUnit => Right(LineMagicOutput)
+ case cmo: CellMagicOutput => Left(cmo)
+ case unknown =>
+ val message =
+ s"""Magic ${name} did not return proper magic output
+ |type. Expected ${classOf[CellMagicOutput].getName} or
+ |${classOf[LineMagicOutput].getName}, but found type of
+ |${unknown.getClass.getName}.""".trim.stripMargin
+ logger.warn(message)
+ Left(CellMagicOutput("text/plain" -> message))
+ }
+ case Failure(t) =>
+ val message = s"Magic ${name} failed to execute with error: \n${t.getMessage}"
+ logger.warn(message)
+ Left(CellMagicOutput("text/plain" -> message))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/MagicNotFoundException.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/MagicNotFoundException.scala b/kernel-api/src/main/scala/org/apache/toree/magic/MagicNotFoundException.scala
new file mode 100644
index 0000000..c5e7d85
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/MagicNotFoundException.scala
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.toree.magic
+
+class MagicNotFoundException(name: String) extends Exception(
+ s"Magic not found with name $name!"
+)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/DependencyMap.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/DependencyMap.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/DependencyMap.scala
deleted file mode 100644
index d136811..0000000
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/DependencyMap.scala
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.toree.magic.dependencies
-
-import java.io.OutputStream
-
-import org.apache.toree.interpreter.Interpreter
-import org.apache.toree.kernel.api.KernelLike
-import org.apache.toree.magic.{MagicLoader, Magic}
-import com.typesafe.config.Config
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.SQLContext
-
-import scala.reflect.runtime.universe._
-import org.apache.toree.dependencies.DependencyDownloader
-
-/**
- * Represents a mapping of dependency types to implementations.
- *
- * TODO: Explore Scala macros to avoid duplicate code.
- */
-class DependencyMap {
- val internalMap =
- scala.collection.mutable.Map[Type, PartialFunction[Magic, Unit]]()
-
- /**
- * Sets the Interpreter for this map.
- * @param interpreter The new Interpreter
- */
- def setInterpreter(interpreter: Interpreter) = {
- internalMap(typeOf[IncludeInterpreter]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeInterpreter].interpreter_=(interpreter)
- )
-
- this
- }
-
- /**
- * Sets the Interpreter for this map.
- * @param interpreter The new Interpreter
- */
- //@deprecated("Use setInterpreter with IncludeInterpreter!", "2015.05.06")
- def setKernelInterpreter(interpreter: Interpreter) = {
- internalMap(typeOf[IncludeKernelInterpreter]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeKernelInterpreter].kernelInterpreter_=(interpreter)
- )
-
- this
- }
-
- /**
- * Sets the SparkContext for this map.
- * @param sparkContext The new SparkContext
- */
- def setSparkContext(sparkContext: SparkContext) = {
- internalMap(typeOf[IncludeSparkContext]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeSparkContext].sparkContext_=(sparkContext)
- )
-
- this
- }
-
- /**
- * Sets the SQLContext for this map.
- * @param sqlContext The new SQLContext
- */
- def setSQLContext(sqlContext: SQLContext) = {
- internalMap(typeOf[IncludeSQLContext]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeSQLContext].sqlContext_=(sqlContext)
- )
-
- this
- }
-
- /**
- * Sets the OutputStream for this map.
- * @param outputStream The new OutputStream
- */
- def setOutputStream(outputStream: OutputStream) = {
- internalMap(typeOf[IncludeOutputStream]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeOutputStream].outputStream_=(outputStream)
- )
-
- this
- }
-
- /**
- * Sets the DependencyDownloader for this map.
- * @param dependencyDownloader The new DependencyDownloader
- */
- def setDependencyDownloader(dependencyDownloader: DependencyDownloader) = {
- internalMap(typeOf[IncludeDependencyDownloader]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeDependencyDownloader]
- .dependencyDownloader_=(dependencyDownloader)
- )
-
- this
- }
-
- /**
- * Sets the Kernel Object for this map.
- * @param kernel The new Kernel
- */
- def setKernel(kernel: KernelLike) = {
- internalMap(typeOf[IncludeKernel]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeKernel]
- .kernel_=(kernel)
- )
-
- this
- }
-
- /**
- * Sets the MagicLoader for this map.
- * @param magicLoader The new MagicLoader
- */
- def setMagicLoader(magicLoader: MagicLoader) = {
- internalMap(typeOf[IncludeMagicLoader]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeMagicLoader]
- .magicLoader_=(magicLoader)
- )
-
- this
- }
-
- /**
- * Sets the Config Object for this map.
- * @param config The config for the kernel
- */
- def setConfig(config: Config) = {
- internalMap(typeOf[IncludeConfig]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeConfig]
- .config=(config)
- )
-
- this
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeConfig.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeConfig.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeConfig.scala
index 44c4d06..af8c593 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeConfig.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeConfig.scala
@@ -19,13 +19,15 @@ package org.apache.toree.magic.dependencies
import org.apache.toree.magic.{Magic}
import com.typesafe.config.Config
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.Init
-trait IncludeConfig {
+trait IncludeConfig extends Plugin {
this: Magic =>
+ @Init protected def init(config: Config) = _config = config
+
private var _config: Config = _
def config: Config = _config
- def config_= (newConfig: Config) =
- _config = newConfig
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeDependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeDependencyDownloader.scala
index 6f39137..1d4fb15 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeDependencyDownloader.scala
@@ -19,12 +19,15 @@ package org.apache.toree.magic.dependencies
import org.apache.toree.dependencies.DependencyDownloader
import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.Init
-trait IncludeDependencyDownloader {
+trait IncludeDependencyDownloader extends Plugin {
this: Magic =>
+ @Init protected def init(newDependencyDownloader: DependencyDownloader) =
+ _dependencyDownloader = newDependencyDownloader
+
private var _dependencyDownloader: DependencyDownloader = _
def dependencyDownloader: DependencyDownloader = _dependencyDownloader
- def dependencyDownloader_=(newDependencyDownloader: DependencyDownloader) =
- _dependencyDownloader = newDependencyDownloader
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeInterpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeInterpreter.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeInterpreter.scala
index cf60540..78be9ce 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeInterpreter.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeInterpreter.scala
@@ -19,13 +19,15 @@ package org.apache.toree.magic.dependencies
import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.Init
-trait IncludeInterpreter {
+trait IncludeInterpreter extends Plugin {
this: Magic =>
- //val interpreter: Interpreter
+ @Init protected def init(newInterpreter: Interpreter) =
+ _interpreter = newInterpreter
+
private var _interpreter: Interpreter = _
def interpreter: Interpreter = _interpreter
- def interpreter_=(newInterpreter: Interpreter) =
- _interpreter = newInterpreter
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernel.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernel.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernel.scala
index 965769f..af50fb0 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernel.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernel.scala
@@ -19,12 +19,14 @@ package org.apache.toree.magic.dependencies
import org.apache.toree.kernel.api.KernelLike
import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.Init
-trait IncludeKernel {
+trait IncludeKernel extends Plugin {
this: Magic =>
+ @Init protected def init(newKernel: KernelLike) = _kernel = newKernel
+
private var _kernel: KernelLike = _
def kernel: KernelLike = _kernel
- def kernel_=(newKernel: KernelLike) =
- _kernel = newKernel
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernelInterpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernelInterpreter.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernelInterpreter.scala
index 1bc33aa..9cedf10 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernelInterpreter.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernelInterpreter.scala
@@ -19,14 +19,15 @@ package org.apache.toree.magic.dependencies
import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.Init
-//@deprecated("Use IncludeInterpreter instead!", "2015.05.06")
-trait IncludeKernelInterpreter {
+trait IncludeKernelInterpreter extends Plugin {
this: Magic =>
- //val interpreter: Interpreter
+ @Init protected def init(newInterpreter: Interpreter) =
+ _interpreter = newInterpreter
+
private var _interpreter: Interpreter = _
def kernelInterpreter: Interpreter = _interpreter
- def kernelInterpreter_=(newInterpreter: Interpreter) =
- _interpreter = newInterpreter
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeMagicLoader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeMagicLoader.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeMagicLoader.scala
deleted file mode 100644
index b615629..0000000
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeMagicLoader.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.toree.magic.dependencies
-
-import org.apache.toree.magic.{MagicLoader, Magic}
-
-
-trait IncludeMagicLoader {
- this: Magic =>
-
- //val sparkContext: SparkContext
- private var _magicLoader: MagicLoader = _
- def magicLoader: MagicLoader = _magicLoader
- def magicLoader_=(newMagicLoader: MagicLoader) =
- _magicLoader = newMagicLoader
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeOutputStream.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeOutputStream.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeOutputStream.scala
index 2525dcc..51fc35e 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeOutputStream.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeOutputStream.scala
@@ -19,14 +19,18 @@ package org.apache.toree.magic.dependencies
import java.io.OutputStream
+import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.{DepName, Event, Init}
-trait IncludeOutputStream {
+trait IncludeOutputStream extends Plugin {
this: Magic =>
- //val outputStream: OutputStream
+ @Event(name = "newOutputStream") protected def newOutputStream(
+ @DepName(name = "outputStream") newOutputStream: OutputStream
+ ) = _outputStream = newOutputStream
+
private var _outputStream: OutputStream = _
def outputStream: OutputStream = _outputStream
- def outputStream_=(newOutputStream: OutputStream) =
- _outputStream = newOutputStream
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludePluginManager.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludePluginManager.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludePluginManager.scala
new file mode 100644
index 0000000..29b5b84
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludePluginManager.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.toree.magic.dependencies
+
+import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.{PluginManager, Plugin}
+import org.apache.toree.plugins.annotations.Init
+
+trait IncludePluginManager extends Plugin { this: Magic =>
+ @Init protected def init(pluginManager: PluginManager) =
+ _pluginManager = pluginManager
+
+ private var _pluginManager: PluginManager = _
+ def pluginManager: PluginManager = _pluginManager
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSQLContext.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSQLContext.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSQLContext.scala
index 91b8348..36117a7 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSQLContext.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSQLContext.scala
@@ -17,14 +17,18 @@
package org.apache.toree.magic.dependencies
-import org.apache.toree.magic.Magic
import org.apache.spark.sql.SQLContext
+import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.{Event, Init}
-trait IncludeSQLContext {
+trait IncludeSQLContext extends Plugin {
this: Magic =>
+ @Event(name = "sparkReady") protected def sparkReady(
+ newSqlContext: SQLContext
+ ) = _sqlContext = newSqlContext
+
private var _sqlContext: SQLContext = _
def sqlContext: SQLContext = _sqlContext
- def sqlContext_=(newSqlContext: SQLContext) =
- _sqlContext = newSqlContext
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSparkContext.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSparkContext.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSparkContext.scala
index f0f0473..92dfe05 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSparkContext.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSparkContext.scala
@@ -17,15 +17,18 @@
package org.apache.toree.magic.dependencies
-import org.apache.toree.magic.Magic
import org.apache.spark.SparkContext
+import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.{Event, Init}
-trait IncludeSparkContext {
+trait IncludeSparkContext extends Plugin {
this: Magic =>
- //val sparkContext: SparkContext
+ @Event(name = "sparkReady") protected def sparkReady(
+ newSparkContext: SparkContext
+ ) = _sparkContext = newSparkContext
+
private var _sparkContext: SparkContext = _
def sparkContext: SparkContext = _sparkContext
- def sparkContext_=(newSparkContext: SparkContext) =
- _sparkContext = newSparkContext
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/test/scala/org/apache/toree/magic/MagicLoaderSpec.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/test/scala/org/apache/toree/magic/MagicLoaderSpec.scala b/kernel-api/src/test/scala/org/apache/toree/magic/MagicLoaderSpec.scala
deleted file mode 100644
index 8e03881..0000000
--- a/kernel-api/src/test/scala/org/apache/toree/magic/MagicLoaderSpec.scala
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.toree.magic
-
-import java.io.OutputStream
-
-import org.apache.toree.dependencies.DependencyDownloader
-import org.apache.toree.interpreter.Interpreter
-import org.apache.toree.magic.dependencies._
-import org.apache.spark.SparkContext
-import org.scalatest.mock.MockitoSugar
-import org.scalatest.{FunSpec, Matchers}
-
-
-/**
-* Used for verification of dependency injection. Calls toString on each
-* dependency to assert that they are provided.
-*/
-class LineMagicWithDependencies extends LineMagic
- with IncludeDependencyDownloader
- with IncludeSparkContext
- with IncludeInterpreter
- with IncludeOutputStream
-{
- override def execute(code: String): Unit = {
- sparkContext.cancelAllJobs()
- interpreter.classServerURI
- outputStream.close()
- dependencyDownloader.setPrintStream(null)
- }
-}
-
-class MockLineMagic extends LineMagic {
- override def execute(code: String): Unit = {}
-}
-
-class MockCellMagic extends CellMagic {
- override def execute(code: String): CellMagicOutput =
- CellMagicOutput()
-}
-
-class MagicLoaderSpec extends FunSpec with Matchers with MockitoSugar {
- describe("MagicLoader") {
- describe("#hasLineMagic") {
- it("should return false if a class with the magic name is not found") {
- val magicLoader = new MagicLoader() {
- override def findClass(name: String): Class[_] =
- throw new ClassNotFoundException()
- }
-
- magicLoader.hasLineMagic("potato") should be (false)
- }
-
- it("should return true if a class with the magic name is found") {
- val magicLoader = new MagicLoader() {
- override def findClass(name: String): Class[_] =
- classOf[MockLineMagic]
- }
-
- magicLoader.hasLineMagic("potato") should be (true)
- }
-
- it("should return true if a class with the magic name is found regardless of case"){
- // Only loads a class named "Potato"
- val classLoader = new ClassLoader() {
- override def findClass(name: String) =
- if (name == "Potato") classOf[MockLineMagic]
- else throw new ClassNotFoundException
- }
-
- // Case insensitive matching should be performed on "Potato"
- val magicLoader = new MagicLoader(parentLoader = classLoader) {
- override def magicClassNames = List("Potato")
- }
-
- magicLoader.hasLineMagic("Potato") should be (true)
- magicLoader.hasLineMagic("potato") should be (true)
- magicLoader.hasLineMagic("pOTatO") should be (true)
- }
- }
-
- describe("#hasCellMagic") {
- it("should return false if a class with the magic name is not found") {
- val magicLoader = new MagicLoader() {
- override def findClass(name: String): Class[_] =
- throw new ClassNotFoundException()
- }
-
- magicLoader.hasCellMagic("potato") should be (false)
- }
-
- it("should return true if a class with the magic name is found") {
- val magicLoader = new MagicLoader() {
- override def findClass(name: String): Class[_] =
- classOf[MockCellMagic]
- }
-
- magicLoader.hasCellMagic("potato") should be (true)
- }
-
- it("should return true if a class with the magic name is found regardless of case"){
- // Only loads a class named "Potato"
- val classLoader = new ClassLoader() {
- override def findClass(name: String) =
- if (name == "Potato") classOf[MockCellMagic]
- else throw new ClassNotFoundException
- }
-
- // Case insensitive matching should be performed on "Potato"
- val magicLoader = new MagicLoader(parentLoader = classLoader) {
- override def magicClassNames = List("Potato")
- }
-
- magicLoader.hasCellMagic("Potato") should be (true)
- magicLoader.hasCellMagic("potato") should be (true)
- magicLoader.hasCellMagic("pOTatO") should be (true)
- }
- }
-
- describe("#magicClassName"){
- it("should return the correctly-cased version of the requested magic name") {
- val magicLoader = new MagicLoader() {
- override def magicClassNames = List("Potato")
- }
-
- magicLoader.magicClassName("Potato") should be ("Potato")
- magicLoader.magicClassName("potato") should be ("Potato")
- magicLoader.magicClassName("pOTatO") should be ("Potato")
- }
-
- it("should return the query if a corresponding magic class does not exist") {
- val magicLoader = new MagicLoader() {
- override def magicClassNames = List()
- }
-
- magicLoader.magicClassName("dne") should be ("dne")
- magicLoader.magicClassName("dNE") should be ("dNE")
- }
- }
-
- describe("#createMagicInstance") {
- it("should correctly insert dependencies into a class") {
- val mockInterpreter = mock[Interpreter]
- val mockSparkContext = mock[SparkContext]
- val mockOutputStream = mock[OutputStream]
- val mockDependencyDownloader = mock[DependencyDownloader]
-
- val dependencyMap = new DependencyMap()
- .setInterpreter(mockInterpreter)
- .setSparkContext(mockSparkContext)
- .setOutputStream(mockOutputStream)
- .setDependencyDownloader(mockDependencyDownloader)
-
- val magicLoader = new MagicLoader(
- dependencyMap = dependencyMap,
- parentLoader = new InternalClassLoader(getClass.getClassLoader)
- )
-
- val magicName = "LineMagicWithDependencies"
- val instance = magicLoader.createMagicInstance(magicName)
- .asInstanceOf[LineMagicWithDependencies]
- instance.interpreter should be(mockInterpreter)
- instance.outputStream should be(mockOutputStream)
- instance.sparkContext should be(mockSparkContext)
- instance.dependencyDownloader should be(mockDependencyDownloader)
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/test/scala/org/apache/toree/magic/MagicManagerSpec.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/test/scala/org/apache/toree/magic/MagicManagerSpec.scala b/kernel-api/src/test/scala/org/apache/toree/magic/MagicManagerSpec.scala
new file mode 100644
index 0000000..5874a86
--- /dev/null
+++ b/kernel-api/src/test/scala/org/apache/toree/magic/MagicManagerSpec.scala
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.toree.magic
+
+import org.apache.toree.plugins.dependencies.Dependency
+import org.apache.toree.plugins._
+import org.mockito.Mockito._
+import org.mockito.Matchers.{eq => mockEq, _}
+import org.scalatest.mock.MockitoSugar
+import org.scalatest.{FunSpec, Matchers, OneInstancePerTest}
+import test.utils
+
+import MagicManagerSpec._
+
+import scala.runtime.BoxedUnit
+
+object MagicManagerSpec {
+ val TestCellMagicOutput = CellMagicOutput("test" -> "value")
+}
+
+class SomeLineMagic extends LineMagic {
+ override def execute(code: String): Unit = {}
+}
+
+class SomeCellMagic extends CellMagic {
+ override def execute(code: String): CellMagicOutput = TestCellMagicOutput
+}
+
+private class SomePlugin extends Plugin
+
+private class SomeMagic extends Magic {
+ override def execute(code: String): Any = ???
+}
+
+class LineMagicException extends Exception
+
+private class ExceptionLineMagic extends LineMagic {
+ override def execute(code: String): Unit = throw new LineMagicException
+}
+
+class CellMagicException extends Exception
+
+private class ExceptionCellMagic extends CellMagic {
+ override def execute(code: String): CellMagicOutput = throw new CellMagicException
+}
+
+class MagicManagerSpec
+ extends FunSpec with Matchers with MockitoSugar with OneInstancePerTest
+{
+ private val TestPluginName = "SomePlugin"
+ private val TestMagicName = "SomeMagic"
+ private val mockPluginManager = mock[PluginManager]
+ private val magicManager = spy(new MagicManager(mockPluginManager))
+
+ describe("MagicManager") {
+ describe("#isLineMagic") {
+ it("should return true if the magic extends the line magic interface") {
+ val expected = true
+
+ val mockLineMagic = mock[LineMagic]
+ val actual = magicManager.isLineMagic(mockLineMagic)
+
+ actual should be (expected)
+ }
+
+ it("should return false if the magic does not extend the line magic interface") {
+ val expected = false
+
+ val mockMagic = mock[Magic]
+ val actual = magicManager.isLineMagic(mockMagic)
+
+ actual should be (expected)
+ }
+
+ it("should throw an exception if provided null") {
+ intercept[NullPointerException] {
+ magicManager.isLineMagic(null)
+ }
+ }
+ }
+
+ describe("#isCellMagic") {
+ it("should return true if the magic extends the cell magic interface") {
+ val expected = true
+
+ val mockCellMagic = mock[CellMagic]
+ val actual = magicManager.isCellMagic(mockCellMagic)
+
+ actual should be (expected)
+ }
+
+ it("should return false if the magic does not extend the cell magic interface") {
+ val expected = false
+
+ val mockMagic = mock[Magic]
+ val actual = magicManager.isCellMagic(mockMagic)
+
+ actual should be (expected)
+ }
+
+ it("should throw an exception if provided null") {
+ intercept[NullPointerException] {
+ magicManager.isCellMagic(null)
+ }
+ }
+ }
+
+ describe("#findMagic") {
+ it("should throw a MagicNotFoundException if no magic matches the name") {
+ intercept[MagicNotFoundException] {
+ doReturn(Seq(new Plugin {}).toIterable).when(mockPluginManager).plugins
+
+ magicManager.findMagic(TestMagicName)
+ }
+ }
+
+ it("should throw a MagicNotFoundException if there are no loaded plugins") {
+ intercept[MagicNotFoundException] {
+ doReturn(Nil).when(mockPluginManager).plugins
+
+ magicManager.findMagic(TestMagicName)
+ }
+ }
+
+ it("should throw a MagicNotFoundException if a plugin matches but is not a magic") {
+ intercept[MagicNotFoundException] {
+ doReturn(Seq(new SomePlugin).toIterable).when(mockPluginManager).plugins
+
+ magicManager.findMagic(TestPluginName)
+ }
+ }
+
+ it("should return the magic if exactly one is found") {
+ val expected = new SomeMagic
+
+ doReturn(Seq(expected).toIterable).when(mockPluginManager).plugins
+ val actual = magicManager.findMagic(TestMagicName)
+
+ actual should be (expected)
+ }
+
+ it("should return a magic whose name matches even if casing is different") {
+ val expected = new SomeMagic
+
+ doReturn(Seq(expected).toIterable).when(mockPluginManager).plugins
+ val actual = magicManager.findMagic(TestMagicName.toUpperCase())
+
+ actual should be (expected)
+ }
+
+ it("should return the first match if more than one magic matches the name") {
+ val expected = new SomeMagic
+
+ doReturn(Seq(expected, new utils.SomeMagic).toIterable)
+ .when(mockPluginManager).plugins
+ val actual = magicManager.findMagic(TestMagicName)
+
+ actual should be (expected)
+ }
+ }
+
+ describe("#applyDynamic") {
+ it("should return CellMagicOutput if the invocation of a magic throws an exception") {
+ doReturn(Some(FailurePluginMethodResult(
+ mock[PluginMethod],
+ new LineMagicException()
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic("TEST")()
+
+ result.isLeft should be(true)
+ }
+
+ it("should fire an event with the lowercase of the magic name") {
+ val arg: java.lang.String = "some arg"
+ val pluginName = "TEST"
+ val expected = Dependency.fromValueWithName("input", arg)
+
+ doReturn(Some(FailurePluginMethodResult(
+ mock[PluginMethod],
+ new LineMagicException()
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ magicManager.applyDynamic(pluginName)(arg :: Nil: _*)
+ verify(mockPluginManager).fireEventFirstResult(mockEq(pluginName.toLowerCase), any())
+ }
+
+ it("should take the first argument and convert it to a string to pass to the magic") {
+ val arg: java.lang.String = "some arg"
+ val pluginName = "TEST"
+ val expected = Dependency.fromValueWithName("input", arg)
+
+ doReturn(Some(FailurePluginMethodResult(
+ mock[PluginMethod],
+ new LineMagicException()
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ magicManager.applyDynamic(pluginName)(arg :: Nil: _*)
+ verify(mockPluginManager).fireEventFirstResult(anyString(), mockEq(Seq(expected)): _*)
+ }
+
+ it("should pass an empty string to the line magic if no arguments are provided") {
+ val arg: java.lang.String = ""
+ val pluginName = "TEST"
+ val expected = Dependency.fromValueWithName("input", arg)
+
+ doReturn(Some(FailurePluginMethodResult(
+ mock[PluginMethod],
+ new LineMagicException()
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ magicManager.applyDynamic(pluginName)(Nil: _*)
+ verify(mockPluginManager).fireEventFirstResult(anyString(), mockEq(Seq(expected)): _*)
+ }
+
+ it("should return a Right[LineMagicOutput] if line magic execution is successful and returns null") {
+ val pluginName = "TEST"
+ val expected = Right(LineMagicOutput)
+
+ doReturn(Some(SuccessPluginMethodResult(
+ mock[PluginMethod],
+ null
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic(pluginName)(Nil: _*)
+ result should be(expected)
+ }
+
+ it("should return a Right[LineMagicOutput] if line magic execution is successful and returns BoxedUnit") {
+ val pluginName = "TEST"
+ val expected = Right(LineMagicOutput)
+
+ doReturn(Some(SuccessPluginMethodResult(
+ mock[PluginMethod],
+ BoxedUnit.UNIT
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic(pluginName)(Nil: _*)
+ result should be(expected)
+ }
+
+ it("should return a Left[CellMagicOutput] if cell magic execution is successful") {
+ val pluginName = "TEST"
+ val cellMagicOutput = CellMagicOutput("our/type" -> "TEST CONTENT")
+ doReturn(Some(SuccessPluginMethodResult(
+ mock[PluginMethod],
+ cellMagicOutput
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic(pluginName)(Nil: _*)
+ result.left.get should be(cellMagicOutput)
+ }
+
+ it("should return a Left[CellMagicOutput] if is a magic but not a line or cell") {
+ val pluginName = "TEST"
+
+ doReturn(Some(SuccessPluginMethodResult(
+ mock[PluginMethod],
+ new AnyRef
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic(pluginName)(Nil: _*)
+ result.left.get("text/plain") should not be (empty)
+
+ }
+
+ it("should return a Left[CellMagicOutput] if magic fails") {
+ val pluginName = "TEST"
+
+ doReturn(Some(FailurePluginMethodResult(
+ mock[PluginMethod],
+ new Throwable
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic(pluginName)(Nil: _*)
+ result.left.get("text/plain") should not be (empty)
+ }
+
+ it("should throw a MagicNotFoundException when a magic cannot be found") {
+ val pluginName = "THISMAGICDOESN'TEXIST"
+
+ doReturn(None).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+ intercept[MagicNotFoundException] {
+ magicManager.applyDynamic(pluginName)(Nil: _*)
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/test/scala/test/utils/SomeMagic.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/test/scala/test/utils/SomeMagic.scala b/kernel-api/src/test/scala/test/utils/SomeMagic.scala
new file mode 100644
index 0000000..ca3a382
--- /dev/null
+++ b/kernel-api/src/test/scala/test/utils/SomeMagic.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package test.utils
+
+import org.apache.toree.magic.Magic
+
+class SomeMagic extends Magic {
+ override def execute(code: String): Any = ???
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala b/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
index b391f20..84a2853 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
@@ -85,7 +85,7 @@ class KernelBootstrap(config: Config) extends LogLike {
// Initialize components needed elsewhere
val (commStorage, commRegistrar, commManager, interpreter,
kernel, dependencyDownloader,
- magicLoader, responseMap) =
+ magicManager, pluginManager, responseMap) =
initializeComponents(
config = config,
appName = DefaultAppName,
@@ -104,7 +104,8 @@ class KernelBootstrap(config: Config) extends LogLike {
interpreter = interpreter,
commRegistrar = commRegistrar,
commStorage = commStorage,
- magicLoader = magicLoader,
+ pluginManager = pluginManager,
+ magicManager = magicManager,
responseMap = responseMap
)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
index e328ceb..d361708 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
@@ -18,30 +18,22 @@
package org.apache.toree.boot.layer
import java.io.File
-import java.util
import java.util.concurrent.ConcurrentHashMap
import akka.actor.ActorRef
-import org.apache.toree.comm.{CommManager, KernelCommManager, CommRegistrar, CommStorage}
-import org.apache.toree.dependencies.{CoursierDependencyDownloader, DependencyDownloader, IvyDependencyDownloader}
-import org.apache.toree.global
+import com.typesafe.config.Config
+import org.apache.toree.comm.{CommManager, CommRegistrar, CommStorage, KernelCommManager}
+import org.apache.toree.dependencies.{CoursierDependencyDownloader, DependencyDownloader}
import org.apache.toree.interpreter._
-import org.apache.toree.kernel.api.{KernelLike, Kernel}
+import org.apache.toree.kernel.api.Kernel
import org.apache.toree.kernel.protocol.v5.KMBuilder
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
-import org.apache.toree.kernel.protocol.v5.stream.KernelOutputStream
-import org.apache.toree.magic.MagicLoader
-import org.apache.toree.magic.builtin.BuiltinLoader
-import org.apache.toree.magic.dependencies.DependencyMap
-import org.apache.toree.utils.{MultiClassLoader, TaskManager, KeyValuePairUtils, LogLike}
-import com.typesafe.config.Config
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkContext, SparkConf}
+import org.apache.toree.magic.MagicManager
+import org.apache.toree.plugins.PluginManager
+import org.apache.toree.utils.LogLike
import scala.collection.JavaConverters._
-import scala.util.Try
-
/**
* Represents the component initialization. All component-related pieces of the
* kernel (non-actors) should be created here. Limited items should be exposed.
@@ -57,7 +49,7 @@ trait ComponentInitialization {
def initializeComponents(
config: Config, appName: String, actorLoader: ActorLoader
): (CommStorage, CommRegistrar, CommManager, Interpreter,
- Kernel, DependencyDownloader, MagicLoader,
+ Kernel, DependencyDownloader, MagicManager, PluginManager,
collection.mutable.Map[String, ActorRef])
}
@@ -84,20 +76,22 @@ trait StandardComponentInitialization extends ComponentInitialization {
val scalaInterpreter = manager.interpreters.get("Scala").orNull
val dependencyDownloader = initializeDependencyDownloader(config)
- val magicLoader = initializeMagicLoader(
+ val pluginManager = createPluginManager(
config, scalaInterpreter, dependencyDownloader)
val kernel = initializeKernel(
- config, actorLoader, manager, commManager, magicLoader
+ config, actorLoader, manager, commManager, pluginManager
)
+ initializePlugins(config, pluginManager)
+
val responseMap = initializeResponseMap()
initializeSparkContext(config, kernel, appName)
(commStorage, commRegistrar, commManager,
manager.defaultInterpreter.orNull, kernel,
- dependencyDownloader, magicLoader, responseMap)
+ dependencyDownloader, kernel.magics, pluginManager, responseMap)
}
@@ -142,14 +136,14 @@ trait StandardComponentInitialization extends ComponentInitialization {
actorLoader: ActorLoader,
interpreterManager: InterpreterManager,
commManager: CommManager,
- magicLoader: MagicLoader
+ pluginManager: PluginManager
) = {
val kernel = new Kernel(
config,
actorLoader,
interpreterManager,
commManager,
- magicLoader
+ pluginManager
)
/*
interpreter.doQuietly {
@@ -159,48 +153,55 @@ trait StandardComponentInitialization extends ComponentInitialization {
)
}
*/
- magicLoader.dependencyMap.setKernel(kernel)
+ pluginManager.dependencyManager.add(kernel)
kernel
}
- private def initializeMagicLoader(
+ private def createPluginManager(
config: Config, interpreter: Interpreter,
dependencyDownloader: DependencyDownloader
) = {
- logger.debug("Constructing magic loader")
+ logger.debug("Constructing plugin manager")
+ val pluginManager = new PluginManager()
logger.debug("Building dependency map")
- val dependencyMap = new DependencyMap()
- .setInterpreter(interpreter)
- .setKernelInterpreter(interpreter) // This is deprecated
- .setDependencyDownloader(dependencyDownloader)
- .setConfig(config)
+ pluginManager.dependencyManager.add(interpreter)
+ pluginManager.dependencyManager.add(dependencyDownloader)
+ pluginManager.dependencyManager.add(config)
+
+ pluginManager.dependencyManager.add(pluginManager)
- logger.debug("Creating BuiltinLoader")
- val builtinLoader = new BuiltinLoader()
+ pluginManager
+ }
+ private def initializePlugins(
+ config: Config,
+ pluginManager: PluginManager
+ ) = {
val magicUrlArray = config.getStringList("magic_urls").asScala
.map(s => new java.net.URL(s)).toArray
if (magicUrlArray.isEmpty)
- logger.warn("No external magics provided to MagicLoader!")
+ logger.warn("No external magics provided to PluginManager!")
else
logger.info("Using magics from the following locations: " +
magicUrlArray.map(_.getPath).mkString(","))
- val multiClassLoader = new MultiClassLoader(
- builtinLoader,
- interpreter.classLoader
- )
+ // Load internal plugins under kernel module
+ logger.debug("Loading internal plugins")
+ val internalPlugins = pluginManager.initialize()
+ logger.info(internalPlugins.size + " internal plugins loaded")
- logger.debug("Creating MagicLoader")
- val magicLoader = new MagicLoader(
- dependencyMap = dependencyMap,
- urls = magicUrlArray,
- parentLoader = multiClassLoader
- )
- magicLoader.dependencyMap.setMagicLoader(magicLoader)
- magicLoader
+ // Load external plugins if provided
+ logger.debug("Loading external plugins")
+ val externalPlugins = if (magicUrlArray.nonEmpty) {
+ val externalPlugins = pluginManager.loadPlugins(
+ magicUrlArray.map(_.getFile).map(new File(_)): _*
+ )
+ pluginManager.initializePlugins(externalPlugins)
+ externalPlugins
+ } else Nil
+ logger.info(externalPlugins.size + " external plugins loaded")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
index 2de5eb2..ea2ebd5 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
@@ -30,7 +30,8 @@ import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.magic.{MagicParser, PostProcessor}
import org.apache.toree.kernel.protocol.v5.relay.ExecuteRequestRelay
import org.apache.toree.kernel.protocol.v5.{MessageType, SocketType, SystemActorType}
-import org.apache.toree.magic.MagicLoader
+import org.apache.toree.magic.MagicManager
+import org.apache.toree.plugins.PluginManager
import org.apache.toree.utils.LogLike
/**
@@ -45,15 +46,16 @@ trait HandlerInitialization {
* @param actorLoader The actor loader needed for registration
* @param kernel The kernel api needed for registration
* @param interpreter The main interpreter needed for registration
- * @param magicLoader The magic loader needed for registration
+ * @param magicManager The magic manager needed for registration
* @param commRegistrar The comm registrar needed for registration
* @param commStorage The comm storage needed for registration
*/
def initializeHandlers(
actorSystem: ActorSystem, actorLoader: ActorLoader,
kernel: Kernel,
- interpreter: Interpreter, magicLoader: MagicLoader,
- commRegistrar: CommRegistrar, commStorage: CommStorage,
+ interpreter: Interpreter, pluginManager: PluginManager,
+ magicManager: MagicManager, commRegistrar: CommRegistrar,
+ commStorage: CommStorage,
responseMap: collection.mutable.Map[String, ActorRef]
): Unit
}
@@ -71,26 +73,30 @@ trait StandardHandlerInitialization extends HandlerInitialization {
* @param actorLoader The actor loader needed for registration
* @param kernel The kernel api needed for registration
* @param interpreter The main interpreter needed for registration
- * @param magicLoader The magic loader needed for registration
+ * @param pluginManager The plugin manager needed for registration
* @param commRegistrar The comm registrar needed for registration
* @param commStorage The comm storage needed for registration
*/
def initializeHandlers(
actorSystem: ActorSystem, actorLoader: ActorLoader,
kernel: Kernel,
- interpreter: Interpreter, magicLoader: MagicLoader,
- commRegistrar: CommRegistrar, commStorage: CommStorage,
+ interpreter: Interpreter, pluginManager: PluginManager,
+ magicManager: MagicManager, commRegistrar: CommRegistrar,
+ commStorage: CommStorage,
responseMap: collection.mutable.Map[String, ActorRef]
): Unit = {
initializeKernelHandlers(
actorSystem, actorLoader, kernel, commRegistrar, commStorage, responseMap
)
- initializeSystemActors(actorSystem, actorLoader, interpreter, magicLoader)
+ initializeSystemActors(
+ actorSystem, actorLoader, interpreter, pluginManager, magicManager
+ )
}
private def initializeSystemActors(
actorSystem: ActorSystem, actorLoader: ActorLoader,
- interpreter: Interpreter, magicLoader: MagicLoader
+ interpreter: Interpreter, pluginManager: PluginManager,
+ magicManager: MagicManager
): Unit = {
logger.debug("Creating interpreter actor")
val interpreterActor = actorSystem.actorOf(
@@ -100,10 +106,10 @@ trait StandardHandlerInitialization extends HandlerInitialization {
logger.debug("Creating execute request relay actor")
val postProcessor = new PostProcessor(interpreter)
- val magicParser = new MagicParser(magicLoader)
+ val magicParser = new MagicParser(magicManager)
val executeRequestRelayActor = actorSystem.actorOf(
Props(classOf[ExecuteRequestRelay],
- actorLoader, magicLoader, magicParser, postProcessor
+ actorLoader, pluginManager, magicParser, postProcessor
),
name = SystemActorType.ExecuteRequestRelay.toString
)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
index 87a0e94..ae2c478 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
@@ -94,14 +94,10 @@ trait StandardHookInitialization extends HookInitialization {
private def registerShutdownHook(): Unit = {
logger.debug("Registering shutdown hook")
val self = this
- val mainThread = Thread.currentThread()
Runtime.getRuntime.addShutdownHook(new Thread() {
override def run() = {
logger.info("Shutting down kernel")
self.shutdown()
- // TODO: Check if you can magically access the spark context to stop it
- // TODO: inside a different thread
- if (mainThread.isAlive) mainThread.join()
}
})
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
index c08467a..2d67189 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
@@ -17,32 +17,32 @@
package org.apache.toree.kernel.api
-import java.io.{OutputStream, InputStream, PrintStream}
+import java.io.{InputStream, PrintStream}
import java.util.concurrent.ConcurrentHashMap
+import com.typesafe.config.Config
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.{SparkConf, SparkContext}
import org.apache.toree.annotations.Experimental
import org.apache.toree.boot.layer.InterpreterManager
import org.apache.toree.comm.CommManager
import org.apache.toree.global
+import org.apache.toree.global.ExecuteRequestState
import org.apache.toree.interpreter.Results.Result
import org.apache.toree.interpreter._
import org.apache.toree.kernel.protocol.v5
-import org.apache.toree.kernel.protocol.v5.{KMBuilder, KernelMessage}
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.magic.MagicParser
-import org.apache.toree.kernel.protocol.v5.stream.{KernelOutputStream, KernelInputStream}
-import org.apache.toree.magic.{MagicLoader, MagicExecutor}
+import org.apache.toree.kernel.protocol.v5.stream.KernelOutputStream
+import org.apache.toree.kernel.protocol.v5.{KMBuilder, KernelMessage}
+import org.apache.toree.magic.MagicManager
+import org.apache.toree.plugins.PluginManager
import org.apache.toree.utils.{KeyValuePairUtils, LogLike}
-import com.typesafe.config.Config
-import org.apache.spark.api.java.JavaSparkContext
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkContext, SparkConf}
-import scala.util.{Try, DynamicVariable}
-
-import scala.reflect.runtime.universe._
import scala.language.dynamics
-import org.apache.toree.global.ExecuteRequestState
+import scala.reflect.runtime.universe._
+import scala.util.{DynamicVariable, Try}
/**
* Represents the main kernel API to be used for interaction.
@@ -58,7 +58,7 @@ class Kernel (
private val actorLoader: ActorLoader,
val interpreterManager: InterpreterManager,
val comm: CommManager,
- val magicLoader: MagicLoader
+ val pluginManager: PluginManager
) extends KernelLike with LogLike {
/**
@@ -96,12 +96,12 @@ class Kernel (
/**
* Represents magics available through the kernel.
*/
- val magics = new MagicExecutor(magicLoader)
+ val magics = new MagicManager(pluginManager)
/**
* Represents magic parsing functionality.
*/
- val magicParser = new MagicParser(magicLoader)
+ val magicParser = new MagicParser(magics)
/**
* Represents the data that can be shared using the kernel as the middleman.
@@ -117,6 +117,7 @@ class Kernel (
/**
* Handles the output of interpreting code.
+ *
* @param output the output of the interpreter
* @return (success, message) or (failure, message)
*/
@@ -175,7 +176,6 @@ class Kernel (
*
* @param parentMessage The message to serve as the parent of outgoing
* messages sent as a result of using streaming methods
- *
* @return The collection of streaming methods
*/
private[toree] def stream(
@@ -198,7 +198,6 @@ class Kernel (
*
* @param parentMessage The message to serve as the parent of outgoing
* messages sent as a result of using streaming methods
- *
* @return The collection of streaming methods
*/
private[toree] def display(
@@ -225,7 +224,6 @@ class Kernel (
* by the factory methods
* @param kmBuilder The builder to be used by objects created by factory
* methods
- *
* @return The collection of factory methods
*/
private[toree] def factory(
@@ -341,7 +339,6 @@ class Kernel (
* Retrieves the last kernel message received by the kernel.
*
* @throws IllegalArgumentException If no kernel message has been received
- *
* @return The kernel message instance
*/
private def lastKernelMessage() = {
@@ -362,8 +359,13 @@ class Kernel (
updateInterpreterWithSparkContext(interpreter, sparkContext)
updateInterpreterWithSqlContext(interpreter, sqlContext)
- magicLoader.dependencyMap =
- magicLoader.dependencyMap.setSparkContext(_sparkContext)
+ // TODO: Convert to events
+ pluginManager.dependencyManager.add(_sparkConf)
+ pluginManager.dependencyManager.add(_sparkContext)
+ pluginManager.dependencyManager.add(_javaSparkContext)
+ pluginManager.dependencyManager.add(_sqlContext)
+
+ pluginManager.fireEvent("sparkReady")
_sparkContext
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParser.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParser.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParser.scala
index f86d76a..bfe518d 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParser.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParser.scala
@@ -17,14 +17,17 @@
package org.apache.toree.kernel.protocol.v5.magic
-import org.apache.toree.magic.MagicLoader
+import org.apache.toree.magic.MagicManager
-class MagicParser(magicLoader: MagicLoader) {
+import scala.util.Try
+
+class MagicParser(private val magicManager: MagicManager) {
private val magicRegex = """^[%]{1,2}(\w+)""".r
protected[magic] val kernelObjectName = "kernel.magics"
/**
* Determines whether a given line of code represents a line magic.
+ *
* @param codeLine a single line of code
* @return
*/
@@ -33,6 +36,7 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Determines whether a given string of code represents a cell magic.
+ *
* @param codeBlob a string of code separated by newlines
* @return
*/
@@ -45,6 +49,7 @@ class MagicParser(magicLoader: MagicLoader) {
*
* E.g.
* "%magic foo bar" -> ("magic", "foo bar")
+ *
* @param codeBlob a string of code separated by newlines
* @return (magicName, args)
*/
@@ -61,13 +66,16 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Given a line of code representing a magic invocation determines whether
* the magic has an implementation.
+ *
* @param codeLine a single line of code
* @return true if the magic exists and is a line magic
*/
protected[magic] def isValidLineMagic(codeLine: String): Boolean = {
parseMagic(codeLine) match {
case Some((magicName, _)) =>
- isLineMagic(codeLine) && magicLoader.hasLineMagic(magicName)
+ isLineMagic(codeLine) && Try(magicManager.isLineMagic(
+ magicManager.findMagic(magicName))
+ ).getOrElse(false)
case None => false
}
}
@@ -75,6 +83,7 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Given a blob of code, finds any magic invocations of magics that don't
* exist.
+ *
* @param codeBlob a string of code separated by newlines
* @return invalid magic names from the given code blob
*/
@@ -88,6 +97,7 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Formats a given magic name and args to code for a kernel method call.
+ *
* @param magicName the name of the magic
* @param args the arguments to the magic
* @return equivalent kernel method call
@@ -98,6 +108,7 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Formats a given line of code representing a line magic invocation into an
* equivalent kernel object call if the magic invocation is valid.
+ *
* @param codeLine the line of code to convert.
* @return a substituted line of code if valid else the original line
*/
@@ -114,13 +125,15 @@ class MagicParser(magicLoader: MagicLoader) {
* Formats a given code blob representing a cell magic invocation into an
* equivalent kernel object call if the cell magic invocation is valid. An
* error message is returned if not.
+ *
* @param codeBlob the blob of code representing a cell magic invocation
* @return Left(the substituted code) or Right(error message)
*/
protected[magic] def parseCell(codeBlob: String): Either[String, String] = {
parseMagic(codeBlob.trim) match {
case Some((cellMagicName, args)) =>
- magicLoader.hasCellMagic(cellMagicName) match {
+ val m = Try(magicManager.findMagic(cellMagicName))
+ m.map(magicManager.isCellMagic).getOrElse(false) match {
case true => Left(substitute(cellMagicName, args))
case false => Right(s"Magic $cellMagicName does not exist!")
}
@@ -132,6 +145,7 @@ class MagicParser(magicLoader: MagicLoader) {
* Parses all lines in a given code blob and either substitutes equivalent
* kernel object calls for each line magic in the code blob OR returns
* an error message if any of the line magic invocations were invalid.
+ *
* @param codeBlob a string of code separated by newlines
* @return Left(code blob with substitutions) or Right(error message)
*/
@@ -149,6 +163,7 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Parses a given code blob and returns an equivalent blob with substitutions
* for magic invocations, if any, or an error string.
+ *
* @param codeBlob the blob of code to parse
* @return Left(parsed code) or Right(error message)
*/
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelay.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelay.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelay.scala
index 022a426..5ac3af9 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelay.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelay.scala
@@ -27,7 +27,7 @@ import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content._
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.magic.{PostProcessor, MagicParser}
-import org.apache.toree.magic.MagicLoader
+import org.apache.toree.plugins.PluginManager
import org.apache.toree.utils.LogLike
import scala.concurrent.Future
@@ -35,7 +35,7 @@ import scala.concurrent.duration._
case class ExecuteRequestRelay(
actorLoader: ActorLoader,
- magicLoader: MagicLoader,
+ pluginManager: PluginManager,
magicParser: MagicParser,
postProcessor: PostProcessor
)
@@ -47,6 +47,7 @@ case class ExecuteRequestRelay(
/**
* Takes an ExecuteFailure and (ExecuteReply, ExecuteResult) with contents
* dictated by the type of failure (either an error or an abort).
+ *
* @param failure the failure
* @return (ExecuteReply, ExecuteResult)
*/
@@ -68,6 +69,7 @@ case class ExecuteRequestRelay(
/**
* Packages the response into an ExecuteReply,ExecuteResult tuple.
+ *
* @param future The future containing either the output or failure
* @return The tuple representing the proper response
*/
@@ -96,7 +98,11 @@ case class ExecuteRequestRelay(
val oldSender = sender()
// Sets the outputStream for this particular ExecuteRequest
- magicLoader.dependencyMap.setOutputStream(outputStream)
+ import org.apache.toree.plugins.Implicits._
+ pluginManager.fireEventFirstResult(
+ "newOutputStream",
+ "outputStream" -> outputStream
+ )
// Parse the code for magics before sending it to the interpreter and
// pipe the response to sender