You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lb...@apache.org on 2016/03/29 20:03:09 UTC
[2/2] incubator-toree git commit: Converted magics to plugins
Converted magics to plugins
Fixed plugin test classpath to run script
Updated internal plugin loading to use only Toree-related classes
Updated compilation to use Java 1.6 to avoid issues on JDK 8
Fixed kernel hanging on shutdown
Added JVM forking options for sbt
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/9c8824ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/9c8824ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/9c8824ff
Branch: refs/heads/master
Commit: 9c8824fff063ff6e84b2e4298088954ae610612b
Parents: 429c74c
Author: Chip Senkbeil and Corey Stubbs <chip.senkbeil@gmail.com & cas5542@gmail.com>
Authored: Fri Mar 25 15:03:22 2016 -0500
Committer: Chip Senkbeil and Corey Stubbs <chip.senkbeil@gmail.com & cas5542@gmail.com>
Committed: Tue Mar 29 12:20:04 2016 -0500
----------------------------------------------------------------------
.jvmopts | 11 +
Makefile | 1 +
etc/bin/run.sh | 5 +-
.../toree/interpreter/broker/BrokerState.scala | 4 +-
.../scala/org/apache/toree/magic/Magic.scala | 24 +-
.../org/apache/toree/magic/MagicExecutor.scala | 51 ---
.../org/apache/toree/magic/MagicLoader.scala | 138 --------
.../org/apache/toree/magic/MagicManager.scala | 111 +++++++
.../toree/magic/MagicNotFoundException.scala | 22 ++
.../magic/dependencies/DependencyMap.scala | 171 ----------
.../magic/dependencies/IncludeConfig.scala | 8 +-
.../IncludeDependencyDownloader.scala | 9 +-
.../magic/dependencies/IncludeInterpreter.scala | 10 +-
.../magic/dependencies/IncludeKernel.scala | 8 +-
.../dependencies/IncludeKernelInterpreter.scala | 11 +-
.../magic/dependencies/IncludeMagicLoader.scala | 31 --
.../dependencies/IncludeOutputStream.scala | 12 +-
.../dependencies/IncludePluginManager.scala | 30 ++
.../magic/dependencies/IncludeSQLContext.scala | 12 +-
.../dependencies/IncludeSparkContext.scala | 13 +-
.../apache/toree/magic/MagicLoaderSpec.scala | 184 -----------
.../apache/toree/magic/MagicManagerSpec.scala | 324 +++++++++++++++++++
.../src/test/scala/test/utils/SomeMagic.scala | 25 ++
.../org/apache/toree/boot/KernelBootstrap.scala | 5 +-
.../boot/layer/ComponentInitialization.scala | 89 ++---
.../boot/layer/HandlerInitialization.scala | 28 +-
.../toree/boot/layer/HookInitialization.scala | 4 -
.../org/apache/toree/kernel/api/Kernel.scala | 44 +--
.../kernel/protocol/v5/magic/MagicParser.scala | 23 +-
.../protocol/v5/relay/ExecuteRequestRelay.scala | 12 +-
.../apache/toree/magic/builtin/AddDeps.scala | 5 +-
.../org/apache/toree/magic/builtin/AddJar.scala | 26 +-
.../org/apache/toree/magic/builtin/Html.scala | 6 +-
.../apache/toree/magic/builtin/JavaScript.scala | 6 +-
.../apache/toree/magic/builtin/LSMagic.scala | 4 +-
.../org/apache/toree/magic/builtin/RDD.scala | 2 +
.../apache/toree/magic/builtin/ShowTypes.scala | 4 +-
.../apache/toree/magic/builtin/Truncation.scala | 4 +-
.../apache/toree/kernel/api/KernelSpec.scala | 15 +-
.../protocol/v5/magic/MagicParserSpec.scala | 86 +++--
.../v5/relay/ExecuteRequestRelaySpec.scala | 29 +-
.../apache/toree/magic/builtin/AddJarSpec.scala | 15 +-
.../scala/system/KernelCommSpecForSystem.scala | 13 +-
.../src/test/scala/system/SuiteForSystem.scala | 8 +-
.../src/test/scala/system/TruncationTests.scala | 19 +-
.../test/utils/NoArgSparkKernelTestKit.scala | 1 -
.../scala/test/utils/SparkKernelDeployer.scala | 19 +-
.../scala/org/apache/toree/plugins/Plugin.scala | 28 +-
.../apache/toree/plugins/PluginManager.scala | 4 +-
.../org/apache/toree/plugins/PluginMethod.scala | 3 -
.../apache/toree/plugins/PluginSearcher.scala | 26 +-
.../PluginManagerSpecForIntegration.scala | 9 +
.../toree/plugins/PluginManagerSpec.scala | 8 +-
.../org/apache/toree/plugins/PluginSpec.scala | 28 +-
project/Build.scala | 14 +-
project/Common.scala | 29 +-
project/plugins.sbt | 2 +-
.../apache/toree/magic/builtin/PySpark.scala | 2 +
resources/compile/log4j.properties | 1 +
.../org/apache/toree/magic/builtin/Scala.scala | 4 +-
.../org/apache/toree/magic/builtin/SparkR.scala | 3 +
.../org/apache/toree/magic/builtin/Sql.scala | 3 +
62 files changed, 964 insertions(+), 882 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/.jvmopts
----------------------------------------------------------------------
diff --git a/.jvmopts b/.jvmopts
new file mode 100644
index 0000000..f22a607
--- /dev/null
+++ b/.jvmopts
@@ -0,0 +1,11 @@
+-Xms1024M
+-Xmx4096M
+-Xss2m
+-XX:MaxPermSize=1024M
+-XX:ReservedCodeCacheSize=256M
+-XX:+TieredCompilation
+-XX:+CMSPermGenSweepingEnabled
+-XX:+CMSClassUnloadingEnabled
+-XX:+UseConcMarkSweepGC
+-XX:+HeapDumpOnOutOfMemoryError
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/Makefile
----------------------------------------------------------------------
diff --git a/Makefile b/Makefile
index 530b41c..cdf525a 100644
--- a/Makefile
+++ b/Makefile
@@ -72,6 +72,7 @@ clean-dist:
clean: VM_WORKDIR=/src/toree-kernel
clean: clean-dist
$(call RUN,$(ENV_OPTS) sbt clean)
+ rm -r `find . -name target -type d`
.example-image: EXTRA_CMD?=pip install jupyter_declarativewidgets==0.4.0; jupyter declarativewidgets install --user; jupyter declarativewidgets activate; pip install jupyter_dashboards; jupyter dashboards install --user; jupyter dashboards activate; apt-get update; apt-get install --yes curl; curl --silent --location https://deb.nodesource.com/setup_0.12 | sudo bash -; apt-get install --yes nodejs; npm install -g bower;
.example-image:
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/etc/bin/run.sh
----------------------------------------------------------------------
diff --git a/etc/bin/run.sh b/etc/bin/run.sh
index bdeb402..67b5a29 100755
--- a/etc/bin/run.sh
+++ b/etc/bin/run.sh
@@ -30,7 +30,8 @@ KERNEL_ASSEMBLY=`(cd ${PROG_HOME}/lib; ls -1 toree-kernel-assembly-*.jar;)`
# disable randomized hash for string in Python 3.3+
export PYTHONHASHSEED=0
-
+TOREE_ASSEMBLY=${PROG_HOME}/lib/${KERNEL_ASSEMBLY}
exec "$SPARK_HOME"/bin/spark-submit \
${SPARK_OPTS} \
- --class org.apache.toree.Main $PROG_HOME/lib/${KERNEL_ASSEMBLY} "$@"
+ --driver-class-path ${TOREE_ASSEMBLY} \
+ --class org.apache.toree.Main ${TOREE_ASSEMBLY} "$@"
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
index a07f724..f97152d 100644
--- a/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
import org.apache.toree.interpreter.broker.BrokerTypes._
import org.slf4j.LoggerFactory
-import scala.concurrent.{Future, promise}
+import scala.concurrent.{Promise, Future, promise}
/**
* Represents the state structure of broker.
@@ -57,7 +57,7 @@ class BrokerState(private val maxQueuedCode: Int) {
// Generate our promise that will be fulfilled when the code is executed
// and the results are sent back
- val codeExecutionPromise = promise[CodeResults]()
+ val codeExecutionPromise = Promise[CodeResults]()
// Build the code representation to send to Broker
val uniqueId = java.util.UUID.randomUUID().toString
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/Magic.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/Magic.scala b/kernel-api/src/main/scala/org/apache/toree/magic/Magic.scala
index 584202b..5f18a8f 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/Magic.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/Magic.scala
@@ -17,14 +17,18 @@
package org.apache.toree.magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.{Event, DepName}
+
/**
- * Represents the base structure for a magic that is loaded and executed.
- */
-trait Magic {
- /**
- * Execute a magic.
- * @param code The code
- * @return The output of the magic
- */
- def execute(code: String): Any
- }
+ * Represents the base structure for a magic that is loaded and executed.
+ */
+trait Magic extends Plugin {
+ /**
+ * Execute a magic.
+ *
+ * @param code The code
+ * @return The output of the magic
+ */
+ def execute(code: String): Any
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/MagicExecutor.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/MagicExecutor.scala b/kernel-api/src/main/scala/org/apache/toree/magic/MagicExecutor.scala
deleted file mode 100644
index e1e1372..0000000
--- a/kernel-api/src/main/scala/org/apache/toree/magic/MagicExecutor.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.toree.magic
-
-import org.apache.toree.utils.DynamicReflectionSupport
-
-import scala.language.dynamics
-
-class MagicExecutor(magicLoader: MagicLoader) extends Dynamic {
-
- val executeMethod = classOf[Magic].getDeclaredMethods.head.getName
-
- def applyDynamic(name: String)(args: Any*): Either[CellMagicOutput, LineMagicOutput] = {
- val className = magicLoader.magicClassName(name)
- val isCellMagic = magicLoader.hasCellMagic(className)
- val isLineMagic = magicLoader.hasLineMagic(className)
-
- (isCellMagic, isLineMagic) match {
- case (true, false) =>
- val result = executeMagic(className, args)
- Left(result.asInstanceOf[CellMagicOutput])
- case (false, true) =>
- executeMagic(className, args)
- Right(LineMagicOutput)
- case (_, _) =>
- Left(CellMagicOutput("text/plain" ->
- s"Magic ${className} could not be executed."))
- }
- }
-
- private def executeMagic(className: String, args: Seq[Any]) = {
- val inst = magicLoader.createMagicInstance(className)
- val dynamicSupport = new DynamicReflectionSupport(inst.getClass, inst)
- dynamicSupport.applyDynamic(executeMethod)(args)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/MagicLoader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/MagicLoader.scala b/kernel-api/src/main/scala/org/apache/toree/magic/MagicLoader.scala
deleted file mode 100644
index e8b706c..0000000
--- a/kernel-api/src/main/scala/org/apache/toree/magic/MagicLoader.scala
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.toree.magic
-
-import java.net.{URL, URLClassLoader}
-
-import com.google.common.reflect.ClassPath
-import org.apache.toree.magic.dependencies.DependencyMap
-
-import scala.reflect.runtime.{universe => runtimeUniverse}
-import scala.collection.JavaConversions._
-
-class MagicLoader(
- var dependencyMap: DependencyMap = new DependencyMap(),
- urls: Array[URL] = Array(),
- parentLoader: ClassLoader = null
-) extends URLClassLoader(urls, parentLoader) {
- private val magicPackage = "org.apache.toree.magic.builtin"
-
- /**
- * Checks whether a magic with a given name, implementing a given interface,
- * exists.
- * @param name case insensitive magic name
- * @param interface interface
- * @return true if a magic with the given name and interface exists
- */
- private def hasSpecificMagic(name: String, interface: Class[_]) : Boolean = {
- val className = magicClassName(name)
- try {
- val clazz = loadClass(className)
- clazz.getInterfaces.contains(interface)
- } catch {
- case _: Throwable => false
- }
- }
-
- /**
- * Checks whether a line magic exists.
- * @param name case insensitive line magic name
- * @return true if the line magic exists
- */
- def hasLineMagic(name: String): Boolean =
- hasSpecificMagic(name, classOf[LineMagic])
-
- /**
- * Checks whether a cell magic exists.
- * @param name case insensitive cell magic name
- * @return true if the cell magic exists
- */
- def hasCellMagic(name: String): Boolean =
- hasSpecificMagic(name, classOf[CellMagic])
-
- /**
- * Attempts to load a class with a given name from a package.
- * @param name the name of the class
- * @param resolve whether to resolve the class or not
- * @return the class if found
- */
- override def loadClass(name: String, resolve: Boolean): Class[_] =
- try {
- super.loadClass(magicPackage + "." + name, resolve)
- } catch {
- case ex: ClassNotFoundException =>
- super.loadClass(name, resolve)
- }
-
- /**
- * Returns the class name for a case insensitive magic name query.
- * If no match is found, returns the query.
- * @param query a magic name, e.g. jAvasCRipt
- * @return the queried magic name's corresponding class, e.g. JavaScript
- */
- def magicClassName(query: String): String = {
- lowercaseClassMap(magicClassNames).getOrElse(query.toLowerCase, query)
- }
-
- /**
- * @return list of magic class names in magicPackage.
- */
- protected def magicClassNames : List[String] = {
- val classPath: ClassPath = ClassPath.from(this)
- val classes = classPath.getTopLevelClasses(magicPackage)
- classes.asList.map(_.getSimpleName).toList
- }
-
- /**
- * @param names list of class names
- * @return map of lowercase class names to class names
- */
- private def lowercaseClassMap(names: List[String]): Map[String, String] = {
- names.map(n => (n.toLowerCase, n)).toMap
- }
-
- def addJar(jar: URL) = addURL(jar)
- /**
- * Creates a instance of the specified magic with dependencies added.
- * @param name name of magic class
- * @return instance of the Magic corresponding to the given name
- */
- protected[magic] def createMagicInstance(name: String): Any = {
- val magicClass = loadClass(name) // Checks parent loadClass first
-
- val runtimeMirror = runtimeUniverse.runtimeMirror(this)
- val classSymbol = runtimeMirror.staticClass(magicClass.getCanonicalName)
- val classMirror = runtimeMirror.reflectClass(classSymbol)
- val selfType = classSymbol.selfType
-
- val classConstructorSymbol =
- selfType.declaration(runtimeUniverse.nme.CONSTRUCTOR).asMethod
- val classConstructorMethod =
- classMirror.reflectConstructor(classConstructorSymbol)
-
- val magicInstance = classConstructorMethod()
-
-
- // Add all of our dependencies to the new instance
- dependencyMap.internalMap.filter(selfType <:< _._1).values.foreach(
- _(magicInstance.asInstanceOf[Magic])
- )
-
- magicInstance
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/MagicManager.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/MagicManager.scala b/kernel-api/src/main/scala/org/apache/toree/magic/MagicManager.scala
new file mode 100644
index 0000000..51a38c2
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/MagicManager.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.toree.magic
+
+import org.apache.toree.plugins.{Plugin, PluginMethodResult, PluginManager}
+import org.slf4j.LoggerFactory
+
+import scala.annotation.tailrec
+import scala.language.dynamics
+import scala.runtime.BoxedUnit
+import scala.util.{Try, Failure, Success}
+
+class MagicManager(private val pluginManager: PluginManager) extends Dynamic {
+ protected val logger = LoggerFactory.getLogger(this.getClass.getName)
+ /**
+ * Checks if the provided magic is a line magic.
+ *
+ * @param magic The magic instance
+ * @return True if the magic is an instance of a line magic
+ */
+ def isLineMagic(magic: Magic): Boolean =
+ magic.getClass.getInterfaces.contains(classOf[LineMagic])
+
+ /**
+ * Checks if the provided magic is a cell magic.
+ *
+ * @param magic The magic instance
+ * @return True if the magic is an instance of a cell magic
+ */
+ def isCellMagic(magic: Magic): Boolean =
+ magic.getClass.getInterfaces.contains(classOf[CellMagic])
+
+ /**
+ * Finds a magic whose class name ends with a case insensitive name.
+ *
+ * @param name The name to search for
+ * @return The magic
+ * @throws MagicNotFoundException when no magics match name
+ */
+ @throws[MagicNotFoundException]
+ def findMagic(name: String): Magic = {
+ @tailrec def inheritsMagic(klass: Class[_]): Boolean = {
+ if (klass == null) false
+ else if (klass.getInterfaces.exists(classOf[Magic].isAssignableFrom)) true
+ else inheritsMagic(klass.getSuperclass)
+ }
+
+ val magics = pluginManager.plugins
+ .filter(p => inheritsMagic(p.getClass))
+ .filter(_.simpleName.split("\\.").last.toLowerCase == name.toLowerCase)
+
+ if (magics.size <= 0){
+ logger.error(s"No magic found for $name!")
+ throw new MagicNotFoundException(name)
+ } else if (magics.size > 1) {
+ logger.warn(s"More than one magic found for $name!")
+ }
+
+ magics.head.asInstanceOf[Magic]
+ }
+
+ @throws[MagicNotFoundException]
+ def applyDynamic(name: String)(args: Any*): Either[CellMagicOutput, LineMagicOutput] = {
+ val arg = args.headOption.map(_.toString).getOrElse("")
+
+ import org.apache.toree.plugins.Implicits._
+ val result = pluginManager.fireEventFirstResult(
+ name.toLowerCase(),
+ "input" -> arg
+ )
+
+ result match {
+ case Some(r: PluginMethodResult) => handleMagicResult(name, r.toTry)
+ case None => throw new MagicNotFoundException(name)
+ }
+ }
+
+ private def handleMagicResult(name: String, result: Try[Any]) = result match {
+ case Success(magicOutput) => magicOutput match {
+ case null | _: BoxedUnit => Right(LineMagicOutput)
+ case cmo: CellMagicOutput => Left(cmo)
+ case unknown =>
+ val message =
+ s"""Magic ${name} did not return proper magic output
+ |type. Expected ${classOf[CellMagicOutput].getName} or
+ |${classOf[LineMagicOutput].getName}, but found type of
+ |${unknown.getClass.getName}.""".trim.stripMargin
+ logger.warn(message)
+ Left(CellMagicOutput("text/plain" -> message))
+ }
+ case Failure(t) =>
+ val message = s"Magic ${name} failed to execute with error: \n${t.getMessage}"
+ logger.warn(message)
+ Left(CellMagicOutput("text/plain" -> message))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/MagicNotFoundException.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/MagicNotFoundException.scala b/kernel-api/src/main/scala/org/apache/toree/magic/MagicNotFoundException.scala
new file mode 100644
index 0000000..c5e7d85
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/MagicNotFoundException.scala
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.toree.magic
+
+class MagicNotFoundException(name: String) extends Exception(
+ s"Magic not found with name $name!"
+)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/DependencyMap.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/DependencyMap.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/DependencyMap.scala
deleted file mode 100644
index d136811..0000000
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/DependencyMap.scala
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.toree.magic.dependencies
-
-import java.io.OutputStream
-
-import org.apache.toree.interpreter.Interpreter
-import org.apache.toree.kernel.api.KernelLike
-import org.apache.toree.magic.{MagicLoader, Magic}
-import com.typesafe.config.Config
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.SQLContext
-
-import scala.reflect.runtime.universe._
-import org.apache.toree.dependencies.DependencyDownloader
-
-/**
- * Represents a mapping of dependency types to implementations.
- *
- * TODO: Explore Scala macros to avoid duplicate code.
- */
-class DependencyMap {
- val internalMap =
- scala.collection.mutable.Map[Type, PartialFunction[Magic, Unit]]()
-
- /**
- * Sets the Interpreter for this map.
- * @param interpreter The new Interpreter
- */
- def setInterpreter(interpreter: Interpreter) = {
- internalMap(typeOf[IncludeInterpreter]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeInterpreter].interpreter_=(interpreter)
- )
-
- this
- }
-
- /**
- * Sets the Interpreter for this map.
- * @param interpreter The new Interpreter
- */
- //@deprecated("Use setInterpreter with IncludeInterpreter!", "2015.05.06")
- def setKernelInterpreter(interpreter: Interpreter) = {
- internalMap(typeOf[IncludeKernelInterpreter]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeKernelInterpreter].kernelInterpreter_=(interpreter)
- )
-
- this
- }
-
- /**
- * Sets the SparkContext for this map.
- * @param sparkContext The new SparkContext
- */
- def setSparkContext(sparkContext: SparkContext) = {
- internalMap(typeOf[IncludeSparkContext]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeSparkContext].sparkContext_=(sparkContext)
- )
-
- this
- }
-
- /**
- * Sets the SQLContext for this map.
- * @param sqlContext The new SQLContext
- */
- def setSQLContext(sqlContext: SQLContext) = {
- internalMap(typeOf[IncludeSQLContext]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeSQLContext].sqlContext_=(sqlContext)
- )
-
- this
- }
-
- /**
- * Sets the OutputStream for this map.
- * @param outputStream The new OutputStream
- */
- def setOutputStream(outputStream: OutputStream) = {
- internalMap(typeOf[IncludeOutputStream]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeOutputStream].outputStream_=(outputStream)
- )
-
- this
- }
-
- /**
- * Sets the DependencyDownloader for this map.
- * @param dependencyDownloader The new DependencyDownloader
- */
- def setDependencyDownloader(dependencyDownloader: DependencyDownloader) = {
- internalMap(typeOf[IncludeDependencyDownloader]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeDependencyDownloader]
- .dependencyDownloader_=(dependencyDownloader)
- )
-
- this
- }
-
- /**
- * Sets the Kernel Object for this map.
- * @param kernel The new Kernel
- */
- def setKernel(kernel: KernelLike) = {
- internalMap(typeOf[IncludeKernel]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeKernel]
- .kernel_=(kernel)
- )
-
- this
- }
-
- /**
- * Sets the MagicLoader for this map.
- * @param magicLoader The new MagicLoader
- */
- def setMagicLoader(magicLoader: MagicLoader) = {
- internalMap(typeOf[IncludeMagicLoader]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeMagicLoader]
- .magicLoader_=(magicLoader)
- )
-
- this
- }
-
- /**
- * Sets the Config Object for this map.
- * @param config The config for the kernel
- */
- def setConfig(config: Config) = {
- internalMap(typeOf[IncludeConfig]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeConfig]
- .config=(config)
- )
-
- this
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeConfig.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeConfig.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeConfig.scala
index 44c4d06..af8c593 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeConfig.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeConfig.scala
@@ -19,13 +19,15 @@ package org.apache.toree.magic.dependencies
import org.apache.toree.magic.{Magic}
import com.typesafe.config.Config
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.Init
-trait IncludeConfig {
+trait IncludeConfig extends Plugin {
this: Magic =>
+ @Init protected def init(config: Config) = _config = config
+
private var _config: Config = _
def config: Config = _config
- def config_= (newConfig: Config) =
- _config = newConfig
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeDependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeDependencyDownloader.scala
index 6f39137..1d4fb15 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeDependencyDownloader.scala
@@ -19,12 +19,15 @@ package org.apache.toree.magic.dependencies
import org.apache.toree.dependencies.DependencyDownloader
import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.Init
-trait IncludeDependencyDownloader {
+trait IncludeDependencyDownloader extends Plugin {
this: Magic =>
+ @Init protected def init(newDependencyDownloader: DependencyDownloader) =
+ _dependencyDownloader = newDependencyDownloader
+
private var _dependencyDownloader: DependencyDownloader = _
def dependencyDownloader: DependencyDownloader = _dependencyDownloader
- def dependencyDownloader_=(newDependencyDownloader: DependencyDownloader) =
- _dependencyDownloader = newDependencyDownloader
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeInterpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeInterpreter.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeInterpreter.scala
index cf60540..78be9ce 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeInterpreter.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeInterpreter.scala
@@ -19,13 +19,15 @@ package org.apache.toree.magic.dependencies
import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.Init
-trait IncludeInterpreter {
+trait IncludeInterpreter extends Plugin {
this: Magic =>
- //val interpreter: Interpreter
+ @Init protected def init(newInterpreter: Interpreter) =
+ _interpreter = newInterpreter
+
private var _interpreter: Interpreter = _
def interpreter: Interpreter = _interpreter
- def interpreter_=(newInterpreter: Interpreter) =
- _interpreter = newInterpreter
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernel.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernel.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernel.scala
index 965769f..af50fb0 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernel.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernel.scala
@@ -19,12 +19,14 @@ package org.apache.toree.magic.dependencies
import org.apache.toree.kernel.api.KernelLike
import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.Init
-trait IncludeKernel {
+trait IncludeKernel extends Plugin {
this: Magic =>
+ @Init protected def init(newKernel: KernelLike) = _kernel = newKernel
+
private var _kernel: KernelLike = _
def kernel: KernelLike = _kernel
- def kernel_=(newKernel: KernelLike) =
- _kernel = newKernel
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernelInterpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernelInterpreter.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernelInterpreter.scala
index 1bc33aa..9cedf10 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernelInterpreter.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeKernelInterpreter.scala
@@ -19,14 +19,15 @@ package org.apache.toree.magic.dependencies
import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.Init
-//@deprecated("Use IncludeInterpreter instead!", "2015.05.06")
-trait IncludeKernelInterpreter {
+trait IncludeKernelInterpreter extends Plugin {
this: Magic =>
- //val interpreter: Interpreter
+ @Init protected def init(newInterpreter: Interpreter) =
+ _interpreter = newInterpreter
+
private var _interpreter: Interpreter = _
def kernelInterpreter: Interpreter = _interpreter
- def kernelInterpreter_=(newInterpreter: Interpreter) =
- _interpreter = newInterpreter
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeMagicLoader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeMagicLoader.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeMagicLoader.scala
deleted file mode 100644
index b615629..0000000
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeMagicLoader.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.toree.magic.dependencies
-
-import org.apache.toree.magic.{MagicLoader, Magic}
-
-
-trait IncludeMagicLoader {
- this: Magic =>
-
- //val sparkContext: SparkContext
- private var _magicLoader: MagicLoader = _
- def magicLoader: MagicLoader = _magicLoader
- def magicLoader_=(newMagicLoader: MagicLoader) =
- _magicLoader = newMagicLoader
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeOutputStream.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeOutputStream.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeOutputStream.scala
index 2525dcc..51fc35e 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeOutputStream.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeOutputStream.scala
@@ -19,14 +19,18 @@ package org.apache.toree.magic.dependencies
import java.io.OutputStream
+import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.{DepName, Event, Init}
-trait IncludeOutputStream {
+trait IncludeOutputStream extends Plugin {
this: Magic =>
- //val outputStream: OutputStream
+ @Event(name = "newOutputStream") protected def newOutputStream(
+ @DepName(name = "outputStream") newOutputStream: OutputStream
+ ) = _outputStream = newOutputStream
+
private var _outputStream: OutputStream = _
def outputStream: OutputStream = _outputStream
- def outputStream_=(newOutputStream: OutputStream) =
- _outputStream = newOutputStream
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludePluginManager.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludePluginManager.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludePluginManager.scala
new file mode 100644
index 0000000..29b5b84
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludePluginManager.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.toree.magic.dependencies
+
+import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.{PluginManager, Plugin}
+import org.apache.toree.plugins.annotations.Init
+
+trait IncludePluginManager extends Plugin { this: Magic =>
+ @Init protected def init(pluginManager: PluginManager) =
+ _pluginManager = pluginManager
+
+ private var _pluginManager: PluginManager = _
+ def pluginManager: PluginManager = _pluginManager
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSQLContext.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSQLContext.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSQLContext.scala
index 91b8348..36117a7 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSQLContext.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSQLContext.scala
@@ -17,14 +17,18 @@
package org.apache.toree.magic.dependencies
-import org.apache.toree.magic.Magic
import org.apache.spark.sql.SQLContext
+import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.{Event, Init}
-trait IncludeSQLContext {
+trait IncludeSQLContext extends Plugin {
this: Magic =>
+ @Event(name = "sparkReady") protected def sparkReady(
+ newSqlContext: SQLContext
+ ) = _sqlContext = newSqlContext
+
private var _sqlContext: SQLContext = _
def sqlContext: SQLContext = _sqlContext
- def sqlContext_=(newSqlContext: SQLContext) =
- _sqlContext = newSqlContext
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSparkContext.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSparkContext.scala b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSparkContext.scala
index f0f0473..92dfe05 100644
--- a/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSparkContext.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/magic/dependencies/IncludeSparkContext.scala
@@ -17,15 +17,18 @@
package org.apache.toree.magic.dependencies
-import org.apache.toree.magic.Magic
import org.apache.spark.SparkContext
+import org.apache.toree.magic.Magic
+import org.apache.toree.plugins.Plugin
+import org.apache.toree.plugins.annotations.{Event, Init}
-trait IncludeSparkContext {
+trait IncludeSparkContext extends Plugin {
this: Magic =>
- //val sparkContext: SparkContext
+ @Event(name = "sparkReady") protected def sparkReady(
+ newSparkContext: SparkContext
+ ) = _sparkContext = newSparkContext
+
private var _sparkContext: SparkContext = _
def sparkContext: SparkContext = _sparkContext
- def sparkContext_=(newSparkContext: SparkContext) =
- _sparkContext = newSparkContext
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/test/scala/org/apache/toree/magic/MagicLoaderSpec.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/test/scala/org/apache/toree/magic/MagicLoaderSpec.scala b/kernel-api/src/test/scala/org/apache/toree/magic/MagicLoaderSpec.scala
deleted file mode 100644
index 8e03881..0000000
--- a/kernel-api/src/test/scala/org/apache/toree/magic/MagicLoaderSpec.scala
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-
-package org.apache.toree.magic
-
-import java.io.OutputStream
-
-import org.apache.toree.dependencies.DependencyDownloader
-import org.apache.toree.interpreter.Interpreter
-import org.apache.toree.magic.dependencies._
-import org.apache.spark.SparkContext
-import org.scalatest.mock.MockitoSugar
-import org.scalatest.{FunSpec, Matchers}
-
-
-/**
-* Used for verification of dependency injection. Calls toString on each
-* dependency to assert that they are provided.
-*/
-class LineMagicWithDependencies extends LineMagic
- with IncludeDependencyDownloader
- with IncludeSparkContext
- with IncludeInterpreter
- with IncludeOutputStream
-{
- override def execute(code: String): Unit = {
- sparkContext.cancelAllJobs()
- interpreter.classServerURI
- outputStream.close()
- dependencyDownloader.setPrintStream(null)
- }
-}
-
-class MockLineMagic extends LineMagic {
- override def execute(code: String): Unit = {}
-}
-
-class MockCellMagic extends CellMagic {
- override def execute(code: String): CellMagicOutput =
- CellMagicOutput()
-}
-
-class MagicLoaderSpec extends FunSpec with Matchers with MockitoSugar {
- describe("MagicLoader") {
- describe("#hasLineMagic") {
- it("should return false if a class with the magic name is not found") {
- val magicLoader = new MagicLoader() {
- override def findClass(name: String): Class[_] =
- throw new ClassNotFoundException()
- }
-
- magicLoader.hasLineMagic("potato") should be (false)
- }
-
- it("should return true if a class with the magic name is found") {
- val magicLoader = new MagicLoader() {
- override def findClass(name: String): Class[_] =
- classOf[MockLineMagic]
- }
-
- magicLoader.hasLineMagic("potato") should be (true)
- }
-
- it("should return true if a class with the magic name is found regardless of case"){
- // Only loads a class named "Potato"
- val classLoader = new ClassLoader() {
- override def findClass(name: String) =
- if (name == "Potato") classOf[MockLineMagic]
- else throw new ClassNotFoundException
- }
-
- // Case insensitive matching should be performed on "Potato"
- val magicLoader = new MagicLoader(parentLoader = classLoader) {
- override def magicClassNames = List("Potato")
- }
-
- magicLoader.hasLineMagic("Potato") should be (true)
- magicLoader.hasLineMagic("potato") should be (true)
- magicLoader.hasLineMagic("pOTatO") should be (true)
- }
- }
-
- describe("#hasCellMagic") {
- it("should return false if a class with the magic name is not found") {
- val magicLoader = new MagicLoader() {
- override def findClass(name: String): Class[_] =
- throw new ClassNotFoundException()
- }
-
- magicLoader.hasCellMagic("potato") should be (false)
- }
-
- it("should return true if a class with the magic name is found") {
- val magicLoader = new MagicLoader() {
- override def findClass(name: String): Class[_] =
- classOf[MockCellMagic]
- }
-
- magicLoader.hasCellMagic("potato") should be (true)
- }
-
- it("should return true if a class with the magic name is found regardless of case"){
- // Only loads a class named "Potato"
- val classLoader = new ClassLoader() {
- override def findClass(name: String) =
- if (name == "Potato") classOf[MockCellMagic]
- else throw new ClassNotFoundException
- }
-
- // Case insensitive matching should be performed on "Potato"
- val magicLoader = new MagicLoader(parentLoader = classLoader) {
- override def magicClassNames = List("Potato")
- }
-
- magicLoader.hasCellMagic("Potato") should be (true)
- magicLoader.hasCellMagic("potato") should be (true)
- magicLoader.hasCellMagic("pOTatO") should be (true)
- }
- }
-
- describe("#magicClassName"){
- it("should return the correctly-cased version of the requested magic name") {
- val magicLoader = new MagicLoader() {
- override def magicClassNames = List("Potato")
- }
-
- magicLoader.magicClassName("Potato") should be ("Potato")
- magicLoader.magicClassName("potato") should be ("Potato")
- magicLoader.magicClassName("pOTatO") should be ("Potato")
- }
-
- it("should return the query if a corresponding magic class does not exist") {
- val magicLoader = new MagicLoader() {
- override def magicClassNames = List()
- }
-
- magicLoader.magicClassName("dne") should be ("dne")
- magicLoader.magicClassName("dNE") should be ("dNE")
- }
- }
-
- describe("#createMagicInstance") {
- it("should correctly insert dependencies into a class") {
- val mockInterpreter = mock[Interpreter]
- val mockSparkContext = mock[SparkContext]
- val mockOutputStream = mock[OutputStream]
- val mockDependencyDownloader = mock[DependencyDownloader]
-
- val dependencyMap = new DependencyMap()
- .setInterpreter(mockInterpreter)
- .setSparkContext(mockSparkContext)
- .setOutputStream(mockOutputStream)
- .setDependencyDownloader(mockDependencyDownloader)
-
- val magicLoader = new MagicLoader(
- dependencyMap = dependencyMap,
- parentLoader = new InternalClassLoader(getClass.getClassLoader)
- )
-
- val magicName = "LineMagicWithDependencies"
- val instance = magicLoader.createMagicInstance(magicName)
- .asInstanceOf[LineMagicWithDependencies]
- instance.interpreter should be(mockInterpreter)
- instance.outputStream should be(mockOutputStream)
- instance.sparkContext should be(mockSparkContext)
- instance.dependencyDownloader should be(mockDependencyDownloader)
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/test/scala/org/apache/toree/magic/MagicManagerSpec.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/test/scala/org/apache/toree/magic/MagicManagerSpec.scala b/kernel-api/src/test/scala/org/apache/toree/magic/MagicManagerSpec.scala
new file mode 100644
index 0000000..5874a86
--- /dev/null
+++ b/kernel-api/src/test/scala/org/apache/toree/magic/MagicManagerSpec.scala
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.toree.magic
+
+import org.apache.toree.plugins.dependencies.Dependency
+import org.apache.toree.plugins._
+import org.mockito.Mockito._
+import org.mockito.Matchers.{eq => mockEq, _}
+import org.scalatest.mock.MockitoSugar
+import org.scalatest.{FunSpec, Matchers, OneInstancePerTest}
+import test.utils
+
+import MagicManagerSpec._
+
+import scala.runtime.BoxedUnit
+
+object MagicManagerSpec {
+ val TestCellMagicOutput = CellMagicOutput("test" -> "value")
+}
+
+class SomeLineMagic extends LineMagic {
+ override def execute(code: String): Unit = {}
+}
+
+class SomeCellMagic extends CellMagic {
+ override def execute(code: String): CellMagicOutput = TestCellMagicOutput
+}
+
+private class SomePlugin extends Plugin
+
+private class SomeMagic extends Magic {
+ override def execute(code: String): Any = ???
+}
+
+class LineMagicException extends Exception
+
+private class ExceptionLineMagic extends LineMagic {
+ override def execute(code: String): Unit = throw new LineMagicException
+}
+
+class CellMagicException extends Exception
+
+private class ExceptionCellMagic extends CellMagic {
+ override def execute(code: String): CellMagicOutput = throw new CellMagicException
+}
+
+class MagicManagerSpec
+ extends FunSpec with Matchers with MockitoSugar with OneInstancePerTest
+{
+ private val TestPluginName = "SomePlugin"
+ private val TestMagicName = "SomeMagic"
+ private val mockPluginManager = mock[PluginManager]
+ private val magicManager = spy(new MagicManager(mockPluginManager))
+
+ describe("MagicManager") {
+ describe("#isLineMagic") {
+ it("should return true if the magic extends the line magic interface") {
+ val expected = true
+
+ val mockLineMagic = mock[LineMagic]
+ val actual = magicManager.isLineMagic(mockLineMagic)
+
+ actual should be (expected)
+ }
+
+ it("should return false if the magic does not extend the line magic interface") {
+ val expected = false
+
+ val mockMagic = mock[Magic]
+ val actual = magicManager.isLineMagic(mockMagic)
+
+ actual should be (expected)
+ }
+
+ it("should throw an exception if provided null") {
+ intercept[NullPointerException] {
+ magicManager.isLineMagic(null)
+ }
+ }
+ }
+
+ describe("#isCellMagic") {
+ it("should return true if the magic extends the cell magic interface") {
+ val expected = true
+
+ val mockCellMagic = mock[CellMagic]
+ val actual = magicManager.isCellMagic(mockCellMagic)
+
+ actual should be (expected)
+ }
+
+ it("should return false if the magic does not extend the cell magic interface") {
+ val expected = false
+
+ val mockMagic = mock[Magic]
+ val actual = magicManager.isCellMagic(mockMagic)
+
+ actual should be (expected)
+ }
+
+ it("should throw an exception if provided null") {
+ intercept[NullPointerException] {
+ magicManager.isCellMagic(null)
+ }
+ }
+ }
+
+ describe("#findMagic") {
+ it("should throw a MagicNotFoundException if no magic matches the name") {
+ intercept[MagicNotFoundException] {
+ doReturn(Seq(new Plugin {}).toIterable).when(mockPluginManager).plugins
+
+ magicManager.findMagic(TestMagicName)
+ }
+ }
+
+ it("should throw a MagicNotFoundException if there are no loaded plugins") {
+ intercept[MagicNotFoundException] {
+ doReturn(Nil).when(mockPluginManager).plugins
+
+ magicManager.findMagic(TestMagicName)
+ }
+ }
+
+ it("should throw a MagicNotFoundException if a plugin matches but is not a magic") {
+ intercept[MagicNotFoundException] {
+ doReturn(Seq(new SomePlugin).toIterable).when(mockPluginManager).plugins
+
+ magicManager.findMagic(TestPluginName)
+ }
+ }
+
+ it("should return the magic if exactly one is found") {
+ val expected = new SomeMagic
+
+ doReturn(Seq(expected).toIterable).when(mockPluginManager).plugins
+ val actual = magicManager.findMagic(TestMagicName)
+
+ actual should be (expected)
+ }
+
+ it("should return a magic whose name matches even if casing is different") {
+ val expected = new SomeMagic
+
+ doReturn(Seq(expected).toIterable).when(mockPluginManager).plugins
+ val actual = magicManager.findMagic(TestMagicName.toUpperCase())
+
+ actual should be (expected)
+ }
+
+ it("should return the first match if more than one magic matches the name") {
+ val expected = new SomeMagic
+
+ doReturn(Seq(expected, new utils.SomeMagic).toIterable)
+ .when(mockPluginManager).plugins
+ val actual = magicManager.findMagic(TestMagicName)
+
+ actual should be (expected)
+ }
+ }
+
+ describe("#applyDynamic") {
+ it("should return CellMagicOutput if the invocation of a magic throws an exception") {
+ doReturn(Some(FailurePluginMethodResult(
+ mock[PluginMethod],
+ new LineMagicException()
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic("TEST")()
+
+ result.isLeft should be(true)
+ }
+
+ it("should fire an event with the lowercase of the magic name") {
+ val arg: java.lang.String = "some arg"
+ val pluginName = "TEST"
+ val expected = Dependency.fromValueWithName("input", arg)
+
+ doReturn(Some(FailurePluginMethodResult(
+ mock[PluginMethod],
+ new LineMagicException()
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ magicManager.applyDynamic(pluginName)(arg :: Nil: _*)
+ verify(mockPluginManager).fireEventFirstResult(mockEq(pluginName.toLowerCase), any())
+ }
+
+ it("should take the first argument and convert it to a string to pass to the magic") {
+ val arg: java.lang.String = "some arg"
+ val pluginName = "TEST"
+ val expected = Dependency.fromValueWithName("input", arg)
+
+ doReturn(Some(FailurePluginMethodResult(
+ mock[PluginMethod],
+ new LineMagicException()
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ magicManager.applyDynamic(pluginName)(arg :: Nil: _*)
+ verify(mockPluginManager).fireEventFirstResult(anyString(), mockEq(Seq(expected)): _*)
+ }
+
+ it("should pass an empty string to the line magic if no arguments are provided") {
+ val arg: java.lang.String = ""
+ val pluginName = "TEST"
+ val expected = Dependency.fromValueWithName("input", arg)
+
+ doReturn(Some(FailurePluginMethodResult(
+ mock[PluginMethod],
+ new LineMagicException()
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ magicManager.applyDynamic(pluginName)(Nil: _*)
+ verify(mockPluginManager).fireEventFirstResult(anyString(), mockEq(Seq(expected)): _*)
+ }
+
+ it("should return a Right[LineMagicOutput] if line magic execution is successful and returns null") {
+ val pluginName = "TEST"
+ val expected = Right(LineMagicOutput)
+
+ doReturn(Some(SuccessPluginMethodResult(
+ mock[PluginMethod],
+ null
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic(pluginName)(Nil: _*)
+ result should be(expected)
+ }
+
+ it("should return a Right[LineMagicOutput] if line magic execution is successful and returns BoxedUnit") {
+ val pluginName = "TEST"
+ val expected = Right(LineMagicOutput)
+
+ doReturn(Some(SuccessPluginMethodResult(
+ mock[PluginMethod],
+ BoxedUnit.UNIT
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic(pluginName)(Nil: _*)
+ result should be(expected)
+ }
+
+ it("should return a Left[CellMagicOutput] if cell magic execution is successful") {
+ val pluginName = "TEST"
+ val cellMagicOutput = CellMagicOutput("our/type" -> "TEST CONTENT")
+ doReturn(Some(SuccessPluginMethodResult(
+ mock[PluginMethod],
+ cellMagicOutput
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic(pluginName)(Nil: _*)
+ result.left.get should be(cellMagicOutput)
+ }
+
+ it("should return a Left[CellMagicOutput] if is a magic but not a line or cell") {
+ val pluginName = "TEST"
+
+ doReturn(Some(SuccessPluginMethodResult(
+ mock[PluginMethod],
+ new AnyRef
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic(pluginName)(Nil: _*)
+ result.left.get("text/plain") should not be (empty)
+
+ }
+
+ it("should return a Left[CellMagicOutput] if magic fails") {
+ val pluginName = "TEST"
+
+ doReturn(Some(FailurePluginMethodResult(
+ mock[PluginMethod],
+ new Throwable
+ ))).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+
+ val result = magicManager.applyDynamic(pluginName)(Nil: _*)
+ result.left.get("text/plain") should not be (empty)
+ }
+
+ it("should throw a MagicNotFoundException when a magic cannot be found") {
+ val pluginName = "THISMAGICDOESN'TEXIST"
+
+ doReturn(None).when(mockPluginManager).fireEventFirstResult(
+ anyString(), any(classOf[Dependency[_ <: AnyRef]])
+ )
+ intercept[MagicNotFoundException] {
+ magicManager.applyDynamic(pluginName)(Nil: _*)
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel-api/src/test/scala/test/utils/SomeMagic.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/test/scala/test/utils/SomeMagic.scala b/kernel-api/src/test/scala/test/utils/SomeMagic.scala
new file mode 100644
index 0000000..ca3a382
--- /dev/null
+++ b/kernel-api/src/test/scala/test/utils/SomeMagic.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package test.utils
+
+import org.apache.toree.magic.Magic
+
+class SomeMagic extends Magic {
+ override def execute(code: String): Any = ???
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala b/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
index b391f20..84a2853 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
@@ -85,7 +85,7 @@ class KernelBootstrap(config: Config) extends LogLike {
// Initialize components needed elsewhere
val (commStorage, commRegistrar, commManager, interpreter,
kernel, dependencyDownloader,
- magicLoader, responseMap) =
+ magicManager, pluginManager, responseMap) =
initializeComponents(
config = config,
appName = DefaultAppName,
@@ -104,7 +104,8 @@ class KernelBootstrap(config: Config) extends LogLike {
interpreter = interpreter,
commRegistrar = commRegistrar,
commStorage = commStorage,
- magicLoader = magicLoader,
+ pluginManager = pluginManager,
+ magicManager = magicManager,
responseMap = responseMap
)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
index e328ceb..d361708 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
@@ -18,30 +18,22 @@
package org.apache.toree.boot.layer
import java.io.File
-import java.util
import java.util.concurrent.ConcurrentHashMap
import akka.actor.ActorRef
-import org.apache.toree.comm.{CommManager, KernelCommManager, CommRegistrar, CommStorage}
-import org.apache.toree.dependencies.{CoursierDependencyDownloader, DependencyDownloader, IvyDependencyDownloader}
-import org.apache.toree.global
+import com.typesafe.config.Config
+import org.apache.toree.comm.{CommManager, CommRegistrar, CommStorage, KernelCommManager}
+import org.apache.toree.dependencies.{CoursierDependencyDownloader, DependencyDownloader}
import org.apache.toree.interpreter._
-import org.apache.toree.kernel.api.{KernelLike, Kernel}
+import org.apache.toree.kernel.api.Kernel
import org.apache.toree.kernel.protocol.v5.KMBuilder
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
-import org.apache.toree.kernel.protocol.v5.stream.KernelOutputStream
-import org.apache.toree.magic.MagicLoader
-import org.apache.toree.magic.builtin.BuiltinLoader
-import org.apache.toree.magic.dependencies.DependencyMap
-import org.apache.toree.utils.{MultiClassLoader, TaskManager, KeyValuePairUtils, LogLike}
-import com.typesafe.config.Config
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkContext, SparkConf}
+import org.apache.toree.magic.MagicManager
+import org.apache.toree.plugins.PluginManager
+import org.apache.toree.utils.LogLike
import scala.collection.JavaConverters._
-import scala.util.Try
-
/**
* Represents the component initialization. All component-related pieces of the
* kernel (non-actors) should be created here. Limited items should be exposed.
@@ -57,7 +49,7 @@ trait ComponentInitialization {
def initializeComponents(
config: Config, appName: String, actorLoader: ActorLoader
): (CommStorage, CommRegistrar, CommManager, Interpreter,
- Kernel, DependencyDownloader, MagicLoader,
+ Kernel, DependencyDownloader, MagicManager, PluginManager,
collection.mutable.Map[String, ActorRef])
}
@@ -84,20 +76,22 @@ trait StandardComponentInitialization extends ComponentInitialization {
val scalaInterpreter = manager.interpreters.get("Scala").orNull
val dependencyDownloader = initializeDependencyDownloader(config)
- val magicLoader = initializeMagicLoader(
+ val pluginManager = createPluginManager(
config, scalaInterpreter, dependencyDownloader)
val kernel = initializeKernel(
- config, actorLoader, manager, commManager, magicLoader
+ config, actorLoader, manager, commManager, pluginManager
)
+ initializePlugins(config, pluginManager)
+
val responseMap = initializeResponseMap()
initializeSparkContext(config, kernel, appName)
(commStorage, commRegistrar, commManager,
manager.defaultInterpreter.orNull, kernel,
- dependencyDownloader, magicLoader, responseMap)
+ dependencyDownloader, kernel.magics, pluginManager, responseMap)
}
@@ -142,14 +136,14 @@ trait StandardComponentInitialization extends ComponentInitialization {
actorLoader: ActorLoader,
interpreterManager: InterpreterManager,
commManager: CommManager,
- magicLoader: MagicLoader
+ pluginManager: PluginManager
) = {
val kernel = new Kernel(
config,
actorLoader,
interpreterManager,
commManager,
- magicLoader
+ pluginManager
)
/*
interpreter.doQuietly {
@@ -159,48 +153,55 @@ trait StandardComponentInitialization extends ComponentInitialization {
)
}
*/
- magicLoader.dependencyMap.setKernel(kernel)
+ pluginManager.dependencyManager.add(kernel)
kernel
}
- private def initializeMagicLoader(
+ private def createPluginManager(
config: Config, interpreter: Interpreter,
dependencyDownloader: DependencyDownloader
) = {
- logger.debug("Constructing magic loader")
+ logger.debug("Constructing plugin manager")
+ val pluginManager = new PluginManager()
logger.debug("Building dependency map")
- val dependencyMap = new DependencyMap()
- .setInterpreter(interpreter)
- .setKernelInterpreter(interpreter) // This is deprecated
- .setDependencyDownloader(dependencyDownloader)
- .setConfig(config)
+ pluginManager.dependencyManager.add(interpreter)
+ pluginManager.dependencyManager.add(dependencyDownloader)
+ pluginManager.dependencyManager.add(config)
+
+ pluginManager.dependencyManager.add(pluginManager)
- logger.debug("Creating BuiltinLoader")
- val builtinLoader = new BuiltinLoader()
+ pluginManager
+ }
+ private def initializePlugins(
+ config: Config,
+ pluginManager: PluginManager
+ ) = {
val magicUrlArray = config.getStringList("magic_urls").asScala
.map(s => new java.net.URL(s)).toArray
if (magicUrlArray.isEmpty)
- logger.warn("No external magics provided to MagicLoader!")
+ logger.warn("No external magics provided to PluginManager!")
else
logger.info("Using magics from the following locations: " +
magicUrlArray.map(_.getPath).mkString(","))
- val multiClassLoader = new MultiClassLoader(
- builtinLoader,
- interpreter.classLoader
- )
+ // Load internal plugins under kernel module
+ logger.debug("Loading internal plugins")
+ val internalPlugins = pluginManager.initialize()
+ logger.info(internalPlugins.size + " internal plugins loaded")
- logger.debug("Creating MagicLoader")
- val magicLoader = new MagicLoader(
- dependencyMap = dependencyMap,
- urls = magicUrlArray,
- parentLoader = multiClassLoader
- )
- magicLoader.dependencyMap.setMagicLoader(magicLoader)
- magicLoader
+ // Load external plugins if provided
+ logger.debug("Loading external plugins")
+ val externalPlugins = if (magicUrlArray.nonEmpty) {
+ val externalPlugins = pluginManager.loadPlugins(
+ magicUrlArray.map(_.getFile).map(new File(_)): _*
+ )
+ pluginManager.initializePlugins(externalPlugins)
+ externalPlugins
+ } else Nil
+ logger.info(externalPlugins.size + " external plugins loaded")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
index 2de5eb2..ea2ebd5 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
@@ -30,7 +30,8 @@ import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.magic.{MagicParser, PostProcessor}
import org.apache.toree.kernel.protocol.v5.relay.ExecuteRequestRelay
import org.apache.toree.kernel.protocol.v5.{MessageType, SocketType, SystemActorType}
-import org.apache.toree.magic.MagicLoader
+import org.apache.toree.magic.MagicManager
+import org.apache.toree.plugins.PluginManager
import org.apache.toree.utils.LogLike
/**
@@ -45,15 +46,16 @@ trait HandlerInitialization {
* @param actorLoader The actor loader needed for registration
* @param kernel The kernel api needed for registration
* @param interpreter The main interpreter needed for registration
- * @param magicLoader The magic loader needed for registration
+ * @param magicManager The magic manager needed for registration
* @param commRegistrar The comm registrar needed for registration
* @param commStorage The comm storage needed for registration
*/
def initializeHandlers(
actorSystem: ActorSystem, actorLoader: ActorLoader,
kernel: Kernel,
- interpreter: Interpreter, magicLoader: MagicLoader,
- commRegistrar: CommRegistrar, commStorage: CommStorage,
+ interpreter: Interpreter, pluginManager: PluginManager,
+ magicManager: MagicManager, commRegistrar: CommRegistrar,
+ commStorage: CommStorage,
responseMap: collection.mutable.Map[String, ActorRef]
): Unit
}
@@ -71,26 +73,30 @@ trait StandardHandlerInitialization extends HandlerInitialization {
* @param actorLoader The actor loader needed for registration
* @param kernel The kernel api needed for registration
* @param interpreter The main interpreter needed for registration
- * @param magicLoader The magic loader needed for registration
+ * @param pluginManager The plugin manager needed for registration
* @param commRegistrar The comm registrar needed for registration
* @param commStorage The comm storage needed for registration
*/
def initializeHandlers(
actorSystem: ActorSystem, actorLoader: ActorLoader,
kernel: Kernel,
- interpreter: Interpreter, magicLoader: MagicLoader,
- commRegistrar: CommRegistrar, commStorage: CommStorage,
+ interpreter: Interpreter, pluginManager: PluginManager,
+ magicManager: MagicManager, commRegistrar: CommRegistrar,
+ commStorage: CommStorage,
responseMap: collection.mutable.Map[String, ActorRef]
): Unit = {
initializeKernelHandlers(
actorSystem, actorLoader, kernel, commRegistrar, commStorage, responseMap
)
- initializeSystemActors(actorSystem, actorLoader, interpreter, magicLoader)
+ initializeSystemActors(
+ actorSystem, actorLoader, interpreter, pluginManager, magicManager
+ )
}
private def initializeSystemActors(
actorSystem: ActorSystem, actorLoader: ActorLoader,
- interpreter: Interpreter, magicLoader: MagicLoader
+ interpreter: Interpreter, pluginManager: PluginManager,
+ magicManager: MagicManager
): Unit = {
logger.debug("Creating interpreter actor")
val interpreterActor = actorSystem.actorOf(
@@ -100,10 +106,10 @@ trait StandardHandlerInitialization extends HandlerInitialization {
logger.debug("Creating execute request relay actor")
val postProcessor = new PostProcessor(interpreter)
- val magicParser = new MagicParser(magicLoader)
+ val magicParser = new MagicParser(magicManager)
val executeRequestRelayActor = actorSystem.actorOf(
Props(classOf[ExecuteRequestRelay],
- actorLoader, magicLoader, magicParser, postProcessor
+ actorLoader, pluginManager, magicParser, postProcessor
),
name = SystemActorType.ExecuteRequestRelay.toString
)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
index 87a0e94..ae2c478 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
@@ -94,14 +94,10 @@ trait StandardHookInitialization extends HookInitialization {
private def registerShutdownHook(): Unit = {
logger.debug("Registering shutdown hook")
val self = this
- val mainThread = Thread.currentThread()
Runtime.getRuntime.addShutdownHook(new Thread() {
override def run() = {
logger.info("Shutting down kernel")
self.shutdown()
- // TODO: Check if you can magically access the spark context to stop it
- // TODO: inside a different thread
- if (mainThread.isAlive) mainThread.join()
}
})
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
index c08467a..2d67189 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
@@ -17,32 +17,32 @@
package org.apache.toree.kernel.api
-import java.io.{OutputStream, InputStream, PrintStream}
+import java.io.{InputStream, PrintStream}
import java.util.concurrent.ConcurrentHashMap
+import com.typesafe.config.Config
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.{SparkConf, SparkContext}
import org.apache.toree.annotations.Experimental
import org.apache.toree.boot.layer.InterpreterManager
import org.apache.toree.comm.CommManager
import org.apache.toree.global
+import org.apache.toree.global.ExecuteRequestState
import org.apache.toree.interpreter.Results.Result
import org.apache.toree.interpreter._
import org.apache.toree.kernel.protocol.v5
-import org.apache.toree.kernel.protocol.v5.{KMBuilder, KernelMessage}
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.magic.MagicParser
-import org.apache.toree.kernel.protocol.v5.stream.{KernelOutputStream, KernelInputStream}
-import org.apache.toree.magic.{MagicLoader, MagicExecutor}
+import org.apache.toree.kernel.protocol.v5.stream.KernelOutputStream
+import org.apache.toree.kernel.protocol.v5.{KMBuilder, KernelMessage}
+import org.apache.toree.magic.MagicManager
+import org.apache.toree.plugins.PluginManager
import org.apache.toree.utils.{KeyValuePairUtils, LogLike}
-import com.typesafe.config.Config
-import org.apache.spark.api.java.JavaSparkContext
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkContext, SparkConf}
-import scala.util.{Try, DynamicVariable}
-
-import scala.reflect.runtime.universe._
import scala.language.dynamics
-import org.apache.toree.global.ExecuteRequestState
+import scala.reflect.runtime.universe._
+import scala.util.{DynamicVariable, Try}
/**
* Represents the main kernel API to be used for interaction.
@@ -58,7 +58,7 @@ class Kernel (
private val actorLoader: ActorLoader,
val interpreterManager: InterpreterManager,
val comm: CommManager,
- val magicLoader: MagicLoader
+ val pluginManager: PluginManager
) extends KernelLike with LogLike {
/**
@@ -96,12 +96,12 @@ class Kernel (
/**
* Represents magics available through the kernel.
*/
- val magics = new MagicExecutor(magicLoader)
+ val magics = new MagicManager(pluginManager)
/**
* Represents magic parsing functionality.
*/
- val magicParser = new MagicParser(magicLoader)
+ val magicParser = new MagicParser(magics)
/**
* Represents the data that can be shared using the kernel as the middleman.
@@ -117,6 +117,7 @@ class Kernel (
/**
* Handles the output of interpreting code.
+ *
* @param output the output of the interpreter
* @return (success, message) or (failure, message)
*/
@@ -175,7 +176,6 @@ class Kernel (
*
* @param parentMessage The message to serve as the parent of outgoing
* messages sent as a result of using streaming methods
- *
* @return The collection of streaming methods
*/
private[toree] def stream(
@@ -198,7 +198,6 @@ class Kernel (
*
* @param parentMessage The message to serve as the parent of outgoing
* messages sent as a result of using streaming methods
- *
* @return The collection of streaming methods
*/
private[toree] def display(
@@ -225,7 +224,6 @@ class Kernel (
* by the factory methods
* @param kmBuilder The builder to be used by objects created by factory
* methods
- *
* @return The collection of factory methods
*/
private[toree] def factory(
@@ -341,7 +339,6 @@ class Kernel (
* Retrieves the last kernel message received by the kernel.
*
* @throws IllegalArgumentException If no kernel message has been received
- *
* @return The kernel message instance
*/
private def lastKernelMessage() = {
@@ -362,8 +359,13 @@ class Kernel (
updateInterpreterWithSparkContext(interpreter, sparkContext)
updateInterpreterWithSqlContext(interpreter, sqlContext)
- magicLoader.dependencyMap =
- magicLoader.dependencyMap.setSparkContext(_sparkContext)
+ // TODO: Convert to events
+ pluginManager.dependencyManager.add(_sparkConf)
+ pluginManager.dependencyManager.add(_sparkContext)
+ pluginManager.dependencyManager.add(_javaSparkContext)
+ pluginManager.dependencyManager.add(_sqlContext)
+
+ pluginManager.fireEvent("sparkReady")
_sparkContext
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParser.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParser.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParser.scala
index f86d76a..bfe518d 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParser.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/magic/MagicParser.scala
@@ -17,14 +17,17 @@
package org.apache.toree.kernel.protocol.v5.magic
-import org.apache.toree.magic.MagicLoader
+import org.apache.toree.magic.MagicManager
-class MagicParser(magicLoader: MagicLoader) {
+import scala.util.Try
+
+class MagicParser(private val magicManager: MagicManager) {
private val magicRegex = """^[%]{1,2}(\w+)""".r
protected[magic] val kernelObjectName = "kernel.magics"
/**
* Determines whether a given line of code represents a line magic.
+ *
* @param codeLine a single line of code
* @return
*/
@@ -33,6 +36,7 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Determines whether a given string of code represents a cell magic.
+ *
* @param codeBlob a string of code separated by newlines
* @return
*/
@@ -45,6 +49,7 @@ class MagicParser(magicLoader: MagicLoader) {
*
* E.g.
* "%magic foo bar" -> ("magic", "foo bar")
+ *
* @param codeBlob a string of code separated by newlines
* @return (magicName, args)
*/
@@ -61,13 +66,16 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Given a line of code representing a magic invocation determines whether
* the magic has an implementation.
+ *
* @param codeLine a single line of code
* @return true if the magic exists and is a line magic
*/
protected[magic] def isValidLineMagic(codeLine: String): Boolean = {
parseMagic(codeLine) match {
case Some((magicName, _)) =>
- isLineMagic(codeLine) && magicLoader.hasLineMagic(magicName)
+ isLineMagic(codeLine) && Try(magicManager.isLineMagic(
+ magicManager.findMagic(magicName))
+ ).getOrElse(false)
case None => false
}
}
@@ -75,6 +83,7 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Given a blob of code, finds any magic invocations of magics that don't
* exist.
+ *
* @param codeBlob a string of code separated by newlines
* @return invalid magic names from the given code blob
*/
@@ -88,6 +97,7 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Formats a given magic name and args to code for a kernel method call.
+ *
* @param magicName the name of the magic
* @param args the arguments to the magic
* @return equivalent kernel method call
@@ -98,6 +108,7 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Formats a given line of code representing a line magic invocation into an
* equivalent kernel object call if the magic invocation is valid.
+ *
* @param codeLine the line of code to convert.
* @return a substituted line of code if valid else the original line
*/
@@ -114,13 +125,15 @@ class MagicParser(magicLoader: MagicLoader) {
* Formats a given code blob representing a cell magic invocation into an
* equivalent kernel object call if the cell magic invocation is valid. An
* error message is returned if not.
+ *
* @param codeBlob the blob of code representing a cell magic invocation
* @return Left(the substituted code) or Right(error message)
*/
protected[magic] def parseCell(codeBlob: String): Either[String, String] = {
parseMagic(codeBlob.trim) match {
case Some((cellMagicName, args)) =>
- magicLoader.hasCellMagic(cellMagicName) match {
+ val m = Try(magicManager.findMagic(cellMagicName))
+ m.map(magicManager.isCellMagic).getOrElse(false) match {
case true => Left(substitute(cellMagicName, args))
case false => Right(s"Magic $cellMagicName does not exist!")
}
@@ -132,6 +145,7 @@ class MagicParser(magicLoader: MagicLoader) {
* Parses all lines in a given code blob and either substitutes equivalent
* kernel object calls for each line magic in the code blob OR returns
* an error message if any of the line magic invocations were invalid.
+ *
* @param codeBlob a string of code separated by newlines
* @return Left(code blob with substitutions) or Right(error message)
*/
@@ -149,6 +163,7 @@ class MagicParser(magicLoader: MagicLoader) {
/**
* Parses a given code blob and returns an equivalent blob with substitutions
* for magic invocations, if any, or an error string.
+ *
* @param codeBlob the blob of code to parse
* @return Left(parsed code) or Right(error message)
*/
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c8824ff/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelay.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelay.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelay.scala
index 022a426..5ac3af9 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelay.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/relay/ExecuteRequestRelay.scala
@@ -27,7 +27,7 @@ import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content._
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.magic.{PostProcessor, MagicParser}
-import org.apache.toree.magic.MagicLoader
+import org.apache.toree.plugins.PluginManager
import org.apache.toree.utils.LogLike
import scala.concurrent.Future
@@ -35,7 +35,7 @@ import scala.concurrent.duration._
case class ExecuteRequestRelay(
actorLoader: ActorLoader,
- magicLoader: MagicLoader,
+ pluginManager: PluginManager,
magicParser: MagicParser,
postProcessor: PostProcessor
)
@@ -47,6 +47,7 @@ case class ExecuteRequestRelay(
/**
* Takes an ExecuteFailure and (ExecuteReply, ExecuteResult) with contents
* dictated by the type of failure (either an error or an abort).
+ *
* @param failure the failure
* @return (ExecuteReply, ExecuteResult)
*/
@@ -68,6 +69,7 @@ case class ExecuteRequestRelay(
/**
* Packages the response into an ExecuteReply,ExecuteResult tuple.
+ *
* @param future The future containing either the output or failure
* @return The tuple representing the proper response
*/
@@ -96,7 +98,11 @@ case class ExecuteRequestRelay(
val oldSender = sender()
// Sets the outputStream for this particular ExecuteRequest
- magicLoader.dependencyMap.setOutputStream(outputStream)
+ import org.apache.toree.plugins.Implicits._
+ pluginManager.fireEventFirstResult(
+ "newOutputStream",
+ "outputStream" -> outputStream
+ )
// Parse the code for magics before sending it to the interpreter and
// pipe the response to sender