You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lb...@apache.org on 2016/01/22 23:07:25 UTC
[17/51] [abbrv] incubator-toree git commit: Moved scala files to new
locations based on new package
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
new file mode 100644
index 0000000..1d16ac4
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
@@ -0,0 +1,185 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.boot.layer
+
+import akka.actor.{ActorRef, ActorSystem, Props}
+import com.ibm.spark.comm.{CommRegistrar, CommStorage}
+import com.ibm.spark.interpreter.Interpreter
+import com.ibm.spark.kernel.api.Kernel
+import com.ibm.spark.kernel.protocol.v5.MessageType.MessageType
+import com.ibm.spark.kernel.protocol.v5.SocketType.SocketType
+import com.ibm.spark.kernel.protocol.v5.handler._
+import com.ibm.spark.kernel.protocol.v5.interpreter.InterpreterActor
+import com.ibm.spark.kernel.protocol.v5.interpreter.tasks.InterpreterTaskFactory
+import com.ibm.spark.kernel.protocol.v5.kernel.ActorLoader
+import com.ibm.spark.kernel.protocol.v5.magic.{MagicParser, PostProcessor}
+import com.ibm.spark.kernel.protocol.v5.relay.ExecuteRequestRelay
+import com.ibm.spark.kernel.protocol.v5.{MessageType, SocketType, SystemActorType}
+import com.ibm.spark.magic.MagicLoader
+import com.ibm.spark.utils.LogLike
+
+/**
+ * Represents the Akka handler initialization. All actors (not needed in bare
+ * initialization) should be constructed here.
+ */
+trait HandlerInitialization {
+ /**
+ * Initializes and registers all handlers.
+ *
+ * @param actorSystem The actor system needed for registration
+ * @param actorLoader The actor loader needed for registration
+ * @param kernel The kernel api needed for registration
+ * @param interpreter The main interpreter needed for registration
+ * @param magicLoader The magic loader needed for registration
+ * @param commRegistrar The comm registrar needed for registration
+ * @param commStorage The comm storage needed for registration
+ */
+ def initializeHandlers(
+ actorSystem: ActorSystem, actorLoader: ActorLoader,
+ kernel: Kernel,
+ interpreter: Interpreter, magicLoader: MagicLoader,
+ commRegistrar: CommRegistrar, commStorage: CommStorage,
+ responseMap: collection.mutable.Map[String, ActorRef]
+ ): Unit
+}
+
+/**
+ * Represents the standard implementation of HandlerInitialization.
+ */
+trait StandardHandlerInitialization extends HandlerInitialization {
+ this: LogLike =>
+
+ /**
+ * Initializes and registers all handlers.
+ *
+ * @param actorSystem The actor system needed for registration
+ * @param actorLoader The actor loader needed for registration
+ * @param kernel The kernel api needed for registration
+ * @param interpreter The main interpreter needed for registration
+ * @param magicLoader The magic loader needed for registration
+ * @param commRegistrar The comm registrar needed for registration
+ * @param commStorage The comm storage needed for registration
+ */
+ def initializeHandlers(
+ actorSystem: ActorSystem, actorLoader: ActorLoader,
+ kernel: Kernel,
+ interpreter: Interpreter, magicLoader: MagicLoader,
+ commRegistrar: CommRegistrar, commStorage: CommStorage,
+ responseMap: collection.mutable.Map[String, ActorRef]
+ ): Unit = {
+ initializeKernelHandlers(
+ actorSystem, actorLoader, kernel, commRegistrar, commStorage, responseMap
+ )
+ initializeSystemActors(actorSystem, actorLoader, interpreter, magicLoader)
+ }
+
+ private def initializeSystemActors(
+ actorSystem: ActorSystem, actorLoader: ActorLoader,
+ interpreter: Interpreter, magicLoader: MagicLoader
+ ): Unit = {
+ logger.debug("Creating interpreter actor")
+ val interpreterActor = actorSystem.actorOf(
+ Props(classOf[InterpreterActor], new InterpreterTaskFactory(interpreter)),
+ name = SystemActorType.Interpreter.toString
+ )
+
+ logger.debug("Creating execute request relay actor")
+ val postProcessor = new PostProcessor(interpreter)
+ val magicParser = new MagicParser(magicLoader)
+ val executeRequestRelayActor = actorSystem.actorOf(
+ Props(classOf[ExecuteRequestRelay],
+ actorLoader, magicLoader, magicParser, postProcessor
+ ),
+ name = SystemActorType.ExecuteRequestRelay.toString
+ )
+ }
+
+ private def initializeKernelHandlers(
+ actorSystem: ActorSystem, actorLoader: ActorLoader,
+ kernel: Kernel,
+ commRegistrar: CommRegistrar, commStorage: CommStorage,
+ responseMap: collection.mutable.Map[String, ActorRef]
+ ): Unit = {
+ def initializeRequestHandler[T](clazz: Class[T], messageType: MessageType, extraArguments: AnyRef*) = {
+ logger.debug("Creating %s handler".format(messageType.toString))
+ actorSystem.actorOf(
+ Props(clazz, actorLoader +: extraArguments: _*),
+ name = messageType.toString
+ )
+ }
+
+ def initializeInputHandler[T](
+ clazz: Class[T],
+ messageType: MessageType
+ ): Unit = {
+ logger.debug("Creating %s handler".format(messageType.toString))
+ actorSystem.actorOf(
+ Props(clazz, actorLoader, responseMap),
+ name = messageType.toString
+ )
+ }
+
+ // TODO: Figure out how to pass variable number of arguments to actor
+ def initializeCommHandler[T](clazz: Class[T], messageType: MessageType) = {
+ logger.debug("Creating %s handler".format(messageType.toString))
+ actorSystem.actorOf(
+ Props(clazz, actorLoader, commRegistrar, commStorage),
+ name = messageType.toString
+ )
+ }
+
+ def initializeSocketHandler(socketType: SocketType, messageType: MessageType): Unit = {
+ logger.debug("Creating %s to %s socket handler ".format(messageType.toString ,socketType.toString))
+ actorSystem.actorOf(
+ Props(classOf[GenericSocketMessageHandler], actorLoader, socketType),
+ name = messageType.toString
+ )
+ }
+
+ // These are the handlers for messages coming into the
+ initializeRequestHandler(classOf[ExecuteRequestHandler],
+ MessageType.Incoming.ExecuteRequest, kernel)
+ initializeRequestHandler(classOf[KernelInfoRequestHandler],
+ MessageType.Incoming.KernelInfoRequest)
+ initializeRequestHandler(classOf[CodeCompleteHandler],
+ MessageType.Incoming.CompleteRequest)
+ initializeInputHandler(classOf[InputRequestReplyHandler],
+ MessageType.Incoming.InputReply)
+ initializeCommHandler(classOf[CommOpenHandler],
+ MessageType.Incoming.CommOpen)
+ initializeCommHandler(classOf[CommMsgHandler],
+ MessageType.Incoming.CommMsg)
+ initializeCommHandler(classOf[CommCloseHandler],
+ MessageType.Incoming.CommClose)
+
+ // These are handlers for messages leaving the kernel through the sockets
+ initializeSocketHandler(SocketType.Shell, MessageType.Outgoing.KernelInfoReply)
+ initializeSocketHandler(SocketType.Shell, MessageType.Outgoing.ExecuteReply)
+ initializeSocketHandler(SocketType.Shell, MessageType.Outgoing.CompleteReply)
+
+ initializeSocketHandler(SocketType.StdIn, MessageType.Outgoing.InputRequest)
+
+ initializeSocketHandler(SocketType.IOPub, MessageType.Outgoing.ExecuteResult)
+ initializeSocketHandler(SocketType.IOPub, MessageType.Outgoing.Stream)
+ initializeSocketHandler(SocketType.IOPub, MessageType.Outgoing.ExecuteInput)
+ initializeSocketHandler(SocketType.IOPub, MessageType.Outgoing.Status)
+ initializeSocketHandler(SocketType.IOPub, MessageType.Outgoing.Error)
+ initializeSocketHandler(SocketType.IOPub, MessageType.Outgoing.CommOpen)
+ initializeSocketHandler(SocketType.IOPub, MessageType.Outgoing.CommMsg)
+ initializeSocketHandler(SocketType.IOPub, MessageType.Outgoing.CommClose)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
new file mode 100644
index 0000000..c590394
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/HookInitialization.scala
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.boot.layer
+
+import com.ibm.spark.boot.KernelBootstrap
+import com.ibm.spark.interpreter.Interpreter
+import com.ibm.spark.utils.LogLike
+
+/**
+ * Represents the hook (interrupt/shutdown) initialization. All JVM-related
+ * hooks should be constructed here.
+ */
+trait HookInitialization {
+ /**
+ * Initializes and registers all hooks except shutdown.
+ *
+ * @param interpreter The main interpreter
+ */
+ def initializeHooks(interpreter: Interpreter): Unit
+
+ /**
+ * Initializes the shutdown hook.
+ */
+ def initializeShutdownHook(): Unit
+}
+
+/**
+ * Represents the standard implementation of HookInitialization.
+ */
+trait StandardHookInitialization extends HookInitialization {
+ this: KernelBootstrap with LogLike =>
+
+ /**
+ * Initializes and registers all hooks.
+ *
+ * @param interpreter The main interpreter
+ */
+ def initializeHooks(interpreter: Interpreter): Unit = {
+ registerInterruptHook(interpreter)
+ }
+
+ /**
+ * Initializes the shutdown hook.
+ */
+ def initializeShutdownHook(): Unit = {
+ registerShutdownHook()
+ }
+
+ private def registerInterruptHook(interpreter: Interpreter): Unit = {
+ val self = this
+
+ import sun.misc.{Signal, SignalHandler}
+
+ // TODO: Signals are not a good way to handle this since JVM only has the
+ // proprietary sun API that is not necessarily available on all platforms
+ Signal.handle(new Signal("INT"), new SignalHandler() {
+ private val MaxSignalTime: Long = 3000 // 3 seconds
+ var lastSignalReceived: Long = 0
+
+ def handle(sig: Signal) = {
+ val currentTime = System.currentTimeMillis()
+ if (currentTime - lastSignalReceived > MaxSignalTime) {
+ logger.info("Resetting code execution!")
+ interpreter.interrupt()
+
+ // TODO: Cancel group representing current code execution
+ //sparkContext.cancelJobGroup()
+
+ logger.info("Enter Ctrl-C twice to shutdown!")
+ lastSignalReceived = currentTime
+ } else {
+ logger.info("Shutting down kernel")
+ self.shutdown()
+ }
+ }
+ })
+ }
+
+ private def registerShutdownHook(): Unit = {
+ logger.debug("Registering shutdown hook")
+ val self = this
+ val mainThread = Thread.currentThread()
+ Runtime.getRuntime.addShutdownHook(new Thread() {
+ override def run() = {
+ logger.info("Shutting down kernel")
+ self.shutdown()
+ // TODO: Check if you can magically access the spark context to stop it
+ // TODO: inside a different thread
+ if (mainThread.isAlive) mainThread.join()
+ }
+ })
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/boot/layer/InterpreterManager.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/InterpreterManager.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/InterpreterManager.scala
new file mode 100644
index 0000000..520d68c
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/InterpreterManager.scala
@@ -0,0 +1,66 @@
+package com.ibm.spark.boot.layer
+
+import com.ibm.spark.kernel.api.KernelLike
+import com.typesafe.config.Config
+import com.ibm.spark.interpreter._
+import scala.collection.JavaConverters._
+
+import org.slf4j.LoggerFactory
+
+case class InterpreterManager(
+ default: String = "Scala",
+ interpreters: Map[String, Interpreter] = Map[String, Interpreter]()
+) {
+
+
+ def initializeInterpreters(kernel: KernelLike): Unit = {
+ interpreters.values.foreach(interpreter =>
+ interpreter.init(kernel)
+ )
+ }
+
+ def addInterpreter(
+ name:String,
+ interpreter: Interpreter
+ ): InterpreterManager = {
+ copy(interpreters = interpreters + (name -> interpreter))
+ }
+
+ def defaultInterpreter: Option[Interpreter] = {
+ interpreters.get(default)
+ }
+}
+
+object InterpreterManager {
+
+ protected val logger = LoggerFactory.getLogger(this.getClass.getName)
+
+ def apply(config: Config): InterpreterManager = {
+ val ip = config.getStringList("interpreter_plugins").asScala ++
+ config.getStringList("default_interpreter_plugin").asScala
+
+ val m = ip.foldLeft(Map[String, Interpreter]())( (acc, v) => {
+ v.split(":") match {
+ case Array(name, className) =>
+ try {
+ val i = Class
+ .forName(className)
+ .newInstance()
+ .asInstanceOf[Interpreter]
+ acc + (name -> i)
+ }
+ catch {
+ case e:Throwable =>
+ logger.error("Error loading interpreter class " + className)
+ logger.error(e.getMessage())
+ acc
+ }
+ case _ => acc
+ }
+ })
+
+ val default = config.getString("default_interpreter")
+
+ InterpreterManager(interpreters = m, default = default)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/comm/KernelCommManager.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/comm/KernelCommManager.scala b/kernel/src/main/scala/org/apache/toree/comm/KernelCommManager.scala
new file mode 100644
index 0000000..9706ce7
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/comm/KernelCommManager.scala
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.comm
+
+import com.ibm.spark.annotations.Experimental
+import com.ibm.spark.kernel.protocol.v5.kernel.ActorLoader
+import com.ibm.spark.kernel.protocol.v5.{KMBuilder, UUID}
+
+/**
+ * Represents a CommManager that uses a KernelCommWriter for its underlying
+ * open implementation.
+ *
+ * @param actorLoader The actor loader to use with the ClientCommWriter
+ * @param kmBuilder The KMBuilder to use with the ClientCommWriter
+ * @param commRegistrar The registrar to use for callback registration
+ */
+@Experimental
+class KernelCommManager(
+ private val actorLoader: ActorLoader,
+ private val kmBuilder: KMBuilder,
+ private val commRegistrar: CommRegistrar
+) extends CommManager(commRegistrar)
+{
+ /**
+ * Creates a new CommWriter instance given the Comm id.
+ *
+ * @param commId The Comm id to use with the Comm writer
+ *
+ * @return The new CommWriter instance
+ */
+ override protected def newCommWriter(commId: UUID): CommWriter =
+ new KernelCommWriter(actorLoader, kmBuilder, commId)
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/comm/KernelCommWriter.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/comm/KernelCommWriter.scala b/kernel/src/main/scala/org/apache/toree/comm/KernelCommWriter.scala
new file mode 100644
index 0000000..e494663
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/comm/KernelCommWriter.scala
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.comm
+
+import com.ibm.spark.annotations.Experimental
+import com.ibm.spark.kernel.protocol.v5
+import com.ibm.spark.kernel.protocol.v5.content.{CommMsg, CommOpen, CommClose, CommContent}
+import com.ibm.spark.kernel.protocol.v5._
+import com.ibm.spark.kernel.protocol.v5.kernel.ActorLoader
+
+/**
+ * Represents a CommWriter to send messages from the Kernel to the Client.
+ *
+ * @param actorLoader The actor loader to use for loading actors responsible for
+ * communication
+ * @param kmBuilder The kernel message builder used to construct kernel messages
+ * @param commId The comm id associated with this writer (defaults to a
+ * random UUID)
+ */
+@Experimental
+class KernelCommWriter(
+ private val actorLoader: ActorLoader,
+ private val kmBuilder: KMBuilder,
+ override private[comm] val commId: v5.UUID
+) extends CommWriter(commId) {
+ /**
+ * Sends the comm message (open/msg/close) to the actor responsible for
+ * relaying messages.
+ *
+ * @param commContent The message to relay (will be packaged)
+ *
+ * @tparam T Either CommOpen, CommMsg, or CommClose
+ */
+ override protected[comm] def sendCommKernelMessage[
+ T <: KernelMessageContent with CommContent
+ ](commContent: T): Unit = {
+ val messageType = commContent match {
+ case _: CommOpen => CommOpen.toTypeString
+ case _: CommMsg => CommMsg.toTypeString
+ case _: CommClose => CommClose.toTypeString
+ case _ => throw new Throwable("Invalid kernel message type!")
+ }
+ actorLoader.load(SystemActorType.KernelMessageRelay) !
+ kmBuilder.withHeader(messageType).withContentString(commContent).build
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/global/ExecuteRequestState.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/global/ExecuteRequestState.scala b/kernel/src/main/scala/org/apache/toree/global/ExecuteRequestState.scala
new file mode 100644
index 0000000..389cf82
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/global/ExecuteRequestState.scala
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.global
+
+import com.ibm.spark.kernel.api.Kernel
+import com.ibm.spark.kernel.protocol.v5.KernelMessage
+
+/**
+ * Represents the state of the kernel messages being received containing
+ * execute requests.
+ */
+object ExecuteRequestState {
+ private var _lastKernelMessage: Option[KernelMessage] = None
+
+ /**
+ * Processes the incoming kernel message and updates any associated state.
+ *
+ * @param kernelMessage The kernel message to process
+ */
+ def processIncomingKernelMessage(kernelMessage: KernelMessage) =
+ _lastKernelMessage = Some(kernelMessage)
+
+ /**
+ * Returns the last kernel message funneled through the KernelMessageRelay
+ * if any.
+ *
+ * @return Some KernelMessage instance if the relay has processed one,
+ * otherwise None
+ */
+ def lastKernelMessage: Option[KernelMessage] = _lastKernelMessage
+
+ /**
+ * Resets the state of the ExecuteRequestState to the default.
+ */
+ def reset() = _lastKernelMessage = None
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/global/ExecutionCounter.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/global/ExecutionCounter.scala b/kernel/src/main/scala/org/apache/toree/global/ExecutionCounter.scala
new file mode 100644
index 0000000..c00f937
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/global/ExecutionCounter.scala
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.global
+
+import com.ibm.spark.kernel.protocol.v5.UUID
+
+import scala.collection.concurrent._
+
+/**
+ * A class to keep track of execution counts for sessions.
+ */
+object ExecutionCounter {
+ private val executionCounts: Map[UUID, Int] = TrieMap[UUID, Int]()
+
+ /**
+ * This function will increase, by 1, an integer value associated with the given key. The incremented value
+ * will be returned. If the key has no value associated value, 1 will be returned.
+ * @param key The key for incrementing the value.
+ * @return The incremented value
+ */
+ def incr(key: UUID): Int = {
+ (executionCounts += (key -> (executionCounts.getOrElse(key, 0) + 1))) (key)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/global/ScheduledTaskManager.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/global/ScheduledTaskManager.scala b/kernel/src/main/scala/org/apache/toree/global/ScheduledTaskManager.scala
new file mode 100644
index 0000000..83d2cea
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/global/ScheduledTaskManager.scala
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.global
+
+import com.ibm.spark.utils
+
+object ScheduledTaskManager {
+ lazy val instance = new utils.ScheduledTaskManager
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/api/FactoryMethods.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/api/FactoryMethods.scala b/kernel/src/main/scala/org/apache/toree/kernel/api/FactoryMethods.scala
new file mode 100644
index 0000000..36c5c5c
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/api/FactoryMethods.scala
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.api
+
+import java.io.{InputStream, OutputStream}
+
+import com.ibm.spark.kernel.protocol.v5
+import com.ibm.spark.kernel.protocol.v5.{KMBuilder, KernelMessage}
+import com.ibm.spark.kernel.protocol.v5.kernel.ActorLoader
+import com.ibm.spark.kernel.protocol.v5.stream.{KernelOutputStream, KernelInputStream}
+import com.typesafe.config.Config
+
+/**
+ * Represents the methods available to stream data from the kernel to the
+ * client.
+ *
+ * @param config The kernel configuration to use during object creation
+ * @param actorLoader The actor loader to use when retrieve actors needed for
+ * object creation
+ * @param parentMessage The parent message to use when needed for object
+ * creation
+ * @param kernelMessageBuilder The builder to use when constructing kernel
+ * messages inside objects created
+ */
+class FactoryMethods(
+ private val config: Config,
+ private val actorLoader: ActorLoader,
+ private val parentMessage: KernelMessage,
+ private val kernelMessageBuilder: KMBuilder
+) extends FactoryMethodsLike {
+ require(parentMessage != null, "Parent message cannot be null!")
+
+ private[api] val kmBuilder = kernelMessageBuilder.withParent(parentMessage)
+
+ /**
+ * Creates a new kernel input stream.
+ *
+ * @param prompt The text to use as a prompt
+ * @param password If true, should treat input as a password field
+ *
+ * @return The new KernelInputStream instance
+ */
+ override def newKernelInputStream(
+ prompt: String = KernelInputStream.DefaultPrompt,
+ password: Boolean = KernelInputStream.DefaultPassword
+ ): InputStream = {
+ new KernelInputStream(
+ actorLoader,
+ kmBuilder.withIds(parentMessage.ids),
+ prompt = prompt,
+ password = password
+ )
+ }
+
+ /**
+ * Creates a new kernel output stream.
+ *
+ * @param streamType The type of output stream (stdout/stderr)
+ * @param sendEmptyOutput If true, will send message even if output is empty
+ *
+ * @return The new KernelOutputStream instance
+ */
+ override def newKernelOutputStream(
+ streamType: String = KernelOutputStream.DefaultStreamType,
+ sendEmptyOutput: Boolean = config.getBoolean("send_empty_output")
+ ): OutputStream = {
+ new v5.stream.KernelOutputStream(
+ actorLoader,
+ kmBuilder,
+ com.ibm.spark.global.ScheduledTaskManager.instance,
+ streamType = streamType,
+ sendEmptyOutput = sendEmptyOutput
+ )
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
new file mode 100644
index 0000000..219804a
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
@@ -0,0 +1,448 @@
+/*
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.api
+
+import java.io.{OutputStream, InputStream, PrintStream}
+import java.util.concurrent.ConcurrentHashMap
+
+import com.ibm.spark.annotations.Experimental
+import com.ibm.spark.boot.layer.InterpreterManager
+import com.ibm.spark.comm.CommManager
+import com.ibm.spark.global
+import com.ibm.spark.interpreter.Results.Result
+import com.ibm.spark.interpreter._
+import com.ibm.spark.kernel.protocol.v5
+import com.ibm.spark.kernel.protocol.v5.{KMBuilder, KernelMessage}
+import com.ibm.spark.kernel.protocol.v5.kernel.ActorLoader
+import com.ibm.spark.kernel.protocol.v5.magic.MagicParser
+import com.ibm.spark.kernel.protocol.v5.stream.{KernelOutputStream, KernelInputStream}
+import com.ibm.spark.magic.{MagicLoader, MagicExecutor}
+import com.ibm.spark.utils.{KeyValuePairUtils, LogLike}
+import com.typesafe.config.Config
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.{SparkContext, SparkConf}
+import scala.util.{Try, DynamicVariable}
+
+import scala.reflect.runtime.universe._
+
+import scala.language.dynamics
+import com.ibm.spark.global.ExecuteRequestState
+
+/**
+ * Represents the main kernel API to be used for interaction.
+ *
+ * @param _config The configuration used when starting the kernel
+ * @param interpreterManager The interpreter manager to expose in this instance
+ * @param comm The Comm manager to expose in this instance
+ * @param actorLoader The actor loader to use for message relaying
+ */
+@Experimental
+class Kernel (
+ private val _config: Config,
+ private val actorLoader: ActorLoader,
+ val interpreterManager: InterpreterManager,
+ val comm: CommManager,
+ val magicLoader: MagicLoader
+) extends KernelLike with LogLike {
+
+ /**
+ * Represents the current input stream used by the kernel for the specific
+ * thread.
+ */
+ private val currentInputStream =
+ new DynamicVariable[InputStream](null)
+ private val currentInputKernelMessage =
+ new DynamicVariable[KernelMessage](null)
+
+ /**
+ * Represents the current output stream used by the kernel for the specific
+ * thread.
+ */
+ private val currentOutputStream =
+ new DynamicVariable[PrintStream](null)
+ private val currentOutputKernelMessage =
+ new DynamicVariable[KernelMessage](null)
+
+ /**
+ * Represents the current error stream used by the kernel for the specific
+ * thread.
+ */
+ private val currentErrorStream =
+ new DynamicVariable[PrintStream](null)
+ private val currentErrorKernelMessage =
+ new DynamicVariable[KernelMessage](null)
+
+ private var _sparkContext:SparkContext = null;
+ private var _sparkConf:SparkConf = null;
+ private var _javaSparkContext:JavaSparkContext = null;
+ private var _sqlContext:SQLContext = null;
+
+ /**
+ * Represents magics available through the kernel.
+ */
+ val magics = new MagicExecutor(magicLoader)
+
+ /**
+ * Represents magic parsing functionality.
+ */
+ val magicParser = new MagicParser(magicLoader)
+
+ /**
+ * Represents the data that can be shared using the kernel as the middleman.
+ *
+ * @note Using Java structure to enable other languages to have easy access!
+ */
+ val data: java.util.Map[String, Any] = new ConcurrentHashMap[String, Any]()
+
+
+ interpreterManager.initializeInterpreters(this)
+
+ val interpreter = interpreterManager.defaultInterpreter.get
+
+ /**
+ * Handles the output of interpreting code.
+ * @param output the output of the interpreter
+ * @return (success, message) or (failure, message)
+ */
+ private def handleInterpreterOutput(
+ output: (Result, Either[ExecuteOutput, ExecuteFailure])
+ ): (Boolean, String) = {
+ val (success, result) = output
+ success match {
+ case Results.Success =>
+ (true, result.left.getOrElse("").asInstanceOf[String])
+ case Results.Error =>
+ (false, result.right.getOrElse("").toString)
+ case Results.Aborted =>
+ (false, "Aborted!")
+ case Results.Incomplete =>
+ // If we get an incomplete it's most likely a syntax error, so
+ // let the user know.
+ (false, "Syntax Error!")
+ }
+ }
+
+ override def config:Config = {
+ _config
+ }
+
+ /**
+ * Executes a block of code represented as a string and returns the result.
+ *
+ * @param code The code as an option to execute
+ * @return A tuple containing the result (true/false) and the output as a
+ * string
+ */
+ def eval(code: Option[String]): (Boolean, String) = {
+ code.map(c => {
+ magicParser.parse(c) match {
+ case Left(parsedCode) =>
+ val output = interpreter.interpret(parsedCode)
+ handleInterpreterOutput(output)
+ case Right(errMsg) =>
+ (false, errMsg)
+ }
+ }).getOrElse((false, "Error!"))
+ }
+
+ /**
+ * Constructs a new instance of the stream methods using the latest
+ * kernel message instance.
+ *
+ * @return The collection of stream methods
+ */
+ override def stream: StreamMethods = stream()
+
+ /**
+ * Constructs a new instance of the stream methods using the specified
+ * kernel message instance.
+ *
+ * @param parentMessage The message to serve as the parent of outgoing
+ * messages sent as a result of using streaming methods
+ *
+ * @return The collection of streaming methods
+ */
+ private[spark] def stream(
+ parentMessage: v5.KernelMessage = lastKernelMessage()
+ ): StreamMethods = {
+ new StreamMethods(actorLoader, parentMessage)
+ }
+
+ /**
+ * Constructs a new instance of the factory methods using the latest
+ * kernel message instance.
+ *
+ * @return The collection of factory methods
+ */
+ override def factory: FactoryMethods = factory()
+
+ /**
+ * Constructs a new instance of the factory methods using the specified
+ * kernel message and kernel message builder.
+ *
+ * @param parentMessage The message to serve as the parent of outgoing
+ * messages sent as a result of using an object created
+ * by the factory methods
+ * @param kmBuilder The builder to be used by objects created by factory
+ * methods
+ *
+ * @return The collection of factory methods
+ */
+ private[spark] def factory(
+ parentMessage: v5.KernelMessage = lastKernelMessage(),
+ kmBuilder: v5.KMBuilder = v5.KMBuilder()
+ ): FactoryMethods = {
+ new FactoryMethods(_config, actorLoader, parentMessage, kmBuilder)
+ }
+
+ /**
+ * Returns a print stream to be used for communication back to clients
+ * via standard out.
+ *
+ * @return The print stream instance or an error if the stream info is
+ * not found
+ */
+ override def out: PrintStream = {
+ val kernelMessage = lastKernelMessage()
+
+ constructStream(currentOutputStream, currentOutputKernelMessage, kernelMessage, { kernelMessage =>
+ val outputStream = this.factory(parentMessage = kernelMessage)
+ .newKernelOutputStream("stdout")
+
+ new PrintStream(outputStream)
+ })
+ }
+
+ /**
+ * Returns a print stream to be used for communication back to clients
+ * via standard error.
+ *
+ * @return The print stream instance or an error if the stream info is
+ * not found
+ */
+ override def err: PrintStream = {
+ val kernelMessage = lastKernelMessage()
+
+ constructStream(currentErrorStream, currentErrorKernelMessage, kernelMessage, { kernelMessage =>
+ val outputStream = this.factory(parentMessage = kernelMessage)
+ .newKernelOutputStream("stderr")
+
+ new PrintStream(outputStream)
+ })
+ }
+
+ /**
+ * Returns an input stream to be used to receive information from the client.
+ *
+ * @return The input stream instance or an error if the stream info is
+ * not found
+ */
+ override def in: InputStream = {
+ val kernelMessage = lastKernelMessage()
+
+ constructStream(currentInputStream, currentInputKernelMessage, kernelMessage, { kernelMessage =>
+ this.factory(parentMessage = kernelMessage).newKernelInputStream()
+ })
+ }
+
+ /**
+ * Constructs or uses an existing stream.
+ *
+ * @param dynamicStream The DynamicVariable containing the stream to modify
+ * or use
+ * @param dynamicKernelMessage The DynamicVariable containing the KernelMessage to
+ * check against the new KernelMessage
+ * @param newKernelMessage The potentially-new KernelMessage
+ * @param streamConstructionFunc The function used to create a new stream
+ * @param typeTag The type information associated with the stream
+ * @tparam T The stream type
+ * @return The new stream or existing stream
+ */
+ private def constructStream[T](
+ dynamicStream: DynamicVariable[T],
+ dynamicKernelMessage: DynamicVariable[KernelMessage],
+ newKernelMessage: KernelMessage,
+ streamConstructionFunc: (KernelMessage) => T
+ )(implicit typeTag: TypeTag[T]) = {
+ // Update the stream being used only if the information has changed
+ // or if the stream has not been initialized
+ if (updateKernelMessage(dynamicKernelMessage, newKernelMessage) ||
+ dynamicStream.value == null)
+ {
+ logger.trace("Creating new kernel " + typeTag.tpe.toString + "!")
+ dynamicStream.value = streamConstructionFunc(newKernelMessage)
+ }
+
+ dynamicStream.value
+ }
+
+ /**
+ * Updates the last stream info returning the status of whether or not the
+ * new stream info was different than the last stream info.
+ *
+ * @param dynamicKernelMessage The dynamic variable containing the current
+ * stream info
+ * @param kernelMessage The new stream info
+ * @return True if the new stream info is different from the last (therefore
+ * replaced), otherwise false
+ */
+ private def updateKernelMessage(
+ dynamicKernelMessage: DynamicVariable[KernelMessage],
+ kernelMessage: KernelMessage
+ ): Boolean =
+ if (kernelMessage != null && !kernelMessage.equals(dynamicKernelMessage.value)) {
+ dynamicKernelMessage.value = kernelMessage
+ true
+ } else {
+ false
+ }
+
+ /**
+ * Retrieves the last kernel message received by the kernel.
+ *
+ * @throws IllegalArgumentException If no kernel message has been received
+ *
+ * @return The kernel message instance
+ */
+ private def lastKernelMessage() = {
+ val someKernelMessage = ExecuteRequestState.lastKernelMessage
+ require(someKernelMessage.nonEmpty, "No kernel message received!")
+ someKernelMessage.get
+ }
+
+ override def createSparkContext(conf: SparkConf): SparkContext = {
+ _sparkConf = createSparkConf(conf)
+ _sparkContext = initializeSparkContext(sparkConf)
+ _javaSparkContext = new JavaSparkContext(_sparkContext)
+ _sqlContext = initializeSqlContext(_sparkContext)
+
+ val sparkMaster = _sparkConf.getOption("spark.master").getOrElse("not_set")
+ logger.info( s"Connecting to spark.master $sparkMaster")
+
+ updateInterpreterWithSparkContext(interpreter, sparkContext)
+ updateInterpreterWithSqlContext(interpreter, sqlContext)
+
+ magicLoader.dependencyMap =
+ magicLoader.dependencyMap.setSparkContext(_sparkContext)
+
+ _sparkContext
+ }
+
+ override def createSparkContext(
+ master: String, appName: String
+ ): SparkContext = {
+ createSparkContext(new SparkConf().setMaster(master).setAppName(appName))
+ }
+
+ // TODO: Think of a better way to test without exposing this
+ protected[kernel] def createSparkConf(conf: SparkConf) = {
+
+ logger.info("Setting deployMode to client")
+ conf.set("spark.submit.deployMode", "client")
+
+ KeyValuePairUtils.stringToKeyValuePairSeq(
+ _config.getString("spark_configuration")
+ ).foreach { keyValuePair =>
+ logger.info(s"Setting ${keyValuePair.key} to ${keyValuePair.value}")
+ Try(conf.set(keyValuePair.key, keyValuePair.value))
+ }
+
+ // TODO: Move SparkIMain to private and insert in a different way
+ logger.warn("Locked to Scala interpreter with SparkIMain until decoupled!")
+
+ // TODO: Construct class server outside of SparkIMain
+ logger.warn("Unable to control initialization of REPL class server!")
+ logger.info("REPL Class Server Uri: " + interpreter.classServerURI)
+ conf.set("spark.repl.class.uri", interpreter.classServerURI)
+
+ conf
+ }
+
+ // TODO: Think of a better way to test without exposing this
+ protected[kernel] def initializeSparkContext(sparkConf: SparkConf): SparkContext = {
+
+ logger.debug("Constructing new Spark Context")
+ // TODO: Inject stream redirect headers in Spark dynamically
+ var sparkContext: SparkContext = null
+ val outStream = new KernelOutputStream(
+ actorLoader, KMBuilder(), global.ScheduledTaskManager.instance,
+ sendEmptyOutput = _config.getBoolean("send_empty_output")
+ )
+
+ // Update global stream state and use it to set the Console local variables
+ // for threads in the Spark threadpool
+ global.StreamState.setStreams(System.in, outStream, outStream)
+ global.StreamState.withStreams {
+ sparkContext = new SparkContext(sparkConf)
+ }
+
+ sparkContext
+ }
+
+ // TODO: Think of a better way to test without exposing this
+ protected[kernel] def updateInterpreterWithSparkContext(
+ interpreter: Interpreter, sparkContext: SparkContext
+ ) = {
+
+ interpreter.bindSparkContext(sparkContext)
+ }
+
+ protected[kernel] def initializeSqlContext(
+ sparkContext: SparkContext
+ ): SQLContext = {
+ val sqlContext: SQLContext = try {
+ logger.info("Attempting to create Hive Context")
+ val hiveContextClassString =
+ "org.apache.spark.sql.hive.HiveContext"
+
+ logger.debug(s"Looking up $hiveContextClassString")
+ val hiveContextClass = Class.forName(hiveContextClassString)
+
+ val sparkContextClass = classOf[SparkContext]
+ val sparkContextClassName = sparkContextClass.getName
+
+ logger.debug(s"Searching for constructor taking $sparkContextClassName")
+ val hiveContextContructor =
+ hiveContextClass.getConstructor(sparkContextClass)
+
+ logger.debug("Invoking Hive Context constructor")
+ hiveContextContructor.newInstance(sparkContext).asInstanceOf[SQLContext]
+ } catch {
+ case _: Throwable =>
+ logger.warn("Unable to create Hive Context! Defaulting to SQL Context!")
+ new SQLContext(sparkContext)
+ }
+
+ sqlContext
+ }
+
+ protected[kernel] def updateInterpreterWithSqlContext(
+ interpreter: Interpreter, sqlContext: SQLContext
+ ): Unit = {
+ interpreter.bindSqlContext(sqlContext)
+ }
+
+ override def interpreter(name: String): Option[Interpreter] = {
+ interpreterManager.interpreters.get(name)
+ }
+
+ override def sparkContext: SparkContext = _sparkContext
+ override def sparkConf: SparkConf = _sparkConf
+ override def javaSparkContext: JavaSparkContext = _javaSparkContext
+ override def sqlContext: SQLContext = _sqlContext
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/api/StreamMethods.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/api/StreamMethods.scala b/kernel/src/main/scala/org/apache/toree/kernel/api/StreamMethods.scala
new file mode 100644
index 0000000..35c4960
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/api/StreamMethods.scala
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.api
+
+import com.ibm.spark.kernel.protocol.v5
+import com.ibm.spark.kernel.protocol.v5.KernelMessage
+import com.ibm.spark.kernel.protocol.v5.kernel.ActorLoader
+
+/**
+ * Represents the methods available to stream data from the kernel to the
+ * client.
+ */
+class StreamMethods(actorLoader: ActorLoader, parentMessage: KernelMessage)
+ extends StreamMethodsLike
+{
+ private[api] val kmBuilder = v5.KMBuilder()
+ .withParent(parentMessage)
+ .withIds(Seq(v5.content.StreamContent.toTypeString))
+ .withHeader(v5.content.StreamContent.toTypeString)
+
+ /**
+ * Sends all text provided as one stream message to the client.
+ * @param text The text to wrap in a stream message
+ */
+ override def sendAll(text: String): Unit = {
+ val streamContent = v5.content.StreamContent(
+ "stdout", text
+ )
+
+ val kernelMessage = kmBuilder.withContentString(streamContent).build
+
+ actorLoader.load(v5.SystemActorType.KernelMessageRelay) ! kernelMessage
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/dispatch/StatusDispatch.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/dispatch/StatusDispatch.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/dispatch/StatusDispatch.scala
new file mode 100644
index 0000000..144b067
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/dispatch/StatusDispatch.scala
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.dispatch
+
+import akka.actor.Actor
+import com.ibm.spark.kernel.protocol.v5.KernelStatusType.KernelStatusType
+import com.ibm.spark.kernel.protocol.v5._
+import com.ibm.spark.kernel.protocol.v5.content.KernelStatus
+import com.ibm.spark.kernel.protocol.v5.kernel.ActorLoader
+import com.ibm.spark.utils.LogLike
+import play.api.libs.json.Json
+
+class StatusDispatch(actorLoader: ActorLoader) extends Actor with LogLike {
+ private def sendStatusMessage(kernelStatus: KernelStatusType, parentHeader: Header) {
+ // Create the status message and send it to the relay
+ val km : KernelMessage = KMBuilder()
+ .withIds(Seq(MessageType.Outgoing.Status.toString))
+ .withSignature("")
+ .withHeader(MessageType.Outgoing.Status)
+ .withParentHeader(parentHeader)
+ .withContentString(KernelStatus(kernelStatus.toString)).build
+
+ actorLoader.load(SystemActorType.KernelMessageRelay) ! km
+ }
+
+ override def receive: Receive = {
+ case (status: KernelStatusType, null) =>
+ // TODO Determine if this should be null or an empty parent header
+ sendStatusMessage(status, null)
+
+ case (status: KernelStatusType, parentHeader: Header) =>
+ sendStatusMessage( status, parentHeader )
+
+ case status: KernelStatusType =>
+ sendStatusMessage(status , HeaderBuilder.empty)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/BaseHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/BaseHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/BaseHandler.scala
new file mode 100644
index 0000000..abbf6d3
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/BaseHandler.scala
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.handler
+
+import com.ibm.spark.communication.utils.OrderedSupport
+import com.ibm.spark.kernel.protocol.v5._
+import com.ibm.spark.kernel.protocol.v5.kernel.ActorLoader
+import com.ibm.spark.utils.MessageLogSupport
+
+import scala.concurrent.Future
+
+abstract class BaseHandler(actorLoader: ActorLoader) extends OrderedSupport
+ with MessageLogSupport {
+ /**
+ * Implements the receive method, sending a busy message out before
+ * processing the message and sending an idle message out once finished.
+ *
+ * @return The Akka partial function n
+ */
+ final def receive = {
+ // NOTE: Not using withProcessing as the finishedProcessing call is inside
+ // a future (not triggered immediately)
+ case kernelMessage: KernelMessage =>
+ startProcessing()
+ // Send the busy message before we process the message
+ logKernelMessageAction("Sending Busy message for", kernelMessage)
+ actorLoader.load(SystemActorType.StatusDispatch) !
+ Tuple2(KernelStatusType.Busy, kernelMessage.header)
+
+ // Process the message
+ logKernelMessageAction("Processing", kernelMessage)
+ import scala.concurrent.ExecutionContext.Implicits.global
+ val processFuture = process(kernelMessage)
+
+ // Send the idle message since message has been processed
+ logKernelMessageAction("Sending Idle message for", kernelMessage)
+ processFuture onComplete {
+ case _ =>
+ actorLoader.load(SystemActorType.StatusDispatch) !
+ Tuple2(KernelStatusType.Idle, kernelMessage.header)
+ finishedProcessing()
+ }
+ }
+
+ override def orderedTypes() : Seq[Class[_]] = {Seq(classOf[KernelMessage])}
+
+ /**
+ * Processes the provided kernel message.
+ *
+ * @param kernelMessage The kernel message instance to process
+ */
+ def process(kernelMessage: KernelMessage): Future[_]
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CodeCompleteHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CodeCompleteHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CodeCompleteHandler.scala
new file mode 100644
index 0000000..c3b38ee
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CodeCompleteHandler.scala
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.handler
+
+import akka.pattern.ask
+import com.ibm.spark.kernel.protocol.v5._
+import com.ibm.spark.kernel.protocol.v5.content._
+import com.ibm.spark.kernel.protocol.v5.kernel.{ActorLoader, Utilities}
+import Utilities._
+import com.ibm.spark.utils.{MessageLogSupport, LogLike}
+import play.api.data.validation.ValidationError
+import play.api.libs.json.{JsPath, Json}
+
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
+import scala.util.Success
+
+class CodeCompleteHandler(actorLoader: ActorLoader)
+ extends BaseHandler(actorLoader) with MessageLogSupport
+{
+ override def process(kernelMessage: KernelMessage): Future[_] = {
+ logKernelMessageAction("Generating code completion for", kernelMessage)
+ Utilities.parseAndHandle(
+ kernelMessage.contentString,
+ CompleteRequest.completeRequestReads,
+ completeRequest(kernelMessage, _ : CompleteRequest)
+ )
+ }
+
+ private def completeRequest(km: KernelMessage, cr: CompleteRequest):
+ Future[(Int, List[String])] = {
+ val interpreterActor = actorLoader.load(SystemActorType.Interpreter)
+ val codeCompleteFuture = ask(interpreterActor, cr).mapTo[(Int, List[String])]
+ codeCompleteFuture.onComplete {
+ case Success(tuple) =>
+ val reply = CompleteReplyOk(tuple._2, cr.cursor_pos,
+ tuple._1, Metadata())
+ val completeReplyType = MessageType.Outgoing.CompleteReply.toString
+ logKernelMessageAction("Sending code complete reply for", km)
+ actorLoader.load(SystemActorType.KernelMessageRelay) !
+ km.copy(
+ header = HeaderBuilder.create(completeReplyType),
+ parentHeader = km.header,
+ contentString = Json.toJson(reply).toString
+ )
+ case _ =>
+ new Exception("Parse error in CodeCompleteHandler")
+ }
+ codeCompleteFuture
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommCloseHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommCloseHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommCloseHandler.scala
new file mode 100644
index 0000000..2c87dd7
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommCloseHandler.scala
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.handler
+
+import com.ibm.spark.comm.{KernelCommWriter, CommRegistrar, CommWriter, CommStorage}
+import com.ibm.spark.kernel.protocol.v5.content.CommClose
+import com.ibm.spark.kernel.protocol.v5.kernel.{Utilities, ActorLoader}
+import com.ibm.spark.kernel.protocol.v5.{KMBuilder, KernelMessage}
+import com.ibm.spark.utils.MessageLogSupport
+import play.api.data.validation.ValidationError
+import play.api.libs.json.JsPath
+
+import scala.concurrent.Future
+import scala.concurrent.future
+import scala.concurrent.ExecutionContext.Implicits.global
+
+/**
+ * Represents the handler for comm_close messages.
+ *
+ * @param actorLoader The actor loader to use for actor communication
+ * @param commRegistrar The Comm registrar used for unlinking
+ * @param commStorage The Comm storage used for close callbacks
+ */
+class CommCloseHandler(
+ actorLoader: ActorLoader, commRegistrar: CommRegistrar,
+ commStorage: CommStorage
+) extends BaseHandler(actorLoader) with MessageLogSupport
+{
+ override def process(kernelMessage: KernelMessage): Future[_] = future {
+ logKernelMessageAction("Initiating Comm Close for", kernelMessage)
+
+ val kmBuilder = KMBuilder().withParent(kernelMessage)
+
+ Utilities.parseAndHandle(
+ kernelMessage.contentString,
+ CommClose.commCloseReads,
+ handler = handleCommClose(kmBuilder),
+ errHandler = handleParseError
+ )
+ }
+
+ private def handleCommClose(kmBuilder: KMBuilder)(commClose: CommClose) = {
+ val commId = commClose.comm_id
+ val data = commClose.data
+
+ logger.debug(s"Received comm_close with id '$commId'")
+
+ val commWriter = new KernelCommWriter(actorLoader, kmBuilder, commId)
+
+ commStorage.getCommIdCallbacks(commId) match {
+ case None =>
+ logger.warn(s"Received invalid id for Comm Close: $commId")
+ case Some(callbacks) =>
+ logger.debug(s"Executing close callbacks for id '$commId'")
+
+ // TODO: Should we be checking the return values? Probably not.
+ callbacks.executeCloseCallbacks(commWriter, commId, data)
+ .filter(_.isFailure).map(_.failed).foreach(throwable => {
+ logger.error("Comm close callback encountered an error!", throwable)
+ })
+ }
+ }
+
+ private def handleParseError(invalid: Seq[(JsPath, Seq[ValidationError])]) = {
+ // TODO: Determine proper response for a parse failure
+ logger.warn("Parse error for Comm Close! Not responding!")
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommMsgHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommMsgHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommMsgHandler.scala
new file mode 100644
index 0000000..03baef9
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommMsgHandler.scala
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.handler
+
+import com.ibm.spark.comm.{KernelCommWriter, CommRegistrar, CommWriter, CommStorage}
+import com.ibm.spark.kernel.protocol.v5.content.CommMsg
+import com.ibm.spark.kernel.protocol.v5.kernel.{Utilities, ActorLoader}
+import com.ibm.spark.kernel.protocol.v5.{KMBuilder, KernelMessage}
+import com.ibm.spark.utils.MessageLogSupport
+import play.api.data.validation.ValidationError
+import play.api.libs.json.JsPath
+
+import scala.concurrent.Future
+import scala.concurrent.future
+import scala.concurrent.ExecutionContext.Implicits.global
+
+/**
+ * Represents the handler for comm_msg messages.
+ *
+ * @param actorLoader The actor loader to use for actor communication
+ * @param commRegistrar The Comm registrar (unused)
+ * @param commStorage The Comm storage used for msg callbacks
+ */
+class CommMsgHandler(
+ actorLoader: ActorLoader, commRegistrar: CommRegistrar,
+ commStorage: CommStorage
+) extends BaseHandler(actorLoader) with MessageLogSupport
+{
+ override def process(kernelMessage: KernelMessage): Future[_] = future {
+ logKernelMessageAction("Initiating Comm Msg for", kernelMessage)
+
+ val kmBuilder = KMBuilder().withParent(kernelMessage)
+
+ Utilities.parseAndHandle(
+ kernelMessage.contentString,
+ CommMsg.commMsgReads,
+ handler = handleCommMsg(kmBuilder),
+ errHandler = handleParseError
+ )
+ }
+
+ private def handleCommMsg(kmBuilder: KMBuilder)(commMsg: CommMsg) = {
+ val commId = commMsg.comm_id
+ val data = commMsg.data
+
+ logger.debug(s"Received comm_msg with id '$commId'")
+
+ val commWriter = new KernelCommWriter(actorLoader, kmBuilder, commId)
+
+ commStorage.getCommIdCallbacks(commId) match {
+ case None =>
+ logger.warn(s"Received invalid id for Comm Msg: $commId")
+ case Some(callbacks) =>
+ logger.debug(s"Executing msg callbacks for id '$commId'")
+
+ // TODO: Should we be checking the return values? Probably not.
+ callbacks.executeMsgCallbacks(commWriter, commId, data)
+ .filter(_.isFailure).map(_.failed).foreach(throwable => {
+ logger.error("Comm msg callback encountered an error!", throwable)
+ })
+ }
+ }
+
+ private def handleParseError(invalid: Seq[(JsPath, Seq[ValidationError])]) = {
+ // TODO: Determine proper response for a parse failure
+ logger.warn("Parse error for Comm Msg! Not responding!")
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommOpenHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommOpenHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommOpenHandler.scala
new file mode 100644
index 0000000..80e2b35
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/CommOpenHandler.scala
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.handler
+
+import com.ibm.spark.comm.{KernelCommWriter, CommStorage, CommRegistrar, CommWriter}
+import com.ibm.spark.kernel.protocol.v5.content.CommOpen
+import com.ibm.spark.kernel.protocol.v5.kernel.{Utilities, ActorLoader}
+import com.ibm.spark.kernel.protocol.v5.{KMBuilder, KernelMessage}
+import com.ibm.spark.utils.MessageLogSupport
+import play.api.data.validation.ValidationError
+import play.api.libs.json.JsPath
+
+import scala.concurrent.Future
+import scala.concurrent.future
+import scala.concurrent.ExecutionContext.Implicits.global
+
+/**
+ * Represents the handler for comm_open messages.
+ *
+ * @param actorLoader The actor loader to use for actor communication
+ * @param commRegistrar The Comm registrar used for linking
+ * @param commStorage The Comm storage used for open callbacks
+ */
+class CommOpenHandler(
+ actorLoader: ActorLoader, commRegistrar: CommRegistrar,
+ commStorage: CommStorage
+) extends BaseHandler(actorLoader) with MessageLogSupport
+{
+ override def process(kernelMessage: KernelMessage): Future[_] = future {
+ logKernelMessageAction("Initiating Comm Open for", kernelMessage)
+
+ val kmBuilder = KMBuilder().withParent(kernelMessage)
+
+ Utilities.parseAndHandle(
+ kernelMessage.contentString,
+ CommOpen.commOpenReads,
+ handler = handleCommOpen(kmBuilder),
+ errHandler = handleParseError
+ )
+ }
+
+ private def handleCommOpen(kmBuilder: KMBuilder)(commOpen: CommOpen) = {
+ val commId = commOpen.comm_id
+ val targetName = commOpen.target_name
+ val data = commOpen.data
+
+ logger.debug(
+ s"Received comm_open for target '$targetName' with id '$commId'")
+
+ val commWriter = new KernelCommWriter(actorLoader, kmBuilder, commId)
+
+ commStorage.getTargetCallbacks(targetName) match {
+ case None =>
+ logger.warn(s"Received invalid target for Comm Open: $targetName")
+
+ commWriter.close()
+ case Some(callbacks) =>
+ logger.debug(s"Executing open callbacks for id '$commId'")
+
+ // TODO: Should we be checking the return values? Probably not.
+ callbacks.executeOpenCallbacks(commWriter, commId, targetName, data)
+ .filter(_.isFailure).map(_.failed).foreach(throwable => {
+ logger.error("Comm open callback encountered an error!", throwable)
+ })
+ }
+ }
+
+ private def handleParseError(invalid: Seq[(JsPath, Seq[ValidationError])]) = {
+ // TODO: Determine proper response for a parse failure
+ logger.warn("Parse error for Comm Open! Not responding!")
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/ExecuteRequestHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/ExecuteRequestHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/ExecuteRequestHandler.scala
new file mode 100644
index 0000000..c066d98
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/ExecuteRequestHandler.scala
@@ -0,0 +1,163 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.handler
+
+import akka.actor.ActorSelection
+import akka.pattern.ask
+import com.ibm.spark.global.{ExecuteRequestState, ExecutionCounter}
+import com.ibm.spark.kernel.api.{Kernel, KernelLike}
+import com.ibm.spark.kernel.protocol.v5._
+import com.ibm.spark.kernel.protocol.v5.content._
+import com.ibm.spark.kernel.protocol.v5.kernel.{ActorLoader, Utilities}
+import com.ibm.spark.kernel.protocol.v5.stream.KernelOutputStream
+import com.ibm.spark.{global => kernelGlobal}
+import Utilities._
+import com.ibm.spark.utils._
+import play.api.data.validation.ValidationError
+import play.api.libs.json.JsPath
+
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent._
+import scala.util.{Failure, Success}
+
+/**
+ * Receives an ExecuteRequest KernelMessage and forwards the ExecuteRequest
+ * to the interpreter actor.
+ *
+ * @param actorLoader The loader to use when needing to retrieve actors for
+ * code execution and output
+ * @param kernel The kernel whose factory methods to use
+ */
+class ExecuteRequestHandler(
+ private val actorLoader: ActorLoader,
+ private val kernel: Kernel
+) extends BaseHandler(actorLoader) with LogLike {
+ override def process(km: KernelMessage): Future[_] = {
+ // Mark the message as our new incoming kernel message for execution
+ ExecuteRequestState.processIncomingKernelMessage(km)
+
+ val skeletonBuilder = KMBuilder().withParent(km).withIds(km.ids)
+ val executionCount = ExecutionCounter.incr(km.header.session)
+ val relayActor = actorLoader.load(SystemActorType.KernelMessageRelay)
+
+ def handleExecuteRequest(executeRequest: ExecuteRequest):
+ Future[(ExecuteReply, ExecuteResult)] = {
+ // Send an ExecuteInput to the client saying we are executing something
+ val executeInputMessage = skeletonBuilder
+ .withHeader(MessageType.Outgoing.ExecuteInput)
+ .withContentString(ExecuteInput(executeRequest.code, executionCount)).build
+
+ relayMsg(executeInputMessage, relayActor)
+
+ // Construct our new set of streams
+ // TODO: Add support for error streams
+ val outputStream = kernel.factory(
+ parentMessage = km,
+ kmBuilder = skeletonBuilder
+ ).newKernelOutputStream()
+ val executeFuture = ask(
+ actorLoader.load(SystemActorType.ExecuteRequestRelay),
+ (executeRequest, km, outputStream)
+ ).mapTo[(ExecuteReply, ExecuteResult)]
+
+ // Flush the output stream after code execution completes to ensure
+ // stream messages are sent prior to idle status messages.
+ executeFuture andThen { case result =>
+ outputStream.flush()
+ result
+ } andThen {
+ case Success(tuple) =>
+ val (executeReply, executeResult) = updateCount(tuple, executionCount)
+
+ // Send an ExecuteReply to the client
+ val executeReplyMsg = skeletonBuilder
+ .withHeader(MessageType.Outgoing.ExecuteReply)
+ .withContentString(executeReply).build
+ relayMsg(executeReplyMsg, relayActor)
+
+ // Send an ExecuteResult with the result of the code execution
+ if (executeResult.hasContent) {
+ val executeResultMsg = skeletonBuilder
+ .withIds(Seq(MessageType.Outgoing.ExecuteResult.toString))
+ .withHeader(MessageType.Outgoing.ExecuteResult)
+ .withContentString(executeResult).build
+ relayMsg(executeResultMsg, relayActor)
+ }
+
+ case Failure(error: Throwable) =>
+ // Send an ExecuteReplyError to the client on the Shell socket
+ val replyError: ExecuteReply = ExecuteReplyError(
+ executionCount,
+ Option(error.getClass.getCanonicalName),
+ Option(error.getMessage),
+ Option(error.getStackTrace.map(_.toString).toList))
+ relayErrorMessages(relayActor, replyError, skeletonBuilder)
+ }
+ }
+
+ def parseErrorHandler(invalid: Seq[(JsPath, Seq[ValidationError])]) = {
+ val errs = invalid.map (e => s"JSPath ${e._1} has error ${e._2}").toList
+ logger.error(s"Validation errors when parsing ExecuteRequest: ${errs}")
+ val replyError: ExecuteReply = ExecuteReplyError(
+ executionCount,
+ Option("JsonParseException"),
+ Option("Error parsing fields"),
+ Option(errs)
+ )
+ future { relayErrorMessages(relayActor, replyError, skeletonBuilder) }
+ }
+
+ Utilities.parseAndHandle(
+ km.contentString,
+ ExecuteRequest.executeRequestReads,
+ handler = handleExecuteRequest,
+ errHandler = parseErrorHandler)
+ }
+
+ private def updateCount(tuple: (ExecuteReply, ExecuteResult), n: Int) =
+ (tuple._1.copy(execution_count = n), tuple._2.copy(execution_count = n))
+
+ /**
+ * Sends an ExecuteReplyError and and Error message to the given actor.
+ * @param relayActor The relay to send kernelMessages through
+ * @param replyError The reply error to build the error kernelMessages from.
+ * @param skeletonBuilder A builder with common base KernelMessage parameters.
+ */
+ def relayErrorMessages(relayActor: ActorSelection,
+ replyError: ExecuteReply,
+ skeletonBuilder: KMBuilder) {
+ val executeReplyMsg = skeletonBuilder
+ .withHeader(MessageType.Outgoing.ExecuteReply)
+ .withContentString(replyError).build
+
+ val errorContent: ErrorContent = ErrorContent(
+ replyError.ename.get, replyError.evalue.get, replyError.traceback.get)
+
+ val errorMsg = skeletonBuilder
+ .withHeader(MessageType.Outgoing.Error)
+ .withContentString(errorContent).build
+
+ relayMsg(executeReplyMsg, relayActor)
+ // Send the Error to the client on the IOPub socket
+ relayMsg(errorMsg, relayActor)
+ }
+
+ private def relayMsg(km: KernelMessage, relayActor: ActorSelection) = {
+ logKernelMessageAction("Sending to KernelMessageRelay.", km)
+ relayActor ! km
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/GenericSocketMessageHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/GenericSocketMessageHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/GenericSocketMessageHandler.scala
new file mode 100644
index 0000000..834f632
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/GenericSocketMessageHandler.scala
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.handler
+
+import akka.actor.Actor
+import com.ibm.spark.communication.utils.OrderedSupport
+import com.ibm.spark.kernel.protocol.v5.KernelMessage
+import com.ibm.spark.kernel.protocol.v5.kernel.ActorLoader
+import com.ibm.spark.utils.{MessageLogSupport, LogLike}
+
+/**
+ * All KernelMessage leaving the kernel for the client will exit the relay in a similar pattern. This class is meant
+ * to encapsulate this behaviour into one generic method. This class should be used by mapping a
+ * {@link com.ibm.spark.kernel.protocol.MessageType} to the {@link com.ibm.spark.kernel.protocol.SocketType} constructor
+ * parameter. This will map MessageTypes to their corresponding SocketTypes. An example us of this class would be
+ *
+ * actorSystem.actorOf(
+ * // Tells the handler to forward anything it receives to the Control socket
+ * Props(classOf[GenericSocketMessageHandler], actorLoader, SocketType.Control),
+ *
+ * // Creates the Actor with the name of the message type, this allows the Relay to route messages here
+ * name = MessageType.KernelInfoReply.toString
+ * )
+ *
+ * @param actorLoader The ActorLoader used to load the socket actors
+ * @param socketType The type of socket, mapping to an Actor for this class to pass messages along to
+ */
+class GenericSocketMessageHandler(actorLoader: ActorLoader, socketType: Enumeration#Value)
+ extends Actor with LogLike with OrderedSupport {
+ override def receive: Receive = {
+ case message: KernelMessage => withProcessing {
+ logger.debug(s"Sending KernelMessage ${message.header.msg_id} of type " +
+ s"${message.header.msg_type} to ${socketType} socket")
+ actorLoader.load(socketType) ! message
+ }
+ }
+
+ /**
+ * Defines the types that will be stashed by {@link #waiting() waiting}
+ * while the Actor is in processing state.
+ * @return
+ */
+ override def orderedTypes(): Seq[Class[_]] = Seq(classOf[KernelMessage])
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/InputRequestReplyHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/InputRequestReplyHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/InputRequestReplyHandler.scala
new file mode 100644
index 0000000..dba0eef
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/InputRequestReplyHandler.scala
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.handler
+
+import akka.actor.ActorRef
+import com.ibm.spark.comm.{CommRegistrar, CommStorage}
+import com.ibm.spark.communication.utils.OrderedSupport
+import com.ibm.spark.kernel.protocol.v5.{SystemActorType, KernelMessage}
+import com.ibm.spark.kernel.protocol.v5.content.{InputReply, CommOpen}
+import com.ibm.spark.kernel.protocol.v5.kernel.{Utilities, ActorLoader}
+import com.ibm.spark.kernel.protocol.v5
+import com.ibm.spark.utils.MessageLogSupport
+import play.api.libs.json.Json
+
+import scala.concurrent.{Promise, Future}
+
+/**
+ * Represents the handler for both input request and input reply messages. Does
+ * not extend BaseHandler because it does not send busy/idle status messages.
+ *
+ * @param actorLoader The actor loader to use for actor communication
+ * @param responseMap The map used to maintain the links to responses
+ */
+class InputRequestReplyHandler(
+ actorLoader: ActorLoader,
+ responseMap: collection.mutable.Map[String, ActorRef]
+) extends OrderedSupport with MessageLogSupport
+{
+ // TODO: Is there a better way than storing actor refs?
+ def receive = {
+ case kernelMessage: KernelMessage =>
+ startProcessing()
+
+ val kernelMessageType = kernelMessage.header.msg_type
+ val inputRequestType = v5.MessageType.Outgoing.InputRequest.toString
+ val inputReplyType = v5.MessageType.Incoming.InputReply.toString
+
+ // Is this an outgoing message to request data?
+ if (kernelMessageType == inputRequestType) {
+ val session = kernelMessage.parentHeader.session
+ responseMap(session) = sender
+
+ logger.debug("Associating input request with session " + session)
+
+ actorLoader.load(SystemActorType.KernelMessageRelay) ! kernelMessage
+
+ // Is this an incoming response to a previous request for data?
+ } else if (kernelMessageType == inputReplyType) {
+ val session = kernelMessage.header.session
+ val inputReply = Json.parse(kernelMessage.contentString).as[InputReply]
+
+ logger.debug(s"Received input reply for session $session with value " +
+ s"'${inputReply.value}'")
+
+ responseMap(session) ! inputReply.value
+ responseMap.remove(session)
+ }
+
+ finishedProcessing()
+ }
+
+ override def orderedTypes() : Seq[Class[_]] = {Seq(classOf[KernelMessage])}
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
new file mode 100644
index 0000000..854a759
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.handler
+
+import com.ibm.spark.kernel.protocol.v5._
+import com.ibm.spark.kernel.protocol.v5.content.KernelInfoReply
+import com.ibm.spark.kernel.protocol.v5.kernel.ActorLoader
+import com.ibm.spark.utils.LogLike
+import play.api.libs.json.Json
+
+import scala.concurrent._
+
+/**
+ * Receives a KernelInfoRequest KernelMessage and returns a KernelInfoReply
+ * KernelMessage.
+ */
+class KernelInfoRequestHandler(actorLoader: ActorLoader)
+ extends BaseHandler(actorLoader) with LogLike
+{
+ def process(kernelMessage: KernelMessage): Future[_] = {
+ import scala.concurrent.ExecutionContext.Implicits.global
+ future {
+ logger.debug("Sending kernel info reply message")
+
+ val kernelInfo = SparkKernelInfo
+ val kernelInfoReply = KernelInfoReply(
+ kernelInfo.protocolVersion,
+ kernelInfo.implementation,
+ kernelInfo.implementationVersion,
+ kernelInfo.language_info,
+ kernelInfo.languageVersion,
+ kernelInfo.banner
+ )
+
+ // TODO could we use HeaderBuilder here?
+ val replyHeader = Header(
+ java.util.UUID.randomUUID.toString,
+ "",
+ java.util.UUID.randomUUID.toString,
+ MessageType.Outgoing.KernelInfoReply.toString,
+ kernelInfo.protocolVersion
+ )
+
+ val kernelResponseMessage = KMBuilder()
+ .withIds(kernelMessage.ids)
+ .withSignature("")
+ .withHeader(replyHeader)
+ .withParent(kernelMessage)
+ .withContentString(kernelInfoReply).build
+
+ actorLoader.load(SystemActorType.KernelMessageRelay) ! kernelResponseMessage
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/ShutdownHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/ShutdownHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/ShutdownHandler.scala
new file mode 100644
index 0000000..a0a2001
--- /dev/null
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/ShutdownHandler.scala
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.kernel.protocol.v5.handler
+
+import com.ibm.spark.comm.{CommRegistrar, CommStorage, KernelCommWriter}
+import com.ibm.spark.kernel.protocol.v5.content.{ShutdownReply, ShutdownRequest, CommOpen}
+import com.ibm.spark.kernel.protocol.v5.kernel.{ActorLoader, Utilities}
+import com.ibm.spark.kernel.protocol.v5._
+import com.ibm.spark.utils.MessageLogSupport
+import play.api.data.validation.ValidationError
+import play.api.libs.json.JsPath
+
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.{Future, future}
+
+/**
+ * Represents the handler to shutdown the kernel
+ *
+ * @param actorLoader The actor loader to use for actor communication
+ */
+class ShutdownHandler(
+ actorLoader: ActorLoader
+) extends BaseHandler(actorLoader) with MessageLogSupport
+{
+ override def process(kernelMessage: KernelMessage): Future[_] = future {
+ logKernelMessageAction("Initiating Shutdown request for", kernelMessage)
+
+ val shutdownReply = ShutdownReply(false)
+
+ val replyHeader = Header(
+ java.util.UUID.randomUUID.toString,
+ "",
+ java.util.UUID.randomUUID.toString,
+ ShutdownReply.toTypeString,
+ "")
+
+ val kernelResponseMessage = KMBuilder()
+ .withIds(kernelMessage.ids)
+ .withSignature("")
+ .withHeader(replyHeader)
+ .withParent(kernelMessage)
+ .withContentString(shutdownReply).build
+
+ logger.debug("Attempting graceful shutdown.")
+ actorLoader.load(SystemActorType.KernelMessageRelay) ! kernelResponseMessage
+ System.exit(0)
+ }
+
+}
+