You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by ma...@apache.org on 2016/11/07 14:29:24 UTC
[01/15] incubator-toree git commit: Adding python kernel language info
Repository: incubator-toree
Updated Branches:
refs/heads/master 1ed5821bb -> 953cbc046
Adding python kernel language info
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/fe4290dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/fe4290dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/fe4290dc
Branch: refs/heads/master
Commit: fe4290dcbbb44cedcddce5b9bbbd2b2f903724ba
Parents: a0646ad
Author: Liam Fisk <li...@xtra.co.nz>
Authored: Sun Jun 19 11:33:47 2016 +1200
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Thu Oct 20 17:50:56 2016 -0400
----------------------------------------------------------------------
.../toree/kernel/interpreter/pyspark/PySparkInterpreter.scala | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/fe4290dc/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
index 6b6b2fd..388076b 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
@@ -21,7 +21,6 @@ import java.net.URL
import org.apache.toree.interpreter.Results.Result
import org.apache.toree.interpreter._
import org.apache.toree.kernel.api.KernelLike
-import org.apache.toree.kernel.BuildInfo
import org.slf4j.LoggerFactory
import py4j.GatewayServer
@@ -149,6 +148,10 @@ class PySparkInterpreter(
// Unsupported
override def doQuietly[T](body: => T): T = ???
- override def languageInfo: Map[String, String] = Map("name" -> "scala", "version" -> BuildInfo.scalaVersion)
+ // TODO Identify how to plumb python version to here
+ override def languageInfo: Map[String, String] = Map(
+ "name" -> "python",
+ "version" -> "2.7.9",
+ "pygments_lexer" -> "ipython2")
}
[11/15] incubator-toree git commit: Fixed incorrect override.
Posted by ma...@apache.org.
Fixed incorrect override.
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/5f2715d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/5f2715d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/5f2715d6
Branch: refs/heads/master
Commit: 5f2715d656e5a459eae730d0409a3f8eb936b602
Parents: 77565fb
Author: Marius van Niekerk <ma...@maxpoint.com>
Authored: Tue Nov 1 19:15:04 2016 -0400
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Tue Nov 1 19:15:04 2016 -0400
----------------------------------------------------------------------
.../toree/kernel/interpreter/scala/ScalaInterpreter.scala | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5f2715d6/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
index 4c04436..6f347f5 100644
--- a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
+++ b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
@@ -316,7 +316,12 @@ class ScalaInterpreter(private val config:Config = ConfigFactory.load) extends I
}
override def classLoader: ClassLoader = _runtimeClassloader
- }
+
+ /**
+ * Returns the language metadata for syntax highlighting
+ */
+ override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, fileExtension = Some(".scala"))
+}
object ScalaInterpreter {
@@ -337,7 +342,4 @@ object ScalaInterpreter {
outputDir
}
-
- override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, fileExtension = Some(".scala"))
-
}
[14/15] incubator-toree git commit: Added proper version grabbing for
python and R using subprocess calls.
Posted by ma...@apache.org.
Added proper version grabbing for python and R using subprocess calls.
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/15327b58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/15327b58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/15327b58
Branch: refs/heads/master
Commit: 15327b589fcd2fcd64c9689a6e4b491e72b643cc
Parents: 9ccf178
Author: Marius van Niekerk <ma...@gmail.com>
Authored: Wed Nov 2 10:21:15 2016 -0400
Committer: Marius van Niekerk <ma...@gmail.com>
Committed: Wed Nov 2 10:21:15 2016 -0400
----------------------------------------------------------------------
.../toree/interpreter/broker/BrokerState.scala | 9 +-----
.../interpreter/broker/BrokerStateSpec.scala | 6 ++--
.../main/resources/PySpark/pyspark_runner.py | 8 ++++--
.../pyspark/PySparkInterpreter.scala | 29 ++++++++++----------
.../interpreter/sparkr/SparkRInterpreter.scala | 14 +++++++++-
.../interpreter/sparkr/SparkRProcess.scala | 4 ++-
.../interpreter/sparkr/SparkRService.scala | 3 ++
7 files changed, 44 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/15327b58/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
index 43ee65c..3d1e3ab 100644
--- a/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
@@ -38,7 +38,6 @@ class BrokerState(private val maxQueuedCode: Int) {
import scala.collection.JavaConverters._
@volatile private var _isReady: Boolean = false
- @volatile private var _version: String = _
protected val codeQueue: java.util.Queue[BrokerCode] =
new java.util.concurrent.ConcurrentLinkedQueue[BrokerCode]()
@@ -128,17 +127,11 @@ class BrokerState(private val maxQueuedCode: Int) {
* @param version The language version used by the broker service
*/
- def markReady(version: String): Unit = {
+ def markReady(): Unit = {
_isReady = true
- _version = version
}
/**
- * Retrieve the runtime language version used by the broker service
- */
- def getVersion(): String = _version
-
- /**
* Marks the specified code as successfully completed using its id.
*
* @param codeId The id of the code to mark as a success
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/15327b58/kernel-api/src/test/scala/org/apache/toree/interpreter/broker/BrokerStateSpec.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/test/scala/org/apache/toree/interpreter/broker/BrokerStateSpec.scala b/kernel-api/src/test/scala/org/apache/toree/interpreter/broker/BrokerStateSpec.scala
index 3617816..43374f8 100644
--- a/kernel-api/src/test/scala/org/apache/toree/interpreter/broker/BrokerStateSpec.scala
+++ b/kernel-api/src/test/scala/org/apache/toree/interpreter/broker/BrokerStateSpec.scala
@@ -78,7 +78,7 @@ class BrokerStateSpec extends FunSpec with Matchers with OneInstancePerTest {
describe("#isReady") {
it("should return true if the broker state is marked as ready") {
- brokerState.markReady("1.0.0")
+ brokerState.markReady()
brokerState.isReady should be (true)
}
@@ -90,11 +90,11 @@ class BrokerStateSpec extends FunSpec with Matchers with OneInstancePerTest {
describe("#markReady") {
it("should mark the state of the broker as ready") {
// Mark once to make sure that the state gets set
- brokerState.markReady("1.0.0")
+ brokerState.markReady()
brokerState.isReady should be (true)
// Mark a second time to ensure that the state does not change
- brokerState.markReady("1.0.0")
+ brokerState.markReady()
brokerState.isReady should be (true)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/15327b58/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py b/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
index f73805f..e3864f8 100644
--- a/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
+++ b/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
@@ -15,7 +15,11 @@
# limitations under the License.
#
-import sys, getopt, traceback, re, ast, platform
+import sys
+import getopt
+import traceback
+import re
+import ast
print("PYTHON::: Starting imports")
from py4j.java_gateway import java_import, JavaGateway, GatewayClient
@@ -53,7 +57,7 @@ java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
bridge = gateway.entry_point
state = bridge.state()
-state.markReady(platform.python_version())
+state.markReady()
if sparkVersion.startswith("1.2"):
java_import(gateway.jvm, "org.apache.spark.sql.SparkSession")
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/15327b58/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
index a76e1d0..d408217 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
@@ -41,6 +41,7 @@ class PySparkInterpreter(
private val WAIT_DURATION: Long = java.util.concurrent.TimeUnit.SECONDS.toMillis(50)
private val PythonExecEnv = "PYTHON_EXEC"
+ private lazy val pythonExecutable = Option(System.getenv(PythonExecEnv)).getOrElse("python")
private val logger = LoggerFactory.getLogger(this.getClass)
private var _kernel:KernelLike = _
@@ -67,7 +68,7 @@ class PySparkInterpreter(
)
private lazy val pySparkService = new PySparkService(
- Option(System.getenv(PythonExecEnv)).getOrElse("python"),
+ pythonExecutable,
gatewayServer,
pySparkBridge,
pySparkProcessHandler
@@ -155,18 +156,18 @@ class PySparkInterpreter(
override def doQuietly[T](body: => T): T = ???
override def languageInfo: LanguageInfo = {
- if ((!pySparkService.isRunning) || (!pySparkState.isReady)) {
- LanguageInfo(
- "python",
- version = "UNKNOWN",
- fileExtension = Some(".py"),
- pygmentsLexer = Some("python"))
- } else {
- LanguageInfo(
- "python",
- pySparkState.getVersion(),
- fileExtension = Some(".py"),
- pygmentsLexer = Some("python"))
- }
+ import scala.sys.process._
+
+ // Issue a subprocess call to grab the python version. This is better than polling a child process.
+ val version = Seq(
+ pythonExecutable,
+ "-c",
+ "import sys; print('{s.major}.{s.minor}.{s.micro}'.format(s=sys.version_info))").!!
+
+ LanguageInfo(
+ "python",
+ version = version,
+ fileExtension = Some(".py"),
+ pygmentsLexer = Some("python"))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/15327b58/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
index 975dee1..54bf14a 100644
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
+++ b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
@@ -38,6 +38,7 @@ class SparkRInterpreter(
) extends Interpreter {
private val logger = LoggerFactory.getLogger(this.getClass)
private var _kernel: KernelLike = _
+ private val rScriptExecutable = "Rscript"
// TODO: Replace hard-coded maximum queue count
/** Represents the state used by this interpreter's R instance. */
@@ -61,6 +62,7 @@ class SparkRInterpreter(
)
private lazy val sparkRService = new SparkRService(
+ rScriptExecutable,
rBackend,
sparkRBridge,
sparkRProcessHandler
@@ -139,6 +141,16 @@ class SparkRInterpreter(
// Unsupported
override def doQuietly[T](body: => T): T = ???
- override def languageInfo = LanguageInfo("R", "Unknown", fileExtension = Some(".R"), pygmentsLexer = Some("r"))
+ override def languageInfo = {
+ import sys.process._
+
+ // Issue a subprocess call to grab the R version. This is better than polling a child process.
+ val version = Seq(
+ rScriptExecutable,
+ "-e",
+ "cat(R.version$major, '.', R.version$minor, sep='', fill=TRUE)").!!
+
+ LanguageInfo("R", version = version, fileExtension = Some(".R"), pygmentsLexer = Some("r"))
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/15327b58/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRProcess.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRProcess.scala b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRProcess.scala
index 0fa453f..d1c145a 100644
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRProcess.scala
+++ b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRProcess.scala
@@ -22,6 +22,7 @@ import scala.collection.JavaConverters._
/**
* Represents the R process used to evaluate SparkR code.
*
+ * @param processName The name of the Rscript process to run.
* @param sparkRBridge The bridge to use to retrieve kernel output streams
* and the Spark version to be verified
* @param sparkRProcessHandler The handler to use when the process fails or
@@ -30,11 +31,12 @@ import scala.collection.JavaConverters._
* back to the JVM
*/
class SparkRProcess(
+ processName: String,
private val sparkRBridge: SparkRBridge,
private val sparkRProcessHandler: SparkRProcessHandler,
private val port: Int
) extends BrokerProcess(
- processName = "Rscript",
+ processName = processName,
entryResource = "kernelR/sparkr_runner.R",
otherResources = Seq("kernelR/sparkr_runner_utils.R"),
brokerBridge = sparkRBridge,
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/15327b58/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRService.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRService.scala b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRService.scala
index f373ab2..350aee0 100644
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRService.scala
+++ b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRService.scala
@@ -29,12 +29,14 @@ import scala.tools.nsc.interpreter._
* Represents the service that provides the high-level interface between the
* JVM and R.
*
+ * @param processName The name of the Rscript process to run.
* @param rBackend The backend to start to communicate between the JVM and R
* @param sparkRBridge The bridge to use for communication between the JVM and R
* @param sparkRProcessHandler The handler used for events that occur with the
* SparkR process
*/
class SparkRService(
+ processName: String,
private val rBackend: ReflectiveRBackend,
private val sparkRBridge: SparkRBridge,
private val sparkRProcessHandler: SparkRProcessHandler
@@ -47,6 +49,7 @@ class SparkRService(
/** Represents the process used to execute R code via the bridge. */
private lazy val sparkRProcess: SparkRProcess = {
val p = new SparkRProcess(
+ processName,
sparkRBridge,
sparkRProcessHandler,
rBackendPort
[06/15] incubator-toree git commit: Making versions optional on the
LanguageInfo object
Posted by ma...@apache.org.
Making versions optional on the LanguageInfo object
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/a5303b4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/a5303b4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/a5303b4c
Branch: refs/heads/master
Commit: a5303b4c3fd1b971982d1c724b71879faad6eb9b
Parents: c330d3a
Author: Liam Fisk <li...@xtra.co.nz>
Authored: Thu Jun 30 09:05:39 2016 +1200
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Thu Oct 20 17:50:58 2016 -0400
----------------------------------------------------------------------
.../main/scala/org/apache/toree/interpreter/Interpreter.scala | 4 ++--
.../protocol/v5/handler/KernelInfoRequestHandlerSpec.scala | 2 +-
.../scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala | 4 ++--
.../toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala | 2 +-
pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py | 1 -
.../toree/kernel/interpreter/pyspark/PySparkInterpreter.scala | 3 +--
.../apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala | 2 +-
.../toree/kernel/interpreter/sparkr/SparkRInterpreter.scala | 2 +-
.../org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala | 2 +-
9 files changed, 10 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a5303b4c/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
index 5b9d4c2..edf1e71 100644
--- a/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
@@ -25,8 +25,8 @@ import scala.tools.nsc.interpreter._
case class LanguageInfo(
name: String,
- version: String,
- fileExtension: String,
+ version: Option[String] = None,
+ fileExtension: Option[String] = None,
pygmentsLexer: Option[String] = None) {
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a5303b4c/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
index 04a718f..252f64a 100644
--- a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
@@ -42,7 +42,7 @@ class KernelInfoRequestHandlerSpec extends TestKit(
ConfigFactory.parseString(KernelInfoRequestHandlerSpec.config))
) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
val actorLoader: ActorLoader = mock[ActorLoader]
- val actor = system.actorOf(Props(classOf[KernelInfoRequestHandler], actorLoader, LanguageInfo("test", "1.0.0", ".test")))
+ val actor = system.actorOf(Props(classOf[KernelInfoRequestHandler], actorLoader, LanguageInfo("test", Some("1.0.0"), Some(".test"))))
val relayProbe : TestProbe = TestProbe()
val relaySelection : ActorSelection =
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a5303b4c/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
----------------------------------------------------------------------
diff --git a/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala b/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
index 04b85c8..d2d99be 100644
--- a/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
+++ b/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
@@ -19,7 +19,7 @@ package org.apache.toree.kernel.protocol.v5
case class LanguageInfo(
name: String,
- version: String,
- file_extension: String,
+ version: Option[String] = None,
+ file_extension: Option[String] = None,
pygments_lexer: Option[String] = None) {
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a5303b4c/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
----------------------------------------------------------------------
diff --git a/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala b/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
index b849df8..68c901c 100644
--- a/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
+++ b/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
@@ -34,7 +34,7 @@ class KernelInfoReplySpec extends FunSpec with Matchers {
""")
val kernelInfoReply: KernelInfoReply = KernelInfoReply(
- "x.y.z", "<name>", "z.y.x", LanguageInfo("<some language>", "a.b.c", "<some extension>"), "<some banner>"
+ "x.y.z", "<name>", "z.y.x", LanguageInfo("<some language>", Some("a.b.c"), Some("<some extension>")), "<some banner>"
)
describe("KernelInfoReply") {
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a5303b4c/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py b/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
index 8876e18..d84c5c4 100644
--- a/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
+++ b/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
@@ -76,7 +76,6 @@ sc = None
spark = None
code_info = None
-
class Logger(object):
def __init__(self):
self.out = ""
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a5303b4c/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
index 784a9ed..e347ed8 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
@@ -151,8 +151,7 @@ class PySparkInterpreter(
// TODO Identify how to plumb python version to here
override def languageInfo = LanguageInfo(
"python",
- "2.7.9",
- ".py",
+ fileExtension = Some(".py"),
pygmentsLexer = Some("ipython2"))
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a5303b4c/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
index 1cd6d13..464c223 100644
--- a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
+++ b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
@@ -338,6 +338,6 @@ object ScalaInterpreter {
}
- override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, ".scala")
+ override def languageInfo = LanguageInfo("scala", Some(BuildInfo.scalaVersion), Some(".scala"))
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a5303b4c/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
index ff4aa06..c067d5b 100644
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
+++ b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
@@ -139,6 +139,6 @@ class SparkRInterpreter(
// Unsupported
override def doQuietly[T](body: => T): T = ???
- override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, ".scala")
+ override def languageInfo = LanguageInfo("R")
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a5303b4c/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
----------------------------------------------------------------------
diff --git a/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala b/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
index 988ef9e..fa7fc2b 100644
--- a/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
+++ b/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
@@ -107,6 +107,6 @@ class SqlInterpreter() extends Interpreter {
// Unsupported
override def doQuietly[T](body: => T): T = ???
- override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, ".scala")
+ override def languageInfo = LanguageInfo("SQL")
}
[09/15] incubator-toree git commit: Fixed some minor issues with the
Kernel Info message handling
Posted by ma...@apache.org.
Fixed some minor issues with the Kernel Info message handling
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/2d3cea40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/2d3cea40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/2d3cea40
Branch: refs/heads/master
Commit: 2d3cea4062135b38210faae47f77e3fffaf51d1d
Parents: e5d8d0f
Author: Marius van Niekerk <ma...@maxpoint.com>
Authored: Thu Oct 20 17:45:41 2016 -0400
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Thu Oct 20 17:51:00 2016 -0400
----------------------------------------------------------------------
.../pyspark/PySparkInterpreter.scala | 38 +++++++++++---------
.../interpreter/sparkr/SparkRInterpreter.scala | 2 +-
.../kernel/interpreter/sql/SqlInterpreter.scala | 2 +-
3 files changed, 23 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/2d3cea40/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
index 72b0aa3..6e05975 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
@@ -37,6 +37,9 @@ import scala.tools.nsc.interpreter.{InputStream, OutputStream}
*/
class PySparkInterpreter(
) extends Interpreter {
+ /** Maximum time to wait for the python kernel to be readu */
+ private val WAIT_DURATION: Long = java.util.concurrent.TimeUnit.SECONDS.toMillis(50)
+
private val PythonExecEnv = "PYTHON_EXEC"
private val logger = LoggerFactory.getLogger(this.getClass)
private var _kernel:KernelLike = _
@@ -73,7 +76,8 @@ class PySparkInterpreter(
/**
* Initializes the interpreter.
- * @param kernel The kernel
+ *
+ * @param kernel The kernel
* @return The newly initialized interpreter
*/
override def init(kernel: KernelLike): Interpreter = {
@@ -83,7 +87,8 @@ class PySparkInterpreter(
/**
* Executes the provided code with the option to silence output.
- * @param code The code to execute
+ *
+ * @param code The code to execute
* @param silent Whether or not to execute the code silently (no output)
* @return The success/failure of the interpretation and the output from the
* execution or the failure
@@ -101,7 +106,8 @@ class PySparkInterpreter(
/**
* Starts the interpreter, initializing any internal state.
- * @return A reference to the interpreter
+ *
+ * @return A reference to the interpreter
*/
override def start(): Interpreter = {
pySparkService.start()
@@ -111,7 +117,8 @@ class PySparkInterpreter(
/**
* Stops the interpreter, removing any previous internal state.
- * @return A reference to the interpreter
+ *
+ * @return A reference to the interpreter
*/
override def stop(): Interpreter = {
pySparkService.stop()
@@ -148,21 +155,18 @@ class PySparkInterpreter(
override def doQuietly[T](body: => T): T = ???
override def languageInfo: LanguageInfo = {
- if (!pySparkService.isRunning) pySparkService.start()
-
- import scala.util.control.Breaks._
- val waitLimit = System.currentTimeMillis() + java.util.concurrent.TimeUnit.SECONDS.toMillis(5)
- while (!pySparkState.isReady) {
- if (System.currentTimeMillis > waitLimit) {
- logger.warn("Timed out waiting for broker state to become ready")
- break
- }
- }
-
- LanguageInfo(
+ if (!pySparkService.isRunning) or (!pySparkState.isReady) {
+ LanguageInfo(
+ "python",
+ version = "UNKNOWN",
+ fileExtension = Some(".py"),
+ pygmentsLexer = Some("python"))
+ } else {
+ LanguageInfo(
"python",
pySparkState.getVersion(),
fileExtension = Some(".py"),
- pygmentsLexer = Some("ipython2"))
+ pygmentsLexer = Some("python"))
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/2d3cea40/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
index d6be29b..975dee1 100644
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
+++ b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
@@ -139,6 +139,6 @@ class SparkRInterpreter(
// Unsupported
override def doQuietly[T](body: => T): T = ???
- override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, fileExtension = Some(".scala"))
+ override def languageInfo = LanguageInfo("R", "Unknown", fileExtension = Some(".R"), pygmentsLexer = Some("r"))
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/2d3cea40/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
----------------------------------------------------------------------
diff --git a/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala b/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
index b6e272f..df2952b 100644
--- a/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
+++ b/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
@@ -107,6 +107,6 @@ class SqlInterpreter() extends Interpreter {
// Unsupported
override def doQuietly[T](body: => T): T = ???
- override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, fileExtension = Some(".scala"))
+ override def languageInfo = LanguageInfo("sql", BuildInfo.sparkVersion, fileExtension = Some(".sql"), pygmentsLexer = Some("sql"))
}
[13/15] incubator-toree git commit: Simplified Makefile for
system-test task
Posted by ma...@apache.org.
Simplified Makefile for system-test task
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/9ccf1784
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/9ccf1784
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/9ccf1784
Branch: refs/heads/master
Commit: 9ccf1784e1c6dc0eda8c9e3ec44c32ef12fb6b8d
Parents: bb4bf47
Author: Marius van Niekerk <ma...@gmail.com>
Authored: Wed Nov 2 00:22:24 2016 -0400
Committer: Marius van Niekerk <ma...@gmail.com>
Committed: Wed Nov 2 00:22:24 2016 -0400
----------------------------------------------------------------------
Dockerfile.system-test | 50 +++++++++++++++++++++++++++++++++++++++++++++
Makefile | 27 ++----------------------
2 files changed, 52 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9ccf1784/Dockerfile.system-test
----------------------------------------------------------------------
diff --git a/Dockerfile.system-test b/Dockerfile.system-test
new file mode 100644
index 0000000..85426bf
--- /dev/null
+++ b/Dockerfile.system-test
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+#
+# This is a dockerfile used to construct the spark environment used for the
+# integration test suite.
+
+FROM jupyter/all-spark-notebook:07a7c4d6d447
+
+# User escalation
+USER root
+
+# Spark dependencies
+ENV APACHE_SPARK_VERSION 2.0.0
+
+# Temporarily add jessie backports to get openjdk 8, but then remove that source
+RUN echo 'deb http://ftp.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/jessie-backports.list && \
+ apt-get -y update && \
+ apt-get install -y --no-install-recommends openjdk-8-jre-headless && \
+ rm /etc/apt/sources.list.d/jessie-backports.list && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/* && \
+ rm -rf /var/lib/apt/lists/* && \
+ update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
+
+
+# Installing Spark2
+RUN cd /tmp && \
+ wget -q http://d3kbcqa49mib13.cloudfront.net/spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz && \
+ echo "e17d9da4b3ac463ea3ce42289f2a71cefb479d154b1ffd00310c7d7ab207aa2c *spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz" | sha256sum -c - && \
+ tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz -C /usr/local && \
+ rm spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz
+
+# Overwrite symlink
+RUN cd /usr/local && \
+ rm spark && \
+ ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6 spark
+
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9ccf1784/Makefile
----------------------------------------------------------------------
diff --git a/Makefile b/Makefile
index 85aed4d..79d3440 100644
--- a/Makefile
+++ b/Makefile
@@ -90,31 +90,8 @@ clean: clean-dist
touch $@
.system-test-image:
- @-docker rm -f system_test_image
- @docker run -t --user root --name system_test_image \
- $(IMAGE) bash -c "cd /tmp && \
- wget http://apache.claz.org/spark/spark-$(APACHE_SPARK_VERSION)/spark-$(APACHE_SPARK_VERSION)-bin-hadoop2.6.tgz && \
- tar xzf spark-$(APACHE_SPARK_VERSION)-bin-hadoop2.6.tgz -C /usr/local && \
- rm spark-$(APACHE_SPARK_VERSION)-bin-hadoop2.6.tgz && \
- cd /usr/local && \
- rm spark && \
- ln -s spark-$(APACHE_SPARK_VERSION)-bin-hadoop2.6 spark && \
- echo /usr/local/spark/RELEASE && \
- \
- echo \"===> add webupd8 repository...\" && \
- echo \"deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main\" | tee /etc/apt/sources.list.d/webupd8team-java.list && \
- echo \"deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main\" | tee -a /etc/apt/sources.list.d/webupd8team-java.list && \
- apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \
- apt-get update && \
- \
- echo \"===> install Java\" && \
- echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
- echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \
- DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes oracle-java8-installer oracle-java8-set-default && \
- apt-get clean && \
- update-java-alternatives -s java-8-oracle"
- @docker commit system_test_image $(SYSTEM_TEST_IMAGE)
- @-docker rm -f system_test_image
+ @-docker rm -f $(SYSTEM_TEST_IMAGE)
+ @docker build -t $(SYSTEM_TEST_IMAGE) -f Dockerfile.system-test .
touch $@
.binder-image:
[12/15] incubator-toree git commit: Fixed sparkR integration test
Posted by ma...@apache.org.
Fixed sparkR integration test
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/bb4bf47c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/bb4bf47c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/bb4bf47c
Branch: refs/heads/master
Commit: bb4bf47c368617d77e78826fdc703adf6ff6d2f9
Parents: 5f2715d
Author: Marius van Niekerk <ma...@gmail.com>
Authored: Wed Nov 2 00:21:36 2016 -0400
Committer: Marius van Niekerk <ma...@gmail.com>
Committed: Wed Nov 2 00:21:36 2016 -0400
----------------------------------------------------------------------
test_toree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bb4bf47c/test_toree.py
----------------------------------------------------------------------
diff --git a/test_toree.py b/test_toree.py
index 2df1978..db639e8 100644
--- a/test_toree.py
+++ b/test_toree.py
@@ -150,7 +150,7 @@ class ToreeSparkRKernelTests(jupyter_kernel_test.KernelTests):
kernel_name = "apache_toree_sparkr"
# language_info.name in a kernel_info_reply should match this
- language_name = "scala"
+ language_name = "R"
# Optional --------------------------------------
[04/15] incubator-toree git commit: Fixing jupyter tests
Posted by ma...@apache.org.
Fixing jupyter tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/ccd711a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/ccd711a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/ccd711a4
Branch: refs/heads/master
Commit: ccd711a4e51bbaeb15363d5f04f971d2d849cdcb
Parents: fe4290d
Author: Liam Fisk <li...@xtra.co.nz>
Authored: Sun Jun 19 16:41:45 2016 +1200
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Thu Oct 20 17:50:57 2016 -0400
----------------------------------------------------------------------
.../toree/kernel/interpreter/pyspark/PySparkInterpreter.scala | 3 ++-
.../toree/kernel/interpreter/scala/ScalaInterpreter.scala | 6 ++++--
test_toree.py | 2 +-
3 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/ccd711a4/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
index 388076b..a192c78 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
@@ -152,6 +152,7 @@ class PySparkInterpreter(
override def languageInfo: Map[String, String] = Map(
"name" -> "python",
"version" -> "2.7.9",
- "pygments_lexer" -> "ipython2")
+ "pygments_lexer" -> "ipython2",
+ "file_extension" -> ".py")
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/ccd711a4/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
index 4df3158..bb198a7 100644
--- a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
+++ b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
@@ -338,7 +338,9 @@ object ScalaInterpreter {
}
- override def languageInfo: Map[String, String] = Map("name" -> "scala", "version" -> BuildInfo.scalaVersion)
+ override def languageInfo: Map[String, String] = Map(
+ "name" -> "scala",
+ "version" -> BuildInfo.scalaVersion,
+ "file_extension" -> ".scala")
}
-
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/ccd711a4/test_toree.py
----------------------------------------------------------------------
diff --git a/test_toree.py b/test_toree.py
index 2e25324..2df1978 100644
--- a/test_toree.py
+++ b/test_toree.py
@@ -129,7 +129,7 @@ class ToreePythonKernelTests(jupyter_kernel_test.KernelTests):
kernel_name = "apache_toree_pyspark"
# language_info.name in a kernel_info_reply should match this
- language_name = "scala"
+ language_name = "python"
# Optional --------------------------------------
[08/15] incubator-toree git commit: Passing version to broker on
'markReady'
Posted by ma...@apache.org.
Passing version to broker on 'markReady'
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/b3ef81bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/b3ef81bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/b3ef81bb
Branch: refs/heads/master
Commit: b3ef81bb04d35d92469a7b285bb55971f59dff8a
Parents: 6a77283
Author: Liam Fisk <li...@xtra.co.nz>
Authored: Fri Jul 1 09:45:38 2016 +1200
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Thu Oct 20 17:50:59 2016 -0400
----------------------------------------------------------------------
.../org/apache/toree/interpreter/Interpreter.scala | 2 +-
.../toree/interpreter/broker/BrokerState.scala | 17 ++++++++++++++---
.../toree/interpreter/broker/BrokerStateSpec.scala | 6 +++---
.../v5/handler/KernelInfoRequestHandlerSpec.scala | 2 +-
.../toree/kernel/protocol/v5/LanguageInfo.scala | 2 +-
.../protocol/v5/content/KernelInfoReplySpec.scala | 2 +-
.../src/main/resources/PySpark/pyspark_runner.py | 4 ++--
.../interpreter/pyspark/PySparkInterpreter.scala | 1 +
.../interpreter/scala/ScalaInterpreter.scala | 2 +-
.../interpreter/sparkr/SparkRInterpreter.scala | 4 ++--
.../kernel/interpreter/sql/SqlInterpreter.scala | 2 +-
11 files changed, 28 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
index edf1e71..bfff81c 100644
--- a/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
@@ -25,7 +25,7 @@ import scala.tools.nsc.interpreter._
case class LanguageInfo(
name: String,
- version: Option[String] = None,
+ version: String,
fileExtension: Option[String] = None,
pygmentsLexer: Option[String] = None) {
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
index 4595a92..43ee65c 100644
--- a/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/broker/BrokerState.scala
@@ -37,7 +37,9 @@ class BrokerState(private val maxQueuedCode: Int) {
import scala.collection.JavaConverters._
- private var _isReady: Boolean = false
+ @volatile private var _isReady: Boolean = false
+ @volatile private var _version: String = _
+
protected val codeQueue: java.util.Queue[BrokerCode] =
new java.util.concurrent.ConcurrentLinkedQueue[BrokerCode]()
protected val promiseMap: collection.mutable.Map[CodeId, BrokerPromise] =
@@ -123,8 +125,18 @@ class BrokerState(private val maxQueuedCode: Int) {
/**
* Marks the state of broker as ready.
+
+ * @param version The language version used by the broker service
*/
- def markReady(): Unit = _isReady = true
+ def markReady(version: String): Unit = {
+ _isReady = true
+ _version = version
+ }
+
+ /**
+ * Retrieve the runtime language version used by the broker service
+ */
+ def getVersion(): String = _version
/**
* Marks the specified code as successfully completed using its id.
@@ -192,4 +204,3 @@ class BrokerState(private val maxQueuedCode: Int) {
}
}
}
-
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/kernel-api/src/test/scala/org/apache/toree/interpreter/broker/BrokerStateSpec.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/test/scala/org/apache/toree/interpreter/broker/BrokerStateSpec.scala b/kernel-api/src/test/scala/org/apache/toree/interpreter/broker/BrokerStateSpec.scala
index 43374f8..3617816 100644
--- a/kernel-api/src/test/scala/org/apache/toree/interpreter/broker/BrokerStateSpec.scala
+++ b/kernel-api/src/test/scala/org/apache/toree/interpreter/broker/BrokerStateSpec.scala
@@ -78,7 +78,7 @@ class BrokerStateSpec extends FunSpec with Matchers with OneInstancePerTest {
describe("#isReady") {
it("should return true if the broker state is marked as ready") {
- brokerState.markReady()
+ brokerState.markReady("1.0.0")
brokerState.isReady should be (true)
}
@@ -90,11 +90,11 @@ class BrokerStateSpec extends FunSpec with Matchers with OneInstancePerTest {
describe("#markReady") {
it("should mark the state of the broker as ready") {
// Mark once to make sure that the state gets set
- brokerState.markReady()
+ brokerState.markReady("1.0.0")
brokerState.isReady should be (true)
// Mark a second time to ensure that the state does not change
- brokerState.markReady()
+ brokerState.markReady("1.0.0")
brokerState.isReady should be (true)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
index 252f64a..73aa2a3 100644
--- a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
@@ -42,7 +42,7 @@ class KernelInfoRequestHandlerSpec extends TestKit(
ConfigFactory.parseString(KernelInfoRequestHandlerSpec.config))
) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
val actorLoader: ActorLoader = mock[ActorLoader]
- val actor = system.actorOf(Props(classOf[KernelInfoRequestHandler], actorLoader, LanguageInfo("test", Some("1.0.0"), Some(".test"))))
+ val actor = system.actorOf(Props(classOf[KernelInfoRequestHandler], actorLoader, LanguageInfo("test", "1.0.0", Some(".test"))))
val relayProbe : TestProbe = TestProbe()
val relaySelection : ActorSelection =
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
----------------------------------------------------------------------
diff --git a/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala b/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
index d2d99be..161181e 100644
--- a/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
+++ b/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
@@ -19,7 +19,7 @@ package org.apache.toree.kernel.protocol.v5
case class LanguageInfo(
name: String,
- version: Option[String] = None,
+ version: String,
file_extension: Option[String] = None,
pygments_lexer: Option[String] = None) {
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
----------------------------------------------------------------------
diff --git a/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala b/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
index 68c901c..f138725 100644
--- a/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
+++ b/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
@@ -34,7 +34,7 @@ class KernelInfoReplySpec extends FunSpec with Matchers {
""")
val kernelInfoReply: KernelInfoReply = KernelInfoReply(
- "x.y.z", "<name>", "z.y.x", LanguageInfo("<some language>", Some("a.b.c"), Some("<some extension>")), "<some banner>"
+ "x.y.z", "<name>", "z.y.x", LanguageInfo("<some language>", "a.b.c", Some("<some extension>")), "<some banner>"
)
describe("KernelInfoReply") {
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py b/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
index 5073a4d..f73805f 100644
--- a/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
+++ b/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
@@ -15,7 +15,7 @@
# limitations under the License.
#
-import sys, getopt, traceback, re, ast
+import sys, getopt, traceback, re, ast, platform
print("PYTHON::: Starting imports")
from py4j.java_gateway import java_import, JavaGateway, GatewayClient
@@ -53,7 +53,7 @@ java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
bridge = gateway.entry_point
state = bridge.state()
-state.markReady()
+state.markReady(platform.python_version())
if sparkVersion.startswith("1.2"):
java_import(gateway.jvm, "org.apache.spark.sql.SparkSession")
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
index e347ed8..632f52d 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
@@ -151,6 +151,7 @@ class PySparkInterpreter(
// TODO Identify how to plumb python version to here
override def languageInfo = LanguageInfo(
"python",
+ pySparkState.getVersion(),
fileExtension = Some(".py"),
pygmentsLexer = Some("ipython2"))
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
index 464c223..4c04436 100644
--- a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
+++ b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
@@ -338,6 +338,6 @@ object ScalaInterpreter {
}
- override def languageInfo = LanguageInfo("scala", Some(BuildInfo.scalaVersion), Some(".scala"))
+ override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, fileExtension = Some(".scala"))
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
index c067d5b..d6be29b 100644
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
+++ b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
@@ -138,7 +138,7 @@ class SparkRInterpreter(
// Unsupported
override def doQuietly[T](body: => T): T = ???
-
- override def languageInfo = LanguageInfo("R")
+
+ override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, fileExtension = Some(".scala"))
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b3ef81bb/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
----------------------------------------------------------------------
diff --git a/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala b/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
index fa7fc2b..b6e272f 100644
--- a/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
+++ b/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
@@ -107,6 +107,6 @@ class SqlInterpreter() extends Interpreter {
// Unsupported
override def doQuietly[T](body: => T): T = ???
- override def languageInfo = LanguageInfo("SQL")
+ override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, fileExtension = Some(".scala"))
}
[02/15] incubator-toree git commit: Moving language_info to the
interpreter, maintaining current values
Posted by ma...@apache.org.
Moving language_info to the interpreter, maintaining current values
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/a0646adc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/a0646adc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/a0646adc
Branch: refs/heads/master
Commit: a0646adc172e38bfbd60cb710a8a49fc0c6e3fc7
Parents: 1ed5821
Author: Liam Fisk <li...@xtra.co.nz>
Authored: Sun Jun 19 11:19:24 2016 +1200
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Thu Oct 20 17:50:56 2016 -0400
----------------------------------------------------------------------
.../main/scala/org/apache/toree/interpreter/Interpreter.scala | 6 ++++++
.../src/main/scala/org/apache/toree/boot/KernelBootstrap.scala | 2 +-
.../org/apache/toree/boot/layer/HandlerInitialization.scala | 6 +++---
.../kernel/protocol/v5/handler/KernelInfoRequestHandler.scala | 4 ++--
.../protocol/v5/handler/KernelInfoRequestHandlerSpec.scala | 2 +-
kernel/src/test/scala/test/utils/DummyInterpreter.scala | 3 +++
.../org/apache/toree/kernel/protocol/v5/SparkKernelInfo.scala | 4 ++--
.../toree/kernel/interpreter/pyspark/PySparkInterpreter.scala | 3 +++
.../toree/kernel/interpreter/scala/ScalaInterpreter.scala | 4 ++++
.../toree/kernel/interpreter/sparkr/SparkRInterpreter.scala | 4 ++++
.../apache/toree/kernel/interpreter/sql/SqlInterpreter.scala | 4 ++++
11 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
index 84d0e70..986a506 100644
--- a/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
@@ -128,4 +128,10 @@ trait Interpreter {
* @return The runtime class loader used by this interpreter
*/
def classLoader: ClassLoader
+
+ /**
+ * Returns the language metadata for syntax highlighting
+ */
+ def languageInfo: Map[String, String]
+
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala b/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
index 938ccd7..7e393c4 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/KernelBootstrap.scala
@@ -174,7 +174,7 @@ class KernelBootstrap(config: Config) extends LogLike {
@inline private def displayVersionInfo() = {
logger.info("Kernel version: " + SparkKernelInfo.implementationVersion)
- logger.info("Scala version: " + SparkKernelInfo.language_info.get("version"))
+ logger.info("Scala version: " + SparkKernelInfo.scalaVersion)
logger.info("ZeroMQ (JeroMQ) version: " + ZMQ.getVersionString)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
index 572da8d..e4407fd 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
@@ -86,7 +86,7 @@ trait StandardHandlerInitialization extends HandlerInitialization {
responseMap: collection.mutable.Map[String, ActorRef]
): Unit = {
initializeKernelHandlers(
- actorSystem, actorLoader, kernel, commRegistrar, commStorage, responseMap
+ actorSystem, actorLoader, interpreter, kernel, commRegistrar, commStorage, responseMap
)
initializeSystemActors(
actorSystem, actorLoader, interpreter, pluginManager, magicManager
@@ -117,7 +117,7 @@ trait StandardHandlerInitialization extends HandlerInitialization {
private def initializeKernelHandlers(
actorSystem: ActorSystem, actorLoader: ActorLoader,
- kernel: Kernel,
+ interpreter: Interpreter, kernel: Kernel,
commRegistrar: CommRegistrar, commStorage: CommStorage,
responseMap: collection.mutable.Map[String, ActorRef]
): Unit = {
@@ -161,7 +161,7 @@ trait StandardHandlerInitialization extends HandlerInitialization {
initializeRequestHandler(classOf[ExecuteRequestHandler],
MessageType.Incoming.ExecuteRequest, kernel)
initializeRequestHandler(classOf[KernelInfoRequestHandler],
- MessageType.Incoming.KernelInfoRequest)
+ MessageType.Incoming.KernelInfoRequest, interpreter.languageInfo)
initializeRequestHandler(classOf[CommInfoRequestHandler],
MessageType.Incoming.CommInfoRequest, commStorage)
initializeRequestHandler(classOf[CodeCompleteHandler],
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
index f27773e..a4dd329 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
@@ -28,7 +28,7 @@ import scala.concurrent._
* Receives a KernelInfoRequest KernelMessage and returns a KernelInfoReply
* KernelMessage.
*/
-class KernelInfoRequestHandler(actorLoader: ActorLoader)
+class KernelInfoRequestHandler(actorLoader: ActorLoader, languageInfo: Map[String, String])
extends BaseHandler(actorLoader) with LogLike
{
def process(kernelMessage: KernelMessage): Future[_] = {
@@ -41,7 +41,7 @@ class KernelInfoRequestHandler(actorLoader: ActorLoader)
kernelInfo.protocolVersion,
kernelInfo.implementation,
kernelInfo.implementationVersion,
- kernelInfo.language_info,
+ languageInfo, // TODO permit arbitrary JSON serializable object here (rather than Map)
kernelInfo.banner
)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
index b4bd912..a96a22a 100644
--- a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
@@ -42,7 +42,7 @@ class KernelInfoRequestHandlerSpec extends TestKit(
ConfigFactory.parseString(KernelInfoRequestHandlerSpec.config))
) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
val actorLoader: ActorLoader = mock[ActorLoader]
- val actor = system.actorOf(Props(classOf[KernelInfoRequestHandler], actorLoader))
+ val actor = system.actorOf(Props(classOf[KernelInfoRequestHandler], actorLoader, Map.empty))
val relayProbe : TestProbe = TestProbe()
val relaySelection : ActorSelection =
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/kernel/src/test/scala/test/utils/DummyInterpreter.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/test/utils/DummyInterpreter.scala b/kernel/src/test/scala/test/utils/DummyInterpreter.scala
index a4adedd..2dbe7f4 100644
--- a/kernel/src/test/scala/test/utils/DummyInterpreter.scala
+++ b/kernel/src/test/scala/test/utils/DummyInterpreter.scala
@@ -113,4 +113,7 @@ class DummyInterpreter(kernel: KernelLike) extends Interpreter {
* @return The newly initialized interpreter
*/
override def init(kernel: KernelLike): Interpreter = ???
+
+ override def languageInfo: Map[String, String] = Map.empty
+
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/SparkKernelInfo.scala
----------------------------------------------------------------------
diff --git a/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/SparkKernelInfo.scala b/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/SparkKernelInfo.scala
index 622d5ed..52ad0d7 100644
--- a/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/SparkKernelInfo.scala
+++ b/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/SparkKernelInfo.scala
@@ -36,9 +36,9 @@ object SparkKernelInfo {
val implementationVersion = BuildInfo.version
/**
- * Represents the language supported by the kernel.
+ * Represents the scala version.
*/
- val language_info = Map("name" -> "scala", "version" -> BuildInfo.scalaVersion)
+ val scalaVersion = BuildInfo.scalaVersion
/**
* Represents the displayed name of the kernel.
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
index 9b59c73..6b6b2fd 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
@@ -21,6 +21,7 @@ import java.net.URL
import org.apache.toree.interpreter.Results.Result
import org.apache.toree.interpreter._
import org.apache.toree.kernel.api.KernelLike
+import org.apache.toree.kernel.BuildInfo
import org.slf4j.LoggerFactory
import py4j.GatewayServer
@@ -148,4 +149,6 @@ class PySparkInterpreter(
// Unsupported
override def doQuietly[T](body: => T): T = ???
+ override def languageInfo: Map[String, String] = Map("name" -> "scala", "version" -> BuildInfo.scalaVersion)
+
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
index 33fa861..4df3158 100644
--- a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
+++ b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
@@ -31,6 +31,7 @@ import org.apache.toree.interpreter._
import org.apache.toree.kernel.api.{KernelLike, KernelOptions}
import org.apache.toree.utils.{MultiOutputStream, TaskManager}
import org.slf4j.LoggerFactory
+import org.apache.toree.kernel.BuildInfo
import scala.annotation.tailrec
import scala.concurrent.{Await, Future}
@@ -336,5 +337,8 @@ object ScalaInterpreter {
outputDir
}
+
+ override def languageInfo: Map[String, String] = Map("name" -> "scala", "version" -> BuildInfo.scalaVersion)
+
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
index 91b4b7b..2c21a54 100644
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
+++ b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
@@ -21,6 +21,7 @@ import java.net.URL
import org.apache.toree.interpreter.Results.Result
import org.apache.toree.interpreter._
import org.apache.toree.kernel.api.KernelLike
+import org.apache.toree.kernel.BuildInfo
import org.slf4j.LoggerFactory
import scala.concurrent.Await
@@ -137,4 +138,7 @@ class SparkRInterpreter(
// Unsupported
override def doQuietly[T](body: => T): T = ???
+
+ override def languageInfo: Map[String, String] = Map("name" -> "scala", "version" -> BuildInfo.scalaVersion)
+
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/a0646adc/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
----------------------------------------------------------------------
diff --git a/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala b/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
index ad7f684..90d211e 100644
--- a/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
+++ b/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
@@ -21,6 +21,7 @@ import java.net.URL
import org.apache.toree.interpreter.Results.Result
import org.apache.toree.interpreter.{ExecuteFailure, ExecuteOutput, Interpreter}
import org.apache.toree.kernel.api.KernelLike
+import org.apache.toree.kernel.BuildInfo
import scala.concurrent.Await
import scala.concurrent.duration._
@@ -105,4 +106,7 @@ class SqlInterpreter() extends Interpreter {
// Unsupported
override def doQuietly[T](body: => T): T = ???
+
+ override def languageInfo: Map[String, String] = Map("name" -> "scala", "version" -> BuildInfo.scalaVersion)
+
}
[15/15] incubator-toree git commit: [TOREE-352] Merge github PR #78
Posted by ma...@apache.org.
[TOREE-352] Merge github PR #78
This pull request adds support for the `kernel_info_request` message.
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/953cbc04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/953cbc04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/953cbc04
Branch: refs/heads/master
Commit: 953cbc046ef9f71cc879c1d3bf99498d393c9a9c
Parents: 1ed5821 15327b5
Author: Marius van Niekerk <ma...@gmail.com>
Authored: Mon Nov 7 09:22:14 2016 -0500
Committer: Marius van Niekerk <ma...@gmail.com>
Committed: Mon Nov 7 09:22:14 2016 -0500
----------------------------------------------------------------------
Dockerfile.system-test | 50 ++++++++++++++++++++
Makefile | 27 +----------
.../apache/toree/interpreter/Interpreter.scala | 12 +++++
.../toree/interpreter/broker/BrokerState.scala | 10 ++--
.../org/apache/toree/boot/KernelBootstrap.scala | 2 +-
.../boot/layer/HandlerInitialization.scala | 15 ++++--
.../v5/handler/KernelInfoRequestHandler.scala | 6 +--
.../handler/KernelInfoRequestHandlerSpec.scala | 4 +-
.../scala/test/utils/DummyInterpreter.scala | 5 +-
.../toree/kernel/protocol/v5/LanguageInfo.scala | 25 ++++++++++
.../kernel/protocol/v5/SparkKernelInfo.scala | 4 +-
.../protocol/v5/content/KernelInfoReply.scala | 7 ++-
.../v5/content/KernelInfoReplySpec.scala | 6 +--
.../main/resources/PySpark/pyspark_runner.py | 13 ++++-
.../pyspark/PySparkInterpreter.scala | 36 +++++++++++---
.../interpreter/scala/ScalaInterpreter.scala | 9 +++-
.../interpreter/sparkr/SparkRInterpreter.scala | 16 +++++++
.../interpreter/sparkr/SparkRProcess.scala | 4 +-
.../interpreter/sparkr/SparkRService.scala | 3 ++
.../kernel/interpreter/sql/SqlInterpreter.scala | 6 ++-
test_toree.py | 4 +-
21 files changed, 203 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
[10/15] incubator-toree git commit: Fixed typo
Posted by ma...@apache.org.
Fixed typo
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/77565fbe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/77565fbe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/77565fbe
Branch: refs/heads/master
Commit: 77565fbe3f95a60d538c733793c590175935fe9e
Parents: 2d3cea4
Author: Marius van Niekerk <ma...@maxpoint.com>
Authored: Thu Oct 20 18:14:09 2016 -0400
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Thu Oct 20 18:14:09 2016 -0400
----------------------------------------------------------------------
.../toree/kernel/interpreter/pyspark/PySparkInterpreter.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/77565fbe/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
index 6e05975..a76e1d0 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
@@ -155,7 +155,7 @@ class PySparkInterpreter(
override def doQuietly[T](body: => T): T = ???
override def languageInfo: LanguageInfo = {
- if (!pySparkService.isRunning) or (!pySparkState.isReady) {
+ if ((!pySparkService.isRunning) || (!pySparkState.isReady)) {
LanguageInfo(
"python",
version = "UNKNOWN",
[03/15] incubator-toree git commit: Adding an internal + external
LanguageInfo case class
Posted by ma...@apache.org.
Adding an internal + external LanguageInfo case class
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/c330d3af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/c330d3af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/c330d3af
Branch: refs/heads/master
Commit: c330d3affb2d9e41974eaaaee5384c662058650a
Parents: ccd711a
Author: Liam Fisk <li...@xtra.co.nz>
Authored: Sun Jun 19 21:11:07 2016 +1200
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Thu Oct 20 17:50:57 2016 -0400
----------------------------------------------------------------------
.../apache/toree/interpreter/Interpreter.scala | 8 ++++++-
.../boot/layer/HandlerInitialization.scala | 11 +++++++--
.../v5/handler/KernelInfoRequestHandler.scala | 6 ++---
.../handler/KernelInfoRequestHandlerSpec.scala | 4 ++--
.../scala/test/utils/DummyInterpreter.scala | 4 ++--
.../toree/kernel/protocol/v5/LanguageInfo.scala | 25 ++++++++++++++++++++
.../protocol/v5/content/KernelInfoReply.scala | 7 ++++--
.../v5/content/KernelInfoReplySpec.scala | 6 ++---
.../pyspark/PySparkInterpreter.scala | 10 ++++----
.../interpreter/scala/ScalaInterpreter.scala | 5 +---
.../interpreter/sparkr/SparkRInterpreter.scala | 4 ++--
.../kernel/interpreter/sql/SqlInterpreter.scala | 4 ++--
12 files changed, 66 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
index 986a506..5b9d4c2 100644
--- a/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
@@ -23,6 +23,12 @@ import org.apache.toree.kernel.api.KernelLike
import scala.tools.nsc.interpreter._
+case class LanguageInfo(
+ name: String,
+ version: String,
+ fileExtension: String,
+ pygmentsLexer: Option[String] = None) {
+}
trait Interpreter {
@@ -132,6 +138,6 @@ trait Interpreter {
/**
* Returns the language metadata for syntax highlighting
*/
- def languageInfo: Map[String, String]
+ def languageInfo: LanguageInfo
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
index e4407fd..8739079 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/HandlerInitialization.scala
@@ -29,7 +29,7 @@ import org.apache.toree.kernel.protocol.v5.interpreter.tasks.InterpreterTaskFact
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.magic.{MagicParser, PostProcessor}
import org.apache.toree.kernel.protocol.v5.relay.ExecuteRequestRelay
-import org.apache.toree.kernel.protocol.v5.{MessageType, SocketType, SystemActorType}
+import org.apache.toree.kernel.protocol.v5.{MessageType, SocketType, SystemActorType, LanguageInfo}
import org.apache.toree.magic.MagicManager
import org.apache.toree.plugins.PluginManager
import org.apache.toree.utils.LogLike
@@ -157,11 +157,18 @@ trait StandardHandlerInitialization extends HandlerInitialization {
)
}
+ val langInfo = interpreter.languageInfo
+ val internalInfo = LanguageInfo(
+ name=langInfo.name,
+ version=langInfo.version,
+ file_extension=langInfo.fileExtension,
+ pygments_lexer=langInfo.pygmentsLexer)
+
// These are the handlers for messages coming into the
initializeRequestHandler(classOf[ExecuteRequestHandler],
MessageType.Incoming.ExecuteRequest, kernel)
initializeRequestHandler(classOf[KernelInfoRequestHandler],
- MessageType.Incoming.KernelInfoRequest, interpreter.languageInfo)
+ MessageType.Incoming.KernelInfoRequest, internalInfo)
initializeRequestHandler(classOf[CommInfoRequestHandler],
MessageType.Incoming.CommInfoRequest, commStorage)
initializeRequestHandler(classOf[CodeCompleteHandler],
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
index a4dd329..f65266f 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandler.scala
@@ -28,7 +28,7 @@ import scala.concurrent._
* Receives a KernelInfoRequest KernelMessage and returns a KernelInfoReply
* KernelMessage.
*/
-class KernelInfoRequestHandler(actorLoader: ActorLoader, languageInfo: Map[String, String])
+class KernelInfoRequestHandler(actorLoader: ActorLoader, languageInfo: LanguageInfo)
extends BaseHandler(actorLoader) with LogLike
{
def process(kernelMessage: KernelMessage): Future[_] = {
@@ -41,7 +41,7 @@ class KernelInfoRequestHandler(actorLoader: ActorLoader, languageInfo: Map[Strin
kernelInfo.protocolVersion,
kernelInfo.implementation,
kernelInfo.implementationVersion,
- languageInfo, // TODO permit arbitrary JSON serializable object here (rather than Map)
+ languageInfo,
kernelInfo.banner
)
@@ -64,4 +64,4 @@ class KernelInfoRequestHandler(actorLoader: ActorLoader, languageInfo: Map[Strin
actorLoader.load(SystemActorType.KernelMessageRelay) ! kernelResponseMessage
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
index a96a22a..04a718f 100644
--- a/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/kernel/protocol/v5/handler/KernelInfoRequestHandlerSpec.scala
@@ -20,7 +20,7 @@ import akka.actor.{ActorSelection, ActorSystem, Props}
import akka.testkit.{TestProbe, ImplicitSender, TestKit}
import org.apache.toree.kernel.protocol.v5.content.KernelInfoReply
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
-import org.apache.toree.kernel.protocol.v5.{SystemActorType, Header, KernelMessage}
+import org.apache.toree.kernel.protocol.v5.{SystemActorType, Header, KernelMessage, LanguageInfo}
import org.mockito.AdditionalMatchers.{not => mockNot}
import org.mockito.Matchers.{eq => mockEq}
import com.typesafe.config.ConfigFactory
@@ -42,7 +42,7 @@ class KernelInfoRequestHandlerSpec extends TestKit(
ConfigFactory.parseString(KernelInfoRequestHandlerSpec.config))
) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
val actorLoader: ActorLoader = mock[ActorLoader]
- val actor = system.actorOf(Props(classOf[KernelInfoRequestHandler], actorLoader, Map.empty))
+ val actor = system.actorOf(Props(classOf[KernelInfoRequestHandler], actorLoader, LanguageInfo("test", "1.0.0", ".test")))
val relayProbe : TestProbe = TestProbe()
val relaySelection : ActorSelection =
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/kernel/src/test/scala/test/utils/DummyInterpreter.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/test/utils/DummyInterpreter.scala b/kernel/src/test/scala/test/utils/DummyInterpreter.scala
index 2dbe7f4..d7c7fab 100644
--- a/kernel/src/test/scala/test/utils/DummyInterpreter.scala
+++ b/kernel/src/test/scala/test/utils/DummyInterpreter.scala
@@ -20,7 +20,7 @@ package test.utils
import java.net.URL
import org.apache.toree.interpreter.Results.Result
-import org.apache.toree.interpreter.{ExecuteFailure, ExecuteOutput, Interpreter}
+import org.apache.toree.interpreter.{ExecuteFailure, ExecuteOutput, Interpreter, LanguageInfo}
import org.apache.toree.kernel.api.KernelLike
import scala.tools.nsc.interpreter.{InputStream, OutputStream}
@@ -114,6 +114,6 @@ class DummyInterpreter(kernel: KernelLike) extends Interpreter {
*/
override def init(kernel: KernelLike): Interpreter = ???
- override def languageInfo: Map[String, String] = Map.empty
+ override def languageInfo: LanguageInfo = ???
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
----------------------------------------------------------------------
diff --git a/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala b/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
new file mode 100644
index 0000000..04b85c8
--- /dev/null
+++ b/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/LanguageInfo.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.toree.kernel.protocol.v5
+
+case class LanguageInfo(
+ name: String,
+ version: String,
+ file_extension: String,
+ pygments_lexer: Option[String] = None) {
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReply.scala
----------------------------------------------------------------------
diff --git a/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReply.scala b/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReply.scala
index 51c1a6d..31898ed 100644
--- a/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReply.scala
+++ b/protocol/src/main/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReply.scala
@@ -18,13 +18,14 @@
package org.apache.toree.kernel.protocol.v5.content
import org.apache.toree.kernel.protocol.v5.KernelMessageContent
+import org.apache.toree.kernel.protocol.v5.LanguageInfo
import play.api.libs.json.Json
case class KernelInfoReply (
protocol_version: String,
implementation: String,
implementation_version: String,
- language_info: Map[String, String],
+ language_info: LanguageInfo,
banner: String
) extends KernelMessageContent {
override def content: String =
@@ -32,6 +33,8 @@ case class KernelInfoReply (
}
object KernelInfoReply extends TypeString {
+ implicit val languageInfoReads = Json.reads[LanguageInfo]
+ implicit val languageInfoWrites = Json.writes[LanguageInfo]
implicit val kernelInfoReplyReads = Json.reads[KernelInfoReply]
implicit val kernelInfoReplyWrites = Json.writes[KernelInfoReply]
@@ -41,4 +44,4 @@ object KernelInfoReply extends TypeString {
* @return The type as a string
*/
override def toTypeString: String = "kernel_info_reply"
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
----------------------------------------------------------------------
diff --git a/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala b/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
index 12ca226..b849df8 100644
--- a/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
+++ b/protocol/src/test/scala/org/apache/toree/kernel/protocol/v5/content/KernelInfoReplySpec.scala
@@ -17,6 +17,7 @@
package org.apache.toree.kernel.protocol.v5.content
+import org.apache.toree.kernel.protocol.v5.LanguageInfo
import org.scalatest.{FunSpec, Matchers}
import play.api.data.validation.ValidationError
import play.api.libs.json._
@@ -27,13 +28,13 @@ class KernelInfoReplySpec extends FunSpec with Matchers {
"protocol_version": "x.y.z",
"implementation": "<name>",
"implementation_version": "z.y.x",
- "language_info": { "name": "<some language>", "version": "a.b.c" },
+ "language_info": { "name": "<some language>", "version": "a.b.c", "file_extension": "<some extension>" },
"banner": "<some banner>"
}
""")
val kernelInfoReply: KernelInfoReply = KernelInfoReply(
- "x.y.z", "<name>", "z.y.x", Map("name" -> "<some language>", "version" -> "a.b.c"), "<some banner>"
+ "x.y.z", "<name>", "z.y.x", LanguageInfo("<some language>", "a.b.c", "<some extension>"), "<some banner>"
)
describe("KernelInfoReply") {
@@ -73,4 +74,3 @@ class KernelInfoReplySpec extends FunSpec with Matchers {
}
}
}
-
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
index a192c78..784a9ed 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
@@ -149,10 +149,10 @@ class PySparkInterpreter(
override def doQuietly[T](body: => T): T = ???
// TODO Identify how to plumb python version to here
- override def languageInfo: Map[String, String] = Map(
- "name" -> "python",
- "version" -> "2.7.9",
- "pygments_lexer" -> "ipython2",
- "file_extension" -> ".py")
+ override def languageInfo = LanguageInfo(
+ "python",
+ "2.7.9",
+ ".py",
+ pygmentsLexer = Some("ipython2"))
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
index bb198a7..1cd6d13 100644
--- a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
+++ b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
@@ -338,9 +338,6 @@ object ScalaInterpreter {
}
- override def languageInfo: Map[String, String] = Map(
- "name" -> "scala",
- "version" -> BuildInfo.scalaVersion,
- "file_extension" -> ".scala")
+ override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, ".scala")
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
index 2c21a54..ff4aa06 100644
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
+++ b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRInterpreter.scala
@@ -138,7 +138,7 @@ class SparkRInterpreter(
// Unsupported
override def doQuietly[T](body: => T): T = ???
-
- override def languageInfo: Map[String, String] = Map("name" -> "scala", "version" -> BuildInfo.scalaVersion)
+
+ override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, ".scala")
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c330d3af/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
----------------------------------------------------------------------
diff --git a/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala b/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
index 90d211e..988ef9e 100644
--- a/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
+++ b/sql-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sql/SqlInterpreter.scala
@@ -19,7 +19,7 @@ package org.apache.toree.kernel.interpreter.sql
import java.net.URL
import org.apache.toree.interpreter.Results.Result
-import org.apache.toree.interpreter.{ExecuteFailure, ExecuteOutput, Interpreter}
+import org.apache.toree.interpreter._
import org.apache.toree.kernel.api.KernelLike
import org.apache.toree.kernel.BuildInfo
@@ -107,6 +107,6 @@ class SqlInterpreter() extends Interpreter {
// Unsupported
override def doQuietly[T](body: => T): T = ???
- override def languageInfo: Map[String, String] = Map("name" -> "scala", "version" -> BuildInfo.scalaVersion)
+ override def languageInfo = LanguageInfo("scala", BuildInfo.scalaVersion, ".scala")
}
[05/15] incubator-toree git commit: Ensuring the contexts are cleaned
up correctly
Posted by ma...@apache.org.
Ensuring the contexts are cleaned up correctly
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/6a77283c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/6a77283c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/6a77283c
Branch: refs/heads/master
Commit: 6a77283c6064ac3e6f4e3d0c86bf86925e4cc686
Parents: a5303b4
Author: Liam Fisk <li...@xtra.co.nz>
Authored: Thu Jun 30 11:57:23 2016 +1200
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Thu Oct 20 17:50:58 2016 -0400
----------------------------------------------------------------------
.../src/main/resources/PySpark/pyspark_runner.py | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/6a77283c/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py b/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
index d84c5c4..5073a4d 100644
--- a/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
+++ b/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
@@ -107,10 +107,16 @@ class Kernel(object):
return parent + [x for x in self._jvm_kernel.__dir__() if x not in parent]
def createSparkContext(self, config):
+ global conf, sc, sqlContext
+
jconf = gateway.jvm.org.apache.spark.SparkConf(False)
for key,value in config.getAll():
jconf.set(key, value)
self._jvm_kernel.createSparkContext(jconf)
+ conf = None
+ sc = None
+ sqlContext = None
+
self.refreshContext()
def refreshContext(self):
[07/15] incubator-toree git commit: Ensuring the broker service is
started before LanguageInfo is requested
Posted by ma...@apache.org.
Ensuring the broker service is started before LanguageInfo is requested
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/e5d8d0f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/e5d8d0f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/e5d8d0f0
Branch: refs/heads/master
Commit: e5d8d0f01565c7eea99cbacb8c86df078a50e8a0
Parents: b3ef81b
Author: Liam Fisk <li...@xtra.co.nz>
Authored: Fri Jul 1 10:06:11 2016 +1200
Committer: Marius van Niekerk <ma...@maxpoint.com>
Committed: Thu Oct 20 17:50:59 2016 -0400
----------------------------------------------------------------------
.../pyspark/PySparkInterpreter.scala | 26 ++++++++++++++------
1 file changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/e5d8d0f0/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
index 632f52d..72b0aa3 100644
--- a/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
+++ b/pyspark-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/pyspark/PySparkInterpreter.scala
@@ -89,8 +89,7 @@ class PySparkInterpreter(
* execution or the failure
*/
override def interpret(code: String, silent: Boolean, output: Option[OutputStream]):
- (Result, Either[ExecuteOutput, ExecuteFailure]) =
- {
+ (Result, Either[ExecuteOutput, ExecuteFailure]) = {
if (!pySparkService.isRunning) pySparkService.start()
val futureResult = pySparkTransformer.transformToInterpreterResult(
@@ -148,11 +147,22 @@ class PySparkInterpreter(
// Unsupported
override def doQuietly[T](body: => T): T = ???
- // TODO Identify how to plumb python version to here
- override def languageInfo = LanguageInfo(
- "python",
- pySparkState.getVersion(),
- fileExtension = Some(".py"),
- pygmentsLexer = Some("ipython2"))
+ override def languageInfo: LanguageInfo = {
+ if (!pySparkService.isRunning) pySparkService.start()
+ import scala.util.control.Breaks._
+ val waitLimit = System.currentTimeMillis() + java.util.concurrent.TimeUnit.SECONDS.toMillis(5)
+ while (!pySparkState.isReady) {
+ if (System.currentTimeMillis > waitLimit) {
+ logger.warn("Timed out waiting for broker state to become ready")
+ break
+ }
+ }
+
+ LanguageInfo(
+ "python",
+ pySparkState.getVersion(),
+ fileExtension = Some(".py"),
+ pygmentsLexer = Some("ipython2"))
+ }
}