You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lb...@apache.org on 2016/01/22 23:07:32 UTC
[24/51] [abbrv] incubator-toree git commit: Moved scala files to new
locations based on new package
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/LineMagic.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/LineMagic.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/LineMagic.scala
deleted file mode 100644
index 0a54e85..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/LineMagic.scala
+++ /dev/null
@@ -1,9 +0,0 @@
-package com.ibm.spark.magic
-
-/**
- * Line Magics perform some function and don't return anything. I.e. you cannot
- * do `val x = %runMyCode 1 2 3` or alter the MIMEType of the cell.
- */
-trait LineMagic extends Magic {
- override def execute(code: String): Unit
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/Magic.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/Magic.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/Magic.scala
deleted file mode 100644
index 0e41b35..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/Magic.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic
-
-/**
- * Represents the base structure for a magic that is loaded and executed.
- */
-trait Magic {
- /**
- * Execute a magic.
- * @param code The code
- * @return The output of the magic
- */
- def execute(code: String): Any
- }
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/MagicExecutor.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/MagicExecutor.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/MagicExecutor.scala
deleted file mode 100644
index f74c9f6..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/MagicExecutor.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-package com.ibm.spark.magic
-
-import com.ibm.spark.utils.DynamicReflectionSupport
-
-import scala.language.dynamics
-
-class MagicExecutor(magicLoader: MagicLoader) extends Dynamic {
-
- val executeMethod = classOf[Magic].getDeclaredMethods.head.getName
-
- def applyDynamic(name: String)(args: Any*): Either[CellMagicOutput, LineMagicOutput] = {
- val className = magicLoader.magicClassName(name)
- val isCellMagic = magicLoader.hasCellMagic(className)
- val isLineMagic = magicLoader.hasLineMagic(className)
-
- (isCellMagic, isLineMagic) match {
- case (true, false) =>
- val result = executeMagic(className, args)
- Left(result.asInstanceOf[CellMagicOutput])
- case (false, true) =>
- executeMagic(className, args)
- Right(LineMagicOutput)
- case (_, _) =>
- Left(CellMagicOutput("text/plain" ->
- s"Magic ${className} could not be executed."))
- }
- }
-
- private def executeMagic(className: String, args: Seq[Any]) = {
- val inst = magicLoader.createMagicInstance(className)
- val dynamicSupport = new DynamicReflectionSupport(inst.getClass, inst)
- dynamicSupport.applyDynamic(executeMethod)(args)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/MagicLoader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/MagicLoader.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/MagicLoader.scala
deleted file mode 100644
index c700c9e..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/MagicLoader.scala
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic
-
-import java.net.{URL, URLClassLoader}
-
-import com.google.common.reflect.ClassPath
-import com.ibm.spark.magic.dependencies.DependencyMap
-
-import scala.reflect.runtime.{universe => runtimeUniverse}
-import scala.collection.JavaConversions._
-
-class MagicLoader(
- var dependencyMap: DependencyMap = new DependencyMap(),
- urls: Array[URL] = Array(),
- parentLoader: ClassLoader = null
-) extends URLClassLoader(urls, parentLoader) {
- private val magicPackage = "com.ibm.spark.magic.builtin"
-
- /**
- * Checks whether a magic with a given name, implementing a given interface,
- * exists.
- * @param name case insensitive magic name
- * @param interface interface
- * @return true if a magic with the given name and interface exists
- */
- private def hasSpecificMagic(name: String, interface: Class[_]) : Boolean = {
- val className = magicClassName(name)
- try {
- val clazz = loadClass(className)
- clazz.getInterfaces.contains(interface)
- } catch {
- case _: Throwable => false
- }
- }
-
- /**
- * Checks whether a line magic exists.
- * @param name case insensitive line magic name
- * @return true if the line magic exists
- */
- def hasLineMagic(name: String): Boolean =
- hasSpecificMagic(name, classOf[LineMagic])
-
- /**
- * Checks whether a cell magic exists.
- * @param name case insensitive cell magic name
- * @return true if the cell magic exists
- */
- def hasCellMagic(name: String): Boolean =
- hasSpecificMagic(name, classOf[CellMagic])
-
- /**
- * Attempts to load a class with a given name from a package.
- * @param name the name of the class
- * @param resolve whether to resolve the class or not
- * @return the class if found
- */
- override def loadClass(name: String, resolve: Boolean): Class[_] =
- try {
- super.loadClass(magicPackage + "." + name, resolve)
- } catch {
- case ex: ClassNotFoundException =>
- super.loadClass(name, resolve)
- }
-
- /**
- * Returns the class name for a case insensitive magic name query.
- * If no match is found, returns the query.
- * @param query a magic name, e.g. jAvasCRipt
- * @return the queried magic name's corresponding class, e.g. JavaScript
- */
- def magicClassName(query: String): String = {
- lowercaseClassMap(magicClassNames).getOrElse(query.toLowerCase, query)
- }
-
- /**
- * @return list of magic class names in magicPackage.
- */
- protected def magicClassNames : List[String] = {
- val classPath: ClassPath = ClassPath.from(this)
- val classes = classPath.getTopLevelClasses(magicPackage)
- classes.asList.map(_.getSimpleName).toList
- }
-
- /**
- * @param names list of class names
- * @return map of lowercase class names to class names
- */
- private def lowercaseClassMap(names: List[String]): Map[String, String] = {
- names.map(n => (n.toLowerCase, n)).toMap
- }
-
- def addJar(jar: URL) = addURL(jar)
- /**
- * Creates a instance of the specified magic with dependencies added.
- * @param name name of magic class
- * @return instance of the Magic corresponding to the given name
- */
- protected[magic] def createMagicInstance(name: String): Any = {
- val magicClass = loadClass(name) // Checks parent loadClass first
-
- val runtimeMirror = runtimeUniverse.runtimeMirror(this)
- val classSymbol = runtimeMirror.staticClass(magicClass.getCanonicalName)
- val classMirror = runtimeMirror.reflectClass(classSymbol)
- val selfType = classSymbol.selfType
-
- val classConstructorSymbol =
- selfType.declaration(runtimeUniverse.nme.CONSTRUCTOR).asMethod
- val classConstructorMethod =
- classMirror.reflectConstructor(classConstructorSymbol)
-
- val magicInstance = classConstructorMethod()
-
-
- // Add all of our dependencies to the new instance
- dependencyMap.internalMap.filter(selfType <:< _._1).values.foreach(
- _(magicInstance.asInstanceOf[Magic])
- )
-
- magicInstance
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/DependencyMap.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/DependencyMap.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/DependencyMap.scala
deleted file mode 100644
index f641a50..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/DependencyMap.scala
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Copyright 2015 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic.dependencies
-
-import java.io.OutputStream
-
-import com.ibm.spark.interpreter.Interpreter
-import com.ibm.spark.kernel.api.KernelLike
-import com.ibm.spark.magic.{MagicLoader, Magic}
-import com.typesafe.config.Config
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.SQLContext
-
-import scala.reflect.runtime.universe._
-import com.ibm.spark.dependencies.DependencyDownloader
-
-/**
- * Represents a mapping of dependency types to implementations.
- *
- * TODO: Explore Scala macros to avoid duplicate code.
- */
-class DependencyMap {
- val internalMap =
- scala.collection.mutable.Map[Type, PartialFunction[Magic, Unit]]()
-
- /**
- * Sets the Interpreter for this map.
- * @param interpreter The new Interpreter
- */
- def setInterpreter(interpreter: Interpreter) = {
- internalMap(typeOf[IncludeInterpreter]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeInterpreter].interpreter_=(interpreter)
- )
-
- this
- }
-
- /**
- * Sets the Interpreter for this map.
- * @param interpreter The new Interpreter
- */
- //@deprecated("Use setInterpreter with IncludeInterpreter!", "2015.05.06")
- def setKernelInterpreter(interpreter: Interpreter) = {
- internalMap(typeOf[IncludeKernelInterpreter]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeKernelInterpreter].kernelInterpreter_=(interpreter)
- )
-
- this
- }
-
- /**
- * Sets the SparkContext for this map.
- * @param sparkContext The new SparkContext
- */
- def setSparkContext(sparkContext: SparkContext) = {
- internalMap(typeOf[IncludeSparkContext]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeSparkContext].sparkContext_=(sparkContext)
- )
-
- this
- }
-
- /**
- * Sets the SQLContext for this map.
- * @param sqlContext The new SQLContext
- */
- def setSQLContext(sqlContext: SQLContext) = {
- internalMap(typeOf[IncludeSQLContext]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeSQLContext].sqlContext_=(sqlContext)
- )
-
- this
- }
-
- /**
- * Sets the OutputStream for this map.
- * @param outputStream The new OutputStream
- */
- def setOutputStream(outputStream: OutputStream) = {
- internalMap(typeOf[IncludeOutputStream]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeOutputStream].outputStream_=(outputStream)
- )
-
- this
- }
-
- /**
- * Sets the DependencyDownloader for this map.
- * @param dependencyDownloader The new DependencyDownloader
- */
- def setDependencyDownloader(dependencyDownloader: DependencyDownloader) = {
- internalMap(typeOf[IncludeDependencyDownloader]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeDependencyDownloader]
- .dependencyDownloader_=(dependencyDownloader)
- )
-
- this
- }
-
- /**
- * Sets the Kernel Object for this map.
- * @param kernel The new Kernel
- */
- def setKernel(kernel: KernelLike) = {
- internalMap(typeOf[IncludeKernel]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeKernel]
- .kernel_=(kernel)
- )
-
- this
- }
-
- /**
- * Sets the MagicLoader for this map.
- * @param magicLoader The new MagicLoader
- */
- def setMagicLoader(magicLoader: MagicLoader) = {
- internalMap(typeOf[IncludeMagicLoader]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeMagicLoader]
- .magicLoader_=(magicLoader)
- )
-
- this
- }
-
- /**
- * Sets the Config Object for this map.
- * @param config The config for the kernel
- */
- def setConfig(config: Config) = {
- internalMap(typeOf[IncludeConfig]) =
- PartialFunction[Magic, Unit](
- magic =>
- magic.asInstanceOf[IncludeConfig]
- .config=(config)
- )
-
- this
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeConfig.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeConfig.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeConfig.scala
deleted file mode 100644
index 675c084..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeConfig.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2015 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic.dependencies
-
-import com.ibm.spark.magic.{Magic}
-import com.typesafe.config.Config
-
-trait IncludeConfig {
- this: Magic =>
-
- private var _config: Config = _
- def config: Config = _config
- def config_= (newConfig: Config) =
- _config = newConfig
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeDependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeDependencyDownloader.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeDependencyDownloader.scala
deleted file mode 100644
index 109fbd1..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeDependencyDownloader.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic.dependencies
-
-import com.ibm.spark.dependencies.DependencyDownloader
-import com.ibm.spark.magic.Magic
-
-trait IncludeDependencyDownloader {
- this: Magic =>
-
- private var _dependencyDownloader: DependencyDownloader = _
- def dependencyDownloader: DependencyDownloader = _dependencyDownloader
- def dependencyDownloader_=(newDependencyDownloader: DependencyDownloader) =
- _dependencyDownloader = newDependencyDownloader
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeInterpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeInterpreter.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeInterpreter.scala
deleted file mode 100644
index fb01131..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeInterpreter.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic.dependencies
-
-import com.ibm.spark.interpreter.Interpreter
-import com.ibm.spark.magic.Magic
-
-trait IncludeInterpreter {
- this: Magic =>
-
- //val interpreter: Interpreter
- private var _interpreter: Interpreter = _
- def interpreter: Interpreter = _interpreter
- def interpreter_=(newInterpreter: Interpreter) =
- _interpreter = newInterpreter
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeKernel.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeKernel.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeKernel.scala
deleted file mode 100644
index fca3cb1..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeKernel.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic.dependencies
-
-import com.ibm.spark.kernel.api.KernelLike
-import com.ibm.spark.magic.Magic
-
-trait IncludeKernel {
- this: Magic =>
-
- private var _kernel: KernelLike = _
- def kernel: KernelLike = _kernel
- def kernel_=(newKernel: KernelLike) =
- _kernel = newKernel
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeKernelInterpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeKernelInterpreter.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeKernelInterpreter.scala
deleted file mode 100644
index de19c07..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeKernelInterpreter.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic.dependencies
-
-import com.ibm.spark.interpreter.Interpreter
-import com.ibm.spark.magic.Magic
-
-//@deprecated("Use IncludeInterpreter instead!", "2015.05.06")
-trait IncludeKernelInterpreter {
- this: Magic =>
-
- //val interpreter: Interpreter
- private var _interpreter: Interpreter = _
- def kernelInterpreter: Interpreter = _interpreter
- def kernelInterpreter_=(newInterpreter: Interpreter) =
- _interpreter = newInterpreter
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeMagicLoader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeMagicLoader.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeMagicLoader.scala
deleted file mode 100644
index 0a78508..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeMagicLoader.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic.dependencies
-
-import com.ibm.spark.magic.{MagicLoader, Magic}
-
-
-trait IncludeMagicLoader {
- this: Magic =>
-
- //val sparkContext: SparkContext
- private var _magicLoader: MagicLoader = _
- def magicLoader: MagicLoader = _magicLoader
- def magicLoader_=(newMagicLoader: MagicLoader) =
- _magicLoader = newMagicLoader
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeOutputStream.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeOutputStream.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeOutputStream.scala
deleted file mode 100644
index a3e679e..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeOutputStream.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic.dependencies
-
-import java.io.OutputStream
-
-import com.ibm.spark.magic.Magic
-
-trait IncludeOutputStream {
- this: Magic =>
-
- //val outputStream: OutputStream
- private var _outputStream: OutputStream = _
- def outputStream: OutputStream = _outputStream
- def outputStream_=(newOutputStream: OutputStream) =
- _outputStream = newOutputStream
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeSQLContext.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeSQLContext.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeSQLContext.scala
deleted file mode 100644
index 5a9b26c..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeSQLContext.scala
+++ /dev/null
@@ -1,13 +0,0 @@
-package com.ibm.spark.magic.dependencies
-
-import com.ibm.spark.magic.Magic
-import org.apache.spark.sql.SQLContext
-
-trait IncludeSQLContext {
- this: Magic =>
-
- private var _sqlContext: SQLContext = _
- def sqlContext: SQLContext = _sqlContext
- def sqlContext_=(newSqlContext: SQLContext) =
- _sqlContext = newSqlContext
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeSparkContext.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeSparkContext.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeSparkContext.scala
deleted file mode 100644
index df5e245..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/dependencies/IncludeSparkContext.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.magic.dependencies
-
-import com.ibm.spark.magic.Magic
-import org.apache.spark.SparkContext
-
-trait IncludeSparkContext {
- this: Magic =>
-
- //val sparkContext: SparkContext
- private var _sparkContext: SparkContext = _
- def sparkContext: SparkContext = _sparkContext
- def sparkContext_=(newSparkContext: SparkContext) =
- _sparkContext = newSparkContext
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/magic/package.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/magic/package.scala b/kernel-api/src/main/scala/com/ibm/spark/magic/package.scala
deleted file mode 100644
index 292d641..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/magic/package.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark
-
-package object magic {
- /**
- * Represents the output of a magic execution.
- */
- // TODO: This is a duplicate of Data in kernel protocol, needs to be given
- // a type/val that can be translated into a specific protocol via
- // implicits - or some other transformation - to separate this from
- // the protocol type
- type CellMagicOutput = Map[String, String]
- val CellMagicOutput = Map
-
- type LineMagicOutput = Unit
- val LineMagicOutput : LineMagicOutput = ()
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/security/KernelSecurityManager.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/security/KernelSecurityManager.scala b/kernel-api/src/main/scala/com/ibm/spark/security/KernelSecurityManager.scala
deleted file mode 100644
index 2529866..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/security/KernelSecurityManager.scala
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.security
-
-import java.security.Permission
-import java.util.UUID
-
-import scala.collection.immutable.HashMap
-
-object KernelSecurityManager {
- val RestrictedGroupName = "restricted-" + UUID.randomUUID().toString
-
- /**
- * Special case for this permission since the name changes with each status
- * code.
- */
- private val SystemExitPermissionName = "exitVM." // + status
-
- /**
- * Used to indicate which permissions to check. Only checks if the permission
- * is found in the keys and the value for that permission is true.
- */
- private val permissionsToCheck: Map[String, Boolean] = HashMap(
- "modifyThreadGroup" -> true
- )
-
- /**
- * Checks whether the permission with the provided name is listed to be
- * checked.
- *
- * @param name The name of the permission
- *
- * @return True if the permission is listed to be checked, false otherwise
- */
- private def shouldCheckPermission(name: String): Boolean =
- permissionsToCheck.getOrElse(name, shouldCheckPermissionSpecialCases(name))
-
- /**
- * Checks whether the permission with the provided name is one of the special
- * cases that don't exist in the normal name conventions.
- *
- * @param name The name of the permission
- *
- * @return True if the permission is to be checked, false otherwise
- */
- private def shouldCheckPermissionSpecialCases(name: String): Boolean =
- name.startsWith(SystemExitPermissionName)
-}
-
-class KernelSecurityManager extends SecurityManager {
- import KernelSecurityManager._
-
- override def checkPermission(perm: Permission, context: scala.Any): Unit = {
- // TODO: Investigate why the StackOverflowError occurs in IntelliJ without
- // this check for FilePermission related to this class
- // NOTE: The above problem does not happen when built with sbt pack
- if (perm.getActions == "read" &&
- perm.getName.contains(this.getClass.getSimpleName))
- return
-
- if (shouldCheckPermission(perm.getName))
- super.checkPermission(perm, context)
- }
-
- override def checkPermission(perm: Permission): Unit = {
- // TODO: Investigate why the StackOverflowError occurs in IntelliJ without
- // this check for FilePermission related to this class
- // NOTE: The above problem does not happen when built with sbt pack
- if (perm.getActions == "read" &&
- perm.getName.contains(this.getClass.getSimpleName))
- return
-
- if (shouldCheckPermission(perm.getName))
- super.checkPermission(perm)
- }
-
- override def getThreadGroup: ThreadGroup = {
- val currentGroup = Thread.currentThread().getThreadGroup
-
- // For restricted groups, we can only catch them in the checkAccess if we
- // set the current group as the parent (to make sure all groups have a
- // consistent name)
- if (currentGroup.getName == RestrictedGroupName) {
- new ThreadGroup(currentGroup, currentGroup.getName)
- } else {
- super.getThreadGroup
- }
- }
-
- override def checkAccess(g: ThreadGroup): Unit = {
- //super.checkAccess(g)
- if (g == null) return
-
- val parentGroup = g.getParent
-
- if (parentGroup != null &&
- parentGroup.getName == RestrictedGroupName &&
- g.getName != RestrictedGroupName)
- throw new SecurityException("Not allowed to modify ThreadGroups!")
- }
-
- override def checkExit(status: Int): Unit = {
- val currentGroup = Thread.currentThread().getThreadGroup
-
- if (currentGroup.getName == RestrictedGroupName) {
- // TODO: Determine why System.exit(...) is being blocked in the ShutdownHandler
- System.out.println("Unauthorized system.exit detected!")
- //throw new SecurityException("Not allowed to invoke System.exit!")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/utils/ArgumentParsingSupport.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/utils/ArgumentParsingSupport.scala b/kernel-api/src/main/scala/com/ibm/spark/utils/ArgumentParsingSupport.scala
deleted file mode 100644
index a748fe4..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/utils/ArgumentParsingSupport.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.utils
-
-import joptsimple.{OptionSpec, OptionParser}
-import scala.collection.JavaConverters._
-import scala.language.implicitConversions
-import java.io.{PrintStream, OutputStream}
-
-trait ArgumentParsingSupport {
- protected lazy val parser = new OptionParser()
- private var options: joptsimple.OptionSet = _
- parser.allowsUnrecognizedOptions()
-
- /**
- * Parses the arguments provided as a string, updating all internal
- * references to specific arguments.
- *
- * @param args The arguments as a string
- * @param delimiter An optional delimiter for separating arguments
- */
- def parseArgs(args: String, delimiter: String = " ") = {
- options = parser.parse(args.split(delimiter): _*)
-
- options.nonOptionArguments().asScala.map(_.toString)
- }
-
- def printHelp(outputStream: OutputStream, usage: String) = {
- val printStream = new PrintStream(outputStream)
-
- printStream.println(s"Usage: $usage\n")
- parser.printHelpOn(outputStream)
- }
-
- implicit def has[T](spec: OptionSpec[T]): Boolean = {
- require(options != null, "Arguments not parsed yet!")
- options.has(spec)
- }
-
- implicit def get[T](spec: OptionSpec[T]): Option[T] = {
- require(options != null, "Arguments not parsed yet!")
- Some(options.valueOf(spec)).filter(_ != null)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/utils/ConditionalOutputStream.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/utils/ConditionalOutputStream.scala b/kernel-api/src/main/scala/com/ibm/spark/utils/ConditionalOutputStream.scala
deleted file mode 100644
index 65f7650..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/utils/ConditionalOutputStream.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.utils
-
-import java.io.OutputStream
-
-class ConditionalOutputStream(
- private val outputStream: OutputStream,
- condition: => Boolean
-) extends OutputStream {
- require(outputStream != null)
-
- override def write(b: Int): Unit = if (condition) outputStream.write(b)
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/utils/DownloadSupport.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/utils/DownloadSupport.scala b/kernel-api/src/main/scala/com/ibm/spark/utils/DownloadSupport.scala
deleted file mode 100644
index 9d36326..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/utils/DownloadSupport.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.utils
-
-import java.net.URL
-import java.nio.channels._
-import java.io.FileOutputStream
-
-/**
- * A utility for downloading the contents of a file to a specified location.
- */
-trait DownloadSupport {
- /**
- * Download a file located at the given URL to the specified destination file.
- * The file type of the downloadDestination should match the file type
- * of the file located at fileUrl. Throws a FileNotFoundException if the
- * fileUrl or downloadDestination are invalid.
- *
- * @param fileUrl A URL for the file to be downloaded
- * @param destinationUrl Location to download the file to (e.g. /tmp/file.txt)
- *
- * @return The URL representing the location of the downloaded file
- */
- def downloadFile(fileUrl: URL, destinationUrl: URL): URL = {
- val rbc = Channels.newChannel(fileUrl.openStream())
- val fos = new FileOutputStream(destinationUrl.getPath)
- fos.getChannel.transferFrom(rbc, 0, Long.MaxValue)
-
- destinationUrl
- }
-
- /**
- * Download a file given a URL string to the specified downloadDestination.
- *
- * @param fileToDownload A URL in string format (e.g. file:///tmp/foo, http://ibm.com)
- * @param destination Location to download the file to (e.g. /tmp/file.txt)
- *
- * @return The URL representing the location of the downloaded file
- */
- def downloadFile(fileToDownload: String, destination: String): URL = {
- downloadFile(new URL(fileToDownload), new URL(destination))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/utils/DynamicReflectionSupport.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/utils/DynamicReflectionSupport.scala b/kernel-api/src/main/scala/com/ibm/spark/utils/DynamicReflectionSupport.scala
deleted file mode 100644
index 73e02b6..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/utils/DynamicReflectionSupport.scala
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.utils
-
-import language.dynamics
-import language.existentials
-
-import java.lang.reflect.Method
-
-/**
- * Represents dynamic support capabilities for a class. Provides field and
- * method catchers to reflectively execute potentially-missing cases.
- * @param klass The class whose fields and methods to access
- * @param instance The specific instance whose fields and methods to access
- */
-case class DynamicReflectionSupport(
- private val klass: Class[_], private val instance: Any
-) extends Dynamic {
-
- /**
- * Handles cases of field access not found from type-checks. Attempts to
- * reflectively access field from provided class (or instance). Will first
- * check for a method signature matching field name due to Scala
- * transformation process.
- * @param name The name of the field
- * @return The content held by the field
- */
- def selectDynamic(name: String): Any = {
- val method = getMethod(name, Nil)
- method match {
- case Some(m) => invokeMethod(m, Nil)
- case _ =>
- try {
- val field = klass.getDeclaredField(name)
- field.setAccessible(true)
- field.get(instance)
- } catch {
- case ex: NoSuchFieldException =>
- throw new NoSuchFieldException(
- klass.getName + "." + name + " does not exist!"
- )
- case ex: Throwable => throw ex
- }
- }
- }
-
- /**
- * Handles cases of method not found from type-checks. Attempts to
- * reflectively execute the method from the provided class (or instance).
- * Will search for a method signature that has matching parameters with
- * arguments allowed to be subclasses of parameter types.
- *
- * @note Chaining in not supported. You must first cast the result of the
- * execution before attempting to apply a method upon it.
- *
- * @param name The name of the method
- * @param args The list of arguments for the method
- * @return The result of the method's execution
- */
- def applyDynamic(name: String)(args: Any*) = {
- val method = getMethod(name, args.toList)
-
- method match {
- case Some(m) => invokeMethod(m, args.toList)
- case _ =>
- throw new NoSuchMethodException(
- klass.getName + "." + name +
- "(" + args.map(_.getClass.getName).mkString(",") + ")"
- )
- }
- }
-
- private def getMethod(name: String, args: Any*): Option[Method] = {
- val flatArgs = flatten(args)
- val potentialMethods = klass.getDeclaredMethods.filter(_.getName == name)
- val method: Option[Method] =
- potentialMethods.foldLeft[Option[Method]](None) {
- (current, m) =>
- if (current != None) current
- else if (m.getParameterTypes.size != flatArgs.size) current
- else if (!m.getParameterTypes.zipWithIndex.forall {
- case (c, i) => isCompatible(c, flatArgs(i).getClass)
- }) current
- else Some(m)
- }
-
- method
- }
-
- private def invokeMethod(method: Method, args: Any*) = {
- val flatArgs = flatten(args).map(_.asInstanceOf[AnyRef])
- method.invoke(instance, flatArgs: _*)
- }
-
- private def flatten(l: Seq[Any]): Seq[Any] = {
- l.flatMap {
- case newList: List[_] => flatten(newList)
- case newSeq: Seq[_] => flatten(newSeq)
- case x => List(x)
- }
- }
-
- private def isCompatible(klazz1: Class[_], klazz2: Class[_]): Boolean = {
- var result =
- klazz1.isAssignableFrom(klazz2) ||
- klazz1.isInstance(klazz2) ||
- klazz1.isInstanceOf[klazz2.type]
-
- if (!result) {
- try {
- klazz1.asInstanceOf[klazz2.type]
- result = true
- } catch {
- case _: Throwable => result = false
- }
- }
-
- result
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/utils/KeyValuePairUtils.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/utils/KeyValuePairUtils.scala b/kernel-api/src/main/scala/com/ibm/spark/utils/KeyValuePairUtils.scala
deleted file mode 100644
index 7869aa6..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/utils/KeyValuePairUtils.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2015 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.utils
-
-import joptsimple.util.KeyValuePair
-
-/**
- * Provides utility methods for jsimple-opt key pair values.
- */
-object KeyValuePairUtils {
- val DefaultDelimiter = ","
-
- /**
- * Transforms the provided string into a list of KeyValuePair objects.
- * @param keyValuePairString The string representing the series of keys and
- * values
- * @param delimiter The delimiter used for splitting up the string
- * @return The list of KeyValuePair objects
- */
- def stringToKeyValuePairSeq(
- keyValuePairString: String,
- delimiter: String = DefaultDelimiter
- ): Seq[KeyValuePair] = {
- require(keyValuePairString != null, "KeyValuePair string cannot be null!")
-
- keyValuePairString
- .split(delimiter)
- .map(_.trim)
- .filterNot(_.isEmpty)
- .map(KeyValuePair.valueOf)
- .toSeq
- }
-
- /**
- * Transforms the provided list of KeyValuePair elements into a string.
- * @param keyValuePairSeq The sequence of KeyValuePair objects
- * @param delimiter The separator between string KeyValuePair
- * @return The resulting string from the list of KeyValuePair objects
- */
- def keyValuePairSeqToString(
- keyValuePairSeq: Seq[KeyValuePair],
- delimiter: String = DefaultDelimiter
- ): String = {
- require(keyValuePairSeq != null, "KeyValuePair sequence cannot be null!")
-
- keyValuePairSeq
- .map(pair => pair.key.trim + "=" + pair.value.trim)
- .mkString(delimiter)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/utils/MultiClassLoader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/utils/MultiClassLoader.scala b/kernel-api/src/main/scala/com/ibm/spark/utils/MultiClassLoader.scala
deleted file mode 100644
index 9ee4da1..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/utils/MultiClassLoader.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Copyright 2015 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.utils
-
-import java.lang.reflect.Method
-import java.net.{URL, URLClassLoader}
-import java.util
-import org.slf4j.LoggerFactory
-
-import scala.collection.JavaConverters._
-import scala.util.{Failure, Success, Try}
-
-import scala.language.existentials
-
-/**
- * Represents a class loader that supports delegating to multiple class loaders.
- *
- * @note Implements URLClassLoader purely to support the Guava requirement for
- * detecting all classes.
- *
- * @param urls The URLs to use for the underlying URLClassLoader
- * @param classLoaders The class loaders to use as the underlying
- * implementations of this class loader
- */
-class MultiClassLoader(
- private val urls: Seq[URL],
- private val classLoaders: Seq[ClassLoader]
-) extends URLClassLoader(
- classLoaders.flatMap({
- case urlClassLoader: URLClassLoader => urlClassLoader.getURLs.toSeq
- case _ => Nil
- }).distinct.toArray,
- /* Create a parent chain based on a each classloader's parent */ {
- val parents = classLoaders.flatMap(cl => Option(cl.getParent))
-
- // If multiple parents, set the parent to another multi class loader
- if (parents.size > 1) new MultiClassLoader(Nil, parents)
-
- // If a single parent, set the parent to that single parent
- else if (parents.size == 1) parents.head
-
- // If no parent, set to null (default if parent not provided)
- else null
- }: ClassLoader
-) { self =>
- private val logger = LoggerFactory.getLogger(this.getClass)
-
- /**
- * Creates a new multi class loader with no URLs of its own, although it may
- * still expose URLs from provided class loaders.
- *
- * @param classLoaders The class loaders to use as the underlying
- * implementations of this class loader
- */
- def this(classLoaders: ClassLoader*) = {
- this(Nil, classLoaders)
- }
-
- override protected def findClass(name: String): Class[_] = {
- @inline def tryFindClass(classLoader: ClassLoader, name: String) = {
- Try(Class.forName(name, false, classLoader))
- }
-
- // NOTE: Using iterator to evaluate elements one at a time
- classLoaders.toIterator
- .map(classLoader => tryFindClass(classLoader, name))
- .find(_.isSuccess)
- .map(_.get)
- .getOrElse(throw new ClassNotFoundException(name))
- }
-
- override protected def findResource(name: String): URL = {
- // NOTE: Using iterator to evaluate elements one at a time
- classLoaders.toIterator.map(cl => _findResource(cl, name)).find(_ != null)
- .getOrElse(super.findResource(name))
- }
-
- override protected def findResources(name: String): util.Enumeration[URL] = {
- val internalResources = classLoaders
- .flatMap(cl => Try(_findResources(cl, name)).toOption)
- .map(_.asScala)
- .reduce(_ ++ _)
-
- (
- internalResources
- ++
- Try(super.findResources(name)).map(_.asScala).getOrElse(Nil)
- ).asJavaEnumeration
- }
-
- private def _findResource[T <: ClassLoader](classLoader: T, name: String) = {
- _getDeclaredMethod(classLoader.getClass, "findResource", classOf[String])
- .invoke(classLoader, name).asInstanceOf[URL]
- }
-
- private def _findResources[T <: ClassLoader](classLoader: T, name: String) = {
- _getDeclaredMethod(classLoader.getClass, "findResources", classOf[String])
- .invoke(classLoader, name).asInstanceOf[util.Enumeration[URL]]
- }
-
- private def _loadClass[T <: ClassLoader](
- classLoader: T,
- name: String,
- resolve: Boolean
- ) = {
- _getDeclaredMethod(classLoader.getClass, "loadClass",
- classOf[String], classOf[Boolean]
- ).invoke(classLoader, name, resolve: java.lang.Boolean).asInstanceOf[Class[_]]
- }
-
- private def _getDeclaredMethod(
- klass: Class[_],
- name: String,
- classes: Class[_]*
- ): Method = {
- // Attempt to retrieve the method (public/protected/private) for the class,
- // trying the super class if the method is not available
- val potentialMethod = Try(klass.getDeclaredMethod(name, classes: _*))
- .orElse(Try(_getDeclaredMethod(klass.getSuperclass, name, classes: _*)))
-
- // Allow access to protected/private methods
- potentialMethod.foreach(_.setAccessible(true))
-
- potentialMethod match {
- case Success(method) => method
- case Failure(error) => throw error
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/utils/MultiOutputStream.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/utils/MultiOutputStream.scala b/kernel-api/src/main/scala/com/ibm/spark/utils/MultiOutputStream.scala
deleted file mode 100644
index 6ebaf3e..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/utils/MultiOutputStream.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.utils
-
-import java.io.OutputStream
-
-case class MultiOutputStream(val outputStreams: List[OutputStream])
- extends OutputStream
-{
- require(outputStreams != null)
-
- override def write(cbuf: Array[Byte]): Unit =
- outputStreams.foreach(outputStream => outputStream.write(cbuf))
-
- override def write(cbuf: Array[Byte], off: Int, len: Int): Unit =
- outputStreams.foreach(outputStream => outputStream.write(cbuf, off, len))
-
- override def write(b: Int): Unit =
- outputStreams.foreach(outputStream => outputStream.write(b))
-
- override def flush() =
- outputStreams.foreach(outputStream => outputStream.flush())
-
- override def close() =
- outputStreams.foreach(outputStream => outputStream.close())
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/utils/ScheduledTaskManager.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/utils/ScheduledTaskManager.scala b/kernel-api/src/main/scala/com/ibm/spark/utils/ScheduledTaskManager.scala
deleted file mode 100644
index a91e2ba..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/utils/ScheduledTaskManager.scala
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright 2014 IBM Corp.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.ibm.spark.utils
-
-import scala.language.existentials
-import java.util.concurrent._
-import java.util.UUID
-import ScheduledTaskManager._
-
-import scala.util.Try
-
-/**
- * Constructs timing-based events that are periodically executed. Does not
- * support hard-killing tasks that are not interruptable.
- * @param totalThreads The total threads to create for the underlying thread
- * pool
- * @param defaultExecutionDelay The default delay to use before all added tasks
- * @param defaultTimeInterval The default time interval between tasks in
- * milliseconds
- */
-class ScheduledTaskManager(
- private val totalThreads: Int = DefaultMaxThreads,
- private val defaultExecutionDelay: Long = DefaultExecutionDelay,
- private val defaultTimeInterval: Long = DefaultTimeInterval
-) {
- private[utils] val _scheduler = new ScheduledThreadPoolExecutor(totalThreads)
- _scheduler.setRemoveOnCancelPolicy(true)
-
- private val _taskMap = new ConcurrentHashMap[String, ScheduledFuture[_]]()
-
- /**
- * Adds the specified task to the queued list to execute at the specified
- * time interval.
- * @param executionDelay The time delay (in milliseconds) before starting
- * @param timeInterval The time interval (in milliseconds)
- * @param task The task to execute
- * @tparam T The type of return result (currently ignored)
- * @return The id of the task
- */
- def addTask[T](
- executionDelay: Long = defaultExecutionDelay,
- timeInterval: Long = defaultTimeInterval,
- task: => T
- ) = {
- val taskId = UUID.randomUUID().toString
- val runnable: Runnable = new Runnable {
- override def run(): Unit = Try(task)
- }
-
- // Schedule our task at the desired interval
- _taskMap.put(taskId, _scheduler.scheduleAtFixedRate(
- runnable, executionDelay, timeInterval, TimeUnit.MILLISECONDS))
-
- taskId
- }
-
- /**
- * Removes the specified task from the manager.
- * @param taskId The id of the task to remove
- * @return True if the task was removed, otherwise false
- */
- def removeTask(taskId: String): Boolean = {
- // Exit if the task with the given id does not exist
- if (taskId == null || !_taskMap.containsKey(taskId)) return false
-
- val future = _taskMap.remove(taskId)
-
- // Stop the future, but allow the current task to finish
- future.cancel(false)
-
- true
- }
-
- /**
- * Shuts down the thread pool used for task execution.
- */
- def stop() = {
- _taskMap.clear()
- _scheduler.shutdown()
- }
-}
-
-object ScheduledTaskManager {
- val DefaultMaxThreads = 4
- val DefaultExecutionDelay = 10 // 10 milliseconds
- val DefaultTimeInterval = 100 // 100 milliseconds
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/com/ibm/spark/utils/TaskManager.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/com/ibm/spark/utils/TaskManager.scala b/kernel-api/src/main/scala/com/ibm/spark/utils/TaskManager.scala
deleted file mode 100644
index c3a43fc..0000000
--- a/kernel-api/src/main/scala/com/ibm/spark/utils/TaskManager.scala
+++ /dev/null
@@ -1,237 +0,0 @@
-package com.ibm.spark.utils
-
-import java.util.concurrent.atomic.AtomicInteger
-
-import org.slf4j.LoggerFactory
-
-import scala.concurrent.{promise, Future}
-import java.util.concurrent._
-
-import com.ibm.spark.security.KernelSecurityManager._
-import TaskManager._
-
-import scala.util.Try
-
-/**
- * Represents a processor of tasks that has X worker threads dedicated to
- * executing the tasks.
- *
- * @param threadGroup The thread group to use with all worker threads
- * @param minimumWorkers The number of workers to spawn initially and keep
- * alive even when idle
- * @param maximumWorkers The max number of worker threads to spawn, defaulting
- * to the number of processors on the machine
- * @param keepAliveTime The maximum time in milliseconds for workers to remain
- * idle before shutting down
- */
-class TaskManager(
- private val threadGroup: ThreadGroup = DefaultThreadGroup,
- private val maxTasks: Int = DefaultMaxTasks,
- private val minimumWorkers: Int = DefaultMinimumWorkers,
- private val maximumWorkers: Int = DefaultMaximumWorkers,
- private val keepAliveTime: Long = DefaultKeepAliveTime
-) {
- protected val logger = LoggerFactory.getLogger(this.getClass.getName)
-
- private class TaskManagerThreadFactory extends ThreadFactory {
- override def newThread(r: Runnable): Thread = {
- val thread = new Thread(threadGroup, r)
-
- logger.trace(s"Creating new thread named ${thread.getName}!")
-
- thread
- }
- }
-
- private[utils] class ScalingThreadPoolExecutor extends ThreadPoolExecutor(
- minimumWorkers,
- maximumWorkers,
- keepAliveTime,
- TimeUnit.MILLISECONDS,
- taskQueue,
- taskManagerThreadFactory
- ) {
- protected val logger = LoggerFactory.getLogger(this.getClass.getName)
-
- /** Used to keep track of tasks separately from the task queue. */
- private val taskCount = new AtomicInteger(0)
-
- /**
- * Syncs the core pool size of the executor with the current number of
- * tasks, using the minimum worker size and maximum worker size as the
- * bounds.
- */
- private def syncPoolLimits(): Unit = {
- val totalTasks = taskCount.get()
- val newCorePoolSize =
- math.max(minimumWorkers, math.min(totalTasks, maximumWorkers))
-
- logger.trace(Seq(
- s"Task execution count is $totalTasks!",
- s"Updating core pool size to $newCorePoolSize!"
- ).mkString(" "))
- executor.foreach(_.setCorePoolSize(newCorePoolSize))
- }
-
- override def execute(r: Runnable): Unit = {
- synchronized {
- if (taskCount.incrementAndGet() > maximumWorkers)
- logger.warn(s"Exceeded $maximumWorkers workers during processing!")
-
- syncPoolLimits()
- }
-
- super.execute(r)
- }
-
- override def afterExecute(r: Runnable, t: Throwable): Unit = {
- super.afterExecute(r, t)
-
- synchronized {
- taskCount.decrementAndGet()
- syncPoolLimits()
- }
- }
- }
-
- private val taskManagerThreadFactory = new TaskManagerThreadFactory
- private val taskQueue = new ArrayBlockingQueue[Runnable](maxTasks)
-
- @volatile
- private[utils] var executor: Option[ScalingThreadPoolExecutor] = None
-
- /**
- * Adds a new task to the list to execute.
- *
- * @param taskFunction The new task as a block of code
- *
- * @return Future representing the return value (or error) from the task
- */
- def add[T <: Any](taskFunction: => T): Future[T] = {
- assert(executor.nonEmpty, "Task manager not started!")
-
- val taskPromise = promise[T]()
-
- // Construct runnable that completes the promise
- logger.trace(s"Queueing new task to be processed!")
- executor.foreach(_.execute(new Runnable {
- override def run(): Unit = {
- var threadName: String = "???"
- try {
- threadName = Try(Thread.currentThread().getName).getOrElse(threadName)
- logger.trace(s"(Thread $threadName) Executing task!")
- val result = taskFunction
-
- logger.trace(s"(Thread $threadName) Task finished successfully!")
- taskPromise.success(result)
- } catch {
- case ex: Throwable =>
- val exName = ex.getClass.getName
- val exMessage = Option(ex.getLocalizedMessage).getOrElse("???")
- logger.trace(
- s"(Thread $threadName) Task failed: ($exName) = $exMessage")
- taskPromise.tryFailure(ex)
- }
- }
- }))
-
- taskPromise.future
- }
-
- /**
- * Returns the count of tasks including the currently-running ones.
- *
- * @return The count of tasks
- */
- def size: Int = taskQueue.size() + executor.map(_.getActiveCount).getOrElse(0)
-
- /**
- * Returns whether or not there is a task in the queue to be processed.
- *
- * @return True if the internal queue is not empty, otherwise false
- */
- def hasTaskInQueue: Boolean = !taskQueue.isEmpty
-
- /**
- * Whether or not there is a task being executed currently.
- *
- * @return True if there is a task being executed, otherwise false
- */
- def isExecutingTask: Boolean = executor.exists(_.getActiveCount > 0)
-
- /**
- * Block execution (by sleeping) until all tasks currently queued up for
- * execution are processed.
- */
- def await(): Unit =
- while (!taskQueue.isEmpty || isExecutingTask) Thread.sleep(1)
-
- /**
- * Starts the task manager (begins processing tasks). Creates X new threads
- * in the process.
- */
- def start(): Unit = {
- logger.trace(
- s"""
- |Initializing with the following settings:
- |- $minimumWorkers core worker pool
- |- $maximumWorkers maximum workers
- |- $keepAliveTime milliseconds keep alive time
- """.stripMargin.trim)
- executor = Some(new ScalingThreadPoolExecutor)
- }
-
- /**
- * Restarts internal processing of tasks (removing current task).
- */
- def restart(): Unit = {
- stop()
- start()
- }
-
- /**
- * Stops internal processing of tasks.
- */
- def stop(): Unit = {
- executor.foreach(_.shutdownNow())
- executor = None
- }
-}
-
-/**
- * Represents constants associated with the task manager.
- */
-object TaskManager {
- /** The default thread group to use with all worker threads. */
- val DefaultThreadGroup = new ThreadGroup(RestrictedGroupName)
-
- /** The default number of maximum tasks accepted by the task manager. */
- val DefaultMaxTasks = 200
-
- /**
- * The default number of workers to spawn initially and keep alive
- * even when idle.
- */
- val DefaultMinimumWorkers = 1
-
- /** The default maximum number of workers to spawn. */
- val DefaultMaximumWorkers = Runtime.getRuntime.availableProcessors()
-
- /** The default timeout in milliseconds for workers waiting for tasks. */
- val DefaultKeepAliveTime = 1000
-
- /**
- * The default timeout in milliseconds to wait before stopping a thread
- * if it cannot be interrupted.
- */
- val InterruptTimeout = 5000
-
- /** The maximum time to wait to add a task to the queue in milliseconds. */
- val MaximumTaskQueueTimeout = 10000
-
- /**
- * The maximum time in milliseconds to wait to queue up a thread in the
- * thread factory.
- */
- val MaximumThreadQueueTimeout = 10000
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
new file mode 100644
index 0000000..2f31cbe
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.dependencies
+
+import java.io.PrintStream
+import java.net.URL
+
+abstract class DependencyDownloader(repositoryUrl: String, baseDir: String) {
+
+ /**
+ * Retrieves the dependency and all of its dependencies as jars.
+ *
+ * @param groupId The group id associated with the main dependency
+ * @param artifactId The id of the dependency artifact
+ * @param version The version of the main dependency
+ * @param transitive If true, downloads all dependencies of the specified
+ * dependency
+ * @param excludeBaseDependencies If true, will exclude any dependencies
+ * included in the build of the kernel
+ *
+ * @return The sequence of strings pointing to the retrieved dependency jars
+ */
+ def retrieve(
+ groupId: String, artifactId: String, version: String,
+ transitive: Boolean = true, excludeBaseDependencies: Boolean = true
+ ): Seq[URL]
+
+ /**
+ * Sets the printstream to log to.
+ *
+ * @param printStream The new print stream to use for output logging
+ */
+ def setPrintStream(printStream: PrintStream): Unit
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
new file mode 100644
index 0000000..2465b0e
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
@@ -0,0 +1,185 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.dependencies
+
+import java.io.{File, PrintStream}
+import java.net.URL
+
+import org.apache.ivy.Ivy
+import org.apache.ivy.core.module.descriptor._
+import org.apache.ivy.core.module.id.{ArtifactId, ModuleId, ModuleRevisionId}
+import org.apache.ivy.core.resolve.ResolveOptions
+import org.apache.ivy.core.retrieve.RetrieveOptions
+import org.apache.ivy.core.settings.IvySettings
+import org.apache.ivy.plugins.matcher.RegexpPatternMatcher
+import org.apache.ivy.plugins.parser.xml.{XmlModuleDescriptorParser, XmlModuleDescriptorWriter}
+import org.apache.ivy.plugins.resolver.IBiblioResolver
+import org.apache.ivy.util.{DefaultMessageLogger, Message}
+import org.springframework.core.io.support._
+
+
+class IvyDependencyDownloader(repositoryUrl: String, baseDirectory: String)
+ extends DependencyDownloader(repositoryUrl, baseDirectory)
+{
+ private val ivySettings = new IvySettings()
+ private val resolver = new IBiblioResolver
+
+ resolver.setUsepoms(true)
+ resolver.setM2compatible(true)
+ resolver.setName("central")
+
+ // Add our resolver as the main resolver (IBiblio goes to Maven Central)
+ ivySettings.addResolver(resolver)
+
+ // Mark our resolver as the default one to use
+ ivySettings.setDefaultResolver(resolver.getName)
+
+ // Set the destination
+ ivySettings.setBaseDir(new File(baseDirectory))
+ ivySettings.setDefaultResolutionCacheBasedir(baseDirectory)
+ ivySettings.setDefaultRepositoryCacheBasedir(baseDirectory)
+
+ //creates an Ivy instance with settings
+ val ivy = Ivy.newInstance(ivySettings)
+
+ private def getBaseDependencies: Iterable[DependencyDescriptor] = {
+ val xmlModuleDescriptor = XmlModuleDescriptorParser.getInstance()
+ val getDependencies = (url: URL) => xmlModuleDescriptor.parseDescriptor(
+ new IvySettings(), url, false
+ ).getDependencies
+
+ // Find all of the *ivy.xml files on the classpath.
+ val ivyFiles = new PathMatchingResourcePatternResolver().getResources(
+ "classpath*:**/*ivy.xml"
+ )
+ val classpathURLs = ivyFiles.map(_.getURI.toURL)
+
+ // Get all of the dependencies from the *ivy.xml files
+ val dependencies = classpathURLs.map(getDependencies).flatten
+
+ // Remove duplicates based on artifact name
+ val distinctDependencies =
+ dependencies.groupBy(_.getDependencyId.getName).map(_._2.head)
+
+ distinctDependencies
+ }
+
+ override def retrieve(
+ groupId: String, artifactId: String, version: String,
+ transitive: Boolean = true, excludeBaseDependencies: Boolean = true
+ ): Seq[URL] = {
+ // Start building the ivy.xml file
+ val ivyFile = File.createTempFile("ivy-custom", ".xml")
+ ivyFile.deleteOnExit()
+
+ val md = DefaultModuleDescriptor.newDefaultInstance(
+ ModuleRevisionId.newInstance("com.ibm.spark", "spark-kernel", "working")
+ )
+
+ // Exclude all sources artifacts i.e. artifactId-version-sources.jar
+ val moduleId = new ModuleId("*", "*")
+ val sourcesArtifactId = new ArtifactId(moduleId, "*", "source", "*")
+ val sourcesExclusion = new DefaultExcludeRule(
+ sourcesArtifactId, new RegexpPatternMatcher(), null
+ )
+
+ // Exclude all javadoc artifacts i.e. artifactId-version-javadoc.jar
+ val javadocArtifactId = new ArtifactId(moduleId, "*", "javadoc", "*")
+ val javadocExclusion = new DefaultExcludeRule(
+ javadocArtifactId, new RegexpPatternMatcher(), null
+ )
+
+ // TODO: figure out why this is not excluded. It's in our build.sbt file
+ // TODO: and we exclude all deps there. Need to get rid of this hard-code
+ val scalaCompilerModuleId = new ModuleId("org.scala-lang", "*")
+ val scalaCompilerArtifactId = new ArtifactId(
+ scalaCompilerModuleId, "*", "*", "*"
+ )
+ val scalaCompilerExclusion = new DefaultExcludeRule(
+ scalaCompilerArtifactId, new RegexpPatternMatcher(), null
+ )
+
+ // Create our dependency descriptor
+ val dependencyDescriptor = new DefaultDependencyDescriptor(
+ md, ModuleRevisionId.newInstance(groupId, artifactId, version),
+ false, false, true
+ )
+
+ md.addDependency(dependencyDescriptor)
+
+ // Add any and all exclusions
+ md.addExcludeRule(sourcesExclusion)
+ md.addExcludeRule(javadocExclusion)
+ md.addExcludeRule(scalaCompilerExclusion)
+
+ // Exclude our base dependencies if marked to do so
+ if (excludeBaseDependencies) {
+ getBaseDependencies.foreach(dep => {
+ val depRevId = dep.getDependencyRevisionId
+ val moduleId = new ModuleId(depRevId.getOrganisation, depRevId.getName)
+ val artifactId = new ArtifactId(moduleId, "*", "*", "*")
+ val excludeRule = new DefaultExcludeRule(
+ artifactId, new RegexpPatternMatcher(), null)
+ md.addExcludeRule(excludeRule)
+ })
+ }
+
+ // Creates our ivy configuration file
+ XmlModuleDescriptorWriter.write(md, ivyFile)
+
+ // Grab our dependencies (and theirs, etc) recursively
+ val resolveOptions = new ResolveOptions()
+ .setTransitive(transitive)
+ .setDownload(true)
+
+ // Init resolve report (has what was downloaded, etc)
+ val report = ivy.resolve(ivyFile.toURI.toURL, resolveOptions)
+
+ // Get the jar libraries
+ val artifactURLs = report.getAllArtifactsReports
+ .map(report => new URL("file:" + report.getLocalFile.getCanonicalPath))
+
+ val moduleDescriptor = report.getModuleDescriptor
+ ivy.retrieve(
+ moduleDescriptor.getModuleRevisionId,
+ baseDirectory + "/[artifact](-[classifier]).[ext]",
+ new RetrieveOptions().setConfs(Seq("default").toArray)
+ )
+
+ artifactURLs
+ }
+
+ /**
+ * Uses our printstream in Ivy's LoggingEngine
+ * @param printStream the print stream to use
+ */
+ override def setPrintStream(printStream: PrintStream): Unit = {
+ ivy.getLoggerEngine.setDefaultLogger(
+ new DefaultMessageLogger(Message.MSG_INFO) {
+ override def doEndProgress(msg: String): Unit =
+ printStream.println(msg)
+
+ override def doProgress(): Unit =
+ printStream.print(".")
+
+ override def log(msg: String, level: Int): Unit =
+ if (level <= this.getLevel)
+ printStream.println(msg)
+ }
+ )
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/org/apache/toree/global/StreamState.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/global/StreamState.scala b/kernel-api/src/main/scala/org/apache/toree/global/StreamState.scala
new file mode 100644
index 0000000..973c000
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/global/StreamState.scala
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.global
+
+import java.io.{InputStream, OutputStream, PrintStream}
+
+/**
+ * Represents the global state for input and output streams used to communicate
+ * standard input and output.
+ */
+object StreamState {
+ private val _baseInputStream = System.in
+ private val _baseOutputStream = System.out
+ private val _baseErrorStream = System.err
+
+ @volatile private var _inputStream = _baseInputStream
+ @volatile private var _outputStream = _baseOutputStream
+ @volatile private var _errorStream = _baseErrorStream
+
+ private def init(in: InputStream, out: OutputStream, err: OutputStream) =
+ synchronized {
+ System.setIn(in)
+ Console.setIn(in)
+
+ System.setOut(new PrintStream(out))
+ Console.setOut(out)
+
+ System.setErr(new PrintStream(err))
+ Console.setErr(err)
+ }
+
+ private def reset(): Unit = synchronized {
+ System.setIn(_baseInputStream)
+ Console.setIn(_baseInputStream)
+
+ System.setOut(_baseOutputStream)
+ Console.setOut(_baseOutputStream)
+
+ System.setErr(_baseErrorStream)
+ Console.setErr(_baseErrorStream)
+ }
+
+ /**
+ * Sets the internal streams to be used with the stream block.
+ *
+ * @param inputStream The input stream to map standard in
+ * @param outputStream The output stream to map standard out
+ * @param errorStream The output stream to map standard err
+ */
+ def setStreams(
+ inputStream: InputStream = _inputStream,
+ outputStream: OutputStream = _outputStream,
+ errorStream: OutputStream = _errorStream
+ ) = {
+ _inputStream = inputStream
+ _outputStream = new PrintStream(outputStream)
+ _errorStream = new PrintStream(errorStream)
+ }
+
+ /**
+ * Execute code block, mapping all input and output to the provided streams.
+ */
+ def withStreams[T](thunk: => T): T = {
+ init(_inputStream, _outputStream, _errorStream)
+
+ val returnValue = thunk
+
+ reset()
+
+ returnValue
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/org/apache/toree/interpreter/ExecuteFailure.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/ExecuteFailure.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/ExecuteFailure.scala
new file mode 100644
index 0000000..8103536
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/ExecuteFailure.scala
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.interpreter
+
+/**
+ * Represents a generic failure in execution.
+ */
+sealed abstract class ExecuteFailure
+
+/**
+ * Represents an error resulting from interpret execution.
+ * @param name The name of the error
+ * @param value The message provided from the error
+ * @param stackTrace The stack trace as a list of strings representing lines
+ * in the stack trace
+ */
+case class ExecuteError(
+ name: String, value: String, stackTrace: List[String]
+) extends ExecuteFailure {
+ override def toString: String =
+ "Name: " + name + "\n" +
+ "Message: " + value + "\n" +
+ "StackTrace: " + stackTrace.mkString("\n")
+}
+
+// TODO: Replace with object?
+/**
+ * Represents an aborted execution.
+ */
+class ExecuteAborted extends ExecuteFailure
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
new file mode 100644
index 0000000..6200b9b
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/Interpreter.scala
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.interpreter
+
+import java.net.URL
+
+import com.ibm.spark.kernel.api.KernelLike
+import org.apache.spark.SparkContext
+import org.apache.spark.sql.SQLContext
+
+import scala.tools.nsc.interpreter._
+
+trait Interpreter {
+
+ /**
+ * Initializes the interpreter.
+ * @param kernel The kernel
+ * @return The newly initialized interpreter
+ */
+ def init(kernel: KernelLike): Interpreter
+
+ /**
+ * Starts the interpreter, initializing any internal state.
+ * @return A reference to the interpreter
+ */
+ def start(): Interpreter
+
+ /**
+ * Interrupts the current code being interpreted.
+ * @return A reference to the interpreter
+ */
+ def interrupt(): Interpreter
+
+ /**
+ * Stops the interpreter, removing any previous internal state.
+ * @return A reference to the interpreter
+ */
+ def stop(): Interpreter
+
+ /**
+ * Adds external jars to the internal classpaths of the interpreter.
+ * @param jars The list of jar locations
+ */
+ def addJars(jars: URL*): Unit
+
+ /**
+ * Executes the provided code with the option to silence output.
+ * @param code The code to execute
+ * @param silent Whether or not to execute the code silently (no output)
+ * @return The success/failure of the interpretation and the output from the
+ * execution or the failure
+ */
+ def interpret(code: String, silent: Boolean = false):
+ (Results.Result, Either[ExecuteOutput, ExecuteFailure])
+
+ /**
+ * @return Returns a string to reference the URI of where the interpreted class files are created
+ */
+ def classServerURI: String
+
+ /**
+ * Executes body and will not print anything to the console during the execution
+ * @param body The function to execute
+ * @tparam T The return type of body
+ * @return The return value of body
+ */
+ def doQuietly[T](body: => T): T
+
+ /**
+ * Binds the SparkContext instance to the interpreter's namespace.
+ *
+ * @param sparkContext The SparkContext to bind
+ */
+ def bindSparkContext(sparkContext: SparkContext): Unit
+
+ /**
+ * Binds the SQLContext instance to the interpreter's namespace.
+ *
+ * @param sqlContext The SQLContext to bind
+ */
+ def bindSqlContext(sqlContext: SQLContext): Unit
+
+ /**
+ * Binds a variable in the interpreter to a value.
+ * @param variableName The name to expose the value in the interpreter
+ * @param typeName The type of the variable, must be the fully qualified class name
+ * @param value The value of the variable binding
+ * @param modifiers Any annotation, scoping modifiers, etc on the variable
+ */
+ def bind(variableName: String, typeName: String, value: Any, modifiers: List[String])
+
+ /**
+ * Retrieves the contents of the variable with the provided name from the
+ * interpreter.
+ * @param variableName The name of the variable whose contents to read
+ * @return An option containing the variable contents or None if the
+ * variable does not exist
+ */
+ def read(variableName: String): Option[AnyRef]
+
+ /**
+ * Mask the Console and System objects with our wrapper implementations
+ * and dump the Console methods into the public namespace (similar to
+ * the Predef approach).
+ * @param in The new input stream
+ * @param out The new output stream
+ * @param err The new error stream
+ */
+ def updatePrintStreams(in: InputStream, out: OutputStream, err: OutputStream)
+
+ /**
+ * Attempts to perform code completion via the <TAB> command.
+ * @param code The current cell to complete
+ * @param pos The cursor position
+ * @return The cursor position and list of possible completions
+ */
+ def completion(code: String, pos: Int): (Int, List[String] )
+
+ /**
+ * Returns the name of the variable created from the last execution.
+ * @return Some String name if a variable was created, otherwise None
+ */
+ def lastExecutionVariableName: Option[String]
+
+ /**
+ * Returns the class loader used by this interpreter.
+ * @return The runtime class loader used by this interpreter
+ */
+ def classLoader: ClassLoader
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/68f7ddd6/kernel-api/src/main/scala/org/apache/toree/interpreter/InterpreterTypes.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/interpreter/InterpreterTypes.scala b/kernel-api/src/main/scala/org/apache/toree/interpreter/InterpreterTypes.scala
new file mode 100644
index 0000000..7ecee65
--- /dev/null
+++ b/kernel-api/src/main/scala/org/apache/toree/interpreter/InterpreterTypes.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.ibm.spark.interpreter
+
+/**
+ * Contains all types associated with the interpreter interface.
+ */
+object InterpreterTypes {
+ /**
+ * Represents the output from an interpret execution.
+ */
+ type ExecuteOutput = String
+}