You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by ch...@apache.org on 2016/05/26 15:52:03 UTC
[2/5] incubator-toree git commit: Added ability to specify default
repositories to use for resolving at launch
Added ability to specify default repositories to use for resolving at launch
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/d960c86b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/d960c86b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/d960c86b
Branch: refs/heads/master
Commit: d960c86bf6d2f3563f7460b76057b0ceddb4119a
Parents: 6d6b06d
Author: Marius van Niekerk <ma...@maxpoint.com>
Authored: Thu May 19 14:58:08 2016 -0400
Committer: Marius van Niekerk <Marius van Niekerk>
Committed: Wed May 25 20:10:01 2016 -0400
----------------------------------------------------------------------
.../dependencies/DependencyDownloader.scala | 28 ++++++++++++++++++++
.../apache/toree/boot/CommandLineOptions.scala | 18 +++++++++++--
.../boot/layer/ComponentInitialization.scala | 16 +++++++++--
.../apache/toree/magic/builtin/AddDeps.scala | 18 ++++---------
resources/compile/reference.conf | 6 +++++
resources/test/reference.conf | 5 ++++
6 files changed, 74 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
index 5b3d9fc..1859776 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
@@ -20,6 +20,7 @@ package org.apache.toree.dependencies
import java.io.{File, PrintStream}
import java.net.{URI, URL}
import java.nio.file.Files
+import scala.util.Try
abstract class DependencyDownloader {
/**
@@ -98,6 +99,33 @@ abstract class DependencyDownloader {
* @return The directory as a string
*/
def getDownloadDirectory: String
+
+
+ /**
+ * Assigns credentials to the right repository and build tuples
+ *
+ * @param repository Sequence of repository urls
+ * @param credentials Sequence of credential filenames
+ * @return
+ */
+ def resolveRepositoriesAndCredentials(repository: List[String], credentials: List[String]): List[(URL, Option[Credentials])] = {
+ val extraRepositories = repository.map(u => (u, Try(new URL(u))))
+
+ // Print error information
+ // extraRepositories.filter(_._2.isFailure).map(_._1)
+ // .foreach(u => printStream.println(s"Ignoring invalid URL $u"))
+
+ // match up credentials with repositories
+ val repositories = extraRepositories.flatMap(_._2.toOption)
+ val authentication = credentials
+ .map(f => new File(f))
+ .map(Credentials(_))
+ .map(c => (c.host, c)).toMap
+
+ val repositoriesWithCreds = repositories.map(u => (u, authentication.get(u.getHost)))
+ repositoriesWithCreds
+ }
+
}
object DependencyDownloader {
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
index 43cfc8e..6ac6442 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
@@ -71,6 +71,14 @@ class CommandLineOptions(args: Seq[String]) {
parser.accepts("magic-url", "path to a magic jar")
.withRequiredArg().ofType(classOf[String])
+ private val _default_repositories = parser.accepts(
+ "default_repositories", "comma seperated list of additional repositories to resolve"
+ ).withRequiredArg().ofType(classOf[String])
+
+ private val _default_repository_credentials = parser.accepts(
+ "default_repository_credentials", "comma seperated list of credential files to use"
+ ).withRequiredArg().ofType(classOf[String])
+
private val _max_interpreter_threads = parser.accepts(
"max-interpreter-threads",
"total number of worker threads to use to execute code"
@@ -144,7 +152,12 @@ class CommandLineOptions(args: Seq[String]) {
"jar_dir" -> get(_jar_dir),
"default_interpreter" -> get(_default_interpreter),
"nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),
- "interpreter_plugins" -> interpreterPlugins
+ "interpreter_plugins" -> interpreterPlugins,
+ "default_repositories" -> getAll(_default_repositories).map(_.asJava)
+ .flatMap(list => if (list.isEmpty) None else Some(list)),
+ "default_repository_credentials" -> getAll(_default_repository_credentials).map(_.asJava)
+ .flatMap(list => if (list.isEmpty) None else Some(list))
+
).flatMap(removeEmptyOptions).asInstanceOf[Map[String, AnyRef]].asJava)
commandLineConfig.withFallback(profileConfig).withFallback(ConfigFactory.load)
@@ -182,7 +195,8 @@ class CommandLineOptions(args: Seq[String]) {
/**
* Prints the help message to the output stream provided.
- * @param out The output stream to direct the help message
+ *
+ * @param out The output stream to direct the help message
*/
def printHelpOn(out: OutputStream) =
parser.printHelpOn(out)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
index fb7e76d..b90e951 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
@@ -18,14 +18,15 @@
package org.apache.toree.boot.layer
import java.io.File
-import java.nio.file.{Paths, Files}
+import java.net.URL
+import java.nio.file.{Files, Paths}
import java.util.concurrent.ConcurrentHashMap
import akka.actor.ActorRef
import com.typesafe.config.Config
import org.apache.spark.SparkConf
import org.apache.toree.comm.{CommManager, CommRegistrar, CommStorage, KernelCommManager}
-import org.apache.toree.dependencies.{CoursierDependencyDownloader, DependencyDownloader}
+import org.apache.toree.dependencies.{CoursierDependencyDownloader, Credentials, DependencyDownloader}
import org.apache.toree.interpreter._
import org.apache.toree.kernel.api.Kernel
import org.apache.toree.kernel.protocol.v5.KMBuilder
@@ -130,6 +131,17 @@ trait StandardComponentInitialization extends ComponentInitialization {
new File(depsDir)
)
+ if (config.hasPath("default_repositories")) {
+ val repository = config.getStringList("default_repositories").asScala.toList
+
+ val credentials = if (config.hasPath("default_repository_credentials")) {
+ config.getStringList("default_repository_credentials").asScala.toList
+ } else Nil
+
+ dependencyDownloader.resolveRepositoriesAndCredentials(repository, credentials)
+ .foreach{case (u, c) => dependencyDownloader.addMavenRepository(u, c)}
+ }
+
dependencyDownloader
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
index f245851..2f4d812 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
@@ -71,20 +71,10 @@ class AddDeps extends LineMagic with IncludeInterpreter
val nonOptionArgs = parseArgs(code)
dependencyDownloader.setPrintStream(printStream)
- val extraRepositories = getAll(_repository).getOrElse(Nil).map(u => (u, Try(new URL(u))))
+ val repository = getAll(_repository).getOrElse(Nil)
+ val credentials = getAll(_credentials).getOrElse(Nil)
- // Print error information
- extraRepositories.filter(_._2.isFailure).map(_._1)
- .foreach(u => printStream.println(s"Ignoring invalid URL $u"))
-
- // match up credentials with repositories
- val repositories = extraRepositories.flatMap(_._2.toOption)
- val authentication = getAll(_credentials).getOrElse(Nil)
- .map(f => new File(f))
- .map(Credentials(_))
- .map(c => (c.host, c)).toMap
-
- val repositoriesWithCreds = repositories.map(u => (u, authentication.get(u.getHost)))
+ val repositoriesWithCreds = dependencyDownloader.resolveRepositoriesAndCredentials(repository, credentials)
if (nonOptionArgs.size == 3) {
// get the jars and hold onto the paths at which they reside
@@ -106,4 +96,6 @@ class AddDeps extends LineMagic with IncludeInterpreter
printHelp(printStream, """%AddDeps my.company artifact-id version""")
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/resources/compile/reference.conf
----------------------------------------------------------------------
diff --git a/resources/compile/reference.conf b/resources/compile/reference.conf
index 1c23174..25e553e 100644
--- a/resources/compile/reference.conf
+++ b/resources/compile/reference.conf
@@ -61,3 +61,9 @@ default_interpreter_plugin = [
"SparkR:org.apache.toree.kernel.interpreter.sparkr.SparkRInterpreter",
"SQL:org.apache.toree.kernel.interpreter.sql.SqlInterpreter"
]
+
+default_repositories = []
+default_repositories = [${?DEFAULT_REPOSITORIES}]
+
+default_repository_credentials = []
+default_repository_credentials = [${?DEFAULT_REPOSITORY_CREDENTIALS}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/resources/test/reference.conf
----------------------------------------------------------------------
diff --git a/resources/test/reference.conf b/resources/test/reference.conf
index 85e614c..c7c9363 100644
--- a/resources/test/reference.conf
+++ b/resources/test/reference.conf
@@ -61,3 +61,8 @@ default_interpreter_plugin = [
"SQL:org.apache.toree.kernel.interpreter.sql.SqlInterpreter"
]
+default_repositories = []
+default_repositories = [${?DEFAULT_REPOSITORIES}]
+
+default_repository_credentials = []
+default_repository_credentials = [${?DEFAULT_REPOSITORY_CREDENTIALS}]