You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by ch...@apache.org on 2016/05/26 15:52:02 UTC
[1/5] incubator-toree git commit: Update CommandLineOptions.scala
Repository: incubator-toree
Updated Branches:
refs/heads/master b59a600e6 -> 834f8c226
Update CommandLineOptions.scala
Fixed underscores
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/834f8c22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/834f8c22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/834f8c22
Branch: refs/heads/master
Commit: 834f8c22660e82f08e18ee76cd83a64fdd90c269
Parents: 781e238
Author: Marius van Niekerk <ma...@gmail.com>
Authored: Thu May 19 20:16:10 2016 -0400
Committer: Marius van Niekerk <Marius van Niekerk>
Committed: Wed May 25 20:10:01 2016 -0400
----------------------------------------------------------------------
.../main/scala/org/apache/toree/boot/CommandLineOptions.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/834f8c22/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
index 6ac6442..1187bba 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
@@ -72,11 +72,11 @@ class CommandLineOptions(args: Seq[String]) {
.withRequiredArg().ofType(classOf[String])
private val _default_repositories = parser.accepts(
- "default_repositories", "comma seperated list of additional repositories to resolve"
+ "default-repositories", "comma seperated list of additional repositories to resolve"
).withRequiredArg().ofType(classOf[String])
private val _default_repository_credentials = parser.accepts(
- "default_repository_credentials", "comma seperated list of credential files to use"
+ "default-repository-credentials", "comma seperated list of credential files to use"
).withRequiredArg().ofType(classOf[String])
private val _max_interpreter_threads = parser.accepts(
[4/5] incubator-toree git commit: Update reference.conf
Posted by ch...@apache.org.
Update reference.conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/781e2388
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/781e2388
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/781e2388
Branch: refs/heads/master
Commit: 781e2388f60930ba4aaca1eae4dc696ff7f3a078
Parents: d960c86
Author: Marius van Niekerk <ma...@gmail.com>
Authored: Thu May 19 15:00:28 2016 -0400
Committer: Marius van Niekerk <Marius van Niekerk>
Committed: Wed May 25 20:10:01 2016 -0400
----------------------------------------------------------------------
resources/compile/reference.conf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/781e2388/resources/compile/reference.conf
----------------------------------------------------------------------
diff --git a/resources/compile/reference.conf b/resources/compile/reference.conf
index 25e553e..a8a2112 100644
--- a/resources/compile/reference.conf
+++ b/resources/compile/reference.conf
@@ -66,4 +66,4 @@ default_repositories = []
default_repositories = [${?DEFAULT_REPOSITORIES}]
default_repository_credentials = []
-default_repository_credentials = [${?DEFAULT_REPOSITORY_CREDENTIALS}]
\ No newline at end of file
+default_repository_credentials = [${?DEFAULT_REPOSITORY_CREDENTIALS}]
[3/5] incubator-toree git commit: Added credential support for adddeps
Posted by ch...@apache.org.
Added credential support for adddeps
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/4177e9b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/4177e9b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/4177e9b2
Branch: refs/heads/master
Commit: 4177e9b2f1708bd5fcabdc209c20d5fc83267182
Parents: b59a600
Author: Marius van Niekerk <ma...@maxpoint.com>
Authored: Thu May 19 13:40:51 2016 -0400
Committer: Marius van Niekerk <Marius van Niekerk>
Committed: Wed May 25 20:10:01 2016 -0400
----------------------------------------------------------------------
kernel-api/build.sbt | 4 +-
.../CoursierDependencyDownloader.scala | 77 ++++++++++++++++----
.../dependencies/DependencyDownloader.scala | 4 +-
.../dependencies/IvyDependencyDownloader.scala | 4 +-
.../apache/toree/magic/builtin/AddDeps.scala | 20 ++++-
.../toree/magic/builtin/AddDepsSpec.scala | 15 ++--
project/plugins.sbt | 2 +-
7 files changed, 94 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel-api/build.sbt
----------------------------------------------------------------------
diff --git a/kernel-api/build.sbt b/kernel-api/build.sbt
index 331f9bf..8898a9b 100644
--- a/kernel-api/build.sbt
+++ b/kernel-api/build.sbt
@@ -50,8 +50,8 @@ libraryDependencies += "net.sf.jopt-simple" % "jopt-simple" % "4.6" // MIT
libraryDependencies ++= Seq(
// Used to find and download jars from Maven-based repositories
"org.apache.ivy" % "ivy" % "2.4.0-rc1", // Apache v2
- "com.github.alexarchambault" %% "coursier" % "1.0.0-M9", // Apache v2
- "com.github.alexarchambault" %% "coursier-cache" % "1.0.0-M9" // Apache v2
+ "io.get-coursier" %% "coursier" % "1.0.0-M12", // Apache v2
+ "io.get-coursier" %% "coursier-cache" % "1.0.0-M12" // Apache v2
)
// Brought in in order to simplify the reading of each project's ivy.xml file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
index ef9ac02..4751f67 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
@@ -16,14 +16,16 @@
*/
package org.apache.toree.dependencies
-import java.io.{File, PrintStream}
+import java.io.{File, FileInputStream, PrintStream}
import java.net.{URI, URL}
+import java.util.Properties
import java.util.concurrent.ConcurrentHashMap
+import coursier.core.Authentication
import coursier.Cache.Logger
import coursier.Dependency
import coursier.core.Repository
-import coursier.ivy.{IvyXml, IvyRepository}
+import coursier.ivy.{IvyRepository, IvyXml}
import coursier.maven.MavenRepository
import org.springframework.core.io.support.PathMatchingResourcePatternResolver
@@ -40,7 +42,7 @@ class CoursierDependencyDownloader extends DependencyDownloader {
// Initialization
setDownloadDirectory(DependencyDownloader.DefaultDownloadDirectory)
- addMavenRepository(DependencyDownloader.DefaultMavenRepository)
+ addMavenRepository(DependencyDownloader.DefaultMavenRepository, None)
/**
* Retrieves the dependency and all of its dependencies as jars.
@@ -69,7 +71,7 @@ class CoursierDependencyDownloader extends DependencyDownloader {
transitive: Boolean,
excludeBaseDependencies: Boolean,
ignoreResolutionErrors: Boolean,
- extraRepositories: Seq[URL],
+ extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
verbose: Boolean,
trace: Boolean
): Seq[URI] = {
@@ -95,12 +97,9 @@ class CoursierDependencyDownloader extends DependencyDownloader {
lazy val defaultBase = new File(localDirectory).getAbsoluteFile
- lazy val downloadLocations = Seq(
- "http://" -> new File(defaultBase, "http"),
- "https://" -> new File(defaultBase, "https")
- )
+ lazy val downloadLocations = defaultBase
- val allRepositories = extraRepositories.map(urlToMavenRepository) ++ repositories
+ val allRepositories = extraRepositories.map(x => urlToMavenRepository(x._1, x._2.map(_.authentication))) ++ repositories
// Build list of locations to fetch dependencies
val fetchLocations = Seq(ivy2Cache(localDirectory)) ++ allRepositories
@@ -150,10 +149,10 @@ class CoursierDependencyDownloader extends DependencyDownloader {
*
* @param url The string representation of the url
*/
- override def addMavenRepository(url: URL): Unit =
- repositories :+= urlToMavenRepository(url)
+ override def addMavenRepository(url: URL, credentials: Option[Credentials]): Unit =
+ repositories :+= urlToMavenRepository(url, credentials.map(_.authentication))
- private def urlToMavenRepository(url: URL) = MavenRepository(url.toString)
+ private def urlToMavenRepository(url: URL, credentials: Option[Authentication]) = MavenRepository(url.toString, authentication = credentials)
/**
* Remove the specified resolver url from the search options.
@@ -162,7 +161,7 @@ class CoursierDependencyDownloader extends DependencyDownloader {
*/
override def removeMavenRepository(url: URL): Unit = {
repositories = repositories.filterNot {
- case MavenRepository(urlString, _, _) => url.toString == urlString
+ case MavenRepository(urlString, _, _, _) => url.toString == urlString
case _ => false
}
}
@@ -314,8 +313,8 @@ class CoursierDependencyDownloader extends DependencyDownloader {
* @return The resulting URIs
*/
private def repositoriesToURIs(repositories: Seq[Repository]) = repositories.map {
- case IvyRepository(pattern, _, _, _, _, _, _, _) => pattern
- case MavenRepository(root, _, _) => root
+ case IvyRepository(pattern, _, _, _, _, _, _, _, _) => pattern
+ case MavenRepository(root, _, _, _) => root
}.map(new URI(_))
/** Creates new Ivy2 local repository using base home URI. */
@@ -340,3 +339,51 @@ class CoursierDependencyDownloader extends DependencyDownloader {
dropInfoAttributes = true
)
}
+
+
+sealed abstract class Credentials extends Product with Serializable {
+ def user: String
+ def password: String
+ def host: String
+
+ def authentication: Authentication =
+ Authentication(user, password)
+}
+
+object Credentials {
+
+ case class FromFile(file: File) extends Credentials {
+
+ private lazy val props = {
+ val p = new Properties()
+ p.load(new FileInputStream(file))
+ p
+ }
+
+ private def findKey(keys: Seq[String]) = keys
+ .iterator
+ .map(props.getProperty)
+ .filter(_ != null)
+ .toStream
+ .headOption
+ .getOrElse {
+ throw new NoSuchElementException(s"${keys.head} key in $file")
+ }
+
+ lazy val user: String = findKey(FromFile.fileUserKeys)
+ lazy val password: String = findKey(FromFile.filePasswordKeys)
+ lazy val host: String = findKey(FromFile.fileHostKeys)
+ }
+
+ object FromFile {
+ // from sbt.Credentials
+ private val fileUserKeys = Seq("user", "user.name", "username")
+ private val filePasswordKeys = Seq("password", "pwd", "pass", "passwd")
+ private val fileHostKeys = Seq("host", "host.name", "hostname", "domain")
+ }
+
+
+ def apply(file: File): Credentials =
+ FromFile(file)
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
index 560d2a0..5b3d9fc 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
@@ -50,7 +50,7 @@ abstract class DependencyDownloader {
transitive: Boolean = true,
excludeBaseDependencies: Boolean = true,
ignoreResolutionErrors: Boolean = true,
- extraRepositories: Seq[URL] = Nil,
+ extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
verbose: Boolean = false,
trace: Boolean = false
): Seq[URI]
@@ -67,7 +67,7 @@ abstract class DependencyDownloader {
*
* @param url The url of the repository
*/
- def addMavenRepository(url: URL): Unit
+ def addMavenRepository(url: URL, credentials: Option[Credentials]): Unit
/**
* Remove the specified resolver url from the search options.
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
index f3a4155..56940c9 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
@@ -89,7 +89,7 @@ class IvyDependencyDownloader(
transitive: Boolean = true,
excludeBaseDependencies: Boolean,
ignoreResolutionErrors: Boolean,
- extraRepositories: Seq[URL],
+ extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
verbose: Boolean,
trace: Boolean
): Seq[URI] = {
@@ -197,7 +197,7 @@ class IvyDependencyDownloader(
*
* @param url The url of the repository
*/
- override def addMavenRepository(url: URL): Unit = ???
+ override def addMavenRepository(url: URL, credentials: Option[Credentials]): Unit = ???
/**
* Remove the specified resolver url from the search options.
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
index be36c8f..f245851 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
@@ -17,15 +17,18 @@
package org.apache.toree.magic.builtin
-import java.io.PrintStream
+import java.io.{File, PrintStream}
import java.net.URL
+import org.apache.toree.dependencies.Credentials
import org.apache.toree.magic._
import org.apache.toree.magic.dependencies._
import org.apache.toree.utils.ArgumentParsingSupport
+
import scala.util.Try
import org.apache.toree.plugins.annotations.Event
+
class AddDeps extends LineMagic with IncludeInterpreter
with IncludeOutputStream with IncludeSparkContext with ArgumentParsingSupport
with IncludeDependencyDownloader with IncludeKernel
@@ -53,6 +56,10 @@ class AddDeps extends LineMagic with IncludeInterpreter
"repository", "Adds an additional repository to available list"
).withRequiredArg().ofType(classOf[String])
+ private val _credentials = parser.accepts(
+ "credential", "Adds a credential file to be used to the list"
+ ).withRequiredArg().ofType(classOf[String])
+
/**
* Execute a magic representing a line magic.
*
@@ -70,6 +77,15 @@ class AddDeps extends LineMagic with IncludeInterpreter
extraRepositories.filter(_._2.isFailure).map(_._1)
.foreach(u => printStream.println(s"Ignoring invalid URL $u"))
+ // match up credentials with repositories
+ val repositories = extraRepositories.flatMap(_._2.toOption)
+ val authentication = getAll(_credentials).getOrElse(Nil)
+ .map(f => new File(f))
+ .map(Credentials(_))
+ .map(c => (c.host, c)).toMap
+
+ val repositoriesWithCreds = repositories.map(u => (u, authentication.get(u.getHost)))
+
if (nonOptionArgs.size == 3) {
// get the jars and hold onto the paths at which they reside
val urls = dependencyDownloader.retrieve(
@@ -78,7 +94,7 @@ class AddDeps extends LineMagic with IncludeInterpreter
version = nonOptionArgs(2),
transitive = _transitive,
ignoreResolutionErrors = !_abortOnResolutionErrors,
- extraRepositories = extraRepositories.flatMap(_._2.toOption),
+ extraRepositories = repositoriesWithCreds,
verbose = _verbose,
trace = _trace
).map(_.toURL)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala b/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
index 181ca3a..659af42 100644
--- a/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
@@ -20,15 +20,14 @@ package org.apache.toree.magic.builtin
import java.io.{ByteArrayOutputStream, OutputStream}
import java.net.{URI, URL}
-import org.apache.toree.dependencies.DependencyDownloader
+import org.apache.toree.dependencies.{Credentials, DependencyDownloader}
import org.apache.toree.interpreter.Interpreter
import org.apache.toree.utils.ArgumentParsingSupport
import org.apache.spark.SparkContext
import org.scalatest.mock.MockitoSugar
-import org.scalatest.{GivenWhenThen, Matchers, FunSpec}
+import org.scalatest.{FunSpec, GivenWhenThen, Matchers}
import org.mockito.Mockito._
import org.mockito.Matchers._
-
import org.apache.toree.magic._
import org.apache.toree.magic.dependencies._
@@ -71,7 +70,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
verify(mockSC, times(0)).addJar(any())
verify(mockDownloader, times(0)).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
- anyBoolean(), any[Seq[URL]], anyBoolean(), anyBoolean()
+ anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean()
)
actual should be (expected)
}
@@ -80,7 +79,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
val mockDependencyDownloader = mock[DependencyDownloader]
doReturn(Nil).when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
- anyBoolean(), any[Seq[URL]], anyBoolean(), anyBoolean()
+ anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean()
)
val addDepsMagic = new AddDeps
@@ -108,7 +107,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
val mockDependencyDownloader = mock[DependencyDownloader]
doReturn(Nil).when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
- anyBoolean(), any[Seq[URL]], anyBoolean(), anyBoolean()
+ anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean()
)
val addDepsMagic = new AddDeps
@@ -136,7 +135,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
val mockDependencyDownloader = mock[DependencyDownloader]
doReturn(Nil).when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
- anyBoolean(), any[Seq[URL]], anyBoolean(), anyBoolean()
+ anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean()
)
val mockInterpreter = mock[Interpreter]
@@ -166,7 +165,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
doReturn(fakeUri :: fakeUri :: fakeUri :: Nil)
.when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
- anyBoolean(), any[Seq[URL]], anyBoolean(), anyBoolean()
+ anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean()
)
val mockSparkContext = mock[SparkContext]
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 4025fd5..de26adf 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -50,7 +50,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "0.8.1")
addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.3")
// Provides alternative resolving/downloading over sbt
-addSbtPlugin("com.github.alexarchambault" % "coursier-sbt-plugin" % "1.0.0-M10")
+addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-M12")
// Used for signing jars published via `sbt publish-signed`
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
[2/5] incubator-toree git commit: Added ability to specify default
repositories to use for resolving at launch
Posted by ch...@apache.org.
Added ability to specify default repositories to use for resolving at launch
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/d960c86b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/d960c86b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/d960c86b
Branch: refs/heads/master
Commit: d960c86bf6d2f3563f7460b76057b0ceddb4119a
Parents: 6d6b06d
Author: Marius van Niekerk <ma...@maxpoint.com>
Authored: Thu May 19 14:58:08 2016 -0400
Committer: Marius van Niekerk <Marius van Niekerk>
Committed: Wed May 25 20:10:01 2016 -0400
----------------------------------------------------------------------
.../dependencies/DependencyDownloader.scala | 28 ++++++++++++++++++++
.../apache/toree/boot/CommandLineOptions.scala | 18 +++++++++++--
.../boot/layer/ComponentInitialization.scala | 16 +++++++++--
.../apache/toree/magic/builtin/AddDeps.scala | 18 ++++---------
resources/compile/reference.conf | 6 +++++
resources/test/reference.conf | 5 ++++
6 files changed, 74 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
index 5b3d9fc..1859776 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
@@ -20,6 +20,7 @@ package org.apache.toree.dependencies
import java.io.{File, PrintStream}
import java.net.{URI, URL}
import java.nio.file.Files
+import scala.util.Try
abstract class DependencyDownloader {
/**
@@ -98,6 +99,33 @@ abstract class DependencyDownloader {
* @return The directory as a string
*/
def getDownloadDirectory: String
+
+
+ /**
+ * Assigns credentials to the right repository and build tuples
+ *
+ * @param repository Sequence of repository urls
+ * @param credentials Sequence of credential filenames
+ * @return
+ */
+ def resolveRepositoriesAndCredentials(repository: List[String], credentials: List[String]): List[(URL, Option[Credentials])] = {
+ val extraRepositories = repository.map(u => (u, Try(new URL(u))))
+
+ // Print error information
+ // extraRepositories.filter(_._2.isFailure).map(_._1)
+ // .foreach(u => printStream.println(s"Ignoring invalid URL $u"))
+
+ // match up credentials with repositories
+ val repositories = extraRepositories.flatMap(_._2.toOption)
+ val authentication = credentials
+ .map(f => new File(f))
+ .map(Credentials(_))
+ .map(c => (c.host, c)).toMap
+
+ val repositoriesWithCreds = repositories.map(u => (u, authentication.get(u.getHost)))
+ repositoriesWithCreds
+ }
+
}
object DependencyDownloader {
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
index 43cfc8e..6ac6442 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
@@ -71,6 +71,14 @@ class CommandLineOptions(args: Seq[String]) {
parser.accepts("magic-url", "path to a magic jar")
.withRequiredArg().ofType(classOf[String])
+ private val _default_repositories = parser.accepts(
+ "default_repositories", "comma seperated list of additional repositories to resolve"
+ ).withRequiredArg().ofType(classOf[String])
+
+ private val _default_repository_credentials = parser.accepts(
+ "default_repository_credentials", "comma seperated list of credential files to use"
+ ).withRequiredArg().ofType(classOf[String])
+
private val _max_interpreter_threads = parser.accepts(
"max-interpreter-threads",
"total number of worker threads to use to execute code"
@@ -144,7 +152,12 @@ class CommandLineOptions(args: Seq[String]) {
"jar_dir" -> get(_jar_dir),
"default_interpreter" -> get(_default_interpreter),
"nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),
- "interpreter_plugins" -> interpreterPlugins
+ "interpreter_plugins" -> interpreterPlugins,
+ "default_repositories" -> getAll(_default_repositories).map(_.asJava)
+ .flatMap(list => if (list.isEmpty) None else Some(list)),
+ "default_repository_credentials" -> getAll(_default_repository_credentials).map(_.asJava)
+ .flatMap(list => if (list.isEmpty) None else Some(list))
+
).flatMap(removeEmptyOptions).asInstanceOf[Map[String, AnyRef]].asJava)
commandLineConfig.withFallback(profileConfig).withFallback(ConfigFactory.load)
@@ -182,7 +195,8 @@ class CommandLineOptions(args: Seq[String]) {
/**
* Prints the help message to the output stream provided.
- * @param out The output stream to direct the help message
+ *
+ * @param out The output stream to direct the help message
*/
def printHelpOn(out: OutputStream) =
parser.printHelpOn(out)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
index fb7e76d..b90e951 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
@@ -18,14 +18,15 @@
package org.apache.toree.boot.layer
import java.io.File
-import java.nio.file.{Paths, Files}
+import java.net.URL
+import java.nio.file.{Files, Paths}
import java.util.concurrent.ConcurrentHashMap
import akka.actor.ActorRef
import com.typesafe.config.Config
import org.apache.spark.SparkConf
import org.apache.toree.comm.{CommManager, CommRegistrar, CommStorage, KernelCommManager}
-import org.apache.toree.dependencies.{CoursierDependencyDownloader, DependencyDownloader}
+import org.apache.toree.dependencies.{CoursierDependencyDownloader, Credentials, DependencyDownloader}
import org.apache.toree.interpreter._
import org.apache.toree.kernel.api.Kernel
import org.apache.toree.kernel.protocol.v5.KMBuilder
@@ -130,6 +131,17 @@ trait StandardComponentInitialization extends ComponentInitialization {
new File(depsDir)
)
+ if (config.hasPath("default_repositories")) {
+ val repository = config.getStringList("default_repositories").asScala.toList
+
+ val credentials = if (config.hasPath("default_repository_credentials")) {
+ config.getStringList("default_repository_credentials").asScala.toList
+ } else Nil
+
+ dependencyDownloader.resolveRepositoriesAndCredentials(repository, credentials)
+ .foreach{case (u, c) => dependencyDownloader.addMavenRepository(u, c)}
+ }
+
dependencyDownloader
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
index f245851..2f4d812 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
@@ -71,20 +71,10 @@ class AddDeps extends LineMagic with IncludeInterpreter
val nonOptionArgs = parseArgs(code)
dependencyDownloader.setPrintStream(printStream)
- val extraRepositories = getAll(_repository).getOrElse(Nil).map(u => (u, Try(new URL(u))))
+ val repository = getAll(_repository).getOrElse(Nil)
+ val credentials = getAll(_credentials).getOrElse(Nil)
- // Print error information
- extraRepositories.filter(_._2.isFailure).map(_._1)
- .foreach(u => printStream.println(s"Ignoring invalid URL $u"))
-
- // match up credentials with repositories
- val repositories = extraRepositories.flatMap(_._2.toOption)
- val authentication = getAll(_credentials).getOrElse(Nil)
- .map(f => new File(f))
- .map(Credentials(_))
- .map(c => (c.host, c)).toMap
-
- val repositoriesWithCreds = repositories.map(u => (u, authentication.get(u.getHost)))
+ val repositoriesWithCreds = dependencyDownloader.resolveRepositoriesAndCredentials(repository, credentials)
if (nonOptionArgs.size == 3) {
// get the jars and hold onto the paths at which they reside
@@ -106,4 +96,6 @@ class AddDeps extends LineMagic with IncludeInterpreter
printHelp(printStream, """%AddDeps my.company artifact-id version""")
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/resources/compile/reference.conf
----------------------------------------------------------------------
diff --git a/resources/compile/reference.conf b/resources/compile/reference.conf
index 1c23174..25e553e 100644
--- a/resources/compile/reference.conf
+++ b/resources/compile/reference.conf
@@ -61,3 +61,9 @@ default_interpreter_plugin = [
"SparkR:org.apache.toree.kernel.interpreter.sparkr.SparkRInterpreter",
"SQL:org.apache.toree.kernel.interpreter.sql.SqlInterpreter"
]
+
+default_repositories = []
+default_repositories = [${?DEFAULT_REPOSITORIES}]
+
+default_repository_credentials = []
+default_repository_credentials = [${?DEFAULT_REPOSITORY_CREDENTIALS}]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/d960c86b/resources/test/reference.conf
----------------------------------------------------------------------
diff --git a/resources/test/reference.conf b/resources/test/reference.conf
index 85e614c..c7c9363 100644
--- a/resources/test/reference.conf
+++ b/resources/test/reference.conf
@@ -61,3 +61,8 @@ default_interpreter_plugin = [
"SQL:org.apache.toree.kernel.interpreter.sql.SqlInterpreter"
]
+default_repositories = []
+default_repositories = [${?DEFAULT_REPOSITORIES}]
+
+default_repository_credentials = []
+default_repository_credentials = [${?DEFAULT_REPOSITORY_CREDENTIALS}]
[5/5] incubator-toree git commit: Fix up test
Posted by ch...@apache.org.
Fix up test
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/6d6b06d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/6d6b06d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/6d6b06d4
Branch: refs/heads/master
Commit: 6d6b06d4ef9ceabaac265d388d81e3ce65631595
Parents: 4177e9b
Author: Marius van Niekerk <ma...@maxpoint.com>
Authored: Thu May 19 14:06:48 2016 -0400
Committer: Marius van Niekerk <Marius van Niekerk>
Committed: Wed May 25 20:10:01 2016 -0400
----------------------------------------------------------------------
.../toree/dependencies/CoursierDependencyDownloaderSpec.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/6d6b06d4/kernel-api/src/test/scala/org/apache/toree/dependencies/CoursierDependencyDownloaderSpec.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/test/scala/org/apache/toree/dependencies/CoursierDependencyDownloaderSpec.scala b/kernel-api/src/test/scala/org/apache/toree/dependencies/CoursierDependencyDownloaderSpec.scala
index 5445b39..8d5b217 100644
--- a/kernel-api/src/test/scala/org/apache/toree/dependencies/CoursierDependencyDownloaderSpec.scala
+++ b/kernel-api/src/test/scala/org/apache/toree/dependencies/CoursierDependencyDownloaderSpec.scala
@@ -31,7 +31,7 @@ class CoursierDependencyDownloaderSpec extends FunSpec with Matchers
it("should add to the list of repositories") {
val repo = new URL("http://some-repo.com")
- coursierDependencyDownloader.addMavenRepository(repo)
+ coursierDependencyDownloader.addMavenRepository(repo, None)
val repos = coursierDependencyDownloader.getRepositories
@@ -43,7 +43,7 @@ class CoursierDependencyDownloaderSpec extends FunSpec with Matchers
it("should remove from the list of repositories") {
val repo = new URL("http://some-repo.com")
- coursierDependencyDownloader.addMavenRepository(repo)
+ coursierDependencyDownloader.addMavenRepository(repo, None)
coursierDependencyDownloader.removeMavenRepository(repo)
val repos = coursierDependencyDownloader.getRepositories