You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by ch...@apache.org on 2016/05/26 15:52:04 UTC
[3/5] incubator-toree git commit: Added credential support for adddeps
Added credential support for adddeps
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/4177e9b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/4177e9b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/4177e9b2
Branch: refs/heads/master
Commit: 4177e9b2f1708bd5fcabdc209c20d5fc83267182
Parents: b59a600
Author: Marius van Niekerk <ma...@maxpoint.com>
Authored: Thu May 19 13:40:51 2016 -0400
Committer: Marius van Niekerk <Marius van Niekerk>
Committed: Wed May 25 20:10:01 2016 -0400
----------------------------------------------------------------------
kernel-api/build.sbt | 4 +-
.../CoursierDependencyDownloader.scala | 77 ++++++++++++++++----
.../dependencies/DependencyDownloader.scala | 4 +-
.../dependencies/IvyDependencyDownloader.scala | 4 +-
.../apache/toree/magic/builtin/AddDeps.scala | 20 ++++-
.../toree/magic/builtin/AddDepsSpec.scala | 15 ++--
project/plugins.sbt | 2 +-
7 files changed, 94 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel-api/build.sbt
----------------------------------------------------------------------
diff --git a/kernel-api/build.sbt b/kernel-api/build.sbt
index 331f9bf..8898a9b 100644
--- a/kernel-api/build.sbt
+++ b/kernel-api/build.sbt
@@ -50,8 +50,8 @@ libraryDependencies += "net.sf.jopt-simple" % "jopt-simple" % "4.6" // MIT
libraryDependencies ++= Seq(
// Used to find and download jars from Maven-based repositories
"org.apache.ivy" % "ivy" % "2.4.0-rc1", // Apache v2
- "com.github.alexarchambault" %% "coursier" % "1.0.0-M9", // Apache v2
- "com.github.alexarchambault" %% "coursier-cache" % "1.0.0-M9" // Apache v2
+ "io.get-coursier" %% "coursier" % "1.0.0-M12", // Apache v2
+ "io.get-coursier" %% "coursier-cache" % "1.0.0-M12" // Apache v2
)
// Brought in in order to simplify the reading of each project's ivy.xml file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
index ef9ac02..4751f67 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
@@ -16,14 +16,16 @@
*/
package org.apache.toree.dependencies
-import java.io.{File, PrintStream}
+import java.io.{File, FileInputStream, PrintStream}
import java.net.{URI, URL}
+import java.util.Properties
import java.util.concurrent.ConcurrentHashMap
+import coursier.core.Authentication
import coursier.Cache.Logger
import coursier.Dependency
import coursier.core.Repository
-import coursier.ivy.{IvyXml, IvyRepository}
+import coursier.ivy.{IvyRepository, IvyXml}
import coursier.maven.MavenRepository
import org.springframework.core.io.support.PathMatchingResourcePatternResolver
@@ -40,7 +42,7 @@ class CoursierDependencyDownloader extends DependencyDownloader {
// Initialization
setDownloadDirectory(DependencyDownloader.DefaultDownloadDirectory)
- addMavenRepository(DependencyDownloader.DefaultMavenRepository)
+ addMavenRepository(DependencyDownloader.DefaultMavenRepository, None)
/**
* Retrieves the dependency and all of its dependencies as jars.
@@ -69,7 +71,7 @@ class CoursierDependencyDownloader extends DependencyDownloader {
transitive: Boolean,
excludeBaseDependencies: Boolean,
ignoreResolutionErrors: Boolean,
- extraRepositories: Seq[URL],
+ extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
verbose: Boolean,
trace: Boolean
): Seq[URI] = {
@@ -95,12 +97,9 @@ class CoursierDependencyDownloader extends DependencyDownloader {
lazy val defaultBase = new File(localDirectory).getAbsoluteFile
- lazy val downloadLocations = Seq(
- "http://" -> new File(defaultBase, "http"),
- "https://" -> new File(defaultBase, "https")
- )
+ lazy val downloadLocations = defaultBase
- val allRepositories = extraRepositories.map(urlToMavenRepository) ++ repositories
+ val allRepositories = extraRepositories.map(x => urlToMavenRepository(x._1, x._2.map(_.authentication))) ++ repositories
// Build list of locations to fetch dependencies
val fetchLocations = Seq(ivy2Cache(localDirectory)) ++ allRepositories
@@ -150,10 +149,10 @@ class CoursierDependencyDownloader extends DependencyDownloader {
*
* @param url The string representation of the url
*/
- override def addMavenRepository(url: URL): Unit =
- repositories :+= urlToMavenRepository(url)
+ override def addMavenRepository(url: URL, credentials: Option[Credentials]): Unit =
+ repositories :+= urlToMavenRepository(url, credentials.map(_.authentication))
- private def urlToMavenRepository(url: URL) = MavenRepository(url.toString)
+ private def urlToMavenRepository(url: URL, credentials: Option[Authentication]) = MavenRepository(url.toString, authentication = credentials)
/**
* Remove the specified resolver url from the search options.
@@ -162,7 +161,7 @@ class CoursierDependencyDownloader extends DependencyDownloader {
*/
override def removeMavenRepository(url: URL): Unit = {
repositories = repositories.filterNot {
- case MavenRepository(urlString, _, _) => url.toString == urlString
+ case MavenRepository(urlString, _, _, _) => url.toString == urlString
case _ => false
}
}
@@ -314,8 +313,8 @@ class CoursierDependencyDownloader extends DependencyDownloader {
* @return The resulting URIs
*/
private def repositoriesToURIs(repositories: Seq[Repository]) = repositories.map {
- case IvyRepository(pattern, _, _, _, _, _, _, _) => pattern
- case MavenRepository(root, _, _) => root
+ case IvyRepository(pattern, _, _, _, _, _, _, _, _) => pattern
+ case MavenRepository(root, _, _, _) => root
}.map(new URI(_))
/** Creates new Ivy2 local repository using base home URI. */
@@ -340,3 +339,51 @@ class CoursierDependencyDownloader extends DependencyDownloader {
dropInfoAttributes = true
)
}
+
+
+sealed abstract class Credentials extends Product with Serializable {
+ def user: String
+ def password: String
+ def host: String
+
+ def authentication: Authentication =
+ Authentication(user, password)
+}
+
+object Credentials {
+
+ case class FromFile(file: File) extends Credentials {
+
+ private lazy val props = {
+ val p = new Properties()
+ p.load(new FileInputStream(file))
+ p
+ }
+
+ private def findKey(keys: Seq[String]) = keys
+ .iterator
+ .map(props.getProperty)
+ .filter(_ != null)
+ .toStream
+ .headOption
+ .getOrElse {
+ throw new NoSuchElementException(s"${keys.head} key in $file")
+ }
+
+ lazy val user: String = findKey(FromFile.fileUserKeys)
+ lazy val password: String = findKey(FromFile.filePasswordKeys)
+ lazy val host: String = findKey(FromFile.fileHostKeys)
+ }
+
+ object FromFile {
+ // from sbt.Credentials
+ private val fileUserKeys = Seq("user", "user.name", "username")
+ private val filePasswordKeys = Seq("password", "pwd", "pass", "passwd")
+ private val fileHostKeys = Seq("host", "host.name", "hostname", "domain")
+ }
+
+
+ def apply(file: File): Credentials =
+ FromFile(file)
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
index 560d2a0..5b3d9fc 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
@@ -50,7 +50,7 @@ abstract class DependencyDownloader {
transitive: Boolean = true,
excludeBaseDependencies: Boolean = true,
ignoreResolutionErrors: Boolean = true,
- extraRepositories: Seq[URL] = Nil,
+ extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
verbose: Boolean = false,
trace: Boolean = false
): Seq[URI]
@@ -67,7 +67,7 @@ abstract class DependencyDownloader {
*
* @param url The url of the repository
*/
- def addMavenRepository(url: URL): Unit
+ def addMavenRepository(url: URL, credentials: Option[Credentials]): Unit
/**
* Remove the specified resolver url from the search options.
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
index f3a4155..56940c9 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
@@ -89,7 +89,7 @@ class IvyDependencyDownloader(
transitive: Boolean = true,
excludeBaseDependencies: Boolean,
ignoreResolutionErrors: Boolean,
- extraRepositories: Seq[URL],
+ extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
verbose: Boolean,
trace: Boolean
): Seq[URI] = {
@@ -197,7 +197,7 @@ class IvyDependencyDownloader(
*
* @param url The url of the repository
*/
- override def addMavenRepository(url: URL): Unit = ???
+ override def addMavenRepository(url: URL, credentials: Option[Credentials]): Unit = ???
/**
* Remove the specified resolver url from the search options.
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
index be36c8f..f245851 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
@@ -17,15 +17,18 @@
package org.apache.toree.magic.builtin
-import java.io.PrintStream
+import java.io.{File, PrintStream}
import java.net.URL
+import org.apache.toree.dependencies.Credentials
import org.apache.toree.magic._
import org.apache.toree.magic.dependencies._
import org.apache.toree.utils.ArgumentParsingSupport
+
import scala.util.Try
import org.apache.toree.plugins.annotations.Event
+
class AddDeps extends LineMagic with IncludeInterpreter
with IncludeOutputStream with IncludeSparkContext with ArgumentParsingSupport
with IncludeDependencyDownloader with IncludeKernel
@@ -53,6 +56,10 @@ class AddDeps extends LineMagic with IncludeInterpreter
"repository", "Adds an additional repository to available list"
).withRequiredArg().ofType(classOf[String])
+ private val _credentials = parser.accepts(
+ "credential", "Adds a credential file to be used to the list"
+ ).withRequiredArg().ofType(classOf[String])
+
/**
* Execute a magic representing a line magic.
*
@@ -70,6 +77,15 @@ class AddDeps extends LineMagic with IncludeInterpreter
extraRepositories.filter(_._2.isFailure).map(_._1)
.foreach(u => printStream.println(s"Ignoring invalid URL $u"))
+ // match up credentials with repositories
+ val repositories = extraRepositories.flatMap(_._2.toOption)
+ val authentication = getAll(_credentials).getOrElse(Nil)
+ .map(f => new File(f))
+ .map(Credentials(_))
+ .map(c => (c.host, c)).toMap
+
+ val repositoriesWithCreds = repositories.map(u => (u, authentication.get(u.getHost)))
+
if (nonOptionArgs.size == 3) {
// get the jars and hold onto the paths at which they reside
val urls = dependencyDownloader.retrieve(
@@ -78,7 +94,7 @@ class AddDeps extends LineMagic with IncludeInterpreter
version = nonOptionArgs(2),
transitive = _transitive,
ignoreResolutionErrors = !_abortOnResolutionErrors,
- extraRepositories = extraRepositories.flatMap(_._2.toOption),
+ extraRepositories = repositoriesWithCreds,
verbose = _verbose,
trace = _trace
).map(_.toURL)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala b/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
index 181ca3a..659af42 100644
--- a/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
@@ -20,15 +20,14 @@ package org.apache.toree.magic.builtin
import java.io.{ByteArrayOutputStream, OutputStream}
import java.net.{URI, URL}
-import org.apache.toree.dependencies.DependencyDownloader
+import org.apache.toree.dependencies.{Credentials, DependencyDownloader}
import org.apache.toree.interpreter.Interpreter
import org.apache.toree.utils.ArgumentParsingSupport
import org.apache.spark.SparkContext
import org.scalatest.mock.MockitoSugar
-import org.scalatest.{GivenWhenThen, Matchers, FunSpec}
+import org.scalatest.{FunSpec, GivenWhenThen, Matchers}
import org.mockito.Mockito._
import org.mockito.Matchers._
-
import org.apache.toree.magic._
import org.apache.toree.magic.dependencies._
@@ -71,7 +70,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
verify(mockSC, times(0)).addJar(any())
verify(mockDownloader, times(0)).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
- anyBoolean(), any[Seq[URL]], anyBoolean(), anyBoolean()
+ anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean()
)
actual should be (expected)
}
@@ -80,7 +79,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
val mockDependencyDownloader = mock[DependencyDownloader]
doReturn(Nil).when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
- anyBoolean(), any[Seq[URL]], anyBoolean(), anyBoolean()
+ anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean()
)
val addDepsMagic = new AddDeps
@@ -108,7 +107,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
val mockDependencyDownloader = mock[DependencyDownloader]
doReturn(Nil).when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
- anyBoolean(), any[Seq[URL]], anyBoolean(), anyBoolean()
+ anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean()
)
val addDepsMagic = new AddDeps
@@ -136,7 +135,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
val mockDependencyDownloader = mock[DependencyDownloader]
doReturn(Nil).when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
- anyBoolean(), any[Seq[URL]], anyBoolean(), anyBoolean()
+ anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean()
)
val mockInterpreter = mock[Interpreter]
@@ -166,7 +165,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
doReturn(fakeUri :: fakeUri :: fakeUri :: Nil)
.when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
- anyBoolean(), any[Seq[URL]], anyBoolean(), anyBoolean()
+ anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean()
)
val mockSparkContext = mock[SparkContext]
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4177e9b2/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 4025fd5..de26adf 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -50,7 +50,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "0.8.1")
addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.3")
// Provides alternative resolving/downloading over sbt
-addSbtPlugin("com.github.alexarchambault" % "coursier-sbt-plugin" % "1.0.0-M10")
+addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-M12")
// Used for signing jars published via `sbt publish-signed`
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")