You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lr...@apache.org on 2018/01/14 18:57:35 UTC
[2/2] incubator-toree git commit: [TOREE-420] Enhance %AddDeps with
option to exclude modules
[TOREE-420] Enhance %AddDeps with option to exclude modules
Add --exclude option to AddDeps in order to be able to exclude
certain modules. The exclude syntax is follows "organisation:name"
or "organisation" when everything from a certain organisation is
to be excluded.
Closes #146
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/87b99c64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/87b99c64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/87b99c64
Branch: refs/heads/master
Commit: 87b99c64e58f0e265bf40af2d81a322185a25e0f
Parents: bcdea9b
Author: Frank Dekervel <fr...@infrabel.be>
Authored: Sun Jan 14 00:15:55 2018 -0800
Committer: Luciano Resende <lr...@apache.org>
Committed: Sun Jan 14 00:15:55 2018 -0800
----------------------------------------------------------------------
.../dependencies/CoursierDependencyDownloader.scala | 5 +++--
.../toree/dependencies/DependencyDownloader.scala | 5 ++++-
.../toree/dependencies/IvyDependencyDownloader.scala | 12 +++++++++++-
.../scala/org/apache/toree/magic/builtin/AddDeps.scala | 12 ++++++++++++
.../org/apache/toree/magic/builtin/AddDepsSpec.scala | 8 ++++----
5 files changed, 34 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/87b99c64/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
index d83ae89..89a5e9c 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
@@ -76,7 +76,8 @@ class CoursierDependencyDownloader extends DependencyDownloader {
trace: Boolean,
configuration: Option[String] = None,
artifactType: Option[String] = None,
- artifactClassifier: Option[String] = None
+ artifactClassifier: Option[String] = None,
+ excludes: Set[(String,String)] = Set.empty
): Seq[URI] = {
assert(localDirectory != null)
import coursier._
@@ -84,7 +85,7 @@ class CoursierDependencyDownloader extends DependencyDownloader {
// Grab exclusions using base dependencies (always exclude scala lang)
val exclusions: Set[(String, String)] = (if (excludeBaseDependencies) {
getBaseDependencies.map(_.module).map(m => (m.organization, m.name))
- } else Nil).toSet ++ Set(("org.scala-lang", "*"), ("org.scala-lang.modules", "*"))
+ } else Nil).toSet ++ Set(("org.scala-lang", "*"), ("org.scala-lang.modules", "*")) ++ excludes
// Mark dependency that we want to download
val start = Resolution(Set(
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/87b99c64/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
index 2f83966..36f8b50 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
@@ -23,6 +23,8 @@ import java.nio.file.Files
import scala.util.Try
import org.apache.toree.utils.FileUtils
+case class DependencyExclude(groupId: String, artifactId: String, version: String);
+
abstract class DependencyDownloader {
/**
* Retrieves the dependency and all of its dependencies as jars.
@@ -57,7 +59,8 @@ abstract class DependencyDownloader {
trace: Boolean = false,
configuration: Option[String] = None,
artifactType: Option[String] = None,
- artifactClassifier: Option[String] = None
+ artifactClassifier: Option[String] = None,
+ excludes: Set[(String,String)] = Set.empty
): Seq[URI]
/**
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/87b99c64/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
----------------------------------------------------------------------
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
index 85b0862..e5939af 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
@@ -94,7 +94,8 @@ class IvyDependencyDownloader(
trace: Boolean,
configuration: Option[String] = None,
artifactType: Option[String] = None,
- artifactClassifier: Option[String] = None
+ artifactClassifier: Option[String] = None,
+ excludes: Set[(String,String)] = Set.empty
): Seq[URI] = {
// Start building the ivy.xml file
val ivyFile = File.createTempFile("ivy-custom", ".xml")
@@ -127,6 +128,7 @@ class IvyDependencyDownloader(
scalaCompilerArtifactId, new RegexpPatternMatcher(), null
)
+
val scalaLangModuleId = new ModuleId("org.scala-lang.modules", "*")
val scalaLangArtifactId = new ArtifactId(
scalaLangModuleId, "*", "*", "*"
@@ -149,6 +151,14 @@ class IvyDependencyDownloader(
md.addExcludeRule(scalaCompilerExclusion)
md.addExcludeRule(scalaLangExclusion)
+
+ excludes.foreach(x => {
+ val moduleId = new ModuleId(x._1,x._2);
+ val artifactId = new ArtifactId(moduleId,"*","*","*");
+ val exclusion = new DefaultExcludeRule(artifactId, new RegexpPatternMatcher(), null);
+ md.addExcludeRule(exclusion);
+ })
+
// Exclude our base dependencies if marked to do so
if (excludeBaseDependencies) {
getBaseDependencies.foreach(dep => {
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/87b99c64/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
index 90f97c2..913d64e 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
@@ -52,6 +52,8 @@ class AddDeps extends LineMagic with IncludeInterpreter
"abort-on-resolution-errors", "Abort (no downloads) when resolution fails"
)
+ private val _exclude = parser.accepts("exclude", "exclude dependency").withRequiredArg().ofType(classOf[String])
+
private val _repository = parser.accepts(
"repository", "Adds an additional repository to available list"
).withRequiredArg().ofType(classOf[String])
@@ -81,6 +83,15 @@ class AddDeps extends LineMagic with IncludeInterpreter
val repository = getAll(_repository).getOrElse(Nil)
val credentials = getAll(_credentials).getOrElse(Nil)
+ val excludes = getAll(_exclude).getOrElse(Nil)
+
+ val excludesSet = excludes.map((x: String) => {
+ if (x.contains(":")) {
+ (x.split(":")(0), x.split(":")(1))
+ } else {
+ (x, "*")
+ }
+ }: (String, String)).toSet
val repositoriesWithCreds = dependencyDownloader.resolveRepositoriesAndCredentials(repository, credentials)
@@ -95,6 +106,7 @@ class AddDeps extends LineMagic with IncludeInterpreter
extraRepositories = repositoriesWithCreds,
verbose = _verbose,
trace = _trace,
+ excludes = excludesSet,
configuration = get(_configuration),
artifactClassifier = get(_classifier)
)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/87b99c64/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala b/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
index d8f75a5..a346805 100644
--- a/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/magic/builtin/AddDepsSpec.scala
@@ -64,7 +64,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
verify(mockDownloader, times(0)).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean(),
- any[Option[String]], any[Option[String]], any[Option[String]]
+ any[Option[String]], any[Option[String]], any[Option[String]], any[Set[(String,String)]]
)
}
@@ -73,7 +73,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
doReturn(Nil).when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean(),
- any[Option[String]], any[Option[String]], any[Option[String]]
+ any[Option[String]], any[Option[String]], any[Option[String]], any[Set[(String,String)]]
)
val addDepsMagic = new AddDeps
@@ -100,7 +100,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
doReturn(Nil).when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean(),
- any[Option[String]], any[Option[String]], any[Option[String]]
+ any[Option[String]], any[Option[String]], any[Option[String]], any[Set[(String,String)]]
)
val addDepsMagic = new AddDeps
@@ -127,7 +127,7 @@ class AddDepsSpec extends FunSpec with Matchers with MockitoSugar
doReturn(Nil).when(mockDependencyDownloader).retrieve(
anyString(), anyString(), anyString(), anyBoolean(), anyBoolean(),
anyBoolean(), any[Seq[(URL, Option[Credentials])]], anyBoolean(), anyBoolean(),
- any[Option[String]], any[Option[String]], any[Option[String]]
+ any[Option[String]], any[Option[String]], any[Option[String]], any[Set[(String,String)]]
)
val mockKernel = mock[KernelLike]