You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by br...@apache.org on 2017/06/01 05:39:32 UTC
spark git commit: [SPARK-20708][CORE] Make `addExclusionRules`
up-to-date
Repository: spark
Updated Branches:
refs/heads/master 5854f77ce -> 34661d8a5
[SPARK-20708][CORE] Make `addExclusionRules` up-to-date
## What changes were proposed in this pull request?
Since [SPARK-9263](https://issues.apache.org/jira/browse/SPARK-9263), `resolveMavenCoordinates` ignores Spark and Spark's dependencies by using `addExclusionRules`. This PR aims to make [addExclusionRules](https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala#L956-L974) up-to-date to neglect correctly because it fails to neglect some components like the following.
**mllib (correct)**
```
$ bin/spark-shell --packages org.apache.spark:spark-mllib_2.11:2.1.1
...
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 0 | 0 | 0 | 0 || 0 | 0 |
---------------------------------------------------------------------
```
**mllib-local (wrong)**
```
$ bin/spark-shell --packages org.apache.spark:spark-mllib-local_2.11:2.1.1
...
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 15 | 2 | 2 | 0 || 15 | 2 |
---------------------------------------------------------------------
```
## How was this patch tested?
Pass the Jenkins with a updated test case.
Author: Dongjoon Hyun <do...@apache.org>
Closes #17947 from dongjoon-hyun/SPARK-20708.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/34661d8a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/34661d8a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/34661d8a
Branch: refs/heads/master
Commit: 34661d8a5acbeecae9b034a2a6a737f16d8738bb
Parents: 5854f77
Author: Dongjoon Hyun <do...@apache.org>
Authored: Wed May 31 22:39:25 2017 -0700
Committer: Burak Yavuz <br...@gmail.com>
Committed: Wed May 31 22:39:25 2017 -0700
----------------------------------------------------------------------
.../org/apache/spark/deploy/SparkSubmit.scala | 17 ++++++++++-------
.../spark/deploy/SparkSubmitUtilsSuite.scala | 9 +++------
2 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/34661d8a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index c60a2a1..d13fb41 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -879,6 +879,15 @@ private[spark] object SparkSubmitUtils {
// Exposed for testing
var printStream = SparkSubmit.printStream
+ // Exposed for testing.
+ // These components are used to make the default exclusion rules for Spark dependencies.
+ // We need to specify each component explicitly, otherwise we miss spark-streaming-kafka-0-8 and
+ // other spark-streaming utility components. Underscore is there to differentiate between
+ // spark-streaming_2.1x and spark-streaming-kafka-0-8-assembly_2.1x
+ val IVY_DEFAULT_EXCLUDES = Seq("catalyst_", "core_", "graphx_", "launcher_", "mllib_",
+ "mllib-local_", "network-common_", "network-shuffle_", "repl_", "sketch_", "sql_", "streaming_",
+ "tags_", "unsafe_")
+
/**
* Represents a Maven Coordinate
* @param groupId the groupId of the coordinate
@@ -1007,13 +1016,7 @@ private[spark] object SparkSubmitUtils {
// Add scala exclusion rule
md.addExcludeRule(createExclusion("*:scala-library:*", ivySettings, ivyConfName))
- // We need to specify each component explicitly, otherwise we miss spark-streaming-kafka-0-8 and
- // other spark-streaming utility components. Underscore is there to differentiate between
- // spark-streaming_2.1x and spark-streaming-kafka-0-8-assembly_2.1x
- val components = Seq("catalyst_", "core_", "graphx_", "hive_", "mllib_", "repl_",
- "sql_", "streaming_", "yarn_", "network-common_", "network-shuffle_", "network-yarn_")
-
- components.foreach { comp =>
+ IVY_DEFAULT_EXCLUDES.foreach { comp =>
md.addExcludeRule(createExclusion(s"org.apache.spark:spark-$comp*:*", ivySettings,
ivyConfName))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/34661d8a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
index 266c9d3..5702478 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
@@ -187,12 +187,9 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
}
test("neglects Spark and Spark's dependencies") {
- val components = Seq("catalyst_", "core_", "graphx_", "hive_", "mllib_", "repl_",
- "sql_", "streaming_", "yarn_", "network-common_", "network-shuffle_", "network-yarn_")
-
- val coordinates =
- components.map(comp => s"org.apache.spark:spark-${comp}2.10:1.2.0").mkString(",") +
- ",org.apache.spark:spark-core_fake:1.2.0"
+ val coordinates = SparkSubmitUtils.IVY_DEFAULT_EXCLUDES
+ .map(comp => s"org.apache.spark:spark-${comp}2.11:2.1.1")
+ .mkString(",") + ",org.apache.spark:spark-core_fake:1.2.0"
val path = SparkSubmitUtils.resolveMavenCoordinates(
coordinates,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org