You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2016/08/20 20:45:43 UTC
spark git commit: [SPARK-12666][CORE] SparkSubmit packages fix for
when 'default' conf doesn't exist in dependent module
Repository: spark
Updated Branches:
refs/heads/master 9560c8d29 -> 9f37d4eac
[SPARK-12666][CORE] SparkSubmit packages fix for when 'default' conf doesn't exist in dependent module
## What changes were proposed in this pull request?
Adding a "(runtime)" to the dependency configuration will set a fallback configuration to be used if the requested one is not found. E.g. with the setting "default(runtime)", Ivy will look for the conf "default" in the module ivy file and if not found will look for the conf "runtime". This can help with the case when using "sbt publishLocal" which does not write a "default" conf in the published ivy.xml file.
## How was this patch tested?
used spark-submit with --packages option for a package published locally with no default conf, and a package resolved from Maven central.
Author: Bryan Cutler <cu...@gmail.com>
Closes #13428 from BryanCutler/fallback-package-conf-SPARK-12666.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9f37d4ea
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9f37d4ea
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9f37d4ea
Branch: refs/heads/master
Commit: 9f37d4eac28dd179dd523fa7d645be97bb52af9c
Parents: 9560c8d
Author: Bryan Cutler <cu...@gmail.com>
Authored: Sat Aug 20 13:45:26 2016 -0700
Committer: Josh Rosen <jo...@databricks.com>
Committed: Sat Aug 20 13:45:26 2016 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/deploy/SparkSubmit.scala | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9f37d4ea/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 9feafc9..7b6d5a3 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -897,9 +897,12 @@ private[spark] object SparkSubmitUtils {
val localIvyRoot = new File(ivySettings.getDefaultIvyUserDir, "local")
localIvy.setLocal(true)
localIvy.setRepository(new FileRepository(localIvyRoot))
- val ivyPattern = Seq("[organisation]", "[module]", "[revision]", "[type]s",
- "[artifact](-[classifier]).[ext]").mkString(File.separator)
- localIvy.addIvyPattern(localIvyRoot.getAbsolutePath + File.separator + ivyPattern)
+ val ivyPattern = Seq(localIvyRoot.getAbsolutePath, "[organisation]", "[module]", "[revision]",
+ "ivys", "ivy.xml").mkString(File.separator)
+ localIvy.addIvyPattern(ivyPattern)
+ val artifactPattern = Seq(localIvyRoot.getAbsolutePath, "[organisation]", "[module]",
+ "[revision]", "[type]s", "[artifact](-[classifier]).[ext]").mkString(File.separator)
+ localIvy.addArtifactPattern(artifactPattern)
localIvy.setName("local-ivy-cache")
cr.add(localIvy)
@@ -944,7 +947,7 @@ private[spark] object SparkSubmitUtils {
artifacts.foreach { mvn =>
val ri = ModuleRevisionId.newInstance(mvn.groupId, mvn.artifactId, mvn.version)
val dd = new DefaultDependencyDescriptor(ri, false, false)
- dd.addDependencyConfiguration(ivyConfName, ivyConfName)
+ dd.addDependencyConfiguration(ivyConfName, ivyConfName + "(runtime)")
// scalastyle:off println
printStream.println(s"${dd.getDependencyId} added as a dependency")
// scalastyle:on println
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org