You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2015/10/06 19:19:01 UTC
spark git commit: [SPARK-10901] [YARN]
spark.yarn.user.classpath.first doesn't work
Repository: spark
Updated Branches:
refs/heads/master 744f03e70 -> e97836015
[SPARK-10901] [YARN] spark.yarn.user.classpath.first doesn't work
This should go into 1.5.2 also.
The issue is we were no longer adding the __app__.jar to the system classpath.
Author: Thomas Graves <tg...@staydecay.corp.gq1.yahoo.com>
Author: Tom Graves <tg...@yahoo-inc.com>
Closes #8959 from tgravescs/SPARK-10901.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e9783601
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e9783601
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e9783601
Branch: refs/heads/master
Commit: e9783601599758df87418bf61a7b4636f06714fa
Parents: 744f03e
Author: Thomas Graves <tg...@staydecay.corp.gq1.yahoo.com>
Authored: Tue Oct 6 10:18:50 2015 -0700
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Tue Oct 6 10:18:50 2015 -0700
----------------------------------------------------------------------
.../org/apache/spark/deploy/yarn/Client.scala | 39 ++++++++++++++------
1 file changed, 27 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e9783601/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index f8748ef..eb3b7fb 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -1155,13 +1155,24 @@ object Client extends Logging {
}
if (sparkConf.getBoolean("spark.yarn.user.classpath.first", false)) {
- val userClassPath =
+ // in order to properly add the app jar when user classpath is first
+ // we have to do the mainJar separate in order to send the right thing
+ // into addFileToClasspath
+ val mainJar =
if (args != null) {
- getUserClasspath(Option(args.userJar), Option(args.addJars))
+ getMainJarUri(Option(args.userJar))
} else {
- getUserClasspath(sparkConf)
+ getMainJarUri(sparkConf.getOption(CONF_SPARK_USER_JAR))
}
- userClassPath.foreach { x =>
+ mainJar.foreach(addFileToClasspath(sparkConf, _, APP_JAR, env))
+
+ val secondaryJars =
+ if (args != null) {
+ getSecondaryJarUris(Option(args.addJars))
+ } else {
+ getSecondaryJarUris(sparkConf.getOption(CONF_SPARK_YARN_SECONDARY_JARS))
+ }
+ secondaryJars.foreach { x =>
addFileToClasspath(sparkConf, x, null, env)
}
}
@@ -1178,16 +1189,20 @@ object Client extends Logging {
* @param conf Spark configuration.
*/
def getUserClasspath(conf: SparkConf): Array[URI] = {
- getUserClasspath(conf.getOption(CONF_SPARK_USER_JAR),
- conf.getOption(CONF_SPARK_YARN_SECONDARY_JARS))
+ val mainUri = getMainJarUri(conf.getOption(CONF_SPARK_USER_JAR))
+ val secondaryUris = getSecondaryJarUris(conf.getOption(CONF_SPARK_YARN_SECONDARY_JARS))
+ (mainUri ++ secondaryUris).toArray
}
- private def getUserClasspath(
- mainJar: Option[String],
- secondaryJars: Option[String]): Array[URI] = {
- val mainUri = mainJar.orElse(Some(APP_JAR)).map(new URI(_))
- val secondaryUris = secondaryJars.map(_.split(",")).toSeq.flatten.map(new URI(_))
- (mainUri ++ secondaryUris).toArray
+ private def getMainJarUri(mainJar: Option[String]): Option[URI] = {
+ mainJar.flatMap { path =>
+ val uri = new URI(path)
+ if (uri.getScheme == LOCAL_SCHEME) Some(uri) else None
+ }.orElse(Some(new URI(APP_JAR)))
+ }
+
+ private def getSecondaryJarUris(secondaryJars: Option[String]): Seq[URI] = {
+ secondaryJars.map(_.split(",")).toSeq.flatten.map(new URI(_))
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org