You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by tg...@apache.org on 2016/11/08 18:13:17 UTC
spark git commit: [SPARK-18357] Fix yarn files/archive broken issue
andd unit tests
Repository: spark
Updated Branches:
refs/heads/master 9c419698f -> 245e5a2f8
[SPARK-18357] Fix yarn files/archive broken issue andd unit tests
## What changes were proposed in this pull request?
The #15627 broke functionality with yarn --files --archives does not accept any files.
This patch ensures that --files and --archives accept unique files.
## How was this patch tested?
A. I added unit tests.
B. Also, manually tested --files with --archives to throw exception if duplicate files are specified and continue if unique files are specified.
Author: Kishor Patil <kp...@yahoo-inc.com>
Closes #15810 from kishorvpatil/SPARK18357.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/245e5a2f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/245e5a2f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/245e5a2f
Branch: refs/heads/master
Commit: 245e5a2f80e3195b7f8a38b480b29bfc23af66bf
Parents: 9c41969
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Tue Nov 8 12:13:09 2016 -0600
Committer: Tom Graves <tg...@yahoo-inc.com>
Committed: Tue Nov 8 12:13:09 2016 -0600
----------------------------------------------------------------------
.../org/apache/spark/deploy/yarn/Client.scala | 2 +-
.../org/apache/spark/deploy/yarn/ClientSuite.scala | 17 +++++++++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/245e5a2f/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 172fb46..e77fa38 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -604,7 +604,7 @@ private[spark] class Client(
cachedSecondaryJarLinks += localizedPath
}
} else {
- if (localizedPath != null) {
+ if (localizedPath == null) {
throw new IllegalArgumentException(s"Attempt to add ($file) multiple times" +
" to the distributed cache.")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/245e5a2f/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
----------------------------------------------------------------------
diff --git a/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
index 06516c1..7deaf0a 100644
--- a/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
+++ b/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
@@ -322,6 +322,23 @@ class ClientSuite extends SparkFunSuite with Matchers with BeforeAndAfterAll
intercept[IllegalArgumentException] {
clientArchives.prepareLocalResources(new Path(tempDirForArchives.getAbsolutePath()), Nil)
}
+
+ // Case 4: FILES_TO_DISTRIBUTE can have unique file.
+ val sparkConfFilesUniq = new SparkConfWithEnv(Map("SPARK_HOME" -> libs.getAbsolutePath))
+ .set(FILES_TO_DISTRIBUTE, Seq(testJar.getPath))
+
+ val clientFilesUniq = createClient(sparkConfFilesUniq)
+ val tempDirForFilesUniq = Utils.createTempDir()
+ clientFilesUniq.prepareLocalResources(new Path(tempDirForFilesUniq.getAbsolutePath()), Nil)
+
+ // Case 5: ARCHIVES_TO_DISTRIBUTE can have unique file.
+ val sparkConfArchivesUniq = new SparkConfWithEnv(Map("SPARK_HOME" -> libs.getAbsolutePath))
+ .set(ARCHIVES_TO_DISTRIBUTE, Seq(testJar.getPath))
+
+ val clientArchivesUniq = createClient(sparkConfArchivesUniq)
+ val tempDirArchivesUniq = Utils.createTempDir()
+ clientArchivesUniq.prepareLocalResources(new Path(tempDirArchivesUniq.getAbsolutePath()), Nil)
+
}
test("distribute local spark jars") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org