You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by tg...@apache.org on 2014/07/23 04:05:18 UTC
git commit: [YARN] SPARK-2577: File upload to viewfs is broken due to
mount point re...
Repository: spark
Updated Branches:
refs/heads/master ddadf1b00 -> 02e457294
[YARN] SPARK-2577: File upload to viewfs is broken due to mount point re...
Opting to the option 2 defined in SPARK-2577, i.e., retrieve and pass the correct file system object to addResource.
Author: Gera Shegalov <ge...@twitter.com>
Closes #1483 from gerashegalov/master and squashes the following commits:
90c9087 [Gera Shegalov] [YARN] SPARK-2577: File upload to viewfs is broken due to mount point resolution
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/02e45729
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/02e45729
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/02e45729
Branch: refs/heads/master
Commit: 02e45729472a22a31629cdd17dc3836ba6810189
Parents: ddadf1b
Author: Gera Shegalov <ge...@twitter.com>
Authored: Tue Jul 22 21:05:12 2014 -0500
Committer: Thomas Graves <tg...@apache.org>
Committed: Tue Jul 22 21:05:12 2014 -0500
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/02e45729/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
----------------------------------------------------------------------
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 556f493..a1298e8 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -232,7 +232,8 @@ trait ClientBase extends Logging {
if (!ClientBase.LOCAL_SCHEME.equals(localURI.getScheme())) {
val setPermissions = if (destName.equals(ClientBase.APP_JAR)) true else false
val destPath = copyRemoteFile(dst, qualifyForLocal(localURI), replication, setPermissions)
- distCacheMgr.addResource(fs, conf, destPath, localResources, LocalResourceType.FILE,
+ val destFs = FileSystem.get(destPath.toUri(), conf)
+ distCacheMgr.addResource(destFs, conf, destPath, localResources, LocalResourceType.FILE,
destName, statCache)
} else if (confKey != null) {
sparkConf.set(confKey, localPath)