You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2014/12/03 22:56:30 UTC
spark git commit: [HOT FIX] [YARN] Check whether `/lib` exists before
listing its files
Repository: spark
Updated Branches:
refs/heads/master 692f49378 -> 90ec643e9
[HOT FIX] [YARN] Check whether `/lib` exists before listing its files
This is caused by a975dc32799bb8a14f9e1c76defaaa7cfbaf8b53
Author: Andrew Or <an...@databricks.com>
Closes #3589 from andrewor14/yarn-hot-fix and squashes the following commits:
a4fad5f [Andrew Or] Check whether lib directory exists before listing its files
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/90ec643e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/90ec643e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/90ec643e
Branch: refs/heads/master
Commit: 90ec643e9af4c8bbb9000edca08c07afb17939c7
Parents: 692f493
Author: Andrew Or <an...@databricks.com>
Authored: Wed Dec 3 13:56:23 2014 -0800
Committer: Andrew Or <an...@databricks.com>
Committed: Wed Dec 3 13:56:23 2014 -0800
----------------------------------------------------------------------
.../apache/spark/deploy/yarn/ClientBase.scala | 27 +++++++++++---------
1 file changed, 15 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/90ec643e/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
----------------------------------------------------------------------
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 8e4360e..290d994 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -243,18 +243,21 @@ private[spark] trait ClientBase extends Logging {
val libsURI = new URI(libsDir)
val jarLinks = ListBuffer.empty[String]
if (libsURI.getScheme != LOCAL_SCHEME) {
- val localURI = getQualifiedLocalPath(libsURI).toUri()
- val jars = FileSystem.get(localURI, hadoopConf).listFiles(new Path(localURI.getPath), false)
- while (jars.hasNext) {
- val jar = jars.next()
- val name = jar.getPath.getName
- if (name.startsWith("datanucleus-")) {
- // copy to remote and add to classpath
- val src = jar.getPath
- val destPath = copyFileToRemote(dst, src, replication)
- distCacheMgr.addResource(fs, hadoopConf, destPath,
- localResources, LocalResourceType.FILE, name, statCache)
- jarLinks += name
+ val localPath = getQualifiedLocalPath(libsURI)
+ val localFs = FileSystem.get(localPath.toUri, hadoopConf)
+ if (localFs.exists(localPath)) {
+ val jars = localFs.listFiles(localPath, /* recursive */ false)
+ while (jars.hasNext) {
+ val jar = jars.next()
+ val name = jar.getPath.getName
+ if (name.startsWith("datanucleus-")) {
+ // copy to remote and add to classpath
+ val src = jar.getPath
+ val destPath = copyFileToRemote(dst, src, replication)
+ distCacheMgr.addResource(localFs, hadoopConf, destPath,
+ localResources, LocalResourceType.FILE, name, statCache)
+ jarLinks += name
+ }
}
}
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org