You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2014/05/27 07:05:25 UTC
git commit: SPARK-1933: Throw a more meaningful exception when a
directory is passed to addJar/addFile.
Repository: spark
Updated Branches:
refs/heads/master 9ed37190f -> 90e281b55
SPARK-1933: Throw a more meaningful exception when a directory is passed to addJar/addFile.
https://issues.apache.org/jira/browse/SPARK-1933
Author: Reynold Xin <rx...@apache.org>
Closes #888 from rxin/addfile and squashes the following commits:
8c402a3 [Reynold Xin] Updated comment.
ff6c162 [Reynold Xin] SPARK-1933: Throw a more meaningful exception when a directory is passed to addJar/addFile.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/90e281b5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/90e281b5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/90e281b5
Branch: refs/heads/master
Commit: 90e281b55aecbfbe4431ac582311d5790fe7aad3
Parents: 9ed3719
Author: Reynold Xin <rx...@apache.org>
Authored: Mon May 26 22:05:23 2014 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Mon May 26 22:05:23 2014 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/HttpFileServer.scala | 7 +++++++
core/src/main/scala/org/apache/spark/SparkContext.scala | 5 ++---
2 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/90e281b5/core/src/main/scala/org/apache/spark/HttpFileServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/HttpFileServer.scala b/core/src/main/scala/org/apache/spark/HttpFileServer.scala
index a6e300d..0e3750f 100644
--- a/core/src/main/scala/org/apache/spark/HttpFileServer.scala
+++ b/core/src/main/scala/org/apache/spark/HttpFileServer.scala
@@ -59,6 +59,13 @@ private[spark] class HttpFileServer(securityManager: SecurityManager) extends Lo
}
def addFileToDir(file: File, dir: File) : String = {
+ // Check whether the file is a directory. If it is, throw a more meaningful exception.
+ // If we don't catch this, Guava throws a very confusing error message:
+ // java.io.FileNotFoundException: [file] (No such file or directory)
+ // even though the directory ([file]) exists.
+ if (file.isDirectory) {
+ throw new IllegalArgumentException(s"$file cannot be a directory.")
+ }
Files.copy(file, new File(dir, file.getName))
dir + "/" + file.getName
}
http://git-wip-us.apache.org/repos/asf/spark/blob/90e281b5/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 49737fa..03ceff8 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -794,7 +794,7 @@ class SparkContext(config: SparkConf) extends Logging {
addedFiles(key) = System.currentTimeMillis
// Fetch the file locally in case a job is executed using DAGScheduler.runLocally().
- Utils.fetchFile(path, new File(SparkFiles.getRootDirectory), conf, env.securityManager)
+ Utils.fetchFile(path, new File(SparkFiles.getRootDirectory()), conf, env.securityManager)
logInfo("Added file " + path + " at " + key + " with timestamp " + addedFiles(key))
postEnvironmentUpdate()
@@ -932,13 +932,12 @@ class SparkContext(config: SparkConf) extends Logging {
try {
env.httpFileServer.addJar(new File(fileName))
} catch {
- case e: Exception => {
+ case e: Exception =>
// For now just log an error but allow to go through so spark examples work.
// The spark examples don't really need the jar distributed since its also
// the app jar.
logError("Error adding jar (" + e + "), was the --addJars option used?")
null
- }
}
} else {
env.httpFileServer.addJar(new File(uri.getPath))