You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2015/10/12 19:22:24 UTC
spark git commit: [SPARK-11023] [YARN] Avoid creating URIs from local
paths directly.
Repository: spark
Updated Branches:
refs/heads/master 64b1d00e1 -> 149472a01
[SPARK-11023] [YARN] Avoid creating URIs from local paths directly.
The issue is that local paths on Windows, when provided with drive
letters or backslashes, are not valid URIs.
Instead of trying to figure out whether paths are URIs or not, use
Utils.resolveURI() which does that for us.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #9049 from vanzin/SPARK-11023 and squashes the following commits:
77021f2 [Marcelo Vanzin] [SPARK-11023] [yarn] Avoid creating URIs from local paths directly.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/149472a0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/149472a0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/149472a0
Branch: refs/heads/master
Commit: 149472a01d12828c64b0a852982d48c123984182
Parents: 64b1d00
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Mon Oct 12 10:21:57 2015 -0700
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Mon Oct 12 10:21:57 2015 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/deploy/yarn/Client.scala | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/149472a0/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index cec81b9..1fbd18a 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -358,7 +358,8 @@ private[spark] class Client(
destName: Option[String] = None,
targetDir: Option[String] = None,
appMasterOnly: Boolean = false): (Boolean, String) = {
- val localURI = new URI(path.trim())
+ val trimmedPath = path.trim()
+ val localURI = Utils.resolveURI(trimmedPath)
if (localURI.getScheme != LOCAL_SCHEME) {
if (addDistributedUri(localURI)) {
val localPath = getQualifiedLocalPath(localURI, hadoopConf)
@@ -374,7 +375,7 @@ private[spark] class Client(
(false, null)
}
} else {
- (true, path.trim())
+ (true, trimmedPath)
}
}
@@ -595,10 +596,10 @@ private[spark] class Client(
LOCALIZED_PYTHON_DIR)
}
(pySparkArchives ++ pyArchives).foreach { path =>
- val uri = new URI(path)
+ val uri = Utils.resolveURI(path)
if (uri.getScheme != LOCAL_SCHEME) {
pythonPath += buildPath(YarnSparkHadoopUtil.expandEnvironment(Environment.PWD),
- new Path(path).getName())
+ new Path(uri).getName())
} else {
pythonPath += uri.getPath()
}
@@ -1229,7 +1230,7 @@ object Client extends Logging {
private def getMainJarUri(mainJar: Option[String]): Option[URI] = {
mainJar.flatMap { path =>
- val uri = new URI(path)
+ val uri = Utils.resolveURI(path)
if (uri.getScheme == LOCAL_SCHEME) Some(uri) else None
}.orElse(Some(new URI(APP_JAR)))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org