You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2017/01/15 11:13:07 UTC
spark git commit: [SPARK-19042] spark executor can't download the
jars when uber jar's http url contains any query strings
Repository: spark
Updated Branches:
refs/heads/master 9112f31bb -> c9d612f82
[SPARK-19042] spark executor can't download the jars when uber jar's http url contains any query strings
If the uber jars' https contains any query strings, the Executor.updateDependencies method can't can't download the jars correctly. This is because the "localName = name.split("/").last" won't get the expected jar's url. The bug fix is the same as [SPARK-17855]
Author: xiaojian.fxj <xi...@alibaba-inc.com>
Closes #16509 from hustfxj/bug.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c9d612f8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c9d612f8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c9d612f8
Branch: refs/heads/master
Commit: c9d612f82c290fc955cae93150ca5c5d74f12217
Parents: 9112f31
Author: xiaojian.fxj <xi...@alibaba-inc.com>
Authored: Sun Jan 15 11:12:59 2017 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Sun Jan 15 11:12:59 2017 +0000
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/executor/Executor.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c9d612f8/core/src/main/scala/org/apache/spark/executor/Executor.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index b6c0f0c..db5d0d8 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -19,7 +19,7 @@ package org.apache.spark.executor
import java.io.{File, NotSerializableException}
import java.lang.management.ManagementFactory
-import java.net.URL
+import java.net.{URI, URL}
import java.nio.ByteBuffer
import java.util.Properties
import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
@@ -640,7 +640,7 @@ private[spark] class Executor(
currentFiles(name) = timestamp
}
for ((name, timestamp) <- newJars) {
- val localName = name.split("/").last
+ val localName = new URI(name).getPath.split("/").last
val currentTimeStamp = currentJars.get(name)
.orElse(currentJars.get(localName))
.getOrElse(-1L)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org