You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2016/02/01 22:38:41 UTC
spark git commit: [SPARK-12979][MESOS] Don’t resolve paths on the local file system in Mesos scheduler
Repository: spark
Updated Branches:
refs/heads/master a41b68b95 -> c9b89a0a0
[SPARK-12979][MESOS] Don’t resolve paths on the local file system in Mesos scheduler
The driver filesystem is likely different from where the executors will run, so resolving paths (and symlinks, etc.) will lead to invalid paths on executors.
Author: Iulian Dragos <ja...@gmail.com>
Closes #10923 from dragos/issue/canonical-paths.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c9b89a0a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c9b89a0a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c9b89a0a
Branch: refs/heads/master
Commit: c9b89a0a0921ce3d52864afd4feb7f37b90f7b46
Parents: a41b68b
Author: Iulian Dragos <ja...@gmail.com>
Authored: Mon Feb 1 13:38:38 2016 -0800
Committer: Andrew Or <an...@databricks.com>
Committed: Mon Feb 1 13:38:38 2016 -0800
----------------------------------------------------------------------
.../scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala | 2 +-
.../spark/scheduler/cluster/mesos/MesosClusterScheduler.scala | 2 +-
.../spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c9b89a0a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
index 58c30e7..2f095b8 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
@@ -179,7 +179,7 @@ private[spark] class CoarseMesosSchedulerBackend(
.orElse(Option(System.getenv("SPARK_EXECUTOR_URI")))
if (uri.isEmpty) {
- val runScript = new File(executorSparkHome, "./bin/spark-class").getCanonicalPath
+ val runScript = new File(executorSparkHome, "./bin/spark-class").getPath
command.setValue(
"%s \"%s\" org.apache.spark.executor.CoarseGrainedExecutorBackend"
.format(prefixEnv, runScript) +
http://git-wip-us.apache.org/repos/asf/spark/blob/c9b89a0a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
index e77d772..8cda4ff 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
@@ -394,7 +394,7 @@ private[spark] class MesosClusterScheduler(
.getOrElse {
throw new SparkException("Executor Spark home `spark.mesos.executor.home` is not set!")
}
- val cmdExecutable = new File(executorSparkHome, "./bin/spark-submit").getCanonicalPath
+ val cmdExecutable = new File(executorSparkHome, "./bin/spark-submit").getPath
// Sandbox points to the current directory by default with Mesos.
(cmdExecutable, ".")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c9b89a0a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
index a8bf79a..340f29b 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
@@ -125,7 +125,7 @@ private[spark] class MesosSchedulerBackend(
val executorBackendName = classOf[MesosExecutorBackend].getName
if (uri.isEmpty) {
- val executorPath = new File(executorSparkHome, "/bin/spark-class").getCanonicalPath
+ val executorPath = new File(executorSparkHome, "/bin/spark-class").getPath
command.setValue(s"$prefixEnv $executorPath $executorBackendName")
} else {
// Grab everything to the first '.'. We'll use that and '*' to
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org