You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/08/11 10:36:33 UTC
spark git commit: [SPARK-16952] don't lookup spark home directory
when executor uri is set
Repository: spark
Updated Branches:
refs/heads/master 7186e8c31 -> 4d496802f
[SPARK-16952] don't lookup spark home directory when executor uri is set
## What changes were proposed in this pull request?
remove requirement to set spark.mesos.executor.home when spark.executor.uri is used
## How was this patch tested?
unit tests
Author: Michael Gummelt <mg...@mesosphere.io>
Closes #14552 from mgummelt/fix-spark-home.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4d496802
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4d496802
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4d496802
Branch: refs/heads/master
Commit: 4d496802f592dca96dada73b24afc93c668a7f26
Parents: 7186e8c
Author: Michael Gummelt <mg...@mesosphere.io>
Authored: Thu Aug 11 11:36:20 2016 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Thu Aug 11 11:36:20 2016 +0100
----------------------------------------------------------------------
.../MesosCoarseGrainedSchedulerBackend.scala | 10 ++++-----
...esosCoarseGrainedSchedulerBackendSuite.scala | 23 ++++++++++++++++++--
2 files changed, 26 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/4d496802/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
index 0933a03..4a88824 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -163,11 +163,6 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
}
def createCommand(offer: Offer, numCores: Int, taskId: String): CommandInfo = {
- val executorSparkHome = conf.getOption("spark.mesos.executor.home")
- .orElse(sc.getSparkHome())
- .getOrElse {
- throw new SparkException("Executor Spark home `spark.mesos.executor.home` is not set!")
- }
val environment = Environment.newBuilder()
val extraClassPath = conf.getOption("spark.executor.extraClassPath")
extraClassPath.foreach { cp =>
@@ -201,6 +196,11 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
.orElse(Option(System.getenv("SPARK_EXECUTOR_URI")))
if (uri.isEmpty) {
+ val executorSparkHome = conf.getOption("spark.mesos.executor.home")
+ .orElse(sc.getSparkHome())
+ .getOrElse {
+ throw new SparkException("Executor Spark home `spark.mesos.executor.home` is not set!")
+ }
val runScript = new File(executorSparkHome, "./bin/spark-class").getPath
command.setValue(
"%s \"%s\" org.apache.spark.executor.CoarseGrainedExecutorBackend"
http://git-wip-us.apache.org/repos/asf/spark/blob/4d496802/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
index 0e66979..26a3ad4 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
@@ -370,6 +370,21 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
verify(driverEndpoint, never()).askWithRetry(isA(classOf[RemoveExecutor]))(any[ClassTag[_]])
}
+ test("mesos supports spark.executor.uri") {
+ val url = "spark.spark.spark.com"
+ setBackend(Map(
+ "spark.executor.uri" -> url
+ ), false)
+
+ val (mem, cpu) = (backend.executorMemory(sc), 4)
+
+ val offer1 = createOffer("o1", "s1", mem, cpu)
+ backend.resourceOffers(driver, List(offer1).asJava)
+
+ val launchedTasks = verifyTaskLaunched(driver, "o1")
+ assert(launchedTasks.head.getCommand.getUrisList.asScala(0).getValue == url)
+ }
+
private def verifyDeclinedOffer(driver: SchedulerDriver,
offerId: OfferID,
filter: Boolean = false): Unit = {
@@ -435,13 +450,17 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
backend
}
- private def setBackend(sparkConfVars: Map[String, String] = null) {
+ private def setBackend(sparkConfVars: Map[String, String] = null,
+ setHome: Boolean = true) {
sparkConf = (new SparkConf)
.setMaster("local[*]")
.setAppName("test-mesos-dynamic-alloc")
- .setSparkHome("/path")
.set("spark.mesos.driver.webui.url", "http://webui")
+ if (setHome) {
+ sparkConf.setSparkHome("/path")
+ }
+
if (sparkConfVars != null) {
sparkConf.setAll(sparkConfVars)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org