You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2015/11/30 10:26:33 UTC
spark git commit: [SPARK-11859][MESOS] SparkContext accepts invalid
Master URLs in the form zk://host:port for a multi-master Mesos cluster using
ZooKeeper
Repository: spark
Updated Branches:
refs/heads/master 0ddfe7868 -> e07494420
[SPARK-11859][MESOS] SparkContext accepts invalid Master URLs in the form zk://host:port for a multi-master Mesos cluster using ZooKeeper
* According to below doc and validation logic in [SparkSubmit.scala](https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala#L231), master URL for a mesos cluster should always start with `mesos://`
http://spark.apache.org/docs/latest/running-on-mesos.html
`The Master URLs for Mesos are in the form mesos://host:5050 for a single-master Mesos cluster, or mesos://zk://host:2181 for a multi-master Mesos cluster using ZooKeeper.`
* However, [SparkContext.scala](https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/SparkContext.scala#L2749) fails the validation and can receive master URL in the form `zk://host:port`
* For the master URLs in the form `zk:host:port`, the valid form should be `mesos://zk://host:port`
* This PR restrict the validation in `SparkContext.scala`, and now only mesos master URLs prefixed with `mesos://` can be accepted.
* This PR also updated corresponding unit test.
Author: toddwan <ta...@outlook.com>
Closes #9886 from toddwan/S11859.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e0749442
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e0749442
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e0749442
Branch: refs/heads/master
Commit: e0749442051d6e29dae4f4cdcb2937c0b015f98f
Parents: 0ddfe78
Author: toddwan <ta...@outlook.com>
Authored: Mon Nov 30 09:26:29 2015 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Mon Nov 30 09:26:29 2015 +0000
----------------------------------------------------------------------
.../main/scala/org/apache/spark/SparkContext.scala | 16 ++++++++++------
.../spark/SparkContextSchedulerCreationSuite.scala | 5 +++++
2 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e0749442/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index b030d3c..8a62b71 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -2708,15 +2708,14 @@ object SparkContext extends Logging {
scheduler.initialize(backend)
(backend, scheduler)
- case mesosUrl @ MESOS_REGEX(_) =>
+ case MESOS_REGEX(mesosUrl) =>
MesosNativeLibrary.load()
val scheduler = new TaskSchedulerImpl(sc)
val coarseGrained = sc.conf.getBoolean("spark.mesos.coarse", defaultValue = true)
- val url = mesosUrl.stripPrefix("mesos://") // strip scheme from raw Mesos URLs
val backend = if (coarseGrained) {
- new CoarseMesosSchedulerBackend(scheduler, sc, url, sc.env.securityManager)
+ new CoarseMesosSchedulerBackend(scheduler, sc, mesosUrl, sc.env.securityManager)
} else {
- new MesosSchedulerBackend(scheduler, sc, url)
+ new MesosSchedulerBackend(scheduler, sc, mesosUrl)
}
scheduler.initialize(backend)
(backend, scheduler)
@@ -2727,6 +2726,11 @@ object SparkContext extends Logging {
scheduler.initialize(backend)
(backend, scheduler)
+ case zkUrl if zkUrl.startsWith("zk://") =>
+ logWarning("Master URL for a multi-master Mesos cluster managed by ZooKeeper should be " +
+ "in the form mesos://zk://host:port. Current Master URL will stop working in Spark 2.0.")
+ createTaskScheduler(sc, "mesos://" + zkUrl)
+
case _ =>
throw new SparkException("Could not parse Master URL: '" + master + "'")
}
@@ -2745,8 +2749,8 @@ private object SparkMasterRegex {
val LOCAL_CLUSTER_REGEX = """local-cluster\[\s*([0-9]+)\s*,\s*([0-9]+)\s*,\s*([0-9]+)\s*]""".r
// Regular expression for connecting to Spark deploy clusters
val SPARK_REGEX = """spark://(.*)""".r
- // Regular expression for connection to Mesos cluster by mesos:// or zk:// url
- val MESOS_REGEX = """(mesos|zk)://.*""".r
+ // Regular expression for connection to Mesos cluster by mesos:// or mesos://zk:// url
+ val MESOS_REGEX = """mesos://(.*)""".r
// Regular expression for connection to Simr cluster
val SIMR_REGEX = """simr://(.*)""".r
}
http://git-wip-us.apache.org/repos/asf/spark/blob/e0749442/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala
index e5a14a6..d18e078 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala
@@ -175,6 +175,11 @@ class SparkContextSchedulerCreationSuite
}
test("mesos with zookeeper") {
+ testMesos("mesos://zk://localhost:1234,localhost:2345",
+ classOf[MesosSchedulerBackend], coarse = false)
+ }
+
+ test("mesos with zookeeper and Master URL starting with zk://") {
testMesos("zk://localhost:1234,localhost:2345", classOf[MesosSchedulerBackend], coarse = false)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org