You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2019/03/11 00:28:52 UTC
[spark] branch master updated: [SPARK-24621][WEBUI] Show secure
URLs on web pages
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 29d9021 [SPARK-24621][WEBUI] Show secure URLs on web pages
29d9021 is described below
commit 29d902124581b2163e49baa6d379025a6cf1fa1a
Author: Gabor Somogyi <ga...@gmail.com>
AuthorDate: Sun Mar 10 19:28:35 2019 -0500
[SPARK-24621][WEBUI] Show secure URLs on web pages
## What changes were proposed in this pull request?
Web UI URLs are pointing to `http://` targets even if SSL is enabled. In this PR I've changed the code to point to `https://` URLs.
## How was this patch tested?
Existing unit tests + manually by starting standalone master/worker/spark-shell. Please see jira.
Closes #23991 from gaborgsomogyi/SPARK-24621.
Authored-by: Gabor Somogyi <ga...@gmail.com>
Signed-off-by: Sean Owen <se...@databricks.com>
---
.../src/main/scala/org/apache/spark/deploy/master/Master.scala | 2 +-
.../scala/org/apache/spark/deploy/worker/ExecutorRunner.scala | 3 ++-
.../src/main/scala/org/apache/spark/deploy/worker/Worker.scala | 3 ++-
core/src/main/scala/org/apache/spark/ui/WebUI.scala | 10 ++++++++--
.../test/scala/org/apache/spark/deploy/DeployTestUtils.scala | 1 +
.../org/apache/spark/deploy/worker/ExecutorRunnerTest.scala | 4 ++--
6 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 356e52c..50e282f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -143,7 +143,7 @@ private[deploy] class Master(
logInfo(s"Running Spark version ${org.apache.spark.SPARK_VERSION}")
webUi = new MasterWebUI(this, webUiPort)
webUi.bind()
- masterWebUiUrl = "http://" + masterPublicAddress + ":" + webUi.boundPort
+ masterWebUiUrl = s"${webUi.scheme}$masterPublicAddress:${webUi.boundPort}"
if (reverseProxy) {
masterWebUiUrl = conf.get(UI_REVERSE_PROXY_URL).orElse(Some(masterWebUiUrl)).get
webUi.addProxy()
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
index ead28f1..9d2301c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
@@ -45,6 +45,7 @@ private[deploy] class ExecutorRunner(
val memory: Int,
val worker: RpcEndpointRef,
val workerId: String,
+ val webUiScheme: String,
val host: String,
val webUiPort: Int,
val publicAddress: String,
@@ -165,7 +166,7 @@ private[deploy] class ExecutorRunner(
if (conf.get(UI_REVERSE_PROXY)) {
s"/proxy/$workerId/logPage/?appId=$appId&executorId=$execId&logType="
} else {
- s"http://$publicAddress:$webUiPort/logPage/?appId=$appId&executorId=$execId&logType="
+ s"$webUiScheme$publicAddress:$webUiPort/logPage/?appId=$appId&executorId=$execId&logType="
}
builder.environment.put("SPARK_LOG_URL_STDERR", s"${baseUrl}stderr")
builder.environment.put("SPARK_LOG_URL_STDOUT", s"${baseUrl}stdout")
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index 2fc2be5..4bd7aaa 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -211,7 +211,7 @@ private[deploy] class Worker(
webUi = new WorkerWebUI(this, workDir, webUiPort)
webUi.bind()
- workerWebUiUrl = s"http://$publicAddress:${webUi.boundPort}"
+ workerWebUiUrl = s"${webUi.scheme}$publicAddress:${webUi.boundPort}"
registerWithMaster()
metricsSystem.registerSource(workerSource)
@@ -528,6 +528,7 @@ private[deploy] class Worker(
memory_,
self,
workerId,
+ webUi.scheme,
host,
webUi.boundPort,
publicAddress,
diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
index ebf8655..8165942 100644
--- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
@@ -139,11 +139,17 @@ private[spark] abstract class WebUI(
}
}
+ /** @return Whether SSL enabled. Only valid after [[bind]]. */
+ def isSecure: Boolean = serverInfo.map(_.securePort.isDefined).getOrElse(false)
+
+ /** @return The scheme of web interface. Only valid after [[bind]]. */
+ def scheme: String = if (isSecure) "https://" else "http://"
+
/** @return The url of web interface. Only valid after [[bind]]. */
- def webUrl: String = s"http://$publicHostName:$boundPort"
+ def webUrl: String = s"${scheme}$publicHostName:${boundPort}"
/** @return The actual port to which this server is bound. Only valid after [[bind]]. */
- def boundPort: Int = serverInfo.map(_.boundPort).getOrElse(-1)
+ def boundPort: Int = serverInfo.map(si => si.securePort.getOrElse(si.boundPort)).getOrElse(-1)
/** Stops the server behind this web interface. Only valid after [[bind]]. */
def stop(): Unit = {
diff --git a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
index 55a541d..784981e 100644
--- a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
@@ -64,6 +64,7 @@ private[deploy] object DeployTestUtils {
1234,
null,
"workerId",
+ "http://",
"host",
123,
"publicAddress",
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
index 0240bf8..988c65f 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
@@ -29,8 +29,8 @@ class ExecutorRunnerTest extends SparkFunSuite {
val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!"))
val appDesc = new ApplicationDescription("app name", Some(8), 500,
Command("foo", Seq(appId), Map(), Seq(), Seq(), Seq()), "appUiUrl")
- val er = new ExecutorRunner(appId, 1, appDesc, 8, 500, null, "blah", "worker321", 123,
- "publicAddr", new File(sparkHome), new File("ooga"), "blah", conf, Seq("localDir"),
+ val er = new ExecutorRunner(appId, 1, appDesc, 8, 500, null, "blah", "http://", "worker321",
+ 123, "publicAddr", new File(sparkHome), new File("ooga"), "blah", conf, Seq("localDir"),
ExecutorState.RUNNING)
val builder = CommandUtils.buildProcessBuilder(
appDesc.command, new SecurityManager(conf), 512, sparkHome, er.substituteVariables)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org