You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2018/01/06 01:25:44 UTC
spark git commit: [SPARK-22914][DEPLOY] Register history.ui.port
Repository: spark
Updated Branches:
refs/heads/master 930b90a84 -> ea9568330
[SPARK-22914][DEPLOY] Register history.ui.port
## What changes were proposed in this pull request?
Register spark.history.ui.port as a known spark conf to be used in substitution expressions even if it's not set explicitly.
## How was this patch tested?
Added unit test to demonstrate the issue
Author: Gera Shegalov <ge...@apache.org>
Author: Gera Shegalov <gs...@salesforce.com>
Closes #20098 from gerashegalov/gera/register-SHS-port-conf.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ea956833
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ea956833
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ea956833
Branch: refs/heads/master
Commit: ea956833017fcbd8ed2288368bfa2e417a2251c5
Parents: 930b90a
Author: Gera Shegalov <ge...@apache.org>
Authored: Fri Jan 5 17:25:28 2018 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Fri Jan 5 17:25:28 2018 -0800
----------------------------------------------------------------------
.../spark/deploy/history/HistoryServer.scala | 3 +-
.../apache/spark/deploy/history/config.scala | 5 +++
.../spark/deploy/yarn/ApplicationMaster.scala | 17 +++++---
.../deploy/yarn/ApplicationMasterSuite.scala | 43 ++++++++++++++++++++
4 files changed, 62 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ea956833/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index 75484f5..0ec4afa 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -28,6 +28,7 @@ import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.deploy.history.config.HISTORY_SERVER_UI_PORT
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config._
import org.apache.spark.status.api.v1.{ApiRootResource, ApplicationInfo, UIRoot}
@@ -276,7 +277,7 @@ object HistoryServer extends Logging {
.newInstance(conf)
.asInstanceOf[ApplicationHistoryProvider]
- val port = conf.getInt("spark.history.ui.port", 18080)
+ val port = conf.get(HISTORY_SERVER_UI_PORT)
val server = new HistoryServer(conf, provider, securityManager, port)
server.bind()
http://git-wip-us.apache.org/repos/asf/spark/blob/ea956833/core/src/main/scala/org/apache/spark/deploy/history/config.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/config.scala b/core/src/main/scala/org/apache/spark/deploy/history/config.scala
index 22b6d49..efdbf67 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/config.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/config.scala
@@ -44,4 +44,9 @@ private[spark] object config {
.bytesConf(ByteUnit.BYTE)
.createWithDefaultString("10g")
+ val HISTORY_SERVER_UI_PORT = ConfigBuilder("spark.history.ui.port")
+ .doc("Web UI port to bind Spark History Server")
+ .intConf
+ .createWithDefault(18080)
+
}
http://git-wip-us.apache.org/repos/asf/spark/blob/ea956833/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
----------------------------------------------------------------------
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index b2576b0..4d5e3bb 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -427,11 +427,8 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments) extends
uiAddress: Option[String]) = {
val appId = client.getAttemptId().getApplicationId().toString()
val attemptId = client.getAttemptId().getAttemptId().toString()
- val historyAddress =
- _sparkConf.get(HISTORY_SERVER_ADDRESS)
- .map { text => SparkHadoopUtil.get.substituteHadoopVariables(text, yarnConf) }
- .map { address => s"${address}${HistoryServer.UI_PATH_PREFIX}/${appId}/${attemptId}" }
- .getOrElse("")
+ val historyAddress = ApplicationMaster
+ .getHistoryServerAddress(_sparkConf, yarnConf, appId, attemptId)
val driverUrl = RpcEndpointAddress(
_sparkConf.get("spark.driver.host"),
@@ -834,6 +831,16 @@ object ApplicationMaster extends Logging {
master.getAttemptId
}
+ private[spark] def getHistoryServerAddress(
+ sparkConf: SparkConf,
+ yarnConf: YarnConfiguration,
+ appId: String,
+ attemptId: String): String = {
+ sparkConf.get(HISTORY_SERVER_ADDRESS)
+ .map { text => SparkHadoopUtil.get.substituteHadoopVariables(text, yarnConf) }
+ .map { address => s"${address}${HistoryServer.UI_PATH_PREFIX}/${appId}/${attemptId}" }
+ .getOrElse("")
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/ea956833/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ApplicationMasterSuite.scala
----------------------------------------------------------------------
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ApplicationMasterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ApplicationMasterSuite.scala
new file mode 100644
index 0000000..695a82f
--- /dev/null
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ApplicationMasterSuite.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.yarn
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+
+class ApplicationMasterSuite extends SparkFunSuite {
+
+ test("history url with hadoop and spark substitutions") {
+ val host = "rm.host.com"
+ val port = 18080
+ val sparkConf = new SparkConf()
+
+ sparkConf.set("spark.yarn.historyServer.address",
+ "http://${hadoopconf-yarn.resourcemanager.hostname}:${spark.history.ui.port}")
+ val yarnConf = new YarnConfiguration()
+ yarnConf.set("yarn.resourcemanager.hostname", host)
+ val appId = "application_123_1"
+ val attemptId = appId + "_1"
+
+ val shsAddr = ApplicationMaster
+ .getHistoryServerAddress(sparkConf, yarnConf, appId, attemptId)
+
+ assert(shsAddr === s"http://${host}:${port}/history/${appId}/${attemptId}")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org