You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2021/08/06 17:54:03 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #897] Support
stop Spark engine through web ui
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 065c98a [KYUUBI #897] Support stop Spark engine through web ui
065c98a is described below
commit 065c98a259f12ebccdf88900ef51329fe13acc22
Author: ulysses-you <ul...@gmail.com>
AuthorDate: Sat Aug 7 01:53:49 2021 +0800
[KYUUBI #897] Support stop Spark engine through web ui
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
Closes https://github.com/apache/incubator-kyuubi/issues/897
### _How was this patch tested?_
The following two images show the stop feature is enabled:
The following image show the stop feature is disabled:
Closes #898 from ulysses-you/kyuubi-897.
Closes #897
61ad25c1 [ulysses-you] address comment
b63997ec [ulysses-you] maven
dbb36014 [ulysses-you] maven
3c91307c [ulysses-you] simplify
6e9e5bca [ulysses-you] pom
4e143657 [ulysses-you] config
e579e227 [ulysses-you] Support stop Spark engine through web ui
Authored-by: ulysses-you <ul...@gmail.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
docs/deployment/settings.md | 1 +
.../org/apache/spark/kyuubi/ui/EnginePage.scala | 25 ++++++++++++--
.../org/apache/spark/kyuubi/ui/EngineTab.scala | 39 ++++++++++++++++++++--
.../org/apache/kyuubi/config/KyuubiConf.scala | 7 ++++
4 files changed, 68 insertions(+), 4 deletions(-)
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index 605418e..61c7f5d 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -153,6 +153,7 @@ kyuubi\.engine\.session<br>\.initialize\.sql|<div style='width: 65pt;word-wrap:
kyuubi\.engine\.share<br>\.level|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>USER</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Engines will be shared in different levels, available configs are: <ul> <li>CONNECTION: engine will not be shared but only used by the current client connection</li> <li>USER: engine will be shared by all sessions created by a unique username, see also kyuubi.engine.share.level.sub.domain</li> <li>SERVER: the Ap [...]
kyuubi\.engine\.share<br>\.level\.sub\.domain|<div style='width: 65pt;word-wrap: break-word;white-space: normal'><undefined></div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Allow end-users to create a sub-domain for the share level of an engine. A sub-domain is a case-insensitive string values in `^[a-zA-Z_]{1,10}$` form. For example, for `USER` share level, an end-user can share a certain engine within a sub-domain, not for all of its clients. End-users a [...]
kyuubi\.engine\.single<br>\.spark\.session|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>false</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>When set to true, this engine is running in a single session mode. All the JDBC/ODBC connections share the temporary views, function registries, SQL configuration and the current database.</div>|<div style='width: 30pt'>boolean</div>|<div style='width: 20pt'>1.3.0</div>
+kyuubi\.engine\.ui\.stop<br>\.enabled|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>true</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>When true, allows Kyuubi engine to be killed from the Spark Web UI.</div>|<div style='width: 30pt'>boolean</div>|<div style='width: 20pt'>1.3.0</div>
### Frontend
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EnginePage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EnginePage.scala
index 1443d5e..f835ef7 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EnginePage.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EnginePage.scala
@@ -27,8 +27,11 @@ import org.apache.spark.ui.UIUtils.formatDurationVerbose
case class EnginePage(parent: EngineTab) extends WebUIPage("") {
override def render(request: HttpServletRequest): Seq[Node] = {
- val content = generateBasicStats() ++
- <br/> ++
+ val content =
+ generateBasicStats() ++
+ <br/> ++
+ stop(request) ++
+ <br/> ++
<h4>
{parent.engine.backendService.sessionManager.getOpenSessionCount} session(s) are online,
running {parent.engine.backendService.sessionManager.operationManager.getOperationCount}
@@ -62,4 +65,22 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") {
</li>
</ul>
}
+
+ private def stop(request: HttpServletRequest): Seq[Node] = {
+ val basePath = UIUtils.prependBaseUri(request, parent.basePath)
+ if (parent.killEnabled) {
+ val confirm =
+ s"if (window.confirm('Are you sure you want to kill kyuubi engine ?')) " +
+ "{ this.parentNode.submit(); return true; } else { return false; }"
+ val stopLinkUri = s"$basePath/kyuubi/stop"
+ <ul class ="list-unstyled">
+ <li>
+ <strong>Stop kyuubi engine: </strong>
+ <a href={stopLinkUri} onclick={confirm} class="stop-link">(kill)</a>
+ </li>
+ </ul>
+ } else {
+ Seq.empty
+ }
+ }
}
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EngineTab.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EngineTab.scala
index a9d8de9..a183584 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EngineTab.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/ui/EngineTab.scala
@@ -17,22 +17,57 @@
package org.apache.spark.kyuubi.ui
+import javax.servlet.http.HttpServletRequest
+
import org.apache.spark.ui.SparkUITab
+import scala.util.control.NonFatal
-import org.apache.kyuubi.Utils
+import org.apache.kyuubi.{Logging, Utils}
+import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.engine.spark.SparkSQLEngine
+import org.apache.kyuubi.service.ServiceState
/**
* Note that [[SparkUITab]] is private for Spark
*/
case class EngineTab(engine: SparkSQLEngine)
- extends SparkUITab(engine.spark.sparkContext.ui.orNull, "kyuubi") {
+ extends SparkUITab(engine.spark.sparkContext.ui.orNull, "kyuubi") with Logging {
override val name: String = "Kyuubi Query Engine"
+ val killEnabled = engine.getConf.get(KyuubiConf.ENGINE_UI_STOP_ENABLED)
engine.spark.sparkContext.ui.foreach { ui =>
this.attachPage(EnginePage(this))
ui.attachTab(this)
Utils.addShutdownHook(() => ui.detachTab(this))
}
+
+ engine.spark.sparkContext.ui.foreach { ui =>
+ try {
+ // Spark shade the jetty package so here we use reflect
+ Class.forName("org.apache.spark.ui.SparkUI")
+ .getMethod("attachHandler",
+ classOf[org.sparkproject.jetty.servlet.ServletContextHandler])
+ .invoke(ui,
+ Class.forName("org.apache.spark.ui.JettyUtils")
+ .getMethod("createRedirectHandler",
+ classOf[String],
+ classOf[String],
+ classOf[(HttpServletRequest) => Unit],
+ classOf[String],
+ classOf[scala.collection.immutable.Set[String]])
+ .invoke(null, "/kyuubi/stop", "/kyuubi", handleKillRequest _, "", Set("GET", "POST"))
+ )
+ } catch {
+ case NonFatal(e) =>
+ warn("Failed to attach handler using SparkUI, please check the Spark version. " +
+ s"So the config '${KyuubiConf.ENGINE_UI_STOP_ENABLED.key}' does not work.", e)
+ }
+ }
+
+ def handleKillRequest(request: HttpServletRequest): Unit = {
+ if (killEnabled && engine != null && engine.getServiceState != ServiceState.STOPPED) {
+ engine.stop()
+ }
+ }
}
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 95f6afc..aa11fcf 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -702,4 +702,11 @@ object KyuubiConf {
.checkValue(_.toSet.subsetOf(Set("SPARK", "JSON", "JDBC", "CUSTOM")),
"Unsupported event loggers")
.createWithDefault(Nil)
+
+ val ENGINE_UI_STOP_ENABLED: ConfigEntry[Boolean] =
+ buildConf("engine.ui.stop.enabled")
+ .doc("When true, allows Kyuubi engine to be killed from the Spark Web UI.")
+ .version("1.3.0")
+ .booleanConf
+ .createWithDefault(true)
}