You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2022/04/27 03:50:25 UTC
[incubator-kyuubi] branch branch-1.5 updated: [KYUUBI #2484] Add conf to SessionEvent and display it in EngineSessionPage
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/branch-1.5 by this push:
new 87f81e3c9 [KYUUBI #2484] Add conf to SessionEvent and display it in EngineSessionPage
87f81e3c9 is described below
commit 87f81e3c9b6e00c43f160f8903273c44bb51d5b6
Author: wforget <64...@qq.com>
AuthorDate: Wed Apr 27 11:50:04 2022 +0800
[KYUUBI #2484] Add conf to SessionEvent and display it in EngineSessionPage
### _Why are the changes needed?_
close #2484
### _How was this patch tested?_
- [X] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [X] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #2485 from wForget/KYUUBI-2484.
Closes #2484
b527caee [wforget] sorted
88a02820 [wforget] add test
5c3bb304 [wforget] [KYUUBI-2484] Add conf to SessionEvent and display it in EngineSessionPage
Authored-by: wforget <64...@qq.com>
Signed-off-by: Fei Wang <fw...@ebay.com>
(cherry picked from commit 06da8cf8966d0a4d8b29e1884e98db2eec483bce)
Signed-off-by: Fei Wang <fw...@ebay.com>
---
.../kyuubi/engine/spark/events/SessionEvent.scala | 2 ++
.../org/apache/spark/ui/EngineSessionPage.scala | 34 ++++++++++++++++++++++
.../scala/org/apache/spark/ui/EngineTabSuite.scala | 27 +++++++++++++++++
3 files changed, 63 insertions(+)
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SessionEvent.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SessionEvent.scala
index b67a6762b..8d87b1cd9 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SessionEvent.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SessionEvent.scala
@@ -43,6 +43,7 @@ case class SessionEvent(
username: String,
ip: String,
serverIp: String,
+ conf: Map[String, String],
startTime: Long,
var endTime: Long = -1L,
var totalOperations: Int = 0) extends KyuubiEvent with SparkListenerEvent {
@@ -70,6 +71,7 @@ object SessionEvent {
session.user,
session.ipAddress,
session.serverIpAddress,
+ session.conf,
session.createTime)
}
}
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala
index 18c12e3d0..9fe8e5ac8 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala
@@ -23,6 +23,7 @@ import javax.servlet.http.HttpServletRequest
import scala.xml.Node
import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.SECRET_REDACTION_PATTERN
import org.apache.spark.ui.UIUtils._
import org.apache.spark.util.Utils
@@ -31,6 +32,10 @@ case class EngineSessionPage(parent: EngineTab)
extends WebUIPage("session") with Logging {
val store = parent.store
+ private def propertyHeader = Seq("Name", "Value")
+ private def headerClasses = Seq("sorttable_alpha", "sorttable_alpha")
+ private def propertyRow(kv: (String, String)) = <tr><td>{kv._1}</td><td>{kv._2}</td></tr>
+
/** Render the page */
def render(request: HttpServletRequest): Seq[Node] = {
val parameterId = request.getParameter("id")
@@ -40,6 +45,34 @@ case class EngineSessionPage(parent: EngineTab)
val sessionStat = store.getSession(parameterId).getOrElse(null)
require(sessionStat != null, "Invalid sessionID[" + parameterId + "]")
+ val redactionPattern = parent.sparkUI match {
+ case Some(ui) => Some(ui.conf.get(SECRET_REDACTION_PATTERN))
+ case None => SECRET_REDACTION_PATTERN.defaultValue
+ }
+
+ val sessionPropertiesTable =
+ if (sessionStat.conf != null && !sessionStat.conf.isEmpty) {
+ val table = UIUtils.listingTable(
+ propertyHeader,
+ propertyRow,
+ Utils.redact(redactionPattern, sessionStat.conf.toSeq.sorted),
+ fixedWidth = true,
+ headerClasses = headerClasses)
+ <span class="collapse-aggregated-kyuubiSessioinProperties collapse-table"
+ onClick="collapseTable('collapse-aggregated-kyuubiSessioinProperties',
+ 'aggregated-kyuubiSessioinProperties')">
+ <h4>
+ <span class="collapse-table-arrow arrow-open"></span>
+ <a>Session Properties</a>
+ </h4>
+ </span>
+ <div class="aggregated-kyuubiSessioinProperties collapsible-table">
+ {table}
+ </div>
+ } else {
+ Seq()
+ }
+
generateBasicStats() ++
<br/> ++
<h4>
@@ -49,6 +82,7 @@ case class EngineSessionPage(parent: EngineTab)
Session created at {formatDate(sessionStat.startTime)},
Total run {sessionStat.totalOperations} SQL
</h4> ++
+ sessionPropertiesTable ++
generateSQLStatsTable(request, sessionStat.sessionId)
}
UIUtils.headerSparkPage(request, parent.name + " Session", content, parent)
diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala
index cb3ef2403..3d27935de 100644
--- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala
+++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala
@@ -23,6 +23,7 @@ import org.apache.http.util.EntityUtils
import org.apache.kyuubi.engine.spark.WithSparkSQLEngine
import org.apache.kyuubi.operation.HiveJDBCTestHelper
+import org.apache.kyuubi.session.SessionHandle
class EngineTabSuite extends WithSparkSQLEngine with HiveJDBCTestHelper {
override def withKyuubiConf: Map[String, String] = Map(
@@ -164,5 +165,31 @@ class EngineTabSuite extends WithSparkSQLEngine with HiveJDBCTestHelper {
}
}
+ test("session properties for engine tab") {
+ assert(spark.sparkContext.ui.nonEmpty)
+ val redactKey = "kyuubi.test.password"
+ val redactValue = "testPassword"
+ val testKey = "kyuubi.test.key"
+ val testValue = "testValue"
+ withSessionConf(Map(
+ redactKey -> redactValue,
+ testKey -> testValue))(Map.empty)(Map.empty) {
+ withSessionHandle { (_, handle) =>
+ val kyuubiHandle = SessionHandle(handle)
+ val httpClient = HttpClients.createDefault()
+ val req = new HttpGet(spark.sparkContext.uiWebUrl.get +
+ s"/kyuubi/session/?id=${kyuubiHandle.identifier}")
+ val response = httpClient.execute(req)
+ assert(response.getStatusLine.getStatusCode === 200)
+ val resp = EntityUtils.toString(response.getEntity)
+ assert(resp.contains("Session Properties"))
+ assert(resp.contains(redactKey))
+ assert(!resp.contains(redactValue))
+ assert(resp.contains(testKey))
+ assert(resp.contains(testValue))
+ }
+ }
+ }
+
override protected def jdbcUrl: String = getJdbcUrl
}