You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@auron.apache.org by "Copilot (via GitHub)" <gi...@apache.org> on 2026/06/15 16:05:11 UTC
Re: [PR] [AURON #1471] Add the conversion execution list in the SparkUI [auron]
Copilot commented on code in PR #2150:
URL: https://github.com/apache/auron/pull/2150#discussion_r3414870569
##########
auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronAllExecutionsPage.scala:
##########
@@ -77,4 +94,612 @@ private[ui] class AuronAllExecutionsPage(parent: AuronSQLTab) extends WebUIPage(
}
}
+ @sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5")
+ private def buildExecutionsListSummary(
+ executionsList: Seq[AuronSQLExecutionUIData],
+ request: javax.servlet.http.HttpServletRequest): NodeSeq = {
+ val content = {
+ val _content = mutable.ListBuffer[Node]()
+ val executionPage =
+ Option(request.getParameter("auron.page")).map(_.toInt).getOrElse(1)
+
Review Comment:
Parsing the page number with `.toInt` can throw (e.g., `auron.page=abc`) and fail the whole page render. The try/catch below won't catch it because it happens before the `try` block.
##########
auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusListener.scala:
##########
@@ -16,20 +16,27 @@
*/
package org.apache.spark.sql.execution.ui
-import scala.annotation.nowarn
+import scala.collection.mutable
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.internal.Logging
import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
-import org.apache.spark.status.ElementTrackingStore
+import org.apache.spark.sql.internal.StaticSQLConf.UI_RETAINED_EXECUTIONS
+import org.apache.spark.status.{ElementTrackingStore, KVUtils}
-import org.apache.auron.spark.ui.AuronBuildInfoEvent
+import org.apache.auron.spark.ui.{AuronBuildInfoEvent, AuronPlanFallbackEvent}
-@nowarn("cat=unused") // conf temporarily unused
class AuronSQLAppStatusListener(conf: SparkConf, kvstore: ElementTrackingStore)
extends SparkListener
with Logging {
+ private val executionIdToDescription = new mutable.HashMap[Long, String]
+ private val executionIdToFallbackEvent = new mutable.HashMap[Long, AuronPlanFallbackEvent]
+
+ kvstore.addTrigger(classOf[SQLExecutionUIData], conf.get[Int](UI_RETAINED_EXECUTIONS)) {
+ count => cleanupExecutions(count)
+ }
Review Comment:
The KVStore cleanup trigger is registered on SQLExecutionUIData, but cleanupExecutions() deletes AuronSQLExecutionUIData. If AuronSQLExecutionUIData is missing for some SQL executions, this can delete the wrong records (or delete too aggressively). Register the trigger on AuronSQLExecutionUIData so the count passed into cleanup matches the data being deleted.
##########
auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronAllExecutionsPage.scala:
##########
@@ -77,4 +94,612 @@ private[ui] class AuronAllExecutionsPage(parent: AuronSQLTab) extends WebUIPage(
}
}
+ @sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5")
+ private def buildExecutionsListSummary(
+ executionsList: Seq[AuronSQLExecutionUIData],
+ request: javax.servlet.http.HttpServletRequest): NodeSeq = {
+ val content = {
+ val _content = mutable.ListBuffer[Node]()
+ val executionPage =
+ Option(request.getParameter("auron.page")).map(_.toInt).getOrElse(1)
+
+ val auronPageTable =
+ try {
+ new AuronExecutionPagedTable30(
+ request,
+ parent,
+ executionsList,
+ "auron",
+ "auron",
+ UIUtils.prependBaseUri(request, parent.basePath),
+ "auron").table(executionPage)
+ } catch {
+ case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
+ <div class="alert alert-error">
+ <p>Error while rendering execution table:</p>
+ <pre>
+ {Utils.exceptionString(e)}
+ </pre>
+ </div>
+ }
+
+ _content ++=
+ <span id="auron" class="collapse-aggregated-runningExecutions collapse-table"
+ onClick="collapseTable('collapse-aggregated-runningExecutions',
+ 'aggregated-runningExecutions')">
+ <h4>
+ <span class="collapse-table-arrow arrow-open"></span>
+ <a href="#auron">
+ Queries:
+ </a>{executionsList.size}
+ </h4>
+ </span> ++
+ <div class="aggregated-runningExecutions collapsible-table">
+ {auronPageTable}
+ </div>
+
+ _content
+ }
+ content ++=
+ <script>
+ function clickDetail(details) {{
+ details.parentNode.querySelector('.stage-details').classList.toggle('collapsed')
+ }}
+ </script>
+ }
+
+ @sparkver("4.0 / 4.1")
+ private def buildExecutionsListSummary(
+ executionsList: Seq[AuronSQLExecutionUIData],
+ request: jakarta.servlet.http.HttpServletRequest): NodeSeq = {
+ val content = {
+ val _content = mutable.ListBuffer[Node]()
+ val executionPage =
+ Option(request.getParameter("auron.page")).map(_.toInt).getOrElse(1)
+
+ val auronPageTable =
+ try {
+ new AuronExecutionPagedTable40(
+ request,
+ parent,
+ executionsList,
+ "auron",
+ "auron",
+ UIUtils.prependBaseUri(request, parent.basePath),
+ "auron").table(executionPage)
+ } catch {
+ case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
+ <div class="alert alert-error">
+ <p>Error while rendering execution table:</p>
+ <pre>
+ {Utils.exceptionString(e)}
+ </pre>
+ </div>
+ }
+
+ _content ++=
+ <span id="auron" class="collapse-aggregated-runningExecutions collapse-table"
+ onClick="collapseTable('collapse-aggregated-runningExecutions',
+ 'aggregated-runningExecutions')">
+ <h4>
+ <span class="collapse-table-arrow arrow-open"></span>
+ <a href="#auron">
+ Queries:
+ </a>{executionsList.size}
+ </h4>
+ </span> ++
+ <div class="aggregated-runningExecutions collapsible-table">
+ {auronPageTable}
+ </div>
+
+ _content
+ }
+ content ++=
+ <script>
+ function clickDetail(details) {{
+ details.parentNode.querySelector('.stage-details').classList.toggle('collapsed')
+ }}
+ </script>
+ }
+
+ @sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5")
+ class AuronExecutionPagedTable30(
+ request: javax.servlet.http.HttpServletRequest,
+ parent: AuronSQLTab,
+ data: Seq[AuronSQLExecutionUIData],
+ tableHeaderId: String,
+ executionTag: String,
+ basePath: String,
+ subPath: String)
+ extends PagedTable[AuronExecutionTableRowData] {
+
+ private val (sortColumn, desc, pageSize) =
+ getAuronTableParameters(request, executionTag, "ID")
+
+ private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+
+ override val dataSource = new AuronExecutionDataSource(data, pageSize, sortColumn, desc)
+
+ private val parameterPath =
+ s"$basePath/$subPath/?${getAuronParameterOtherTable(request, executionTag)}"
+
+ override def tableId: String = s"$executionTag-table"
+
+ override def tableCssClass: String =
+ "table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"
+
+ override def pageLink(page: Int): String = {
+ parameterPath +
+ s"&$pageNumberFormField=$page" +
+ s"&$executionTag.sort=$encodedSortColumn" +
+ s"&$executionTag.desc=$desc" +
+ s"&$pageSizeFormField=$pageSize" +
+ s"#$tableHeaderId"
+ }
+
+ /**
+ * Returns parameters of other tables in the page.
+ */
+ def getAuronParameterOtherTable(
+ request: javax.servlet.http.HttpServletRequest,
+ tableTag: String): String = {
+ request.getParameterMap.asScala
+ .filterNot(_._1.startsWith(tableTag))
+ .map(parameter => parameter._1 + "=" + parameter._2(0))
+ .mkString("&")
Review Comment:
Query parameters from the request are concatenated into URLs without URL-encoding. Because these values are user-controlled, this can lead to broken links and potential injection/XSS issues when rendered into href attributes.
##########
auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronAllExecutionsPage.scala:
##########
@@ -77,4 +94,612 @@ private[ui] class AuronAllExecutionsPage(parent: AuronSQLTab) extends WebUIPage(
}
}
+ @sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5")
+ private def buildExecutionsListSummary(
+ executionsList: Seq[AuronSQLExecutionUIData],
+ request: javax.servlet.http.HttpServletRequest): NodeSeq = {
+ val content = {
+ val _content = mutable.ListBuffer[Node]()
+ val executionPage =
+ Option(request.getParameter("auron.page")).map(_.toInt).getOrElse(1)
+
+ val auronPageTable =
+ try {
+ new AuronExecutionPagedTable30(
+ request,
+ parent,
+ executionsList,
+ "auron",
+ "auron",
+ UIUtils.prependBaseUri(request, parent.basePath),
+ "auron").table(executionPage)
+ } catch {
+ case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
+ <div class="alert alert-error">
+ <p>Error while rendering execution table:</p>
+ <pre>
+ {Utils.exceptionString(e)}
+ </pre>
+ </div>
+ }
+
+ _content ++=
+ <span id="auron" class="collapse-aggregated-runningExecutions collapse-table"
+ onClick="collapseTable('collapse-aggregated-runningExecutions',
+ 'aggregated-runningExecutions')">
+ <h4>
+ <span class="collapse-table-arrow arrow-open"></span>
+ <a href="#auron">
+ Queries:
+ </a>{executionsList.size}
+ </h4>
+ </span> ++
+ <div class="aggregated-runningExecutions collapsible-table">
+ {auronPageTable}
+ </div>
+
+ _content
+ }
+ content ++=
+ <script>
+ function clickDetail(details) {{
+ details.parentNode.querySelector('.stage-details').classList.toggle('collapsed')
+ }}
+ </script>
+ }
+
+ @sparkver("4.0 / 4.1")
+ private def buildExecutionsListSummary(
+ executionsList: Seq[AuronSQLExecutionUIData],
+ request: jakarta.servlet.http.HttpServletRequest): NodeSeq = {
+ val content = {
+ val _content = mutable.ListBuffer[Node]()
+ val executionPage =
+ Option(request.getParameter("auron.page")).map(_.toInt).getOrElse(1)
+
+ val auronPageTable =
+ try {
+ new AuronExecutionPagedTable40(
+ request,
+ parent,
+ executionsList,
+ "auron",
+ "auron",
+ UIUtils.prependBaseUri(request, parent.basePath),
+ "auron").table(executionPage)
+ } catch {
+ case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
+ <div class="alert alert-error">
+ <p>Error while rendering execution table:</p>
+ <pre>
+ {Utils.exceptionString(e)}
+ </pre>
+ </div>
+ }
+
+ _content ++=
+ <span id="auron" class="collapse-aggregated-runningExecutions collapse-table"
+ onClick="collapseTable('collapse-aggregated-runningExecutions',
+ 'aggregated-runningExecutions')">
+ <h4>
+ <span class="collapse-table-arrow arrow-open"></span>
+ <a href="#auron">
+ Queries:
+ </a>{executionsList.size}
+ </h4>
+ </span> ++
+ <div class="aggregated-runningExecutions collapsible-table">
+ {auronPageTable}
+ </div>
+
+ _content
+ }
+ content ++=
+ <script>
+ function clickDetail(details) {{
+ details.parentNode.querySelector('.stage-details').classList.toggle('collapsed')
+ }}
+ </script>
+ }
+
+ @sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5")
+ class AuronExecutionPagedTable30(
+ request: javax.servlet.http.HttpServletRequest,
+ parent: AuronSQLTab,
+ data: Seq[AuronSQLExecutionUIData],
+ tableHeaderId: String,
+ executionTag: String,
+ basePath: String,
+ subPath: String)
+ extends PagedTable[AuronExecutionTableRowData] {
+
+ private val (sortColumn, desc, pageSize) =
+ getAuronTableParameters(request, executionTag, "ID")
+
+ private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+
+ override val dataSource = new AuronExecutionDataSource(data, pageSize, sortColumn, desc)
+
+ private val parameterPath =
+ s"$basePath/$subPath/?${getAuronParameterOtherTable(request, executionTag)}"
+
+ override def tableId: String = s"$executionTag-table"
+
+ override def tableCssClass: String =
+ "table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"
+
+ override def pageLink(page: Int): String = {
+ parameterPath +
+ s"&$pageNumberFormField=$page" +
+ s"&$executionTag.sort=$encodedSortColumn" +
+ s"&$executionTag.desc=$desc" +
+ s"&$pageSizeFormField=$pageSize" +
+ s"#$tableHeaderId"
+ }
+
+ /**
+ * Returns parameters of other tables in the page.
+ */
+ def getAuronParameterOtherTable(
+ request: javax.servlet.http.HttpServletRequest,
+ tableTag: String): String = {
+ request.getParameterMap.asScala
+ .filterNot(_._1.startsWith(tableTag))
+ .map(parameter => parameter._1 + "=" + parameter._2(0))
+ .mkString("&")
+ }
+
+ /**
+ * Returns parameter of this table.
+ */
+ def getAuronTableParameters(
+ request: javax.servlet.http.HttpServletRequest,
+ tableTag: String,
+ defaultSortColumn: String): (String, Boolean, Int) = {
+ val parameterSortColumn = request.getParameter(s"$tableTag.sort")
+ val parameterSortDesc = request.getParameter(s"$tableTag.desc")
+ val parameterPageSize = request.getParameter(s"$tableTag.pageSize")
+ val sortColumn = Option(parameterSortColumn)
+ .map { sortColumn =>
+ UIUtils.decodeURLParameter(sortColumn)
+ }
+ .getOrElse(defaultSortColumn)
+ val desc =
+ Option(parameterSortDesc).map(_.toBoolean).getOrElse(sortColumn == defaultSortColumn)
+ val pageSize = Option(parameterPageSize).map(_.toInt).getOrElse(100)
+
+ (sortColumn, desc, pageSize)
+ }
+
+ override def pageSizeFormField: String = s"$executionTag.pageSize"
+
+ override def pageNumberFormField: String = s"$executionTag.page"
+
+ override def goButtonFormPath: String =
+ s"$parameterPath&$executionTag.sort=$encodedSortColumn&$executionTag.desc=$desc#$tableHeaderId"
+
+ // Information for each header: title, sortable, tooltip
+ private val headerInfo: Seq[(String, Boolean, Option[String])] = {
+ Seq(
+ ("ID", true, None),
+ ("Description", true, None),
+ ("Num Auron Nodes", true, None),
+ ("Num Fallback Nodes", true, None))
+ }
+
+ override def headers: Seq[Node] = {
+ isAuronSortColumnValid(headerInfo, sortColumn)
+
+ headerAuronRow(
+ headerInfo,
+ desc,
+ pageSize,
+ sortColumn,
+ parameterPath,
+ executionTag,
+ tableHeaderId)
+ }
+
+ def headerAuronRow(
+ headerInfo: Seq[(String, Boolean, Option[String])],
+ desc: Boolean,
+ pageSize: Int,
+ sortColumn: String,
+ parameterPath: String,
+ tableTag: String,
+ headerId: String): Seq[Node] = {
+ val row: Seq[Node] = {
+ headerInfo.map { case (header, sortable, tooltip) =>
+ if (header == sortColumn) {
+ val headerLink = Unparsed(
+ parameterPath +
+ s"&$tableTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
+ s"&$tableTag.desc=${!desc}" +
+ s"&$tableTag.pageSize=$pageSize" +
+ s"#$headerId")
+ val arrow = if (desc) "▾" else "▴" // UP or DOWN
+
+ <th>
+ <a href={headerLink}>
+ <span data-toggle="tooltip" data-placement="top" title={tooltip.getOrElse("")}>
+ {header} {Unparsed(arrow)}
+ </span>
+ </a>
+ </th>
+ } else {
+ if (sortable) {
+ val headerLink = Unparsed(
+ parameterPath +
+ s"&$tableTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
+ s"&$tableTag.pageSize=$pageSize" +
+ s"#$headerId")
+
+ <th>
+ <a href={headerLink}>
+ <span data-toggle="tooltip" data-placement="top" title={tooltip.getOrElse("")}>
+ {header}
+ </span>
+ </a>
+ </th>
+ } else {
+ <th>
+ <span data-toggle="tooltip" data-placement="top" title={tooltip.getOrElse("")}>
+ {header}
+ </span>
+ </th>
+ }
+ }
+ }
+ }
+ <thead>
+ <tr>
+ {row}
+ </tr>
+ </thead>
+ }
+
+ def isAuronSortColumnValid(
+ headerInfo: Seq[(String, Boolean, Option[String])],
+ sortColumn: String): Unit = {
+ if (!headerInfo.filter(_._2).map(_._1).contains(sortColumn)) {
+ throw new IllegalArgumentException(s"Unknown column: $sortColumn")
+ }
+ }
+
+ override def row(executionTableRow: AuronExecutionTableRowData): Seq[Node] = {
+ val executionUIData = executionTableRow.executionUIData
+
+ <tr>
+ <td>
+ {executionUIData.executionId.toString}
+ </td>
+ <td>
+ {descriptionCell(executionUIData)}
+ </td>
+ <td sorttable_customkey={executionUIData.numAuronNodes.toString}>
+ {executionUIData.numAuronNodes.toString}
+ </td>
+ <td sorttable_customkey={executionUIData.numFallbackNodes.toString}>
+ {executionUIData.numFallbackNodes.toString}
+ </td>
+ </tr>
+ }
+
+ private def descriptionCell(execution: AuronSQLExecutionUIData): Seq[Node] = {
+ val details = if (execution.description != null && execution.description.nonEmpty) {
+ val concat = new PlanStringConcat()
+ concat.append("== Fallback Summary ==\n")
+ val fallbackSummary = execution.fallbackNodeToReason
+ .map { case (name, reason) =>
+ val id = name.substring(0, 3)
+ val nodeName = name.substring(4)
+ s"(${id.toInt}) $nodeName: $reason"
+ }
+ .mkString("\n")
+ concat.append(fallbackSummary)
+ if (execution.fallbackNodeToReason.isEmpty) {
+ concat.append("No fallback nodes")
+ }
+ concat.append("\n\n")
+ concat.append(execution.fallbackDescription)
+
+ <span onclick="this.parentNode.querySelector('.stage-details').classList.toggle('collapsed')"
+ class="expand-details">
+ +details
+ </span> ++
+ <div class="stage-details collapsed">
+ <pre>
+ {concat.toString()}
+ </pre>
+ </div>
+ } else {
+ Nil
+ }
+
+ val desc = if (execution.description != null && execution.description.nonEmpty) {
+ <a href={executionURL(execution.executionId)} class="description-input">
+ {execution.description}
+ </a>
+ } else {
+ <a href={executionURL(execution.executionId)}>
+ {execution.executionId}
+ </a>
+ }
+
+ <div>
+ {desc}{details}
+ </div>
+ }
+
+ private def executionURL(executionID: Long): String =
+ s"${UIUtils.prependBaseUri(request, parent.basePath)}/SQL/execution/?id=$executionID"
+ }
+
+ @sparkver("4.0 / 4.1")
+ private[ui] class AuronExecutionPagedTable40(
+ request: jakarta.servlet.http.HttpServletRequest,
+ parent: AuronSQLTab,
+ data: Seq[AuronSQLExecutionUIData],
+ tableHeaderId: String,
+ executionTag: String,
+ basePath: String,
+ subPath: String)
+ extends PagedTable[AuronExecutionTableRowData] {
+
+ private val (sortColumn, desc, pageSize) =
+ getAuronTableParameters(request, executionTag, "ID")
+
+ private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
+
+ override val dataSource = new AuronExecutionDataSource(data, pageSize, sortColumn, desc)
+
+ private val parameterPath =
+ s"$basePath/$subPath/?${getAuronParameterOtherTable(request, executionTag)}"
+
+ override def tableId: String = s"$executionTag-table"
+
+ override def tableCssClass: String =
+ "table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"
+
+ override def pageLink(page: Int): String = {
+ parameterPath +
+ s"&$pageNumberFormField=$page" +
+ s"&$executionTag.sort=$encodedSortColumn" +
+ s"&$executionTag.desc=$desc" +
+ s"&$pageSizeFormField=$pageSize" +
+ s"#$tableHeaderId"
+ }
+
+ /**
+ * Returns parameters of other tables in the page.
+ */
+ def getAuronParameterOtherTable(
+ request: jakarta.servlet.http.HttpServletRequest,
+ tableTag: String): String = {
+ request.getParameterMap.asScala
+ .filterNot(_._1.startsWith(tableTag))
+ .map(parameter => parameter._1 + "=" + parameter._2(0))
+ .mkString("&")
Review Comment:
Query parameters from the request are concatenated into URLs without URL-encoding. Because these values are user-controlled, this can lead to broken links and potential injection/XSS issues when rendered into href attributes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@auron.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org