You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/03/31 13:05:14 UTC
(spark) branch master updated: [SPARK-56137][UI][TESTS] Add regression tests for SQL tab DataTables migration
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a67f1eca72c4 [SPARK-56137][UI][TESTS] Add regression tests for SQL tab DataTables migration
a67f1eca72c4 is described below
commit a67f1eca72c48dd5ece5c4b58b773239b98aa7af
Author: Kent Yao <ke...@microsoft.com>
AuthorDate: Tue Mar 31 06:05:05 2026 -0700
[SPARK-56137][UI][TESTS] Add regression tests for SQL tab DataTables migration
### What changes were proposed in this pull request?
Add regression tests for the SQL tab DataTables migration (SPARK-55875 umbrella).
**`AllExecutionsPageSuite`** — 4 new tests:
- DataTables CSS/JS resource inclusion verification
- `group-sub-exec` config propagation (default and explicit)
- Loading spinner rendering
- REST API integration (run query → verify `/api/v1/.../sql` endpoint)
**`SqlResourceWithActualMetricsSuite`** — 4 new tests:
- ISO date format validation for `submissionTime`
- No artificial result limit (5 queries all returned)
- `planDescription` in detail endpoint
- Multiple query types (DDL, DML, SELECT) all appear with COMPLETED status
### Why are the changes needed?
The SQL tab was migrated from server-side rendered HTML tables to client-side DataTables. These regression tests ensure correctness of:
1. Page skeleton rendering (resources, config, spinner)
2. REST API responses (date format, no limit, status, plan description)
3. Coverage across query types (DDL, DML, SELECT)
### Does this PR introduce _any_ user-facing change?
No. Test-only changes.
### How was this patch tested?
All 12 tests pass:
- `AllExecutionsPageWithInMemoryStoreSuite` (5 tests)
- `AllExecutionsPageWithRocksDBBackendSuite` (5 tests)
- `SqlResourceWithActualMetricsSuite` (8 tests)
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: GitHub Copilot (Claude Opus 4.6)
Closes #55111 from yaooqinn/SPARK-56137.
Authored-by: Kent Yao <ke...@microsoft.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../sql/execution/ui/AllExecutionsPageSuite.scala | 102 ++++++++++++++++-
.../v1/sql/SqlResourceWithActualMetricsSuite.scala | 121 +++++++++++++++++++++
2 files changed, 222 insertions(+), 1 deletion(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
index d7615493cccb..2bc4e53611ba 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
@@ -17,14 +17,18 @@
package org.apache.spark.sql.execution.ui
+import java.net.URI
import java.util.Locale
import jakarta.servlet.http.HttpServletRequest
import org.mockito.Mockito.{mock, when, RETURNS_SMART_NULLS}
import org.scalatest.BeforeAndAfter
+import org.scalatest.time.SpanSugar._
import org.apache.spark.SparkConf
+import org.apache.spark.deploy.history.HistoryServerSuite.getContentAndCode
import org.apache.spark.internal.config.Status.LIVE_UI_LOCAL_STORE_DIR
+import org.apache.spark.internal.config.UI.UI_SQL_GROUP_SUB_EXECUTION_ENABLED
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.status.{AppStatusStore, ElementTrackingStore}
import org.apache.spark.tags.SlowSQLTest
@@ -35,7 +39,9 @@ abstract class AllExecutionsPageSuite extends SharedSparkSession with BeforeAndA
override def sparkConf: SparkConf = {
// Disable async kv store write in the UI, to make tests more stable here.
- super.sparkConf.set(org.apache.spark.internal.config.Status.ASYNC_TRACKING_ENABLED, false)
+ super.sparkConf
+ .set(org.apache.spark.internal.config.Status.ASYNC_TRACKING_ENABLED, false)
+ .set("spark.ui.enabled", "true")
}
var kvstore: ElementTrackingStore = _
@@ -64,6 +70,100 @@ abstract class AllExecutionsPageSuite extends SharedSparkSession with BeforeAndA
assert(html.contains("datatables"))
}
+ test("SPARK-56137: page includes DataTables CSS and JS resources") {
+ val statusStore = createStatusStore
+ val tab = mock(classOf[SQLTab], RETURNS_SMART_NULLS)
+ when(tab.sqlStore).thenReturn(statusStore)
+
+ val request = mock(classOf[HttpServletRequest])
+ when(tab.conf).thenReturn(new SparkConf(false))
+ when(tab.appName).thenReturn("testing")
+ when(tab.headerTabs).thenReturn(Seq.empty)
+
+ val page = new AllExecutionsPage(tab)
+ val html = page.render(request).toString().toLowerCase(Locale.ROOT)
+ // DataTables CSS
+ assert(html.contains("datatables.bootstrap5.min.css"))
+ assert(html.contains("jquery.datatables.min.css"))
+ assert(html.contains("webui-datatables.css"))
+ // DataTables JS
+ assert(html.contains("jquery.datatables.min.js"))
+ assert(html.contains("datatables.bootstrap5.min.js"))
+ // jQuery
+ assert(html.contains("jquery.min.js"))
+ }
+
+ test("SPARK-56137: group-sub-exec config propagation") {
+ val statusStore = createStatusStore
+ val tab = mock(classOf[SQLTab], RETURNS_SMART_NULLS)
+ when(tab.sqlStore).thenReturn(statusStore)
+
+ val request = mock(classOf[HttpServletRequest])
+ when(tab.appName).thenReturn("testing")
+ when(tab.headerTabs).thenReturn(Seq.empty)
+
+ // Default config should propagate true
+ when(tab.conf).thenReturn(new SparkConf(false))
+ val page1 = new AllExecutionsPage(tab)
+ val html1 = page1.render(request).toString()
+ assert(html1.contains("data-value=\"true\""),
+ "Default group-sub-exec config should be true")
+
+ // Explicitly set to false
+ val confFalse = new SparkConf(false)
+ .set(UI_SQL_GROUP_SUB_EXECUTION_ENABLED.key, "false")
+ when(tab.conf).thenReturn(confFalse)
+ val page2 = new AllExecutionsPage(tab)
+ val html2 = page2.render(request).toString()
+ assert(html2.contains("data-value=\"false\""),
+ "Config set to false should propagate as false")
+ }
+
+ test("SPARK-56137: page renders loading spinner") {
+ val statusStore = createStatusStore
+ val tab = mock(classOf[SQLTab], RETURNS_SMART_NULLS)
+ when(tab.sqlStore).thenReturn(statusStore)
+
+ val request = mock(classOf[HttpServletRequest])
+ when(tab.conf).thenReturn(new SparkConf(false))
+ when(tab.appName).thenReturn("testing")
+ when(tab.headerTabs).thenReturn(Seq.empty)
+
+ val page = new AllExecutionsPage(tab)
+ val html = page.render(request).toString().toLowerCase(Locale.ROOT)
+ assert(html.contains("spinner-border"),
+ "Should have spinner element")
+ assert(html.contains("text-primary"),
+ "Spinner should use primary color")
+ assert(html.contains("loading..."),
+ "Should have loading text")
+ }
+
+ test("SPARK-56137: REST API returns data after SQL queries") {
+ spark.sql("SELECT 1 AS spark_56137_rest_test").collect()
+
+ val baseUrl = spark.sparkContext.ui.get.webUrl
+ val appId = spark.sparkContext.applicationId
+ val url =
+ new URI(s"$baseUrl/api/v1/applications/$appId/sql").toURL
+
+ eventually(timeout(10.seconds), interval(50.milliseconds)) {
+ val (code, resultOpt, error) = getContentAndCode(url)
+ assert(code == 200, s"Expected HTTP 200 but got: $code")
+ assert(resultOpt.nonEmpty,
+ "REST response should not be empty")
+ assert(error.isEmpty,
+ s"Error should be empty but got: $error")
+ val result = resultOpt.get
+ assert(result.startsWith("["),
+ "Response should be a JSON array")
+ assert(result.contains("\"status\""),
+ "Response should contain status field")
+ assert(result.contains("COMPLETED"),
+ "Should have completed executions")
+ }
+ }
+
protected def createStatusStore: SQLAppStatusStore
}
diff --git a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
index 5790abce4202..e375678157b2 100644
--- a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.status.api.v1.sql
import java.net.{URI, URL}
import java.text.SimpleDateFormat
+import java.util.Locale
import jakarta.servlet.http.HttpServletResponse
import org.json4s.DefaultFormats
@@ -214,4 +215,124 @@ class SqlResourceWithActualMetricsSuite
}
}
}
+
+ test("SPARK-56137: sqlList returns ISO date format in submissionTime") {
+ withSQLConf(ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+ spark.sql("SELECT 'date_format_test'").collect()
+ }
+
+ val url = new URI(spark.sparkContext.ui.get.webUrl +
+ s"/api/v1/applications/${spark.sparkContext.applicationId}" +
+ "/sql").toURL
+ eventually(timeout(10.seconds), interval(50.milliseconds)) {
+ val result = verifyAndGetSqlRestResult(url)
+ // yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'
+ val datePattern =
+ """"submissionTime"\s*:\s*"\d{4}-\d{2}-\d{2}T""" +
+ """\d{2}:\d{2}:\d{2}\.\d{3}GMT"""
+ assert(datePattern.r.findFirstIn(result).isDefined,
+ "submissionTime should match ISO date format with GMT")
+ }
+ }
+
+ test("SPARK-56137: sqlList returns all executions without limit") {
+ val url = new URI(spark.sparkContext.ui.get.webUrl +
+ s"/api/v1/applications/${spark.sparkContext.applicationId}" +
+ "/sql").toURL
+
+ val startCount = {
+ val result = verifyAndGetSqlRestResult(url)
+ JsonMethods.parse(result).extract[Seq[ExecutionData]].size
+ }
+
+ withSQLConf(ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+ for (i <- 1 to 5) {
+ spark.sql(s"SELECT $i AS limit_test").collect()
+ }
+ }
+
+ eventually(timeout(10.seconds), interval(50.milliseconds)) {
+ val result = verifyAndGetSqlRestResult(url)
+ val executions =
+ JsonMethods.parse(result).extract[Seq[ExecutionData]]
+ assert(executions.size >= startCount + 5,
+ s"Expected at least ${startCount + 5} executions " +
+ s"but got: ${executions.size}")
+ }
+ }
+
+ test("SPARK-56137: execution detail returns planDescription") {
+ withSQLConf(ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+ spark.sql("SELECT 'plan_desc_test'").collect()
+ }
+
+ val baseUrl = spark.sparkContext.ui.get.webUrl
+ val appId = spark.sparkContext.applicationId
+ val listUrl = new URI(
+ s"$baseUrl/api/v1/applications/$appId/sql").toURL
+
+ eventually(timeout(10.seconds), interval(50.milliseconds)) {
+ val listResult = verifyAndGetSqlRestResult(listUrl)
+ val executions = JsonMethods.parse(listResult)
+ .extract[Seq[ExecutionData]]
+ assert(executions.nonEmpty,
+ "Should have at least one execution")
+
+ val execId = executions.head.id
+ val detailUrl = new URI(
+ s"$baseUrl/api/v1/applications/$appId/sql/$execId"
+ ).toURL
+ val detailResult = verifyAndGetSqlRestResult(detailUrl)
+ val execution = JsonMethods.parse(detailResult)
+ .extract[ExecutionData]
+ assert(execution.planDescription != null &&
+ execution.planDescription.nonEmpty,
+ "planDescription should not be empty")
+ assert(execution.nodes.nonEmpty,
+ "nodes should not be empty in detail response")
+ }
+ }
+
+ test("SPARK-56137: multiple query types appear in listing") {
+ withSQLConf(ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+ withTable("spark_56137_multi") {
+ // DDL
+ spark.sql(
+ "CREATE TABLE spark_56137_multi " +
+ "(id INT, name STRING) USING parquet")
+ // DML
+ spark.sql(
+ "INSERT INTO spark_56137_multi VALUES (1, 'test')")
+ // SELECT
+ spark.sql(
+ "SELECT * FROM spark_56137_multi").collect()
+
+ val url = new URI(spark.sparkContext.ui.get.webUrl +
+ s"/api/v1/applications/" +
+ s"${spark.sparkContext.applicationId}/sql").toURL
+ eventually(
+ timeout(20.seconds), interval(50.milliseconds)) {
+ val result = verifyAndGetSqlRestResult(url)
+ val testExecs = JsonMethods.parse(result)
+ .extract[Seq[ExecutionData]]
+ .filter { e =>
+ val desc =
+ Option(e.description).getOrElse("")
+ val plan =
+ Option(e.planDescription).getOrElse("")
+ val text = (desc + " " + plan).toLowerCase(Locale.ROOT)
+ text.contains("spark_56137_multi")
+ }
+ assert(testExecs.size >= 3,
+ s"Expected at least 3 executions " +
+ s"(DDL, DML, SELECT), got: ${testExecs.size}")
+ testExecs.foreach { e =>
+ assert(e.status == "COMPLETED",
+ s"Execution ${e.id} should be COMPLETED " +
+ s"but was ${e.status}")
+ }
+ }
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org