You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2026/10/13 06:39:38 UTC
(spark) branch branch-4.x updated: [SPARK-60192][SQL] Include supported YAML versions in invalid version errors
This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 5cd371c52384 [SPARK-60192][SQL] Include supported YAML versions in invalid version errors
5cd371c52384 is described below
commit 5cd371c5238490959749e2627e75780d920e17a6
Author: Marcus Lin <ml...@gmail.com>
AuthorDate: Tue Oct 13 06:38:20 2026 +0000
[SPARK-60192][SQL] Include supported YAML versions in invalid version errors
### What changes were proposed in this pull request?
This pull request updates invalid metric view YAML version errors to include the versions that
Spark supports.
It also centralizes version-specific deserialization and serialization in one ordered registry so
the supported-version list cannot drift from the actual handlers. Factory-level and SQL-facing
regression tests cover the updated behavior.
### Why are the changes needed?
The existing error reports only the invalid version. Users must otherwise inspect documentation or
source code to discover which versions are accepted. Listing the supported versions makes the error
actionable, while the shared registry keeps that information consistent as versions are added.
### Does this PR introduce _any_ user-facing change?
Yes. An invalid metric view YAML version now produces an error such as:
```
Invalid YAML version: 1.1. Supported YAML versions: 0.1
```
Previously, the error stopped after `Invalid YAML version: 1.1`.
### How was this patch tested?
Added regression coverage to `MetricViewFactorySuite` for deserialization and serialization, and to
`SimpleMetricViewSuite` for the SQL-facing structured error.
```
build/sbt catalyst/scalastyle catalyst/Test/scalastyle sql/scalastyle sql/Test/scalastyle \
'catalyst/testOnly org.apache.spark.sql.metricview.serde.MetricViewFactorySuite' \
'sql/testOnly org.apache.spark.sql.execution.SimpleMetricViewSuite'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (used for code assistance).
Closes #59405 from marcuslin123/codex/spark-60192-yaml-versions.
Authored-by: Marcus Lin <ml...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 0aef0cb14d37a1b344b6670af6218debbf06c37f)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../sql/metricview/serde/MetricViewFactory.scala | 41 ++++++++++++----------
.../sql/metricview/serde/MetricViewSerDeBase.scala | 16 ---------
.../metricview/serde/MetricViewFactorySuite.scala | 17 ++++++++-
.../spark/sql/execution/MetricViewSuite.scala | 18 ++++++++++
4 files changed, 56 insertions(+), 36 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/metricview/serde/MetricViewFactory.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/metricview/serde/MetricViewFactory.scala
index ced4c7e635e3..b8bcee06fcf9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/metricview/serde/MetricViewFactory.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/metricview/serde/MetricViewFactory.scala
@@ -20,18 +20,31 @@ package org.apache.spark.sql.metricview.serde
import scala.util.control.NonFatal
private[sql] object MetricViewFactory {
+ private case class YAMLVersionHandler(
+ deserialize: String => MetricView,
+ serialize: MetricView => String)
+
+ private val versionHandlers = Seq(
+ "0.1" -> YAMLVersionHandler(
+ yamlContent => MetricViewYAMLDeserializerV01.parseYaml(yamlContent).toCanonical,
+ metricView => MetricViewYAMLSerializerV01.toYaml(MetricViewV01.fromCanonical(metricView))))
+
+ private def invalidVersionMessage(version: String): String = {
+ s"Invalid YAML version: $version. " +
+ s"Supported YAML versions: ${versionHandlers.map(_._1).mkString(", ")}"
+ }
+
+ private def versionHandler(version: String): YAMLVersionHandler = {
+ versionHandlers.find(_._1 == version).map(_._2).getOrElse {
+ throw MetricViewValidationException(invalidVersionMessage(version))
+ }
+ }
+
def fromYAML(yamlContent: String): MetricView = {
try {
val yamlVersion =
YamlMapperProviderV01.mapperWithAllFields.readValue(yamlContent, classOf[YAMLVersion])
- yamlVersion.version match {
- case "0.1" =>
- MetricViewYAMLDeserializerV01.parseYaml(yamlContent).toCanonical
- case _ =>
- throw MetricViewValidationException(
- s"Invalid YAML version: ${yamlVersion.version}"
- )
- }
+ versionHandler(yamlVersion.version).deserialize(yamlContent)
} catch {
case e: MetricViewSerdeException =>
throw e
@@ -45,17 +58,7 @@ private[sql] object MetricViewFactory {
def toYAML(metricView: MetricView): String = {
try {
- val versionSpecific = MetricViewBase.fromCanonical(metricView)
- versionSpecific.version match {
- case "0.1" =>
- MetricViewYAMLSerializerV01.toYaml(
- versionSpecific.asInstanceOf[MetricViewV01]
- )
- case _ =>
- throw MetricViewValidationException(
- s"Invalid YAML version: ${metricView.version}"
- )
- }
+ versionHandler(metricView.version).serialize(metricView)
} catch {
case e: MetricViewSerdeException =>
throw e
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/metricview/serde/MetricViewSerDeBase.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/metricview/serde/MetricViewSerDeBase.scala
index 137e34e53a9d..c8dad9cee9fa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/metricview/serde/MetricViewSerDeBase.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/metricview/serde/MetricViewSerDeBase.scala
@@ -171,19 +171,3 @@ private[sql] trait MetricViewBase {
)
}
}
-
-private[sql] object MetricViewBase {
- /**
- * Factory method to create the appropriate version-specific MetricView from canonical form.
- * @param canonical The canonical MetricView to convert from
- * @return The appropriate version-specific MetricView
- */
- def fromCanonical(canonical: MetricView): MetricViewBase = {
- canonical.version match {
- case "0.1" =>
- MetricViewV01.fromCanonical(canonical)
- case _ =>
- throw new IllegalArgumentException(s"Unsupported version: ${canonical.version}")
- }
- }
-}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/metricview/serde/MetricViewFactorySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/metricview/serde/MetricViewFactorySuite.scala
index 77d2b1cfbf3e..6027c5d9fe76 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/metricview/serde/MetricViewFactorySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/metricview/serde/MetricViewFactorySuite.scala
@@ -141,7 +141,8 @@ class MetricViewFactorySuite extends SparkFunSuite {
val exception = intercept[MetricViewValidationException] {
MetricViewFactory.fromYAML(yaml)
}
- assert(exception.getMessage.contains("Invalid YAML version: 99.9"))
+ assert(exception.getMessage ===
+ "Invalid YAML version: 99.9. Supported YAML versions: 0.1")
}
test("fromYAML - malformed YAML") {
@@ -153,6 +154,20 @@ class MetricViewFactorySuite extends SparkFunSuite {
assert(exception.getMessage.contains("Failed to parse YAML"))
}
+ test("toYAML - invalid YAML version") {
+ val metricView = MetricView(
+ version = "99.9",
+ from = AssetSource("my_table"),
+ where = None,
+ select = Seq.empty)
+
+ val exception = intercept[MetricViewValidationException] {
+ MetricViewFactory.toYAML(metricView)
+ }
+ assert(exception.getMessage ===
+ "Invalid YAML version: 99.9. Supported YAML versions: 0.1")
+ }
+
test("toYAML - serialize basic metric view") {
val metricView = MetricView(
version = "0.1",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/MetricViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/MetricViewSuite.scala
index 2c156549142d..b583c336379f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/MetricViewSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/MetricViewSuite.scala
@@ -82,6 +82,24 @@ abstract class MetricViewSuite extends QueryTest {
}
}
+ test("SPARK-60192: invalid metric view YAML version lists supported versions") {
+ val exception = intercept[AnalysisException] {
+ sql(
+ """
+ |CREATE VIEW invalid_version_metric_view
+ |WITH METRICS
+ |LANGUAGE YAML
+ |AS $$ version: 1.1 $$
+ |""".stripMargin)
+ }
+
+ checkError(
+ exception = exception,
+ condition = "INVALID_METRIC_VIEW_YAML",
+ parameters = Map(
+ "message" -> "Invalid YAML version: 1.1. Supported YAML versions: 0.1"))
+ }
+
test("metric view SQL source rejects a temporary variable read via an IDENTIFIER clause") {
// A variable read via an IDENTIFIER clause in the metric-view SQL source is absent from the
// analyzed plan, so it is captured during analysis and must be rejected for a persisted metric
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org