You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/07/29 16:01:10 UTC
(spark) branch master updated: [SPARK-58400][SQL] Fix ANSI mode assumption for views created by Spark 4.0+
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3ff822a4d50e [SPARK-58400][SQL] Fix ANSI mode assumption for views created by Spark 4.0+
3ff822a4d50e is described below
commit 3ff822a4d50ecd5cdc9e46c6157c6d9e33a07157
Author: Cheng Pan <pa...@gmail.com>
AuthorDate: Wed Jul 29 09:01:00 2026 -0700
[SPARK-58400][SQL] Fix ANSI mode assumption for views created by Spark 4.0+
### What changes were proposed in this pull request?
1. `Analyzer.trySetAnsiValue`: replace `createSparkVersion.startsWith("4.")` with `VersionUtils.majorMinorPatchVersion(createSparkVersion).exists { case (major, _, _) => major >= 4 }`. The old check only matched the 4.x major; views created by Spark 5.x (ANSI still the default) wrongly fell to ANSI=false.
2. View resolution now forwards `createSparkVersion` to `View.effectiveSQLConf` on all paths that build the throwaway conf from persisted `viewSQLConfigs` -- `ViewResolver.resolve` (single-pass) and `SessionCatalog.parseMetricViewDefinition` -- matching `SessionCatalog.fromCatalogTable` / the fixed-point `ViewResolution.resolve`, which already forwarded it.
### Why are the changes needed?
ANSI became the default in 4.0 (SPARK-44444) and remains so in 5.x. `trySetAnsiValue`'s `startsWith("4.")` only recognized 4.x, so a 5.x-created view with no persisted ANSI value was resolved as LEGACY instead of ANSI -- its schema-enforcement casts used the wrong eval mode.
The single-pass `ViewResolver` and `parseMetricViewDefinition` compounded this by dropping `createVersion` entirely. This surfaced as a dual-run `LOGICAL_PLAN_COMPARISON_MISMATCH`: the two analyzers' plans differed only in `Cast.evalMode`, which `NormalizePlan` does not strip. On master the failing test passes only by coincidence (`"5.0.0"` and `""` both miss `startsWith("4.")` and both resolve to LEGACY).
### Does this PR introduce _any_ user-facing change?
Bug fix. Views created by Spark 4.0+ without a persisted ANSI value are now resolved with ANSI=true (matching their creating version's default) in both analyzers. Empty/unparseable `createVersion` still maps to ANSI=false, as documented.
### How was this patch tested?
Added `AlwaysPersistedConfigsSuite` -> "ANSI value derived from createSparkVersion when not persisted for views", exercising `View.effectiveSQLConf`/`trySetAnsiValue` for `createSparkVersion` in `{"3.5.0", "4.0.0", "4.0.0-SNAPSHOT", "5.0.0", "", "bogus"}` (ANSI true for major `>= 4`, false otherwise/unparseable). Existing suites: `HiveSQLViewSuite`, `SQLViewSuite`, `ViewResolverSuite`, `HybridAnalyzerSuite`, `MetricViewSuite`, `MetricViewV2CatalogSuite`.
### Was this patch authored or co-authored using generative AI tooling?
Yes. Generated-by: GLM 5.2
Closes #57593 from pan3793/ansi-view-fix.
Authored-by: Cheng Pan <pa...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../apache/spark/sql/catalyst/analysis/Analyzer.scala | 16 ++++++++--------
.../sql/catalyst/analysis/resolver/ViewResolver.scala | 5 ++++-
.../spark/sql/catalyst/catalog/SessionCatalog.scala | 3 ++-
.../spark/sql/AlwaysPersistedConfigsSuite.scala | 19 +++++++++++++++++++
4 files changed, 33 insertions(+), 10 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 981e92ea69ef..d6eac52ebbb2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -67,6 +67,7 @@ import org.apache.spark.sql.internal.connector.V1Function
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.CaseInsensitiveStringMap
import org.apache.spark.util.ArrayImplicits._
+import org.apache.spark.util.VersionUtils
/**
* A trivial [[Analyzer]] with a dummy [[SessionCatalog]] and
@@ -314,18 +315,17 @@ object Analyzer {
/**
* In case ANSI value wasn't persisted for a view or a UDF, we set it to `true` in case Spark
- * version used to create the view is 4.0.0 or higher. We set it to `false` in case Spark version
- * is lower than 4.0.0 or if the Spark version wasn't stored (in that case we assume that the
- * value is `false`)
+ * version used to create the view is 4.0.0 or higher (ANSI SQL mode became the default in
+ * Spark 4.0, see SPARK-44444). We set it to `false` in case Spark version is lower than 4.0.0
+ * or if the Spark version wasn't stored / can't be parsed (in that case we assume that the
+ * value is `false`).
*/
def trySetAnsiValue(sqlConf: SQLConf, createSparkVersion: String = ""): Unit = {
if (conf.getConf(SQLConf.ASSUME_ANSI_FALSE_IF_NOT_PERSISTED) &&
!sqlConf.settings.containsKey(SQLConf.ANSI_ENABLED.key)) {
- if (createSparkVersion.startsWith("4.")) {
- sqlConf.settings.put(SQLConf.ANSI_ENABLED.key, "true")
- } else {
- sqlConf.settings.put(SQLConf.ANSI_ENABLED.key, "false")
- }
+ val assumeAnsiEnabled = VersionUtils.majorMinorPatchVersion(createSparkVersion)
+ .exists { case (major, _, _) => major >= 4 }
+ sqlConf.settings.put(SQLConf.ANSI_ENABLED.key, assumeAnsiEnabled.toString)
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
index a224e521b548..8aa5262c4b59 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
@@ -102,7 +102,10 @@ class ViewResolver(
val (resolvedChild, usedViewResolutionContext) = withViewResolutionContext(unresolvedView) {
SQLConf.withExistingConf(
- View.effectiveSQLConf(unresolvedView.desc.viewSQLConfigs, unresolvedView.isTempView)
+ View.effectiveSQLConf(
+ configs = unresolvedView.desc.viewSQLConfigs,
+ isTempView = unresolvedView.isTempView,
+ createSparkVersion = unresolvedView.desc.createVersion)
) {
checkResolverGuard(unresolvedView)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index 37e03e2284d2..9a08cf1ee49e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -1153,7 +1153,8 @@ class SessionCatalog(
SQLConf.withExistingConf(
View.effectiveSQLConf(
configs = viewConfigs,
- isTempView = false
+ isTempView = false,
+ createSparkVersion = metadata.createVersion
)
) {
CurrentOrigin.withOrigin(origin) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/AlwaysPersistedConfigsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/AlwaysPersistedConfigsSuite.scala
index 5612c831c37a..634eb2b0c028 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/AlwaysPersistedConfigsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/AlwaysPersistedConfigsSuite.scala
@@ -173,6 +173,25 @@ class AlwaysPersistedConfigsSuite extends SharedSparkSession {
assert(sqlConf.settings.get("spark.sql.ansi.enabled") == "false")
}
+ test("SPARK-58400: ANSI value derived from createSparkVersion when not persisted for views") {
+ Seq(
+ "3.5.0" -> false,
+ "4.0.0" -> true,
+ "4.0.0-SNAPSHOT" -> true,
+ "5.0.0" -> true,
+ "" -> false,
+ "bogus" -> false
+ ).foreach { case (version, expected) =>
+ val sqlConf = View.effectiveSQLConf(
+ configs = Map.empty[String, String],
+ isTempView = false,
+ createSparkVersion = version)
+
+ assert(sqlConf.settings.get("spark.sql.ansi.enabled") == expected.toString,
+ s"createSparkVersion='$version'")
+ }
+ }
+
test("Current schema marker is materialized in persisted view path") {
withView(testViewName) {
withSQLConf(SQLConf.PATH_ENABLED.key -> "true") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org