You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/08/26 17:48:33 UTC

(spark) branch branch-4.0 updated: Revert "[SPARK-53348][SQL] Always persist ANSI value when creating a view or assume it when querying if not stored"

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new f3c2d39f32c9 Revert "[SPARK-53348][SQL] Always persist ANSI value when creating a view or assume it when querying if not stored"
f3c2d39f32c9 is described below

commit f3c2d39f32c9122bf0e496082b34572185ce8f38
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Tue Aug 26 10:44:26 2025 -0700

    Revert "[SPARK-53348][SQL] Always persist ANSI value when creating a view or assume it when querying if not stored"
    
    This reverts commit fbedfb1e8c2c5e7b8530becffe75e45382b3b4a3.
---
 .../spark/sql/catalyst/analysis/Analyzer.scala     |  24 +---
 .../sql/catalyst/analysis/ViewResolution.scala     |   8 +-
 .../sql/catalyst/catalog/SessionCatalog.scala      |  15 +--
 .../plans/logical/basicLogicalOperators.scala      |  12 +-
 .../org/apache/spark/sql/internal/SQLConf.scala    |   9 --
 .../command/CreateUserDefinedFunctionCommand.scala |  19 +---
 .../apache/spark/sql/execution/command/views.scala |  15 +--
 .../apache/spark/sql/DefaultANSIValueSuite.scala   | 126 ---------------------
 8 files changed, 13 insertions(+), 215 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 8c16cca97d85..1045e73f2242 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -258,10 +258,7 @@ object Analyzer {
     "spark.sql.expressionTreeChangeLog.level"
   )
 
-  def retainResolutionConfigsForAnalysis(
-      newConf: SQLConf,
-      existingConf: SQLConf,
-      createSparkVersion: String = ""): Unit = {
+  def retainResolutionConfigsForAnalysis(newConf: SQLConf, existingConf: SQLConf): Unit = {
     val retainedConfigs = existingConf.getAllConfs.filter { case (key, _) =>
       // Also apply catalog configs
       RETAINED_ANALYSIS_FLAGS.contains(key) || key.startsWith("spark.sql.catalog.")
@@ -270,25 +267,6 @@ object Analyzer {
     retainedConfigs.foreach { case (k, v) =>
       newConf.settings.put(k, v)
     }
-
-    trySetAnsiValue(newConf, createSparkVersion)
-  }
-
-  /**
-   * In case ANSI value wasn't persisted for a view or a UDF, we set it to `true` in case Spark
-   * version used to create the view is 4.0.0 or higher. We set it to `false` in case Spark version
-   * is lower than 4.0.0 or if the Spark version wasn't stored (in that case we assume that the
-   * value is `false`)
-   */
-  def trySetAnsiValue(sqlConf: SQLConf, createSparkVersion: String = ""): Unit = {
-    if (conf.getConf(SQLConf.ASSUME_ANSI_FALSE_IF_NOT_PERSISTED) &&
-      !sqlConf.settings.containsKey(SQLConf.ANSI_ENABLED.key)) {
-      if (createSparkVersion.startsWith("4.")) {
-        sqlConf.settings.put(SQLConf.ANSI_ENABLED.key, "true")
-      } else {
-        sqlConf.settings.put(SQLConf.ANSI_ENABLED.key, "false")
-      }
-    }
   }
 }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewResolution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewResolution.scala
index 585b8635031e..89ef29ddaaf1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewResolution.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewResolution.scala
@@ -41,13 +41,7 @@ object ViewResolution {
           view
         )
       }
-      SQLConf.withExistingConf(
-        View.effectiveSQLConf(
-          configs = view.desc.viewSQLConfigs,
-          isTempView = view.isTempView,
-          createSparkVersion = view.desc.createVersion
-        )
-      ) {
+      SQLConf.withExistingConf(View.effectiveSQLConf(view.desc.viewSQLConfigs, view.isTempView)) {
         resolveChild(view.child)
       }
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index e34d402ccd36..0e5ff57def89 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -982,13 +982,7 @@ class SessionCatalog(
       objectType = Some("VIEW"),
       objectName = Some(metadata.qualifiedName)
     )
-    val parsedPlan = SQLConf.withExistingConf(
-      View.effectiveSQLConf(
-        configs = viewConfigs,
-        isTempView = isTempView,
-        createSparkVersion = metadata.createVersion
-      )
-    ) {
+    val parsedPlan = SQLConf.withExistingConf(View.effectiveSQLConf(viewConfigs, isTempView)) {
         CurrentOrigin.withOrigin(origin) {
           parser.parseQuery(viewText)
         }
@@ -1016,11 +1010,7 @@ class SessionCatalog(
         // Note that, the column names may have duplication, e.g. `CREATE VIEW v(x, y) AS
         // SELECT 1 col, 2 col`. We need to make sure that the matching attributes have the same
         // number of duplications, and pick the corresponding attribute by ordinal.
-        val viewConf = View.effectiveSQLConf(
-          configs = metadata.viewSQLConfigs,
-          isTempView = isTempView,
-          createSparkVersion = metadata.createVersion
-        )
+        val viewConf = View.effectiveSQLConf(metadata.viewSQLConfigs, isTempView)
         val normalizeColName: String => String = if (viewConf.caseSensitiveAnalysis) {
           identity
         } else {
@@ -1627,7 +1617,6 @@ class SessionCatalog(
     // Use captured SQL configs when parsing a SQL function.
     val conf = new SQLConf()
     function.getSQLConfigs.foreach { case (k, v) => conf.settings.put(k, v) }
-    Analyzer.trySetAnsiValue(conf)
     SQLConf.withExistingConf(conf) {
       val inputParam = function.inputParam
       val returnType = function.getScalarFuncReturnType
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 6fb3bc9e6267..3f1cc65a13e1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -835,10 +835,7 @@ case class View(
 }
 
 object View {
-  def effectiveSQLConf(
-      configs: Map[String, String],
-      isTempView: Boolean,
-      createSparkVersion: String = ""): SQLConf = {
+  def effectiveSQLConf(configs: Map[String, String], isTempView: Boolean): SQLConf = {
     val activeConf = SQLConf.get
     // For temporary view, we always use captured sql configs
     if (activeConf.useCurrentSQLConfigsForView && !isTempView) return activeConf
@@ -847,12 +844,7 @@ object View {
     for ((k, v) <- configs) {
       sqlConf.settings.put(k, v)
     }
-    Analyzer.retainResolutionConfigsForAnalysis(
-      newConf = sqlConf,
-      existingConf = activeConf,
-      createSparkVersion = createSparkVersion
-    )
-
+    Analyzer.retainResolutionConfigsForAnalysis(newConf = sqlConf, existingConf = activeConf)
     sqlConf
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 8427c8320d5c..da39b24813f9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -5594,15 +5594,6 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
-  val ASSUME_ANSI_FALSE_IF_NOT_PERSISTED =
-    buildConf("spark.sql.assumeAnsiFalseIfNotPersisted.enabled")
-      .internal()
-      .doc("If enabled, assume ANSI mode is false if not persisted during view or UDF " +
-        "creation. Otherwise use the default value.")
-      .version("4.0.1")
-      .booleanConf
-      .createWithDefault(true)
-
   /**
    * Holds information about keys that have been deprecated.
    *
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateUserDefinedFunctionCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateUserDefinedFunctionCommand.scala
index 122e7d9b7ef9..1ee3c8a4c388 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateUserDefinedFunctionCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateUserDefinedFunctionCommand.scala
@@ -19,8 +19,6 @@ package org.apache.spark.sql.execution.command
 
 import java.util.Locale
 
-import scala.collection.mutable
-
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.FunctionIdentifier
 import org.apache.spark.sql.catalyst.catalog.{LanguageSQL, RoutineLanguage, UserDefinedFunctionErrors}
@@ -89,25 +87,10 @@ object CreateUserDefinedFunctionCommand {
    * [[org.apache.spark.sql.catalyst.expressions.ExpressionInfo]], all SQL configs and other
    * function properties (such as the function parameters and the function return type)
    * are saved together in a property map.
-   *
-   * Here we only capture the SQL configs that are modifiable and should be captured, i.e. not in
-   * the denyList and in the allowList. Besides mentioned ones we also capture `ANSI_ENABLED`.
-   *
-   * We need to always capture them to make sure we apply the same configs when querying the
-   * function.
    */
   def sqlConfigsToProps(conf: SQLConf): Map[String, String] = {
     val modifiedConfs = ViewHelper.getModifiedConf(conf)
-
-    val alwaysCaptured = Seq(SQLConf.ANSI_ENABLED)
-      .filter(c => !modifiedConfs.contains(c.key))
-      .map(c => (c.key, conf.getConf(c).toString))
-
-    val props = new mutable.HashMap[String, String]
-    for ((key, value) <- modifiedConfs ++ alwaysCaptured) {
-      props.put(s"$SQL_CONFIG_PREFIX$key", value)
-    }
-    props.toMap
+    modifiedConfs.map { case (key, value) => s"$SQL_CONFIG_PREFIX$key" -> value }
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 8ed3509cf726..c73fedc3697e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -481,19 +481,16 @@ object ViewHelper extends SQLConfHelper with Logging {
   }
 
   /**
-   * Convert the view SQL configs to `properties`. Here we only capture the SQL configs that are
-   * modifiable and should be captured, i.e. not in the denyList and in the allowList. We also
-   * capture `SESSION_LOCAL_TIMEZONE` whose default value relies on the JVM system timezone and
-   * the `ANSI_ENABLED` value.
-   *
-   * We need to always capture them to make sure we apply the same configs when querying the view.
+   * Convert the view SQL configs to `properties`.
    */
   private def sqlConfigsToProps(conf: SQLConf): Map[String, String] = {
     val modifiedConfs = getModifiedConf(conf)
-
-    val alwaysCaptured = Seq(SQLConf.SESSION_LOCAL_TIMEZONE, SQLConf.ANSI_ENABLED)
+    // Some configs have dynamic default values, such as SESSION_LOCAL_TIMEZONE whose
+    // default value relies on the JVM system timezone. We need to always capture them to
+    // to make sure we apply the same configs when reading the view.
+    val alwaysCaptured = Seq(SQLConf.SESSION_LOCAL_TIMEZONE)
       .filter(c => !modifiedConfs.contains(c.key))
-      .map(c => (c.key, conf.getConf(c).toString))
+      .map(c => (c.key, conf.getConf(c)))
 
     val props = new mutable.HashMap[String, String]
     for ((key, value) <- modifiedConfs ++ alwaysCaptured) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DefaultANSIValueSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DefaultANSIValueSuite.scala
deleted file mode 100644
index 956c27de619c..000000000000
--- a/sql/core/src/test/scala/org/apache/spark/sql/DefaultANSIValueSuite.scala
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.SQLScalarFunction
-import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, SQLFunction}
-import org.apache.spark.sql.catalyst.expressions.Alias
-import org.apache.spark.sql.catalyst.plans.logical.{OneRowRelation, Project, View}
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.StructType
-
-/**
- * This suite tests if default ANSI value is persisted for views and functions if not explicitly
- * set.
- */
-class DefaultANSIValueSuite extends QueryTest with SharedSparkSession {
-
-  protected override def sparkConf: SparkConf = {
-    super.sparkConf
-      .set(SQLConf.ASSUME_ANSI_FALSE_IF_NOT_PERSISTED.key, "true")
-  }
-
-  private val testViewName = "test_view"
-  private val testFunctionName = "test_function"
-
-  test("Default ANSI value is stored for views") {
-    withView(testViewName) {
-      testView(expectedAnsiValue = true)
-    }
-  }
-
-  test("Explicitly set ANSI value is respected over default one for views") {
-    withView(testViewName) {
-      withSQLConf("spark.sql.ansi.enabled" -> "false") {
-        testView(expectedAnsiValue = false)
-      }
-    }
-
-    withView(testViewName) {
-      withSQLConf("spark.sql.ansi.enabled" -> "true") {
-        testView(expectedAnsiValue = true)
-      }
-    }
-  }
-
-  test("Default ANSI value is stored for functions") {
-    withUserDefinedFunction(testFunctionName -> false) {
-      testFunction(expectedAnsiValue = true)
-    }
-  }
-
-  test("Explicitly set ANSI value is respected over default one for functions") {
-    withUserDefinedFunction(testFunctionName -> false) {
-      withSQLConf("spark.sql.ansi.enabled" -> "false") {
-        testFunction(expectedAnsiValue = false)
-      }
-    }
-
-    withUserDefinedFunction(testFunctionName -> false) {
-      withSQLConf("spark.sql.ansi.enabled" -> "true") {
-        testFunction(expectedAnsiValue = true)
-      }
-    }
-  }
-
-  test("ANSI value is set to false if not persisted for views") {
-    val catalogTable = new CatalogTable(
-      identifier = TableIdentifier(testViewName),
-      tableType = CatalogTableType.VIEW,
-      storage = CatalogStorageFormat(None, None, None, None, false, Map.empty),
-      schema = new StructType(),
-      properties = Map.empty[String, String]
-    )
-    val view = View(desc = catalogTable, isTempView = false, child = OneRowRelation())
-
-    val sqlConf = View.effectiveSQLConf(view.desc.viewSQLConfigs, view.isTempView)
-
-    assert(sqlConf.settings.get("spark.sql.ansi.enabled") == "false")
-  }
-
-  private def testView(expectedAnsiValue: Boolean): Unit = {
-    sql(s"CREATE VIEW $testViewName AS SELECT CAST('string' AS BIGINT) AS alias")
-
-    val viewMetadata = spark.sessionState.catalog.getTableMetadata(TableIdentifier(testViewName))
-
-    assert(
-      viewMetadata.properties("view.sqlConfig.spark.sql.ansi.enabled") == expectedAnsiValue.toString
-    )
-  }
-
-  private def testFunction(expectedAnsiValue: Boolean): Unit = {
-    sql(
-      s"""
-         |CREATE OR REPLACE FUNCTION $testFunctionName()
-         |RETURN SELECT CAST('string' AS BIGINT) AS alias
-         |""".stripMargin)
-
-    val df = sql(s"select $testFunctionName()")
-
-    assert(
-      df.queryExecution.analyzed.asInstanceOf[Project]
-        .projectList.head.asInstanceOf[Alias]
-        .child.asInstanceOf[SQLScalarFunction]
-        .function.asInstanceOf[SQLFunction]
-        .properties.get("sqlConfig.spark.sql.ansi.enabled").get == expectedAnsiValue.toString
-    )
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org