You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/05/23 13:54:35 UTC

(spark) branch master updated: [SPARK-52219][SQL] Schema level collation support for tables

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 858fbccb7bab [SPARK-52219][SQL] Schema level collation support for tables
858fbccb7bab is described below

commit 858fbccb7bab82c55bf020bc8a03b04a920ee27b
Author: ilicmarkodb <ma...@databricks.com>
AuthorDate: Fri May 23 21:54:20 2025 +0800

    [SPARK-52219][SQL] Schema level collation support for tables
    
    ### What changes were proposed in this pull request?
    Added support for setting and altering a default collation at the schema level.
    Tables that do not specify their own collation will now automatically inherit the schema-level default.
    
    `CREATE SCHEMA schema1 DEFAULT COLLATION UTF8_LCASE; CREATE TABLE t1 (c1 STRING);`
    The default collation for table `t1` will be `UTF8_LCASE`. and c1 data type will be `STRING COLLATE UTF8_LCASE`.
    
    Following PRs will add support for Views, UDFs.
    
    ### Why are the changes needed?
    New feature.
    
    ### Does this PR introduce _any_ user-facing change?
    No.
    
    ### How was this patch tested?
    Tests added to `DefaultCollationTestSuite.scala`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No.
    
    Closes #50937 from ilicmarkodb/schema_level_collation.
    
    Authored-by: ilicmarkodb <ma...@databricks.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../src/main/resources/error/error-conditions.json |   5 +
 .../spark/sql/catalyst/parser/SqlBaseParser.g4     |   3 +
 .../sql/connector/catalog/SupportsNamespaces.java  |   6 +
 .../ApplyDefaultCollationToStringType.scala        |  65 +++++-
 .../spark/sql/catalyst/parser/AstBuilder.scala     |  13 ++
 .../sql/connector/catalog/CatalogV2Util.scala      |   1 +
 .../spark/sql/errors/QueryCompilationErrors.scala  |   7 +
 .../org/apache/spark/sql/internal/SQLConf.scala    |  16 ++
 .../spark/sql/execution/SparkSqlParser.scala       |  19 ++
 .../execution/command/SetNamespaceCollation.scala  |  44 ++++
 .../apache/spark/sql/execution/command/ddl.scala   |   1 +
 .../sql/collation/DefaultCollationTestSuite.scala  | 231 ++++++++++++++++++++-
 .../sql/errors/QueryCompilationErrorsSuite.scala   |  14 ++
 .../AlterNamespaceSetPropertiesSuiteBase.scala     |   3 +-
 .../AlterNamespaceUnsetPropertiesSuiteBase.scala   |   3 +-
 15 files changed, 418 insertions(+), 13 deletions(-)

diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json
index 79e8ab4567f0..f5869573af0f 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -6064,6 +6064,11 @@
           "The replace function does not support nested column <colName>."
         ]
       },
+      "SCHEMA_LEVEL_COLLATIONS" : {
+        "message" : [
+          "Default collation for the specified schema."
+        ]
+      },
       "SET_NAMESPACE_PROPERTY" : {
         "message" : [
           "<property> is a reserved namespace property, <msg>."
diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4
index d95a32de3385..08f222b2f412 100644
--- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4
+++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4
@@ -191,11 +191,14 @@ statement
     | CREATE namespace (IF errorCapturingNot EXISTS)? identifierReference
         (commentSpec |
          locationSpec |
+         collationSpec |
          (WITH (DBPROPERTIES | PROPERTIES) propertyList))*             #createNamespace
     | ALTER namespace identifierReference
         SET (DBPROPERTIES | PROPERTIES) propertyList                   #setNamespaceProperties
     | ALTER namespace identifierReference
         UNSET (DBPROPERTIES | PROPERTIES) propertyList                 #unsetNamespaceProperties
+    | ALTER namespace identifierReference
+        collationSpec                                                  #setNamespaceCollation
     | ALTER namespace identifierReference
         SET locationSpec                                               #setNamespaceLocation
     | DROP namespace (IF EXISTS)? identifierReference
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
index c1a4960068d2..49d4b0e6609b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
@@ -54,6 +54,12 @@ public interface SupportsNamespaces extends CatalogPlugin {
    */
   String PROP_COMMENT = "comment";
 
+  /**
+   * A reserved property to specify the collation of the namespace. The collation
+   * will be returned in the result of "DESCRIBE NAMESPACE" command.
+   */
+  String PROP_COLLATION = "collation";
+
   /**
    * A reserved property to specify the owner of the namespace.
    */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ApplyDefaultCollationToStringType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ApplyDefaultCollationToStringType.scala
index 0082c65ab8b2..b0133524ca40 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ApplyDefaultCollationToStringType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ApplyDefaultCollationToStringType.scala
@@ -18,9 +18,10 @@
 package org.apache.spark.sql.catalyst.analysis
 
 import org.apache.spark.sql.catalyst.expressions.{Cast, DefaultStringProducingExpression, Expression, Literal, SubqueryExpression}
-import org.apache.spark.sql.catalyst.plans.logical.{AddColumns, AlterColumns, AlterColumnSpec, AlterViewAs, ColumnDefinition, CreateTable, CreateTempView, CreateView, LogicalPlan, QualifiedColType, ReplaceColumns, ReplaceTable, V2CreateTablePlan}
+import org.apache.spark.sql.catalyst.plans.logical.{AddColumns, AlterColumns, AlterColumnSpec, AlterViewAs, ColumnDefinition, CreateTable, CreateTempView, CreateView, LogicalPlan, QualifiedColType, ReplaceColumns, ReplaceTable, TableSpec, V2CreateTablePlan}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.connector.catalog.{SupportsNamespaces, TableCatalog}
+import org.apache.spark.sql.connector.catalog.SupportsNamespaces.PROP_COLLATION
 import org.apache.spark.sql.types.{DataType, StringType}
 
 /**
@@ -32,10 +33,12 @@ import org.apache.spark.sql.types.{DataType, StringType}
  */
 object ApplyDefaultCollationToStringType extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = {
-    fetchDefaultCollation(plan) match {
+    val planWithResolvedDefaultCollation = resolveDefaultCollation(plan)
+
+    fetchDefaultCollation(planWithResolvedDefaultCollation) match {
       case Some(collation) =>
-        transform(plan, StringType(collation))
-      case None => plan
+        transform(planWithResolvedDefaultCollation, StringType(collation))
+      case None => planWithResolvedDefaultCollation
     }
   }
 
@@ -44,8 +47,8 @@ object ApplyDefaultCollationToStringType extends Rule[LogicalPlan] {
    */
   private def fetchDefaultCollation(plan: LogicalPlan): Option[String] = {
     plan match {
-      case createTable: CreateTable =>
-        createTable.tableSpec.collation
+      case CreateTable(_: ResolvedIdentifier, _, _, tableSpec: TableSpec, _) =>
+        tableSpec.collation
 
       // CreateView also handles CREATE OR REPLACE VIEW
       // Unlike for tables, CreateView also handles CREATE OR REPLACE VIEW
@@ -57,8 +60,8 @@ object ApplyDefaultCollationToStringType extends Rule[LogicalPlan] {
       case createTempView: CreateTempView =>
         createTempView.collation
 
-      case replaceTable: ReplaceTable =>
-        replaceTable.tableSpec.collation
+      case ReplaceTable(_: ResolvedIdentifier, _, _, tableSpec: TableSpec, _) =>
+        tableSpec.collation
 
       // In `transform` we handle these 3 ALTER TABLE commands.
       case cmd: AddColumns => getCollationFromTableProps(cmd.table)
@@ -91,6 +94,50 @@ object ApplyDefaultCollationToStringType extends Rule[LogicalPlan] {
     }
   }
 
+  /**
+   * Determines the default collation for an object in the following order:
+   * 1. Use the object's explicitly defined default collation, if available.
+   * 2. Otherwise, use the default collation defined by the object's schema.
+   * 3. If not defined in the schema, use the default collation from the object's catalog.
+   *
+   * If none of these collations are specified, None will be persisted as the default collation,
+   * which means the system default collation `UTF8_BINARY` will be used and the plan will not be
+   * changed.
+   * This function applies to DDL commands. An object's default collation is persisted at the moment
+   * of its creation, and altering the schema or catalog collation will not affect existing objects.
+   */
+  def resolveDefaultCollation(plan: LogicalPlan): LogicalPlan = {
+    try {
+      plan match {
+        case createTable@CreateTable(ResolvedIdentifier(
+        catalog: SupportsNamespaces, identifier), _, _, tableSpec: TableSpec, _)
+          if tableSpec.collation.isEmpty =>
+          createTable.copy(tableSpec = tableSpec.copy(
+            collation = getCollationFromSchemaMetadata(catalog, identifier.namespace())))
+        case replaceTable@ReplaceTable(
+        ResolvedIdentifier(catalog: SupportsNamespaces, identifier), _, _, tableSpec: TableSpec, _)
+          if tableSpec.collation.isEmpty =>
+          replaceTable.copy(tableSpec = tableSpec.copy(
+            collation = getCollationFromSchemaMetadata(catalog, identifier.namespace())))
+        case other =>
+          other
+      }
+    } catch {
+      case _: NoSuchNamespaceException =>
+        plan
+    }
+  }
+
+  /**
+   Retrieves the schema's default collation from the metadata of the given catalog and schema
+   name. Returns None if the default collation is not specified for the schema.
+   */
+  private def getCollationFromSchemaMetadata(
+      catalog: SupportsNamespaces, schemaName: Array[String]): Option[String] = {
+    val metadata = catalog.loadNamespaceMetadata(schemaName)
+    Option(metadata.get(PROP_COLLATION))
+  }
+
   private def isCreateOrAlterPlan(plan: LogicalPlan): Boolean = plan match {
     // For CREATE TABLE, only v2 CREATE TABLE command is supported.
     // Also, table DEFAULT COLLATION cannot be specified through CREATE TABLE AS SELECT command.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index df348f1f6051..9b9ff2175457 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -4435,6 +4435,10 @@ class AstBuilder extends DataTypeAstBuilder
         throw QueryParsingErrors.cannotCleanReservedNamespacePropertyError(
           PROP_LOCATION, ctx, "please use the LOCATION clause to specify it")
       case (PROP_LOCATION, _) => false
+      case (PROP_COLLATION, _) if !legacyOn =>
+        throw QueryParsingErrors.cannotCleanReservedNamespacePropertyError(
+          PROP_COLLATION, ctx, "please use the DEFAULT COLLATION clause to specify it")
+      case (PROP_COLLATION, _) => false
       case (PROP_OWNER, _) if !legacyOn =>
         throw QueryParsingErrors.cannotCleanReservedNamespacePropertyError(
           PROP_OWNER, ctx, "it will be set to the current user")
@@ -4460,6 +4464,7 @@ class AstBuilder extends DataTypeAstBuilder
   override def visitCreateNamespace(ctx: CreateNamespaceContext): LogicalPlan = withOrigin(ctx) {
     import SupportsNamespaces._
     checkDuplicateClauses(ctx.commentSpec(), "COMMENT", ctx)
+    checkDuplicateClauses(ctx.collationSpec(), "DEFAULT COLLATION", ctx)
     checkDuplicateClauses(ctx.locationSpec, "LOCATION", ctx)
     checkDuplicateClauses(ctx.PROPERTIES, "WITH PROPERTIES", ctx)
     checkDuplicateClauses(ctx.DBPROPERTIES, "WITH DBPROPERTIES", ctx)
@@ -4478,6 +4483,14 @@ class AstBuilder extends DataTypeAstBuilder
       properties += PROP_COMMENT -> _
     }
 
+    // Schema-level default collation is under development and currently not enabled.
+    if (!SQLConf.get.schemaLevelCollationsEnabled && !ctx.collationSpec().isEmpty) {
+      throw QueryCompilationErrors.schemaLevelCollationsNotEnabledError()
+    }
+    visitCollationSpecList(ctx.collationSpec()).foreach {
+      properties += PROP_COLLATION -> _
+    }
+
     visitLocationSpecList(ctx.locationSpec()).foreach {
       properties += PROP_LOCATION -> _
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
index 527972dd0a36..ded42a2b0f87 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
@@ -73,6 +73,7 @@ private[sql] object CatalogV2Util {
    */
   val NAMESPACE_RESERVED_PROPERTIES =
     Seq(SupportsNamespaces.PROP_COMMENT,
+      SupportsNamespaces.PROP_COLLATION,
       SupportsNamespaces.PROP_LOCATION,
       SupportsNamespaces.PROP_OWNER)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index 76a5a7365972..9de52533dbbb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -380,6 +380,13 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat
     )
   }
 
+  def schemaLevelCollationsNotEnabledError(): Throwable = {
+    new AnalysisException(
+      errorClass = "UNSUPPORTED_FEATURE.SCHEMA_LEVEL_COLLATIONS",
+      messageParameters = Map.empty
+    )
+  }
+
   def trimCollationNotEnabledError(): Throwable = {
     new AnalysisException(
       errorClass = "UNSUPPORTED_FEATURE.TRIM_COLLATION",
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index b8b1ef4e135a..7335d525c2a2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -949,6 +949,20 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  lazy val SCHEMA_LEVEL_COLLATIONS_ENABLED =
+    buildConf("spark.sql.collation.schemaLevel.enabled")
+      .internal()
+      .doc(
+        "Schema level collations feature is under development and its use should be done " +
+          "under this feature flag. The feature allows setting default collation for all " +
+          "underlying objects within that schema, except the ones that were previously created." +
+          "An object with an explicitly set collation will not inherit the collation from the " +
+          "schema."
+      )
+      .version("4.0.0")
+      .booleanConf
+      .createWithDefault(false)
+
   lazy val TRIM_COLLATION_ENABLED =
     buildConf("spark.sql.collation.trim.enabled")
       .internal()
@@ -6177,6 +6191,8 @@ class SQLConf extends Serializable with Logging with SqlApiConf {
 
   def objectLevelCollationsEnabled: Boolean = getConf(OBJECT_LEVEL_COLLATIONS_ENABLED)
 
+  def schemaLevelCollationsEnabled: Boolean = getConf(SCHEMA_LEVEL_COLLATIONS_ENABLED)
+
   def trimCollationEnabled: Boolean = getConf(TRIM_COLLATION_ENABLED)
 
   def adaptiveExecutionEnabled: Boolean = getConf(ADAPTIVE_EXECUTION_ENABLED)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index 9565bc9296a6..0d218b6e183c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -1158,6 +1158,25 @@ class SparkSqlAstBuilder extends AstBuilder {
       cleanedProperties)
   }
 
+  /**
+   * Create an [[SetNamespaceCollation]] logical plan.
+   *
+   * For example:
+   * {{{
+   *   ALTER (DATABASE|SCHEMA|NAMESPACE) namespace DEFAULT COLLATION collation;
+   * }}}
+   */
+  override def visitSetNamespaceCollation(ctx: SetNamespaceCollationContext): LogicalPlan = {
+    withOrigin(ctx) {
+      if (!SQLConf.get.schemaLevelCollationsEnabled) {
+        throw QueryCompilationErrors.schemaLevelCollationsNotEnabledError()
+      }
+      SetNamespaceCollationCommand(
+        withIdentClause(ctx.identifierReference, UnresolvedNamespace(_)),
+        visitCollationSpec(ctx.collationSpec()))
+    }
+  }
+
   /**
    * Create a [[DescribeColumn]] or [[DescribeRelation]] or [[DescribeRelationAsJsonCommand]]
    * command.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetNamespaceCollation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetNamespaceCollation.scala
new file mode 100644
index 000000000000..e434ec53ce93
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetNamespaceCollation.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{Row, SparkSession}
+import org.apache.spark.sql.catalyst.analysis.ResolvedNamespace
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.CatalogHelper
+import org.apache.spark.sql.connector.catalog.NamespaceChange
+import org.apache.spark.sql.connector.catalog.SupportsNamespaces.PROP_COLLATION
+
+/**
+ * The logical plan of the ALTER (DATABASE|SCHEMA|NAMESPACE) ... DEFAULT COLLATION command.
+ */
+case class SetNamespaceCollationCommand(
+    namespace: LogicalPlan,
+    collation: String) extends UnaryRunnableCommand {
+  override def child: LogicalPlan = namespace
+  override protected def withNewChildInternal(newChild: LogicalPlan): SetNamespaceCollationCommand =
+    copy(namespace = newChild)
+
+  override def run(sparkSession: SparkSession): Seq[Row] = {
+    val ResolvedNamespace(catalog, ns, _) = child
+    val collationChange = NamespaceChange.setProperty(PROP_COLLATION, collation)
+    catalog.asNamespaceCatalog.alterNamespace(ns.toArray, collationChange)
+
+    Seq.empty
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index 6eb81e6ec670..13994c58f120 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -66,6 +66,7 @@ import org.apache.spark.util.ArrayImplicits._
  * {{{
  *   CREATE (DATABASE|SCHEMA) [IF NOT EXISTS] database_name
  *     [COMMENT database_comment]
+ *     [DEFAULT COLLATION collation]
  *     [LOCATION database_directory]
  *     [WITH DBPROPERTIES (property_name=property_value, ...)];
  * }}}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
index 031af8200d7e..9a2665e3054f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
@@ -18,13 +18,25 @@
 package org.apache.spark.sql.collation
 
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
+import org.apache.spark.sql.catalyst.catalog.SessionCatalog.DEFAULT_DATABASE
 import org.apache.spark.sql.catalyst.util.CollationFactory
 import org.apache.spark.sql.connector.DatasourceV2SQLBase
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.StringType
 
 abstract class DefaultCollationTestSuite extends QueryTest with SharedSparkSession {
 
+  override def beforeEach(): Unit = {
+    super.beforeEach()
+    spark.conf.set(SQLConf.SCHEMA_LEVEL_COLLATIONS_ENABLED, true)
+  }
+
+  override def afterEach(): Unit = {
+    spark.conf.set(SQLConf.SCHEMA_LEVEL_COLLATIONS_ENABLED, false)
+    super.afterEach()
+  }
+
   val defaultStringProducingExpressions: Seq[String] = Seq(
     "current_timezone()", "current_database()", "md5('Spark' collate unicode)",
     "soundex('Spark' collate unicode)", "url_encode('https://spark.apache.org' collate unicode)",
@@ -33,6 +45,7 @@ abstract class DefaultCollationTestSuite extends QueryTest with SharedSparkSessi
   )
 
   def dataSource: String = "parquet"
+  def testSchema: String = "test_schema"
   def testTable: String = "test_tbl"
   def testView: String = "test_view"
   protected val fullyQualifiedPrefix = s"${CollationFactory.CATALOG}.${CollationFactory.SCHEMA}."
@@ -129,6 +142,91 @@ abstract class DefaultCollationTestSuite extends QueryTest with SharedSparkSessi
     }
   }
 
+  Seq(
+    // (schemaDefaultCollation, tableDefaultCollation)
+    ("UTF8_BINARY", None),
+    ("UTF8_LCASE", None),
+    ("UNICODE", None),
+    ("DE", None),
+    ("UTF8_BINARY", Some("UTF8_BINARY")),
+    ("UTF8_BINARY", Some("UTF8_LCASE")),
+    ("UTF8_BINARY", Some("DE")),
+    ("UTF8_LCASE", Some("UTF8_BINARY")),
+    ("UTF8_LCASE", Some("UTF8_LCASE")),
+    ("UTF8_LCASE", Some("DE"))
+  ).foreach {
+    case (schemaDefaultCollation, tableDefaultCollation) =>
+      test(
+        s"""CREATE table with schema level collation
+          | (schema default collation = $schemaDefaultCollation,
+          | table default collation = $tableDefaultCollation)""".stripMargin) {
+        testCreateTableWithSchemaLevelCollation(
+          schemaDefaultCollation, tableDefaultCollation)
+      }
+
+      test(
+        s"""ALTER table with schema level collation
+          | (schema default collation = $schemaDefaultCollation,
+          | table default collation = $tableDefaultCollation)""".stripMargin) {
+        testAlterTableWithSchemaLevelCollation(
+         schemaDefaultCollation, tableDefaultCollation)
+      }
+  }
+
+  Seq(
+    // (schemaOldCollation, schemaNewCollation)
+    (None, "UTF8_BINARY"),
+    (None, "UTF8_LCASE"),
+    (None, "DE"),
+    (Some("UTF8_BINARY"), "UTF8_BINARY"),
+    (Some("UTF8_BINARY"), "UTF8_LCASE"),
+    (Some("UTF8_BINARY"), "DE"),
+    (Some("UTF8_LCASE"), "UTF8_BINARY"),
+    (Some("UTF8_LCASE"), "UTF8_LCASE"),
+    (Some("UTF8_LCASE"), "DE")
+  ).foreach {
+    case (schemaOldCollation, schemaNewCollation) =>
+      val schemaOldCollationDefaultClause =
+        if (schemaOldCollation.isDefined) {
+          s"DEFAULT COLLATION ${schemaOldCollation.get}"
+        } else {
+          ""
+        }
+
+      test(
+        s"""ALTER schema default collation (old schema default collation = $schemaOldCollation,
+          | new schema default collation = $schemaNewCollation)""".stripMargin) {
+        withDatabase(testSchema) {
+          sql(s"CREATE SCHEMA $testSchema $schemaOldCollationDefaultClause")
+          sql(s"USE $testSchema")
+
+          withTable(testTable) {
+            sql(s"CREATE TABLE $testTable (c1 STRING, c2 STRING COLLATE SR_AI)")
+            val tableDefaultCollation =
+              if (schemaOldCollation.isDefined) {
+                schemaOldCollation.get
+              } else {
+                "UTF8_BINARY"
+              }
+
+            // ALTER SCHEMA
+            sql(s"ALTER SCHEMA $testSchema DEFAULT COLLATION $schemaNewCollation")
+
+            // Altering schema default collation should not affect existing objects.
+            addAndAlterColumns(tableDefaultCollation = tableDefaultCollation)
+          }
+
+          withTable(testTable) {
+            sql(s"CREATE TABLE $testTable " +
+              s"(c1 STRING, c2 STRING COLLATE SR_AI, c3 STRING COLLATE UTF8_BINARY)")
+            assertTableColumnCollation(testTable, "c1", schemaNewCollation)
+            assertTableColumnCollation(testTable, "c2", "SR_AI")
+            assertTableColumnCollation(testTable, "c3", "UTF8_BINARY")
+          }
+        }
+      }
+  }
+
   test("create table as select") {
     // literals in select do not pick up session collation
     withTable(testTable) {
@@ -263,6 +361,72 @@ abstract class DefaultCollationTestSuite extends QueryTest with SharedSparkSessi
   }
 
   // endregion
+
+  protected def testCreateTableWithSchemaLevelCollation(
+      schemaDefaultCollation: String,
+      tableDefaultCollation: Option[String] = None,
+      replaceTable: Boolean = false): Unit = {
+    val (tableDefaultCollationClause, resolvedDefaultCollation) =
+      if (tableDefaultCollation.isDefined) {
+        (s"DEFAULT COLLATION ${tableDefaultCollation.get}", tableDefaultCollation.get)
+      } else {
+        ("", schemaDefaultCollation)
+      }
+    val replace = if (replaceTable) "OR REPLACE" else ""
+
+    withDatabase(testSchema) {
+      sql(s"CREATE SCHEMA $testSchema DEFAULT COLLATION $schemaDefaultCollation")
+      sql(s"USE $testSchema")
+      withTable(testTable) {
+        sql(s"CREATE $replace TABLE $testTable " +
+          s"(c1 STRING, c2 STRING COLLATE SR_AI, c3 STRING COLLATE UTF8_BINARY) " +
+          s"$tableDefaultCollationClause")
+        assertTableColumnCollation(testTable, "c1", resolvedDefaultCollation)
+        assertTableColumnCollation(testTable, "c2", "SR_AI")
+        assertTableColumnCollation(testTable, "c3", "UTF8_BINARY")
+      }
+    }
+  }
+
+  def testAlterTableWithSchemaLevelCollation(
+      schemaDefaultCollation: String, tableDefaultCollation: Option[String] = None): Unit = {
+    val (tableDefaultCollationClause, resolvedDefaultCollation) =
+      if (tableDefaultCollation.isDefined) {
+        (s"DEFAULT COLLATION ${tableDefaultCollation.get}", tableDefaultCollation.get)
+      } else {
+        ("", schemaDefaultCollation)
+      }
+
+    withDatabase(testSchema) {
+      sql(s"CREATE SCHEMA $testSchema DEFAULT COLLATION $schemaDefaultCollation")
+      sql(s"USE $testSchema")
+
+      withTable(testTable) {
+        sql(s"CREATE TABLE $testTable (c1 STRING, c2 STRING COLLATE SR_AI) " +
+          s"$tableDefaultCollationClause")
+
+        addAndAlterColumns(tableDefaultCollation = resolvedDefaultCollation)
+      }
+    }
+  }
+
+  private def addAndAlterColumns(tableDefaultCollation: String): Unit = {
+    // ADD COLUMN
+    sql(s"ALTER TABLE $testTable ADD COLUMN c3 STRING")
+    sql(s"ALTER TABLE $testTable ADD COLUMN c4 STRING COLLATE SR_AI")
+    sql(s"ALTER TABLE $testTable ADD COLUMN c5 STRING COLLATE UTF8_BINARY")
+    assertTableColumnCollation(testTable, "c3", tableDefaultCollation)
+    assertTableColumnCollation(testTable, "c4", "SR_AI")
+    assertTableColumnCollation(testTable, "c5", "UTF8_BINARY")
+
+    // ALTER COLUMN
+    sql(s"ALTER TABLE $testTable ALTER COLUMN c1 TYPE STRING COLLATE UNICODE")
+    sql(s"ALTER TABLE $testTable ALTER COLUMN c2 TYPE STRING")
+    sql(s"ALTER TABLE $testTable ALTER COLUMN c3 TYPE STRING COLLATE UTF8_BINARY")
+    assertTableColumnCollation(testTable, "c1", "UNICODE")
+    assertTableColumnCollation(testTable, "c2", tableDefaultCollation)
+    assertTableColumnCollation(testTable, "c3", "UTF8_BINARY")
+  }
 }
 
 class DefaultCollationTestSuiteV1 extends DefaultCollationTestSuite {
@@ -391,8 +555,14 @@ class DefaultCollationTestSuiteV1 extends DefaultCollationTestSuite {
 }
 
 class DefaultCollationTestSuiteV2 extends DefaultCollationTestSuite with DatasourceV2SQLBase {
-  override def testTable: String = s"testcat.${super.testTable}"
-  override def testView: String = s"testcat.${super.testView}"
+  override def testSchema: String = s"testcat.${super.testSchema}"
+
+  override def beforeEach(): Unit = {
+    super.beforeEach()
+    sql("USE testcat")
+    sql(s"CREATE SCHEMA IF NOT EXISTS $DEFAULT_DATABASE")
+    sql(s"USE testcat.$DEFAULT_DATABASE")
+  }
 
   test("inline table in RTAS") {
     withTable(testTable) {
@@ -427,4 +597,61 @@ class DefaultCollationTestSuiteV2 extends DefaultCollationTestSuite with Datasou
       checkAnswer(sql(s"SELECT DISTINCT COLLATION(c2) FROM $testTable"), Row(s"$prefix.UTF8_LCASE"))
     }
   }
+
+  Seq(
+    // (schemaDefaultCollation, tableDefaultCollation)
+    ("UTF8_BINARY", None),
+    ("UTF8_LCASE", None),
+    ("UNICODE", None),
+    ("DE", None),
+    ("UTF8_BINARY", Some("UTF8_BINARY")),
+    ("UTF8_BINARY", Some("UTF8_LCASE")),
+    ("UTF8_BINARY", Some("DE")),
+    ("UTF8_LCASE", Some("UTF8_BINARY")),
+    ("UTF8_LCASE", Some("UTF8_LCASE")),
+    ("UTF8_LCASE", Some("DE"))
+  ).foreach {
+    case (schemaDefaultCollation, tableDefaultCollation) =>
+      test(
+        s"""CREATE OR REPLACE table with schema level collation
+          | (schema default collation = $schemaDefaultCollation,
+          | table default collation = $tableDefaultCollation)""".stripMargin) {
+        testCreateTableWithSchemaLevelCollation(
+          schemaDefaultCollation, tableDefaultCollation, replaceTable = true)
+      }
+
+      test(
+        s"""REPLACE COLUMNS with schema level collation
+          | (schema default collation = $schemaDefaultCollation,
+          | table default collation = $tableDefaultCollation""".stripMargin) {
+        testReplaceColumns(
+          schemaDefaultCollation, tableDefaultCollation)
+      }
+  }
+
+  private def testReplaceColumns(
+      schemaDefaultCollation: String, tableDefaultCollation: Option[String] = None): Unit = {
+    val (tableDefaultCollationClause, resolvedDefaultCollation) =
+      if (tableDefaultCollation.isDefined) {
+        (s"DEFAULT COLLATION ${tableDefaultCollation.get}", tableDefaultCollation.get)
+      } else {
+        ("", schemaDefaultCollation)
+      }
+
+    withDatabase(testSchema) {
+      sql(s"CREATE SCHEMA $testSchema DEFAULT COLLATION $schemaDefaultCollation")
+      sql(s"USE $testSchema")
+
+      withTable(testTable) {
+        sql(s"CREATE TABLE $testTable (c1 STRING, c2 STRING COLLATE SR_AI) " +
+          s"$tableDefaultCollationClause")
+
+        sql(s"ALTER TABLE $testTable REPLACE COLUMNS " +
+          "(c1 STRING COLLATE UNICODE, c2 STRING, c3 STRING COLLATE UTF8_BINARY)")
+        assertTableColumnCollation(testTable, "c1", "UNICODE")
+        assertTableColumnCollation(testTable, "c2", resolvedDefaultCollation)
+        assertTableColumnCollation(testTable, "c3", "UTF8_BINARY")
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
index be83d2c65323..110347e766e8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
@@ -918,6 +918,20 @@ class QueryCompilationErrorsSuite
     }
   }
 
+  test("SPARK-52219: the schema level collations feature is unsupported") {
+    // TODO: when schema level collations are supported, change this test to set the flag to false
+    Seq(
+      "CREATE SCHEMA test_schema DEFAULT COLLATION UNICODE",
+      "ALTER SCHEMA test_schema DEFAULT COLLATION UNICODE"
+    ).foreach {
+      sqlText =>
+        checkError(
+          exception = intercept[AnalysisException](sql(sqlText)),
+          condition = "UNSUPPORTED_FEATURE.SCHEMA_LEVEL_COLLATIONS"
+        )
+    }
+  }
+
   test("UNSUPPORTED_CALL: call the unsupported method update()") {
     checkError(
       exception = intercept[SparkUnsupportedOperationException] {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterNamespaceSetPropertiesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterNamespaceSetPropertiesSuiteBase.scala
index 3b0ac1d40823..f42fd131adc4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterNamespaceSetPropertiesSuiteBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterNamespaceSetPropertiesSuiteBase.scala
@@ -100,7 +100,8 @@ trait AlterNamespaceSetPropertiesSuiteBase extends QueryTest with DDLCommandTest
       }
     }
     withSQLConf((SQLConf.LEGACY_PROPERTY_NON_RESERVED.key, "true")) {
-      CatalogV2Util.NAMESPACE_RESERVED_PROPERTIES.filterNot(_ == PROP_COMMENT).foreach { key =>
+      CatalogV2Util.NAMESPACE_RESERVED_PROPERTIES
+        .filterNot(prop => prop == PROP_COLLATION || prop == PROP_COMMENT).foreach { key =>
         withNamespace(ns) {
           // Set the location explicitly because v2 catalog may not set the default location.
           // Without this, `meta.get(key)` below may return null.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterNamespaceUnsetPropertiesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterNamespaceUnsetPropertiesSuiteBase.scala
index 42550ef84436..593726526e4c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterNamespaceUnsetPropertiesSuiteBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterNamespaceUnsetPropertiesSuiteBase.scala
@@ -102,7 +102,8 @@ trait AlterNamespaceUnsetPropertiesSuiteBase extends QueryTest with DDLCommandTe
       }
     }
     withSQLConf((SQLConf.LEGACY_PROPERTY_NON_RESERVED.key, "true")) {
-      CatalogV2Util.NAMESPACE_RESERVED_PROPERTIES.filterNot(_ == PROP_COMMENT).foreach { key =>
+      CatalogV2Util.NAMESPACE_RESERVED_PROPERTIES
+        .filterNot(prop => prop == PROP_COLLATION || prop == PROP_COMMENT).foreach { key =>
         withNamespace(ns) {
           // Set the location explicitly because v2 catalog may not set the default location.
           // Without this, `meta.get(key)` below may return null.


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org