You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2025/03/25 21:13:17 UTC

(spark) branch master updated: [MINOR][SQL] Desc As JSON test for top-level default collation field

This is an automated email from the ASF dual-hosted git repository.

gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 94535c91f79c [MINOR][SQL] Desc As JSON test for top-level default collation field
94535c91f79c is described below

commit 94535c91f79cee98184e53a7ef71aff8add7879f
Author: Amanda Liu <am...@databricks.com>
AuthorDate: Tue Mar 25 14:13:04 2025 -0700

    [MINOR][SQL] Desc As JSON test for top-level default collation field
    
    ### What changes were proposed in this pull request?
    
    Follow-up to https://github.com/apache/spark/pull/50290. Add a test that Desc As JSON includes a top-level field `collation` when a default table collation is specified.
    
    ### Why are the changes needed?
    
    Test that Desc As JSON supports default table collation in addition to column collations.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No
    
    ### How was this patch tested?
    
    Added test in DescribeTableSuite
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No
    
    Closes #50386 from asl3/asl3/descasjson-testdefaultcollation.
    
    Authored-by: Amanda Liu <am...@databricks.com>
    Signed-off-by: Gengliang Wang <ge...@apache.org>
---
 .../execution/command/DescribeTableSuiteBase.scala | 64 ++++++++++++++++++++
 .../execution/command/v1/DescribeTableSuite.scala  | 69 ++--------------------
 2 files changed, 69 insertions(+), 64 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala
index f8d2e9dd3a3c..2d24fb289d4c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala
@@ -319,3 +319,67 @@ trait DescribeTableSuiteBase extends QueryTest with DDLCommandTestUtils {
     }
   }
 }
+
+/** Represents JSON output of DESCRIBE TABLE AS JSON */
+case class DescribeTableJson(
+    table_name: Option[String] = None,
+    catalog_name: Option[String] = None,
+    namespace: Option[List[String]] = Some(Nil),
+    schema_name: Option[String] = None,
+    columns: Option[List[TableColumn]] = Some(Nil),
+    created_time: Option[String] = None,
+    last_access: Option[String] = None,
+    created_by: Option[String] = None,
+    `type`: Option[String] = None,
+    collation: Option[String] = None,
+    provider: Option[String] = None,
+    bucket_columns: Option[List[String]] = Some(Nil),
+    sort_columns: Option[List[String]] = Some(Nil),
+    comment: Option[String] = None,
+    table_properties: Option[Map[String, String]] = None,
+    location: Option[String] = None,
+    serde_library: Option[String] = None,
+    storage_properties: Option[Map[String, String]] = None,
+    partition_provider: Option[String] = None,
+    partition_columns: Option[List[String]] = Some(Nil),
+    partition_values: Option[Map[String, String]] = None,
+    clustering_columns: Option[List[String]] = None,
+    statistics: Option[Map[String, Any]] = None,
+    view_text: Option[String] = None,
+    view_original_text: Option[String] = None,
+    view_schema_mode: Option[String] = None,
+    view_catalog_and_namespace: Option[String] = None,
+    view_query_output_columns: Option[List[String]] = None
+)
+
+/** Used for columns field of DescribeTableJson */
+case class TableColumn(
+    name: String,
+    `type`: Type,
+    element_nullable: Boolean = true,
+    comment: Option[String] = None,
+    default: Option[String] = None
+)
+
+case class Type(
+    name: String,
+    collation: Option[String] = None,
+    fields: Option[List[Field]] = None,
+    `type`: Option[Type] = None,
+    element_type: Option[Type] = None,
+    key_type: Option[Type] = None,
+    value_type: Option[Type] = None,
+    comment: Option[String] = None,
+    default: Option[String] = None,
+    element_nullable: Option[Boolean] = Some(true),
+    value_nullable: Option[Boolean] = Some(true),
+    nullable: Option[Boolean] = Some(true)
+)
+
+case class Field(
+    name: String,
+    `type`: Type,
+    element_nullable: Boolean = true,
+    comment: Option[String] = None,
+    default: Option[String] = None
+)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala
index 72f1a9d37e40..7bc871193d6c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.SPARK_VERSION
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
 import org.apache.spark.sql.execution.command
+import org.apache.spark.sql.execution.command.{DescribeTableJson, Field, TableColumn, Type}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StringType
 
@@ -436,12 +437,14 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
            |  id INT
            |)
            |USING parquet COMMENT 'table_comment'
+           |DEFAULT COLLATION UTF8_BINARY
            |""".stripMargin
       spark.sql(tableCreationStr)
 
       val descriptionDf = spark.sql(s"DESC EXTENDED $t AS JSON")
       val firstRow = descriptionDf.select("json_metadata").head()
       val jsonValue = firstRow.getString(0)
+      System.out.print("\n *** jsonValue: " + jsonValue + "\n")
       val parsedOutput = parse(jsonValue).extract[DescribeTableJson]
 
       val expectedOutput = DescribeTableJson(
@@ -458,6 +461,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
         last_access = Some("UNKNOWN"),
         created_by = Some(s"Spark $SPARK_VERSION"),
         `type` = Some("MANAGED"),
+        collation = Some("UTF8_BINARY"),
         storage_properties = None,
         provider = Some("parquet"),
         bucket_columns = Some(Nil),
@@ -468,6 +472,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
         } else {
           None
         },
+        // table_properties should not include collation
         table_properties = None
       )
       assert(parsedOutput.location.isDefined)
@@ -846,67 +851,3 @@ class DescribeTableSuite extends DescribeTableSuiteBase with CommandSuiteBase {
     }
   }
 }
-
-/** Represents JSON output of DESCRIBE TABLE AS JSON */
-case class DescribeTableJson(
-    table_name: Option[String] = None,
-    catalog_name: Option[String] = None,
-    namespace: Option[List[String]] = Some(Nil),
-    schema_name: Option[String] = None,
-    columns: Option[List[TableColumn]] = Some(Nil),
-    created_time: Option[String] = None,
-    last_access: Option[String] = None,
-    created_by: Option[String] = None,
-    `type`: Option[String] = None,
-    provider: Option[String] = None,
-    bucket_columns: Option[List[String]] = Some(Nil),
-    sort_columns: Option[List[String]] = Some(Nil),
-    comment: Option[String] = None,
-    table_properties: Option[Map[String, String]] = None,
-    location: Option[String] = None,
-    serde_library: Option[String] = None,
-    storage_properties: Option[Map[String, String]] = None,
-    partition_provider: Option[String] = None,
-    partition_columns: Option[List[String]] = Some(Nil),
-    partition_values: Option[Map[String, String]] = None,
-    clustering_columns: Option[List[String]] = None,
-    statistics: Option[Map[String, Any]] = None,
-    view_text: Option[String] = None,
-    view_original_text: Option[String] = None,
-    view_schema_mode: Option[String] = None,
-    view_catalog_and_namespace: Option[String] = None,
-    view_query_output_columns: Option[List[String]] = None
-  )
-
-/** Used for columns field of DescribeTableJson */
-case class TableColumn(
-  name: String,
-  `type`: Type,
-  element_nullable: Boolean = true,
-  comment: Option[String] = None,
-  default: Option[String] = None
-)
-
-case class Type(
-   name: String,
-   collation: Option[String] = None,
-   length: Option[Int] = None,
-   fields: Option[List[Field]] = None,
-   `type`: Option[Type] = None,
-   element_type: Option[Type] = None,
-   key_type: Option[Type] = None,
-   value_type: Option[Type] = None,
-   comment: Option[String] = None,
-   default: Option[String] = None,
-   element_nullable: Option[Boolean] = Some(true),
-   value_nullable: Option[Boolean] = Some(true),
-   nullable: Option[Boolean] = Some(true)
-)
-
-case class Field(
-  name: String,
-  `type`: Type,
-  element_nullable: Boolean = true,
-  comment: Option[String] = None,
-  default: Option[String] = None
-)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org