You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2025/04/01 03:52:49 UTC
(spark) branch master updated: [SPARK-51657] UTF8_BINARY default table collation shown by default in Desc As JSON (v1)
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 13945c84731f [SPARK-51657] UTF8_BINARY default table collation shown by default in Desc As JSON (v1)
13945c84731f is described below
commit 13945c84731f163b20c3f21fe351242dc2f9dd91
Author: Amanda Liu <am...@databricks.com>
AuthorDate: Mon Mar 31 20:52:38 2025 -0700
[SPARK-51657] UTF8_BINARY default table collation shown by default in Desc As JSON (v1)
### What changes were proposed in this pull request?
Display `UTF8_BINARY` default table collation by default in Desc As JSON for v1 table.
Note this only affects the `desc as json` output and not `desc table extended` -- desc table does NOT always show the top-level table collation (if omitted in the`desc table` output, it is `UTF8_BINARY`)
### Why are the changes needed?
If the CatalogTable `collation` property is None, then it is by default `UTF8_BINARY`. This update is for completeness for the `desc as json` output.
### Does this PR introduce _any_ user-facing change?
Yes, affects the JSON result
### How was this patch tested?
Added tests in `v1/DescribeTableSuite.scala`
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #50451 from asl3/asl3/descasjson-defaultcollation.
Authored-by: Amanda Liu <am...@databricks.com>
Signed-off-by: Gengliang Wang <ge...@apache.org>
---
.../command/DescribeRelationJsonCommand.scala | 4 ++
.../resources/sql-tests/results/describe.sql.out | 6 +--
.../execution/command/v1/DescribeTableSuite.scala | 53 ++++++++++++----------
3 files changed, 36 insertions(+), 27 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DescribeRelationJsonCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DescribeRelationJsonCommand.scala
index 86d2ddad4a5e..ed248ccca67a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DescribeRelationJsonCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DescribeRelationJsonCommand.scala
@@ -99,6 +99,10 @@ case class DescribeRelationJsonCommand(
case _ => throw QueryCompilationErrors.describeAsJsonNotSupportedForV2TablesError()
}
+ // Add default collation if not yet added (addKeyValueToMap only adds unique keys).
+ // Add here to only affect `DESC AS JSON` and not the `DESC TABLE` output.
+ addKeyValueToMap("collation", JString("UTF8_BINARY"), jsonMap)
+
Seq(Row(compact(render(JObject(jsonMap.toList)))))
}
diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
index ca98bd730d45..3659267e8e9f 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
@@ -76,7 +76,7 @@ DESCRIBE EXTENDED t AS JSON
-- !query schema
struct<json_metadata:string>
-- !query output
-{"table_name":"t","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"a","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"b","type":{"name":"int"},"nullable":true},{"name":"c","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"d","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true}],"num_buckets":2,"bucket_columns":["a"],"sort_columns":["b"],"location":"file:[not included i [...]
+{"table_name":"t","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"a","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"b","type":{"name":"int"},"nullable":true},{"name":"c","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"d","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true}],"num_buckets":2,"bucket_columns":["a"],"sort_columns":["b"],"location":"file:[not included i [...]
-- !query
@@ -303,7 +303,7 @@ DESC EXTENDED t PARTITION (c='Us', d=1) AS JSON
-- !query schema
struct<json_metadata:string>
-- !query output
-{"table_name":"t","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"a","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"b","type":{"name":"int"},"nullable":true},{"name":"c","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"d","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true}],"partition_values":{"c":"Us","d":"1"},"location":"file:[not included in comparison]/{warehou [...]
+{"table_name":"t","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"a","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"b","type":{"name":"int"},"nullable":true},{"name":"c","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"d","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true}],"partition_values":{"c":"Us","d":"1"},"location":"file:[not included in comparison]/{warehou [...]
-- !query
@@ -929,7 +929,7 @@ DESC TABLE EXTENDED f PARTITION (B='SPARK', C=TIMESTAMP'2018-11-17 13:33:33') AS
-- !query schema
struct<json_metadata:string>
-- !query output
-{"table_name":"f","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"A","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"B","type":{"name":"binary"},"nullable":true},{"name":"C","type":{"name":"timestamp_ltz"},"nullable":true}],"partition_values":{"B":"SPARK","C":"2018-11-17 13:33:33"},"location":"file:[not included in comparison]/{warehouse_dir}/f/B=SPARK/C=2018-11-17 13%3A33%3A33","partition_parameters":{"numFi [...]
+{"table_name":"f","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"A","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"B","type":{"name":"binary"},"nullable":true},{"name":"C","type":{"name":"timestamp_ltz"},"nullable":true}],"partition_values":{"B":"SPARK","C":"2018-11-17 13:33:33"},"location":"file:[not included in comparison]/{warehouse_dir}/f/B=SPARK/C=2018-11-17 13%3A33%3A33","partition_parameters":{"numFi [...]
-- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala
index 71d63b32023a..f07dcea6b30e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala
@@ -273,6 +273,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
last_access = Some("UNKNOWN"),
created_by = Some(s"Spark $SPARK_VERSION"),
`type` = Some("MANAGED"),
+ collation = Some("UTF8_BINARY"),
provider = Some("parquet"),
bucket_columns = Some(List("employee_id")),
sort_columns = Some(List("employee_name")),
@@ -338,6 +339,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
last_access = Some("UNKNOWN"),
created_by = Some(s"Spark $SPARK_VERSION"),
`type` = Some("MANAGED"),
+ collation = Some("UTF8_BINARY"),
provider = Some("parquet"),
bucket_columns = Some(Nil),
sort_columns = Some(Nil),
@@ -402,6 +404,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
last_access = Some("UNKNOWN"),
created_by = Some(s"Spark $SPARK_VERSION"),
`type` = Some("MANAGED"),
+ collation = Some("UTF8_BINARY"),
provider = Some("parquet"),
bucket_columns = Some(Nil),
sort_columns = Some(Nil),
@@ -437,7 +440,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
| id INT
|)
|USING parquet COMMENT 'table_comment'
- |DEFAULT COLLATION UTF8_BINARY
+ |DEFAULT COLLATION DE
|""".stripMargin
spark.sql(tableCreationStr)
@@ -455,12 +458,12 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
TableColumn("c1", Type("string", collation = Some("UNICODE_CI"))),
TableColumn("c2", Type("string", collation = Some("UNICODE_RTRIM"))),
TableColumn("c3", Type("string", collation = Some("fr"))),
- TableColumn("c4", Type("string", collation = Some("UTF8_BINARY"))),
+ TableColumn("c4", Type("string", collation = Some("de"))),
TableColumn("id", Type("int")))),
last_access = Some("UNKNOWN"),
created_by = Some(s"Spark $SPARK_VERSION"),
`type` = Some("MANAGED"),
- collation = Some("UTF8_BINARY"),
+ collation = Some("de"),
storage_properties = None,
provider = Some("parquet"),
bucket_columns = Some(Nil),
@@ -514,6 +517,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
last_access = Some("UNKNOWN"),
created_by = Some(s"Spark $SPARK_VERSION"),
`type` = Some("MANAGED"),
+ collation = Some("UTF8_BINARY"),
storage_properties = None,
provider = Some("parquet"),
bucket_columns = Some(Nil),
@@ -557,27 +561,27 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
val jsonValue = firstRow.getString(0)
val parsedOutput = parse(jsonValue).extract[DescribeTableJson]
- val expectedOutput = DescribeTableJson(
- table_name = Some("view"),
- catalog_name = if (isTemp) Some("system") else Some("spark_catalog"),
- namespace = if (isTemp) Some(List("session")) else Some(List("default")),
- schema_name = if (isTemp) Some("session") else Some("default"),
- columns = Some(List(
- TableColumn("id", Type("int")),
- TableColumn("name", Type("string", collation = Some("UTF8_BINARY"))),
- TableColumn("created_at", Type("timestamp_ltz"))
- )),
- last_access = Some("UNKNOWN"),
- created_by = Some(s"Spark $SPARK_VERSION"),
- `type` = Some("VIEW"),
- view_text = Some("SELECT * FROM spark_catalog.ns.table"),
- view_original_text =
- if (isTemp) None else Some("SELECT * FROM spark_catalog.ns.table"),
- // TODO: this is unexpected and temp view should also use COMPENSATION mode.
- view_schema_mode = if (isTemp) Some("BINDING") else Some("COMPENSATION"),
- view_catalog_and_namespace = Some("spark_catalog.default"),
- view_query_output_columns = Some(List("id", "name", "created_at"))
- )
+ val expectedOutput = DescribeTableJson(
+ table_name = Some("view"),
+ catalog_name = if (isTemp) Some("system") else Some("spark_catalog"),
+ namespace = if (isTemp) Some(List("session")) else Some(List("default")),
+ schema_name = if (isTemp) Some("session") else Some("default"),
+ columns = Some(List(
+ TableColumn("id", Type("int")),
+ TableColumn("name", Type("string", collation = Some("UTF8_BINARY"))),
+ TableColumn("created_at", Type("timestamp_ltz"))
+ )),
+ last_access = Some("UNKNOWN"),
+ created_by = Some(s"Spark $SPARK_VERSION"),
+ `type` = Some("VIEW"),
+ collation = Some("UTF8_BINARY"),
+ view_text = Some("SELECT * FROM spark_catalog.ns.table"),
+ view_original_text = if (isTemp) None else Some("SELECT * FROM spark_catalog.ns.table"),
+ // TODO: this is unexpected and temp view should also use COMPENSATION mode.
+ view_schema_mode = if (isTemp) Some("BINDING") else Some("COMPENSATION"),
+ view_catalog_and_namespace = Some("spark_catalog.default"),
+ view_query_output_columns = Some(List("id", "name", "created_at"))
+ )
assert(iso8601Regex.matches(parsedOutput.created_time.get))
assert(expectedOutput == parsedOutput.copy(
@@ -763,6 +767,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
last_access = Some("UNKNOWN"),
created_by = Some(s"Spark $SPARK_VERSION"),
`type` = Some("MANAGED"),
+ collation = Some("UTF8_BINARY"),
provider = Some("parquet"),
comment = Some("A table with nested complex types"),
table_properties = Some(Map(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org