You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2025/03/25 21:13:17 UTC
(spark) branch master updated: [MINOR][SQL] Desc As JSON test for top-level default collation field
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 94535c91f79c [MINOR][SQL] Desc As JSON test for top-level default collation field
94535c91f79c is described below
commit 94535c91f79cee98184e53a7ef71aff8add7879f
Author: Amanda Liu <am...@databricks.com>
AuthorDate: Tue Mar 25 14:13:04 2025 -0700
[MINOR][SQL] Desc As JSON test for top-level default collation field
### What changes were proposed in this pull request?
Follow-up to https://github.com/apache/spark/pull/50290. Add a test that Desc As JSON includes a top-level field `collation` when a default table collation is specified.
### Why are the changes needed?
Test that Desc As JSON supports default table collation in addition to column collations.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Added test in DescribeTableSuite
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #50386 from asl3/asl3/descasjson-testdefaultcollation.
Authored-by: Amanda Liu <am...@databricks.com>
Signed-off-by: Gengliang Wang <ge...@apache.org>
---
.../execution/command/DescribeTableSuiteBase.scala | 64 ++++++++++++++++++++
.../execution/command/v1/DescribeTableSuite.scala | 69 ++--------------------
2 files changed, 69 insertions(+), 64 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala
index f8d2e9dd3a3c..2d24fb289d4c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala
@@ -319,3 +319,67 @@ trait DescribeTableSuiteBase extends QueryTest with DDLCommandTestUtils {
}
}
}
+
+/** Represents JSON output of DESCRIBE TABLE AS JSON */
+case class DescribeTableJson(
+ table_name: Option[String] = None,
+ catalog_name: Option[String] = None,
+ namespace: Option[List[String]] = Some(Nil),
+ schema_name: Option[String] = None,
+ columns: Option[List[TableColumn]] = Some(Nil),
+ created_time: Option[String] = None,
+ last_access: Option[String] = None,
+ created_by: Option[String] = None,
+ `type`: Option[String] = None,
+ collation: Option[String] = None,
+ provider: Option[String] = None,
+ bucket_columns: Option[List[String]] = Some(Nil),
+ sort_columns: Option[List[String]] = Some(Nil),
+ comment: Option[String] = None,
+ table_properties: Option[Map[String, String]] = None,
+ location: Option[String] = None,
+ serde_library: Option[String] = None,
+ storage_properties: Option[Map[String, String]] = None,
+ partition_provider: Option[String] = None,
+ partition_columns: Option[List[String]] = Some(Nil),
+ partition_values: Option[Map[String, String]] = None,
+ clustering_columns: Option[List[String]] = None,
+ statistics: Option[Map[String, Any]] = None,
+ view_text: Option[String] = None,
+ view_original_text: Option[String] = None,
+ view_schema_mode: Option[String] = None,
+ view_catalog_and_namespace: Option[String] = None,
+ view_query_output_columns: Option[List[String]] = None
+)
+
+/** Used for columns field of DescribeTableJson */
+case class TableColumn(
+ name: String,
+ `type`: Type,
+ element_nullable: Boolean = true,
+ comment: Option[String] = None,
+ default: Option[String] = None
+)
+
+case class Type(
+ name: String,
+ collation: Option[String] = None,
+ fields: Option[List[Field]] = None,
+ `type`: Option[Type] = None,
+ element_type: Option[Type] = None,
+ key_type: Option[Type] = None,
+ value_type: Option[Type] = None,
+ comment: Option[String] = None,
+ default: Option[String] = None,
+ element_nullable: Option[Boolean] = Some(true),
+ value_nullable: Option[Boolean] = Some(true),
+ nullable: Option[Boolean] = Some(true)
+)
+
+case class Field(
+ name: String,
+ `type`: Type,
+ element_nullable: Boolean = true,
+ comment: Option[String] = None,
+ default: Option[String] = None
+)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala
index 72f1a9d37e40..7bc871193d6c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.SPARK_VERSION
import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
import org.apache.spark.sql.execution.command
+import org.apache.spark.sql.execution.command.{DescribeTableJson, Field, TableColumn, Type}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.StringType
@@ -436,12 +437,14 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
| id INT
|)
|USING parquet COMMENT 'table_comment'
+ |DEFAULT COLLATION UTF8_BINARY
|""".stripMargin
spark.sql(tableCreationStr)
val descriptionDf = spark.sql(s"DESC EXTENDED $t AS JSON")
val firstRow = descriptionDf.select("json_metadata").head()
val jsonValue = firstRow.getString(0)
+ System.out.print("\n *** jsonValue: " + jsonValue + "\n")
val parsedOutput = parse(jsonValue).extract[DescribeTableJson]
val expectedOutput = DescribeTableJson(
@@ -458,6 +461,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
last_access = Some("UNKNOWN"),
created_by = Some(s"Spark $SPARK_VERSION"),
`type` = Some("MANAGED"),
+ collation = Some("UTF8_BINARY"),
storage_properties = None,
provider = Some("parquet"),
bucket_columns = Some(Nil),
@@ -468,6 +472,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
} else {
None
},
+ // table_properties should not include collation
table_properties = None
)
assert(parsedOutput.location.isDefined)
@@ -846,67 +851,3 @@ class DescribeTableSuite extends DescribeTableSuiteBase with CommandSuiteBase {
}
}
}
-
-/** Represents JSON output of DESCRIBE TABLE AS JSON */
-case class DescribeTableJson(
- table_name: Option[String] = None,
- catalog_name: Option[String] = None,
- namespace: Option[List[String]] = Some(Nil),
- schema_name: Option[String] = None,
- columns: Option[List[TableColumn]] = Some(Nil),
- created_time: Option[String] = None,
- last_access: Option[String] = None,
- created_by: Option[String] = None,
- `type`: Option[String] = None,
- provider: Option[String] = None,
- bucket_columns: Option[List[String]] = Some(Nil),
- sort_columns: Option[List[String]] = Some(Nil),
- comment: Option[String] = None,
- table_properties: Option[Map[String, String]] = None,
- location: Option[String] = None,
- serde_library: Option[String] = None,
- storage_properties: Option[Map[String, String]] = None,
- partition_provider: Option[String] = None,
- partition_columns: Option[List[String]] = Some(Nil),
- partition_values: Option[Map[String, String]] = None,
- clustering_columns: Option[List[String]] = None,
- statistics: Option[Map[String, Any]] = None,
- view_text: Option[String] = None,
- view_original_text: Option[String] = None,
- view_schema_mode: Option[String] = None,
- view_catalog_and_namespace: Option[String] = None,
- view_query_output_columns: Option[List[String]] = None
- )
-
-/** Used for columns field of DescribeTableJson */
-case class TableColumn(
- name: String,
- `type`: Type,
- element_nullable: Boolean = true,
- comment: Option[String] = None,
- default: Option[String] = None
-)
-
-case class Type(
- name: String,
- collation: Option[String] = None,
- length: Option[Int] = None,
- fields: Option[List[Field]] = None,
- `type`: Option[Type] = None,
- element_type: Option[Type] = None,
- key_type: Option[Type] = None,
- value_type: Option[Type] = None,
- comment: Option[String] = None,
- default: Option[String] = None,
- element_nullable: Option[Boolean] = Some(true),
- value_nullable: Option[Boolean] = Some(true),
- nullable: Option[Boolean] = Some(true)
-)
-
-case class Field(
- name: String,
- `type`: Type,
- element_nullable: Boolean = true,
- comment: Option[String] = None,
- default: Option[String] = None
-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org