You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2023/03/09 03:30:41 UTC
[spark] branch branch-3.4 updated: [SPARK-42723][SQL] Support parser data type json "timestamp_ltz" as TimestampType
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 006e838edeb [SPARK-42723][SQL] Support parser data type json "timestamp_ltz" as TimestampType
006e838edeb is described below
commit 006e838edebc62ba8a4398dde1b5720b72227c41
Author: Gengliang Wang <ge...@apache.org>
AuthorDate: Wed Mar 8 19:30:09 2023 -0800
[SPARK-42723][SQL] Support parser data type json "timestamp_ltz" as TimestampType
### What changes were proposed in this pull request?
Support parsing `timestamp_ltz` as `TimestampType` in schema JSON string.
It also add tests for both parsing JSON/DDL for "timestamp_ltz" and "timestamp_ntz"
### Why are the changes needed?
`timestamp_ltz` becomes an alias for TimestampType since Spark 3.4
### Does this PR introduce _any_ user-facing change?
No, the new keyword is not released yet.
### How was this patch tested?
New UT
Closes #40345 from gengliangwang/parseJson.
Authored-by: Gengliang Wang <ge...@apache.org>
Signed-off-by: Gengliang Wang <ge...@apache.org>
(cherry picked from commit c4756edde5d71fa6fcb071a071c93c4ca4b5a37a)
Signed-off-by: Gengliang Wang <ge...@apache.org>
---
.../src/main/scala/org/apache/spark/sql/types/DataType.scala | 1 +
.../test/scala/org/apache/spark/sql/types/DataTypeSuite.scala | 9 +++++++++
2 files changed, 10 insertions(+)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
index e4e548127d3..08d6f312066 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -201,6 +201,7 @@ object DataType {
case VARCHAR_TYPE(length) => VarcharType(length.toInt)
// For backwards compatibility, previously the type name of NullType is "null"
case "null" => NullType
+ case "timestamp_ltz" => TimestampType
case other => otherTypes.getOrElse(
other,
throw new IllegalArgumentException(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
index 5d71732f7b0..2a487133b48 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
@@ -191,6 +191,12 @@ class DataTypeSuite extends SparkFunSuite {
assert(DataType.fromJson("\"null\"") == NullType)
}
+ test("SPARK-42723: Parse timestamp_ltz as TimestampType") {
+ assert(DataType.fromJson("\"timestamp_ltz\"") == TimestampType)
+ val expectedStructType = StructType(Seq(StructField("ts", TimestampType)))
+ assert(DataType.fromDDL("ts timestamp_ltz") == expectedStructType)
+ }
+
def checkDataTypeFromJson(dataType: DataType): Unit = {
test(s"from Json - $dataType") {
assert(DataType.fromJson(dataType.json) === dataType)
@@ -241,6 +247,9 @@ class DataTypeSuite extends SparkFunSuite {
checkDataTypeFromJson(TimestampType)
checkDataTypeFromDDL(TimestampType)
+ checkDataTypeFromJson(TimestampNTZType)
+ checkDataTypeFromDDL(TimestampNTZType)
+
checkDataTypeFromJson(StringType)
checkDataTypeFromDDL(StringType)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org