You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2021/10/22 13:32:35 UTC
[spark] branch master updated: [SPARK-37067][SQL] Use ZoneId.of()
to handle timezone string in DatetimeUtils
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c3f6132 [SPARK-37067][SQL] Use ZoneId.of() to handle timezone string in DatetimeUtils
c3f6132 is described below
commit c3f6132167a31dfd858887017bd41eda15a17ce8
Author: Linhong Liu <li...@databricks.com>
AuthorDate: Fri Oct 22 21:31:37 2021 +0800
[SPARK-37067][SQL] Use ZoneId.of() to handle timezone string in DatetimeUtils
### What changes were proposed in this pull request?
Use `ZoneId.of()` to handle timezone string in `DatetimeUtils.stringToTimestamp()`
instead of manually splitting it into hours and minutes.
### Why are the changes needed?
In current implementation. we will split the timezone string into hours and minutes.
But if there is no colon in the timezone, the full digits will be treated as hours, then
we couldn't get the correct value.
For example:
`2021-10-11T03:58:03.000+0700` will return null, but "+0700" is actually correct
format that accepted by `ZoneId.of()`
### Does this PR introduce _any_ user-facing change?
bugfix. this allows Spark to accept a more valid timezone format.
### How was this patch tested?
newly added UT
Closes #34338 from linhongliu-db/SPARK-37067.
Authored-by: Linhong Liu <li...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/catalyst/util/DateTimeUtils.scala | 26 ++++++++--------------
.../sql/catalyst/util/DateTimeUtilsSuite.scala | 10 +++++++++
.../sql/execution/datasources/csv/CSVSuite.scala | 2 +-
.../sql/execution/datasources/json/JsonSuite.scala | 2 +-
4 files changed, 21 insertions(+), 19 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 89b7414..3f60fa0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -51,8 +51,13 @@ object DateTimeUtils {
val TIMEZONE_OPTION = "timeZone"
def getZoneId(timeZoneId: String): ZoneId = {
- // To support the (+|-)h:mm format because it was supported before Spark 3.0.
- ZoneId.of(timeZoneId.replaceFirst("(\\+|\\-)(\\d):", "$10$2:"), ZoneId.SHORT_IDS)
+ val formattedZoneId = timeZoneId
+ // To support the (+|-)h:mm format because it was supported before Spark 3.0.
+ .replaceFirst("(\\+|\\-)(\\d):", "$10$2:")
+ // To support the (+|-)hh:m format because it was supported before Spark 3.0.
+ .replaceFirst("(\\+|\\-)(\\d\\d):(\\d)$", "$1$2:0$3")
+
+ ZoneId.of(formattedZoneId, ZoneId.SHORT_IDS)
}
def getTimeZone(timeZoneId: String): TimeZone = TimeZone.getTimeZone(getZoneId(timeZoneId))
@@ -328,16 +333,7 @@ object DateTimeUtils {
return (Array.empty, None, false)
}
} else if (i == 5 || i == 6) {
- if (b == '-' || b == '+') {
- if (!isValidDigits(i, currentSegmentDigits)) {
- return (Array.empty, None, false)
- }
- segments(i) = currentSegmentValue
- currentSegmentValue = 0
- currentSegmentDigits = 0
- i += 1
- tz = Some(new String(bytes, j, 1))
- } else if (b == '.' && i == 5) {
+ if (b == '.' && i == 5) {
if (!isValidDigits(i, currentSegmentDigits)) {
return (Array.empty, None, false)
}
@@ -397,11 +393,7 @@ object DateTimeUtils {
}
// This step also validates time zone part
- val zoneId = tz.map {
- case "+" => ZoneOffset.ofHoursMinutes(segments(7), segments(8))
- case "-" => ZoneOffset.ofHoursMinutes(-segments(7), -segments(8))
- case zoneName: String => getZoneId(zoneName.trim)
- }
+ val zoneId = tz.map(zoneName => getZoneId(zoneName.trim))
segments(0) *= yearSign.getOrElse(1)
(segments, zoneId, justTime)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index a339f1c..69bb6c1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -197,6 +197,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
checkStringToTimestamp("2015-03-18T12:03:17-13:53", expected)
checkStringToTimestamp("2015-03-18T12:03:17GMT-13:53", expected)
+ checkStringToTimestamp("2015-03-18T12:03:17-1353", expected)
expected = Option(date(2015, 3, 18, 12, 3, 17, zid = UTC))
checkStringToTimestamp("2015-03-18T12:03:17Z", expected)
@@ -208,16 +209,19 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
checkStringToTimestamp("2015-03-18T12:03:17-1:0", expected)
checkStringToTimestamp("2015-03-18T12:03:17-01:00", expected)
checkStringToTimestamp("2015-03-18T12:03:17GMT-01:00", expected)
+ checkStringToTimestamp("2015-03-18T12:03:17-0100", expected)
zoneId = getZoneId("+07:30")
expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
checkStringToTimestamp("2015-03-18T12:03:17+07:30", expected)
checkStringToTimestamp("2015-03-18T12:03:17 GMT+07:30", expected)
+ checkStringToTimestamp("2015-03-18T12:03:17+0730", expected)
zoneId = getZoneId("+07:03")
expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
checkStringToTimestamp("2015-03-18T12:03:17+07:03", expected)
checkStringToTimestamp("2015-03-18T12:03:17GMT+07:03", expected)
+ checkStringToTimestamp("2015-03-18T12:03:17+0703", expected)
// tests for the string including milliseconds.
expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zid))
@@ -236,16 +240,19 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
checkStringToTimestamp("2015-03-18T12:03:17.123-1:0", expected)
checkStringToTimestamp("2015-03-18T12:03:17.123-01:00", expected)
checkStringToTimestamp("2015-03-18T12:03:17.123 GMT-01:00", expected)
+ checkStringToTimestamp("2015-03-18T12:03:17.123-0100", expected)
zoneId = getZoneId("+07:30")
expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
checkStringToTimestamp("2015-03-18T12:03:17.123+07:30", expected)
checkStringToTimestamp("2015-03-18T12:03:17.123 GMT+07:30", expected)
+ checkStringToTimestamp("2015-03-18T12:03:17.123+0730", expected)
zoneId = getZoneId("+07:30")
expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
checkStringToTimestamp("2015-03-18T12:03:17.123+07:30", expected)
checkStringToTimestamp("2015-03-18T12:03:17.123GMT+07:30", expected)
+ checkStringToTimestamp("2015-03-18T12:03:17.123+0730", expected)
expected = Option(date(2015, 3, 18, 12, 3, 17, 123121, zid = zoneId))
checkStringToTimestamp("2015-03-18T12:03:17.123121+7:30", expected)
@@ -255,6 +262,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
expected = Option(date(2015, 3, 18, 12, 3, 17, 123120, zid = zoneId))
checkStringToTimestamp("2015-03-18T12:03:17.12312+7:30", expected)
checkStringToTimestamp("2015-03-18T12:03:17.12312 UT+07:30", expected)
+ checkStringToTimestamp("2015-03-18T12:03:17.12312+0730", expected)
expected = Option(time(18, 12, 15, zid = zid))
checkStringToTimestamp("18:12:15", expected)
@@ -263,11 +271,13 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
expected = Option(time(18, 12, 15, 123120, zid = zoneId))
checkStringToTimestamp("T18:12:15.12312+7:30", expected)
checkStringToTimestamp("T18:12:15.12312 UTC+07:30", expected)
+ checkStringToTimestamp("T18:12:15.12312+0730", expected)
zoneId = getZoneId("+07:30")
expected = Option(time(18, 12, 15, 123120, zid = zoneId))
checkStringToTimestamp("18:12:15.12312+7:30", expected)
checkStringToTimestamp("18:12:15.12312 GMT+07:30", expected)
+ checkStringToTimestamp("18:12:15.12312+0730", expected)
expected = Option(date(2011, 5, 6, 7, 8, 9, 100000, zid = zid))
checkStringToTimestamp("2011-05-06 07:08:09.1000", expected)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index 5b265e8..41f4069 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -2347,7 +2347,7 @@ abstract class CSVSuite
}
test("exception mode for parsing date/timestamp string") {
- val ds = Seq("2020-01-27T20:06:11.847-0800").toDS()
+ val ds = Seq("2020-01-27T20:06:11.847-08000").toDS()
val csv = spark.read
.option("header", false)
.option("timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSz")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index d5e0a3f..3a30e1b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -2692,7 +2692,7 @@ abstract class JsonSuite
}
test("exception mode for parsing date/timestamp string") {
- val ds = Seq("{'t': '2020-01-27T20:06:11.847-0800'}").toDS()
+ val ds = Seq("{'t': '2020-01-27T20:06:11.847-08000'}").toDS()
val json = spark.read
.schema("t timestamp")
.option("timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSz")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org