You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2020/03/18 22:28:15 UTC
[spark] branch master updated: [SPARK-31150][SQL][FOLLOWUP] handle
' as escape for text
This is an automated email from the ASF dual-hosted git repository.
yamamuro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3d69595 [SPARK-31150][SQL][FOLLOWUP] handle ' as escape for text
3d69595 is described below
commit 3d695954e53038f978bebcb3e4444798fa8728d1
Author: Kent Yao <ya...@hotmail.com>
AuthorDate: Thu Mar 19 07:27:06 2020 +0900
[SPARK-31150][SQL][FOLLOWUP] handle ' as escape for text
### What changes were proposed in this pull request?
pattern `''` means literal `'`
```sql
select date_format(to_timestamp("11111904-01-23 15:02:01", 'y-MM-dd HH:mm:ss'), "y-MM-dd HH:mm:ss''SSSSSSSSS");
5377-02-14 06:27:19'000000519
```
https://github.com/apache/spark/commit/0946a9514f56565c78b0555383c1ece14aaf2b7b missed this case and this pr add it back.
### Why are the changes needed?
bugfix
### Does this PR introduce any user-facing change?
no
### How was this patch tested?
add ut
Closes #27949 from yaooqinn/SPARK-31150-2.
Authored-by: Kent Yao <ya...@hotmail.com>
Signed-off-by: Takeshi Yamamuro <ya...@apache.org>
---
.../catalyst/util/DateTimeFormatterHelper.scala | 4 +++
.../test/resources/sql-tests/inputs/datetime.sql | 5 ++++
.../resources/sql-tests/results/datetime.sql.out | 34 +++++++++++++++++++++-
3 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 72bae28..4ed618e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -117,6 +117,10 @@ private object DateTimeFormatterHelper {
pattern: String): DateTimeFormatterBuilder = {
val builder = createBuilder()
pattern.split("'").zipWithIndex.foreach {
+ // Split string starting with the regex itself which is `'` here will produce an extra empty
+ // string at res(0). So when the first element here is empty string we do not need append `'`
+ // literal to the DateTimeFormatterBuilder.
+ case ("", idx) if idx != 0 => builder.appendLiteral("'")
case (pattenPart, idx) if idx % 2 == 0 =>
var rest = pattenPart
while (rest.nonEmpty) {
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index a06cdfd..2c4ed64 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -107,3 +107,8 @@ select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd");
select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee');
select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc');
select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu');
+
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS"); -- middle
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''"); -- tail
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss"); -- head
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss"); -- head but as single quote
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index 714412f..f440b5f 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 73
+-- Number of queries: 77
-- !query
@@ -601,3 +601,35 @@ select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
-- !query output
2019-10-06 Sunday
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'12', 'yyyy-MM-dd\'T\'HH:mm:ss\'\'SSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'', 'yyyy-MM-dd\'T\'HH:mm:ss\'\''):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('\'2019-10-06T10:11:12', '\'\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('P2019-10-06T10:11:12', '\'P\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org