You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2019/07/31 02:37:02 UTC
[spark] branch master updated: [SPARK-28038][SQL][TEST] Port
text.sql
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 261e113 [SPARK-28038][SQL][TEST] Port text.sql
261e113 is described below
commit 261e113449cf1ae84feb073caff01cc27cb5d10f
Author: Yuming Wang <yu...@ebay.com>
AuthorDate: Wed Jul 31 11:36:26 2019 +0900
[SPARK-28038][SQL][TEST] Port text.sql
## What changes were proposed in this pull request?
This PR is to port text.sql from PostgreSQL regression tests. https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/text.sql
The expected results can be found in the link: https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/expected/text.out
When porting the test cases, found a PostgreSQL specific features that do not exist in Spark SQL:
[SPARK-28037](https://issues.apache.org/jira/browse/SPARK-28037): Add built-in String Functions: quote_literal
Also, found three inconsistent behavior:
[SPARK-27930](https://issues.apache.org/jira/browse/SPARK-27930): Spark SQL's format_string can not fully support PostgreSQL's format
[SPARK-28036](https://issues.apache.org/jira/browse/SPARK-28036): Built-in udf left/right has inconsistent behavior
[SPARK-28033](https://issues.apache.org/jira/browse/SPARK-28033): String concatenation should low priority than other operators
## How was this patch tested?
N/A
Closes #24862 from wangyum/SPARK-28038.
Authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
.../test/resources/sql-tests/inputs/pgSQL/text.sql | 137 ++++++++
.../resources/sql-tests/results/pgSQL/text.sql.out | 375 +++++++++++++++++++++
2 files changed, 512 insertions(+)
diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/text.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/text.sql
new file mode 100644
index 0000000..04d3acc
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/text.sql
@@ -0,0 +1,137 @@
+--
+-- Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
+--
+--
+-- TEXT
+-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/text.sql
+
+SELECT string('this is a text string') = string('this is a text string') AS true;
+
+SELECT string('this is a text string') = string('this is a text strin') AS `false`;
+
+CREATE TABLE TEXT_TBL (f1 string) USING parquet;
+
+INSERT INTO TEXT_TBL VALUES ('doh!');
+INSERT INTO TEXT_TBL VALUES ('hi de ho neighbor');
+
+SELECT '' AS two, * FROM TEXT_TBL;
+
+-- As of 8.3 we have removed most implicit casts to text, so that for example
+-- this no longer works:
+-- Spark SQL implicit cast integer to string
+select length(42);
+
+-- But as a special exception for usability's sake, we still allow implicit
+-- casting to text in concatenations, so long as the other input is text or
+-- an unknown literal. So these work:
+-- [SPARK-28033] String concatenation low priority than other arithmeticBinary
+select string('four: ') || 2+2;
+select 'four: ' || 2+2;
+
+-- but not this:
+-- Spark SQL implicit cast both side to string
+select 3 || 4.0;
+
+/*
+ * various string functions
+ */
+select concat('one');
+-- Spark SQL does not support YYYYMMDD, we replace it to yyyyMMdd
+select concat(1,2,3,'hello',true, false, to_date('20100309','yyyyMMdd'));
+select concat_ws('#','one');
+select concat_ws('#',1,2,3,'hello',true, false, to_date('20100309','yyyyMMdd'));
+select concat_ws(',',10,20,null,30);
+select concat_ws('',10,20,null,30);
+select concat_ws(NULL,10,20,null,30) is null;
+select reverse('abcde');
+-- [SPARK-28036] Built-in udf left/right has inconsistent behavior
+-- [SPARK-28479] Parser error when enabling ANSI mode
+set spark.sql.parser.ansi.enabled=false;
+select i, left('ahoj', i), right('ahoj', i) from range(-5, 6) t(i) order by i;
+set spark.sql.parser.ansi.enabled=true;
+-- [SPARK-28037] Add built-in String Functions: quote_literal
+-- select quote_literal('');
+-- select quote_literal('abc''');
+-- select quote_literal(e'\\');
+
+-- Skip these tests because Spark does not support variadic labeled argument
+-- check variadic labeled argument
+-- select concat(variadic array[1,2,3]);
+-- select concat_ws(',', variadic array[1,2,3]);
+-- select concat_ws(',', variadic NULL::int[]);
+-- select concat(variadic NULL::int[]) is NULL;
+-- select concat(variadic '{}'::int[]) = '';
+--should fail
+-- select concat_ws(',', variadic 10);
+
+-- [SPARK-27930] Replace format to format_string
+/*
+ * format
+ */
+select format_string(NULL);
+select format_string('Hello');
+select format_string('Hello %s', 'World');
+select format_string('Hello %%');
+select format_string('Hello %%%%');
+-- should fail
+select format_string('Hello %s %s', 'World');
+select format_string('Hello %s');
+select format_string('Hello %x', 20);
+-- check literal and sql identifiers
+-- [SPARK-27930] format_string can not fully support PostgreSQL's format
+-- select format_string('INSERT INTO %I VALUES(%L,%L)', 'mytab', 10, 'Hello');
+-- select format_string('%s%s%s','Hello', NULL,'World');
+-- select format_string('INSERT INTO %I VALUES(%L,%L)', 'mytab', 10, NULL);
+-- select format_string('INSERT INTO %I VALUES(%L,%L)', 'mytab', NULL, 'Hello');
+-- should fail, sql identifier cannot be NULL
+-- select format_string('INSERT INTO %I VALUES(%L,%L)', NULL, 10, 'Hello');
+-- check positional placeholders
+select format_string('%1$s %3$s', 1, 2, 3);
+select format_string('%1$s %12$s', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+-- should fail
+select format_string('%1$s %4$s', 1, 2, 3);
+select format_string('%1$s %13$s', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+--PostgreSQL throw ERROR: format specifies argument 0, but arguments are numbered from 1
+select format_string('%0$s', 'Hello');
+-- [SPARK-27930] format_string can not fully support PostgreSQL's format
+-- select format_string('%*0$s', 'Hello');
+-- select format_string('%1$', 1);
+-- select format_string('%1$1', 1);
+-- check mix of positional and ordered placeholders
+select format_string('Hello %s %1$s %s', 'World', 'Hello again');
+select format_string('Hello %s %s, %2$s %2$s', 'World', 'Hello again');
+-- Skip these tests because Spark does not support variadic labeled argument
+-- check variadic labeled arguments
+-- select format('%s, %s', variadic array['Hello','World']);
+-- select format('%s, %s', variadic array[1, 2]);
+-- select format('%s, %s', variadic array[true, false]);
+-- select format('%s, %s', variadic array[true, false]::text[]);
+-- check variadic with positional placeholders
+-- select format('%2$s, %1$s', variadic array['first', 'second']);
+-- select format('%2$s, %1$s', variadic array[1, 2]);
+-- variadic argument can be array type NULL, but should not be referenced
+-- select format('Hello', variadic NULL::int[]);
+-- variadic argument allows simulating more than FUNC_MAX_ARGS parameters
+-- select format(string_agg('%s',','), variadic array_agg(i))
+-- from generate_series(1,200) g(i);
+-- check field widths and left, right alignment
+select format_string('>>%10s<<', 'Hello');
+select format_string('>>%10s<<', NULL);
+select format_string('>>%10s<<', '');
+select format_string('>>%-10s<<', '');
+select format_string('>>%-10s<<', 'Hello');
+select format_string('>>%-10s<<', NULL);
+select format_string('>>%1$10s<<', 'Hello');
+-- [SPARK-27930] format_string can not fully support PostgreSQL's format
+-- select format_string('>>%1$-10I<<', 'Hello');
+-- select format_string('>>%2$*1$L<<', 10, 'Hello');
+-- select format_string('>>%2$*1$L<<', 10, NULL);
+-- select format_string('>>%2$*1$L<<', -10, NULL);
+-- select format_string('>>%*s<<', 10, 'Hello');
+-- select format_string('>>%*1$s<<', 10, 'Hello');
+-- select format_string('>>%-s<<', 'Hello');
+-- select format_string('>>%10L<<', NULL);
+-- select format_string('>>%2$*1$L<<', NULL, 'Hello');
+-- select format_string('>>%2$*1$L<<', 0, 'Hello');
+
+DROP TABLE TEXT_TBL;
diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/text.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/text.sql.out
new file mode 100644
index 0000000..cae75b8
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/text.sql.out
@@ -0,0 +1,375 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 44
+
+
+-- !query 0
+SELECT string('this is a text string') = string('this is a text string') AS true
+-- !query 0 schema
+struct<true:boolean>
+-- !query 0 output
+true
+
+
+-- !query 1
+SELECT string('this is a text string') = string('this is a text strin') AS `false`
+-- !query 1 schema
+struct<false:boolean>
+-- !query 1 output
+false
+
+
+-- !query 2
+CREATE TABLE TEXT_TBL (f1 string) USING parquet
+-- !query 2 schema
+struct<>
+-- !query 2 output
+
+
+
+-- !query 3
+INSERT INTO TEXT_TBL VALUES ('doh!')
+-- !query 3 schema
+struct<>
+-- !query 3 output
+
+
+
+-- !query 4
+INSERT INTO TEXT_TBL VALUES ('hi de ho neighbor')
+-- !query 4 schema
+struct<>
+-- !query 4 output
+
+
+
+-- !query 5
+SELECT '' AS two, * FROM TEXT_TBL
+-- !query 5 schema
+struct<two:string,f1:string>
+-- !query 5 output
+doh!
+ hi de ho neighbor
+
+
+-- !query 6
+select length(42)
+-- !query 6 schema
+struct<length(CAST(42 AS STRING)):int>
+-- !query 6 output
+2
+
+
+-- !query 7
+select string('four: ') || 2+2
+-- !query 7 schema
+struct<(CAST(concat(CAST(four: AS STRING), CAST(2 AS STRING)) AS DOUBLE) + CAST(2 AS DOUBLE)):double>
+-- !query 7 output
+NULL
+
+
+-- !query 8
+select 'four: ' || 2+2
+-- !query 8 schema
+struct<(CAST(concat(four: , CAST(2 AS STRING)) AS DOUBLE) + CAST(2 AS DOUBLE)):double>
+-- !query 8 output
+NULL
+
+
+-- !query 9
+select 3 || 4.0
+-- !query 9 schema
+struct<concat(CAST(3 AS STRING), CAST(4.0 AS STRING)):string>
+-- !query 9 output
+34.0
+
+
+-- !query 10
+/*
+ * various string functions
+ */
+select concat('one')
+-- !query 10 schema
+struct<concat(one):string>
+-- !query 10 output
+one
+
+
+-- !query 11
+select concat(1,2,3,'hello',true, false, to_date('20100309','yyyyMMdd'))
+-- !query 11 schema
+struct<concat(CAST(1 AS STRING), CAST(2 AS STRING), CAST(3 AS STRING), hello, CAST(true AS STRING), CAST(false AS STRING), CAST(to_date('20100309', 'yyyyMMdd') AS STRING)):string>
+-- !query 11 output
+123hellotruefalse2010-03-09
+
+
+-- !query 12
+select concat_ws('#','one')
+-- !query 12 schema
+struct<concat_ws(#, one):string>
+-- !query 12 output
+one
+
+
+-- !query 13
+select concat_ws('#',1,2,3,'hello',true, false, to_date('20100309','yyyyMMdd'))
+-- !query 13 schema
+struct<concat_ws(#, CAST(1 AS STRING), CAST(2 AS STRING), CAST(3 AS STRING), hello, CAST(true AS STRING), CAST(false AS STRING), CAST(to_date('20100309', 'yyyyMMdd') AS STRING)):string>
+-- !query 13 output
+1#x#x#hello#true#false#x-03-09
+
+
+-- !query 14
+select concat_ws(',',10,20,null,30)
+-- !query 14 schema
+struct<concat_ws(,, CAST(10 AS STRING), CAST(20 AS STRING), NULL, CAST(30 AS STRING)):string>
+-- !query 14 output
+10,20,30
+
+
+-- !query 15
+select concat_ws('',10,20,null,30)
+-- !query 15 schema
+struct<concat_ws(, CAST(10 AS STRING), CAST(20 AS STRING), NULL, CAST(30 AS STRING)):string>
+-- !query 15 output
+102030
+
+
+-- !query 16
+select concat_ws(NULL,10,20,null,30) is null
+-- !query 16 schema
+struct<(concat_ws(CAST(NULL AS STRING), CAST(10 AS STRING), CAST(20 AS STRING), NULL, CAST(30 AS STRING)) IS NULL):boolean>
+-- !query 16 output
+true
+
+
+-- !query 17
+select reverse('abcde')
+-- !query 17 schema
+struct<reverse(abcde):string>
+-- !query 17 output
+edcba
+
+
+-- !query 18
+set spark.sql.parser.ansi.enabled=false
+-- !query 18 schema
+struct<key:string,value:string>
+-- !query 18 output
+spark.sql.parser.ansi.enabled false
+
+
+-- !query 19
+select i, left('ahoj', i), right('ahoj', i) from range(-5, 6) t(i) order by i
+-- !query 19 schema
+struct<i:bigint,left('ahoj', t.`i`):string,right('ahoj', t.`i`):string>
+-- !query 19 output
+-5
+-4
+-3
+-2
+-1
+0
+1 a j
+2 ah oj
+3 aho hoj
+4 ahoj ahoj
+5 ahoj ahoj
+
+
+-- !query 20
+set spark.sql.parser.ansi.enabled=true
+-- !query 20 schema
+struct<key:string,value:string>
+-- !query 20 output
+spark.sql.parser.ansi.enabled true
+
+
+-- !query 21
+/*
+ * format
+ */
+select format_string(NULL)
+-- !query 21 schema
+struct<format_string(CAST(NULL AS STRING)):string>
+-- !query 21 output
+NULL
+
+
+-- !query 22
+select format_string('Hello')
+-- !query 22 schema
+struct<format_string(Hello):string>
+-- !query 22 output
+Hello
+
+
+-- !query 23
+select format_string('Hello %s', 'World')
+-- !query 23 schema
+struct<format_string(Hello %s, World):string>
+-- !query 23 output
+Hello World
+
+
+-- !query 24
+select format_string('Hello %%')
+-- !query 24 schema
+struct<format_string(Hello %%):string>
+-- !query 24 output
+Hello %
+
+
+-- !query 25
+select format_string('Hello %%%%')
+-- !query 25 schema
+struct<format_string(Hello %%%%):string>
+-- !query 25 output
+Hello %%
+
+
+-- !query 26
+select format_string('Hello %s %s', 'World')
+-- !query 26 schema
+struct<>
+-- !query 26 output
+java.util.MissingFormatArgumentException
+Format specifier '%s'
+
+
+-- !query 27
+select format_string('Hello %s')
+-- !query 27 schema
+struct<>
+-- !query 27 output
+java.util.MissingFormatArgumentException
+Format specifier '%s'
+
+
+-- !query 28
+select format_string('Hello %x', 20)
+-- !query 28 schema
+struct<format_string(Hello %x, 20):string>
+-- !query 28 output
+Hello 14
+
+
+-- !query 29
+select format_string('%1$s %3$s', 1, 2, 3)
+-- !query 29 schema
+struct<format_string(%1$s %3$s, 1, 2, 3):string>
+-- !query 29 output
+1 3
+
+
+-- !query 30
+select format_string('%1$s %12$s', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)
+-- !query 30 schema
+struct<format_string(%1$s %12$s, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12):string>
+-- !query 30 output
+1 12
+
+
+-- !query 31
+select format_string('%1$s %4$s', 1, 2, 3)
+-- !query 31 schema
+struct<>
+-- !query 31 output
+java.util.MissingFormatArgumentException
+Format specifier '%4$s'
+
+
+-- !query 32
+select format_string('%1$s %13$s', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)
+-- !query 32 schema
+struct<>
+-- !query 32 output
+java.util.MissingFormatArgumentException
+Format specifier '%13$s'
+
+
+-- !query 33
+select format_string('%0$s', 'Hello')
+-- !query 33 schema
+struct<format_string(%0$s, Hello):string>
+-- !query 33 output
+Hello
+
+
+-- !query 34
+select format_string('Hello %s %1$s %s', 'World', 'Hello again')
+-- !query 34 schema
+struct<format_string(Hello %s %1$s %s, World, Hello again):string>
+-- !query 34 output
+Hello World World Hello again
+
+
+-- !query 35
+select format_string('Hello %s %s, %2$s %2$s', 'World', 'Hello again')
+-- !query 35 schema
+struct<format_string(Hello %s %s, %2$s %2$s, World, Hello again):string>
+-- !query 35 output
+Hello World Hello again, Hello again Hello again
+
+
+-- !query 36
+select format_string('>>%10s<<', 'Hello')
+-- !query 36 schema
+struct<format_string(>>%10s<<, Hello):string>
+-- !query 36 output
+>> Hello<<
+
+
+-- !query 37
+select format_string('>>%10s<<', NULL)
+-- !query 37 schema
+struct<format_string(>>%10s<<, NULL):string>
+-- !query 37 output
+>> null<<
+
+
+-- !query 38
+select format_string('>>%10s<<', '')
+-- !query 38 schema
+struct<format_string(>>%10s<<, ):string>
+-- !query 38 output
+>> <<
+
+
+-- !query 39
+select format_string('>>%-10s<<', '')
+-- !query 39 schema
+struct<format_string(>>%-10s<<, ):string>
+-- !query 39 output
+>> <<
+
+
+-- !query 40
+select format_string('>>%-10s<<', 'Hello')
+-- !query 40 schema
+struct<format_string(>>%-10s<<, Hello):string>
+-- !query 40 output
+>>Hello <<
+
+
+-- !query 41
+select format_string('>>%-10s<<', NULL)
+-- !query 41 schema
+struct<format_string(>>%-10s<<, NULL):string>
+-- !query 41 output
+>>null <<
+
+
+-- !query 42
+select format_string('>>%1$10s<<', 'Hello')
+-- !query 42 schema
+struct<format_string(>>%1$10s<<, Hello):string>
+-- !query 42 output
+>> Hello<<
+
+
+-- !query 43
+DROP TABLE TEXT_TBL
+-- !query 43 schema
+struct<>
+-- !query 43 output
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org