You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2018/10/21 01:02:45 UTC
spark git commit: [SPARK-25779][SQL][TESTS] Remove SQL query tests
for function documentation by DESCRIBE FUNCTION at SQLQueryTestSuite
Repository: spark
Updated Branches:
refs/heads/master ab5752cb9 -> b8c6ba9e6
[SPARK-25779][SQL][TESTS] Remove SQL query tests for function documentation by DESCRIBE FUNCTION at SQLQueryTestSuite
Currently, there are some tests testing function descriptions:
```bash
$ grep -ir "describe function" sql/core/src/test/resources/sql-tests/inputs
sql/core/src/test/resources/sql-tests/inputs/json-functions.sql:describe function to_json;
sql/core/src/test/resources/sql-tests/inputs/json-functions.sql:describe function extended to_json;
sql/core/src/test/resources/sql-tests/inputs/json-functions.sql:describe function from_json;
sql/core/src/test/resources/sql-tests/inputs/json-functions.sql:describe function extended from_json;
```
Looks there are not quite good points about testing them since we're not going to test documentation itself.
For `DESCRIBE FCUNTION` functionality itself, they are already being tested here and there.
See the test failures in https://github.com/apache/spark/pull/18749 (where I added examples to function descriptions)
We better remove those tests so that people don't add such tests in the SQL tests.
## How was this patch tested?
Manual.
Closes #22776 from HyukjinKwon/SPARK-25779.
Authored-by: hyukjinkwon <gu...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b8c6ba9e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b8c6ba9e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b8c6ba9e
Branch: refs/heads/master
Commit: b8c6ba9e644786e5da7e009eb8030ac375ccd75f
Parents: ab5752c
Author: hyukjinkwon <gu...@apache.org>
Authored: Sat Oct 20 18:02:38 2018 -0700
Committer: Dongjoon Hyun <do...@apache.org>
Committed: Sat Oct 20 18:02:38 2018 -0700
----------------------------------------------------------------------
.../sql-tests/inputs/json-functions.sql | 4 -
.../sql-tests/results/json-functions.sql.out | 296 +++++++------------
2 files changed, 114 insertions(+), 186 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b8c6ba9e/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
index bdd1fe4..8bfd7c0 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
@@ -1,6 +1,4 @@
-- to_json
-describe function to_json;
-describe function extended to_json;
select to_json(named_struct('a', 1, 'b', 2));
select to_json(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy'));
select to_json(array(named_struct('a', 1, 'b', 2)));
@@ -15,8 +13,6 @@ select to_json(named_struct('a', 1, 'b', 2), map('mode', 1));
select to_json();
-- from_json
-describe function from_json;
-describe function extended from_json;
select from_json('{"a":1}', 'a INT');
select from_json('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'));
-- Check if errors handled
http://git-wip-us.apache.org/repos/asf/spark/blob/b8c6ba9e/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
index 868eee8..c70a81e 100644
--- a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
@@ -1,196 +1,128 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 42
+-- Number of queries: 38
-- !query 0
-describe function to_json
--- !query 0 schema
-struct<function_desc:string>
--- !query 0 output
-Class: org.apache.spark.sql.catalyst.expressions.StructsToJson
-Function: to_json
-Usage: to_json(expr[, options]) - Returns a JSON string with a given struct value
-
-
--- !query 1
-describe function extended to_json
--- !query 1 schema
-struct<function_desc:string>
--- !query 1 output
-Class: org.apache.spark.sql.catalyst.expressions.StructsToJson
-Extended Usage:
- Examples:
- > SELECT to_json(named_struct('a', 1, 'b', 2));
- {"a":1,"b":2}
- > SELECT to_json(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy'));
- {"time":"26/08/2015"}
- > SELECT to_json(array(named_struct('a', 1, 'b', 2)));
- [{"a":1,"b":2}]
- > SELECT to_json(map('a', named_struct('b', 1)));
- {"a":{"b":1}}
- > SELECT to_json(map(named_struct('a', 1),named_struct('b', 2)));
- {"[1]":{"b":2}}
- > SELECT to_json(map('a', 1));
- {"a":1}
- > SELECT to_json(array((map('a', 1))));
- [{"a":1}]
-
- Since: 2.2.0
-
-Function: to_json
-Usage: to_json(expr[, options]) - Returns a JSON string with a given struct value
-
-
--- !query 2
select to_json(named_struct('a', 1, 'b', 2))
--- !query 2 schema
+-- !query 0 schema
struct<to_json(named_struct(a, 1, b, 2)):string>
--- !query 2 output
+-- !query 0 output
{"a":1,"b":2}
--- !query 3
+-- !query 1
select to_json(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy'))
--- !query 3 schema
+-- !query 1 schema
struct<to_json(named_struct(time, to_timestamp('2015-08-26', 'yyyy-MM-dd'))):string>
--- !query 3 output
+-- !query 1 output
{"time":"26/08/2015"}
--- !query 4
+-- !query 2
select to_json(array(named_struct('a', 1, 'b', 2)))
--- !query 4 schema
+-- !query 2 schema
struct<to_json(array(named_struct(a, 1, b, 2))):string>
--- !query 4 output
+-- !query 2 output
[{"a":1,"b":2}]
--- !query 5
+-- !query 3
select to_json(map(named_struct('a', 1, 'b', 2), named_struct('a', 1, 'b', 2)))
--- !query 5 schema
+-- !query 3 schema
struct<to_json(map(named_struct(a, 1, b, 2), named_struct(a, 1, b, 2))):string>
--- !query 5 output
+-- !query 3 output
{"[1,2]":{"a":1,"b":2}}
--- !query 6
+-- !query 4
select to_json(map('a', named_struct('a', 1, 'b', 2)))
--- !query 6 schema
+-- !query 4 schema
struct<to_json(map(a, named_struct(a, 1, b, 2))):string>
--- !query 6 output
+-- !query 4 output
{"a":{"a":1,"b":2}}
--- !query 7
+-- !query 5
select to_json(map('a', 1))
--- !query 7 schema
+-- !query 5 schema
struct<to_json(map(a, 1)):string>
--- !query 7 output
+-- !query 5 output
{"a":1}
--- !query 8
+-- !query 6
select to_json(array(map('a',1)))
--- !query 8 schema
+-- !query 6 schema
struct<to_json(array(map(a, 1))):string>
--- !query 8 output
+-- !query 6 output
[{"a":1}]
--- !query 9
+-- !query 7
select to_json(array(map('a',1), map('b',2)))
--- !query 9 schema
+-- !query 7 schema
struct<to_json(array(map(a, 1), map(b, 2))):string>
--- !query 9 output
+-- !query 7 output
[{"a":1},{"b":2}]
--- !query 10
+-- !query 8
select to_json(named_struct('a', 1, 'b', 2), named_struct('mode', 'PERMISSIVE'))
--- !query 10 schema
+-- !query 8 schema
struct<>
--- !query 10 output
+-- !query 8 output
org.apache.spark.sql.AnalysisException
Must use a map() function for options;; line 1 pos 7
--- !query 11
+-- !query 9
select to_json(named_struct('a', 1, 'b', 2), map('mode', 1))
--- !query 11 schema
+-- !query 9 schema
struct<>
--- !query 11 output
+-- !query 9 output
org.apache.spark.sql.AnalysisException
A type of keys and values in map() must be string, but got map<string,int>;; line 1 pos 7
--- !query 12
+-- !query 10
select to_json()
--- !query 12 schema
+-- !query 10 schema
struct<>
--- !query 12 output
+-- !query 10 output
org.apache.spark.sql.AnalysisException
Invalid number of arguments for function to_json. Expected: one of 1 and 2; Found: 0; line 1 pos 7
--- !query 13
-describe function from_json
--- !query 13 schema
-struct<function_desc:string>
--- !query 13 output
-Class: org.apache.spark.sql.catalyst.expressions.JsonToStructs
-Function: from_json
-Usage: from_json(jsonStr, schema[, options]) - Returns a struct value with the given `jsonStr` and `schema`.
-
-
--- !query 14
-describe function extended from_json
--- !query 14 schema
-struct<function_desc:string>
--- !query 14 output
-Class: org.apache.spark.sql.catalyst.expressions.JsonToStructs
-Extended Usage:
- Examples:
- > SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE');
- {"a":1, "b":0.8}
- > SELECT from_json('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'));
- {"time":"2015-08-26 00:00:00.0"}
-
- Since: 2.2.0
-
-Function: from_json
-Usage: from_json(jsonStr, schema[, options]) - Returns a struct value with the given `jsonStr` and `schema`.
-
-
--- !query 15
+-- !query 11
select from_json('{"a":1}', 'a INT')
--- !query 15 schema
+-- !query 11 schema
struct<from_json({"a":1}):struct<a:int>>
--- !query 15 output
+-- !query 11 output
{"a":1}
--- !query 16
+-- !query 12
select from_json('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'))
--- !query 16 schema
+-- !query 12 schema
struct<from_json({"time":"26/08/2015"}):struct<time:timestamp>>
--- !query 16 output
+-- !query 12 output
{"time":2015-08-26 00:00:00.0}
--- !query 17
+-- !query 13
select from_json('{"a":1}', 1)
--- !query 17 schema
+-- !query 13 schema
struct<>
--- !query 17 output
+-- !query 13 output
org.apache.spark.sql.AnalysisException
Schema should be specified in DDL format as a string literal or output of the schema_of_json function instead of 1;; line 1 pos 7
--- !query 18
+-- !query 14
select from_json('{"a":1}', 'a InvalidType')
--- !query 18 schema
+-- !query 14 schema
struct<>
--- !query 18 output
+-- !query 14 output
org.apache.spark.sql.AnalysisException
DataType invalidtype is not supported.(line 1, pos 2)
@@ -201,188 +133,188 @@ a InvalidType
; line 1 pos 7
--- !query 19
+-- !query 15
select from_json('{"a":1}', 'a INT', named_struct('mode', 'PERMISSIVE'))
--- !query 19 schema
+-- !query 15 schema
struct<>
--- !query 19 output
+-- !query 15 output
org.apache.spark.sql.AnalysisException
Must use a map() function for options;; line 1 pos 7
--- !query 20
+-- !query 16
select from_json('{"a":1}', 'a INT', map('mode', 1))
--- !query 20 schema
+-- !query 16 schema
struct<>
--- !query 20 output
+-- !query 16 output
org.apache.spark.sql.AnalysisException
A type of keys and values in map() must be string, but got map<string,int>;; line 1 pos 7
--- !query 21
+-- !query 17
select from_json()
--- !query 21 schema
+-- !query 17 schema
struct<>
--- !query 21 output
+-- !query 17 output
org.apache.spark.sql.AnalysisException
Invalid number of arguments for function from_json. Expected: one of 2 and 3; Found: 0; line 1 pos 7
--- !query 22
+-- !query 18
SELECT json_tuple('{"a" : 1, "b" : 2}', CAST(NULL AS STRING), 'b', CAST(NULL AS STRING), 'a')
--- !query 22 schema
+-- !query 18 schema
struct<c0:string,c1:string,c2:string,c3:string>
--- !query 22 output
+-- !query 18 output
NULL 2 NULL 1
--- !query 23
+-- !query 19
CREATE TEMPORARY VIEW jsonTable(jsonField, a) AS SELECT * FROM VALUES ('{"a": 1, "b": 2}', 'a')
--- !query 23 schema
+-- !query 19 schema
struct<>
--- !query 23 output
+-- !query 19 output
--- !query 24
+-- !query 20
SELECT json_tuple(jsonField, 'b', CAST(NULL AS STRING), a) FROM jsonTable
--- !query 24 schema
+-- !query 20 schema
struct<c0:string,c1:string,c2:string>
--- !query 24 output
+-- !query 20 output
2 NULL 1
--- !query 25
+-- !query 21
DROP VIEW IF EXISTS jsonTable
--- !query 25 schema
+-- !query 21 schema
struct<>
--- !query 25 output
+-- !query 21 output
--- !query 26
+-- !query 22
select from_json('{"a":1, "b":2}', 'map<string, int>')
--- !query 26 schema
+-- !query 22 schema
struct<entries:map<string,int>>
--- !query 26 output
+-- !query 22 output
{"a":1,"b":2}
--- !query 27
+-- !query 23
select from_json('{"a":1, "b":"2"}', 'struct<a:int,b:string>')
--- !query 27 schema
+-- !query 23 schema
struct<from_json({"a":1, "b":"2"}):struct<a:int,b:string>>
--- !query 27 output
+-- !query 23 output
{"a":1,"b":"2"}
--- !query 28
+-- !query 24
select schema_of_json('{"c1":0, "c2":[1]}')
--- !query 28 schema
+-- !query 24 schema
struct<schema_of_json({"c1":0, "c2":[1]}):string>
--- !query 28 output
+-- !query 24 output
struct<c1:bigint,c2:array<bigint>>
--- !query 29
+-- !query 25
select from_json('{"c1":[1, 2, 3]}', schema_of_json('{"c1":[0]}'))
--- !query 29 schema
+-- !query 25 schema
struct<from_json({"c1":[1, 2, 3]}):struct<c1:array<bigint>>>
--- !query 29 output
+-- !query 25 output
{"c1":[1,2,3]}
--- !query 30
+-- !query 26
select from_json('[1, 2, 3]', 'array<int>')
--- !query 30 schema
+-- !query 26 schema
struct<from_json([1, 2, 3]):array<int>>
--- !query 30 output
+-- !query 26 output
[1,2,3]
--- !query 31
+-- !query 27
select from_json('[1, "2", 3]', 'array<int>')
--- !query 31 schema
+-- !query 27 schema
struct<from_json([1, "2", 3]):array<int>>
--- !query 31 output
+-- !query 27 output
NULL
--- !query 32
+-- !query 28
select from_json('[1, 2, null]', 'array<int>')
--- !query 32 schema
+-- !query 28 schema
struct<from_json([1, 2, null]):array<int>>
--- !query 32 output
+-- !query 28 output
[1,2,null]
--- !query 33
+-- !query 29
select from_json('[{"a": 1}, {"a":2}]', 'array<struct<a:int>>')
--- !query 33 schema
+-- !query 29 schema
struct<from_json([{"a": 1}, {"a":2}]):array<struct<a:int>>>
--- !query 33 output
+-- !query 29 output
[{"a":1},{"a":2}]
--- !query 34
+-- !query 30
select from_json('{"a": 1}', 'array<struct<a:int>>')
--- !query 34 schema
+-- !query 30 schema
struct<from_json({"a": 1}):array<struct<a:int>>>
--- !query 34 output
+-- !query 30 output
[{"a":1}]
--- !query 35
+-- !query 31
select from_json('[null, {"a":2}]', 'array<struct<a:int>>')
--- !query 35 schema
+-- !query 31 schema
struct<from_json([null, {"a":2}]):array<struct<a:int>>>
--- !query 35 output
+-- !query 31 output
[null,{"a":2}]
--- !query 36
+-- !query 32
select from_json('[{"a": 1}, {"b":2}]', 'array<map<string,int>>')
--- !query 36 schema
+-- !query 32 schema
struct<from_json([{"a": 1}, {"b":2}]):array<map<string,int>>>
--- !query 36 output
+-- !query 32 output
[{"a":1},{"b":2}]
--- !query 37
+-- !query 33
select from_json('[{"a": 1}, 2]', 'array<map<string,int>>')
--- !query 37 schema
+-- !query 33 schema
struct<from_json([{"a": 1}, 2]):array<map<string,int>>>
--- !query 37 output
+-- !query 33 output
NULL
--- !query 38
+-- !query 34
select to_json(array('1', '2', '3'))
--- !query 38 schema
+-- !query 34 schema
struct<to_json(array(1, 2, 3)):string>
--- !query 38 output
+-- !query 34 output
["1","2","3"]
--- !query 39
+-- !query 35
select to_json(array(array(1, 2, 3), array(4)))
--- !query 39 schema
+-- !query 35 schema
struct<to_json(array(array(1, 2, 3), array(4))):string>
--- !query 39 output
+-- !query 35 output
[[1,2,3],[4]]
--- !query 40
+-- !query 36
select schema_of_json('{"c1":1}', map('primitivesAsString', 'true'))
--- !query 40 schema
+-- !query 36 schema
struct<schema_of_json({"c1":1}):string>
--- !query 40 output
+-- !query 36 output
struct<c1:string>
--- !query 41
+-- !query 37
select schema_of_json('{"c1":01, "c2":0.1}', map('allowNumericLeadingZeros', 'true', 'prefersDecimal', 'true'))
--- !query 41 schema
+-- !query 37 schema
struct<schema_of_json({"c1":01, "c2":0.1}):string>
--- !query 41 output
+-- !query 37 output
struct<c1:bigint,c2:decimal(1,1)>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org