You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2018/04/25 10:10:56 UTC
spark git commit: [SPARK-24012][SQL] Union of map and other
compatible column
Repository: spark
Updated Branches:
refs/heads/master 5fea17b3b -> 64e8408e6
[SPARK-24012][SQL] Union of map and other compatible column
## What changes were proposed in this pull request?
Union of map and other compatible column result in unresolved operator 'Union; exception
Reproduction
`spark-sql>select map(1,2), 'str' union all select map(1,2,3,null), 1`
Output:
```
Error in query: unresolved operator 'Union;;
'Union
:- Project [map(1, 2) AS map(1, 2)#106, str AS str#107]
: +- OneRowRelation$
+- Project [map(1, cast(2 as int), 3, cast(null as int)) AS map(1, CAST(2 AS INT), 3, CAST(NULL AS INT))#109, 1 AS 1#108]
+- OneRowRelation$
```
So, we should cast part of columns to be compatible when appropriate.
## How was this patch tested?
Added a test (query union of map and other columns) to SQLQueryTestSuite's union.sql.
Author: liutang123 <li...@yeah.net>
Closes #21100 from liutang123/SPARK-24012.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/64e8408e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/64e8408e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/64e8408e
Branch: refs/heads/master
Commit: 64e8408e6fa2d74929601b01a29771738f6d8c65
Parents: 5fea17b
Author: liutang123 <li...@yeah.net>
Authored: Wed Apr 25 18:10:51 2018 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Wed Apr 25 18:10:51 2018 +0800
----------------------------------------------------------------------
.../sql/catalyst/analysis/TypeCoercion.scala | 8 ++++
.../test/resources/sql-tests/inputs/union.sql | 11 +++++
.../resources/sql-tests/results/union.sql.out | 42 +++++++++++++++-----
3 files changed, 51 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/64e8408e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index cfcbd8d..25bad28 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -112,6 +112,14 @@ object TypeCoercion {
StructField(f1.name, dataType, nullable = f1.nullable || f2.nullable)
}))
+ case (a1 @ ArrayType(et1, hasNull1), a2 @ ArrayType(et2, hasNull2)) if a1.sameType(a2) =>
+ findTightestCommonType(et1, et2).map(ArrayType(_, hasNull1 || hasNull2))
+
+ case (m1 @ MapType(kt1, vt1, hasNull1), m2 @ MapType(kt2, vt2, hasNull2)) if m1.sameType(m2) =>
+ val keyType = findTightestCommonType(kt1, kt2)
+ val valueType = findTightestCommonType(vt1, vt2)
+ Some(MapType(keyType.get, valueType.get, hasNull1 || hasNull2))
+
case _ => None
}
http://git-wip-us.apache.org/repos/asf/spark/blob/64e8408e/sql/core/src/test/resources/sql-tests/inputs/union.sql
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/inputs/union.sql b/sql/core/src/test/resources/sql-tests/inputs/union.sql
index e57d69e..6da1b9b 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/union.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/union.sql
@@ -35,6 +35,17 @@ FROM (SELECT col AS col
SELECT col
FROM p3) T1) T2;
+-- SPARK-24012 Union of map and other compatible columns.
+SELECT map(1, 2), 'str'
+UNION ALL
+SELECT map(1, 2, 3, NULL), 1;
+
+-- SPARK-24012 Union of array and other compatible columns.
+SELECT array(1, 2), 'str'
+UNION ALL
+SELECT array(1, 2, 3, NULL), 1;
+
+
-- Clean-up
DROP VIEW IF EXISTS t1;
DROP VIEW IF EXISTS t2;
http://git-wip-us.apache.org/repos/asf/spark/blob/64e8408e/sql/core/src/test/resources/sql-tests/results/union.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/union.sql.out b/sql/core/src/test/resources/sql-tests/results/union.sql.out
index d123b7f..b023df8 100644
--- a/sql/core/src/test/resources/sql-tests/results/union.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/union.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 14
+-- Number of queries: 16
-- !query 0
@@ -105,23 +105,29 @@ struct<x:int,col:int>
-- !query 9
-DROP VIEW IF EXISTS t1
+SELECT map(1, 2), 'str'
+UNION ALL
+SELECT map(1, 2, 3, NULL), 1
-- !query 9 schema
-struct<>
+struct<map(1, 2):map<int,int>,str:string>
-- !query 9 output
-
+{1:2,3:null} 1
+{1:2} str
-- !query 10
-DROP VIEW IF EXISTS t2
+SELECT array(1, 2), 'str'
+UNION ALL
+SELECT array(1, 2, 3, NULL), 1
-- !query 10 schema
-struct<>
+struct<array(1, 2):array<int>,str:string>
-- !query 10 output
-
+[1,2,3,null] 1
+[1,2] str
-- !query 11
-DROP VIEW IF EXISTS p1
+DROP VIEW IF EXISTS t1
-- !query 11 schema
struct<>
-- !query 11 output
@@ -129,7 +135,7 @@ struct<>
-- !query 12
-DROP VIEW IF EXISTS p2
+DROP VIEW IF EXISTS t2
-- !query 12 schema
struct<>
-- !query 12 output
@@ -137,8 +143,24 @@ struct<>
-- !query 13
-DROP VIEW IF EXISTS p3
+DROP VIEW IF EXISTS p1
-- !query 13 schema
struct<>
-- !query 13 output
+
+
+-- !query 14
+DROP VIEW IF EXISTS p2
+-- !query 14 schema
+struct<>
+-- !query 14 output
+
+
+
+-- !query 15
+DROP VIEW IF EXISTS p3
+-- !query 15 schema
+struct<>
+-- !query 15 output
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org