You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2014/10/16 19:33:39 UTC
svn commit: r1632393 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/
java/org/apache/hadoop/hive/ql/udf/generic/ test/queries/clientpositive/
test/results/clientpositive/
Author: szehon
Date: Thu Oct 16 17:33:38 2014
New Revision: 1632393
URL: http://svn.apache.org/r1632393
Log:
HIVE-8448 : Union All might not work due to the type conversion issue (Yongzhi Chen via Szehon)
Added:
hive/trunk/ql/src/test/queries/clientpositive/union_date_trim.q
hive/trunk/ql/src/test/results/clientpositive/union_date_trim.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java?rev=1632393&r1=1632392&r2=1632393&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java Thu Oct 16 17:33:38 2014
@@ -80,7 +80,7 @@ public class UnionOperator extends Opera
for (int p = 0; p < parents; p++) {
assert (parentFields[p].size() == columns);
for (int c = 0; c < columns; c++) {
- if (!columnTypeResolvers[c].update(parentFields[p].get(c)
+ if (!columnTypeResolvers[c].updateForUnionAll(parentFields[p].get(c)
.getFieldObjectInspector())) {
// checked in SemanticAnalyzer. Should not happen
throw new HiveException("Incompatible types for union operator");
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java?rev=1632393&r1=1632392&r2=1632393&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java Thu Oct 16 17:33:38 2014
@@ -100,6 +100,26 @@ public final class GenericUDFUtils {
* @return false if there is a type mismatch
*/
public boolean update(ObjectInspector oi) throws UDFArgumentTypeException {
+ return update(oi, false);
+ }
+
+ /**
+ * Update returnObjectInspector and valueInspectorsAreTheSame based on the
+ * ObjectInspector seen for UnionAll.
+ *
+ * @return false if there is a type mismatch
+ */
+ public boolean updateForUnionAll(ObjectInspector oi) throws UDFArgumentTypeException {
+ return update(oi, true);
+ }
+
+ /**
+ * Update returnObjectInspector and valueInspectorsAreTheSame based on the
+ * ObjectInspector seen.
+ *
+ * @return false if there is a type mismatch
+ */
+ private boolean update(ObjectInspector oi, boolean isUnionAll) throws UDFArgumentTypeException {
if (oi instanceof VoidObjectInspector) {
return true;
}
@@ -137,8 +157,14 @@ public final class GenericUDFUtils {
// Types are different, we need to check whether we can convert them to
// a common base class or not.
- TypeInfo commonTypeInfo = FunctionRegistry.getCommonClass(oiTypeInfo,
+ TypeInfo commonTypeInfo = null;
+ if (isUnionAll) {
+ commonTypeInfo = FunctionRegistry.getCommonClassForUnionAll(oiTypeInfo,
rTypeInfo);
+ } else {
+ commonTypeInfo = FunctionRegistry.getCommonClass(oiTypeInfo,
+ rTypeInfo);
+ }
if (commonTypeInfo == null) {
return false;
}
Added: hive/trunk/ql/src/test/queries/clientpositive/union_date_trim.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/union_date_trim.q?rev=1632393&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/union_date_trim.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/union_date_trim.q Thu Oct 16 17:33:38 2014
@@ -0,0 +1,7 @@
+drop table if exists testDate;
+create table testDate(id int, dt date);
+insert into table testDate select 1, '2014-04-07' from src where key=100 limit 1;
+insert into table testDate select 2, '2014-04-08' from src where key=100 limit 1;
+insert into table testDate select 3, '2014-04-09' from src where key=100 limit 1;
+--- without the fix following query will throw HiveException: Incompatible types for union operator
+insert into table testDate select id, tm from (select id, dt as tm from testDate where id = 1 union all select id, dt as tm from testDate where id = 2 union all select id, trim(Cast (dt as string)) as tm from testDate where id = 3 ) a;
Added: hive/trunk/ql/src/test/results/clientpositive/union_date_trim.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union_date_trim.q.out?rev=1632393&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union_date_trim.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/union_date_trim.q.out Thu Oct 16 17:33:38 2014
@@ -0,0 +1,54 @@
+PREHOOK: query: drop table if exists testDate
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists testDate
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table testDate(id int, dt date)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testDate
+POSTHOOK: query: create table testDate(id int, dt date)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testDate
+PREHOOK: query: insert into table testDate select 1, '2014-04-07' from src where key=100 limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@testdate
+POSTHOOK: query: insert into table testDate select 1, '2014-04-07' from src where key=100 limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@testdate
+POSTHOOK: Lineage: testdate.dt EXPRESSION []
+POSTHOOK: Lineage: testdate.id SIMPLE []
+PREHOOK: query: insert into table testDate select 2, '2014-04-08' from src where key=100 limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@testdate
+POSTHOOK: query: insert into table testDate select 2, '2014-04-08' from src where key=100 limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@testdate
+POSTHOOK: Lineage: testdate.dt EXPRESSION []
+POSTHOOK: Lineage: testdate.id SIMPLE []
+PREHOOK: query: insert into table testDate select 3, '2014-04-09' from src where key=100 limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@testdate
+POSTHOOK: query: insert into table testDate select 3, '2014-04-09' from src where key=100 limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@testdate
+POSTHOOK: Lineage: testdate.dt EXPRESSION []
+POSTHOOK: Lineage: testdate.id SIMPLE []
+PREHOOK: query: --- without the fix following query will throw HiveException: Incompatible types for union operator
+insert into table testDate select id, tm from (select id, dt as tm from testDate where id = 1 union all select id, dt as tm from testDate where id = 2 union all select id, trim(Cast (dt as string)) as tm from testDate where id = 3 ) a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testdate
+PREHOOK: Output: default@testdate
+POSTHOOK: query: --- without the fix following query will throw HiveException: Incompatible types for union operator
+insert into table testDate select id, tm from (select id, dt as tm from testDate where id = 1 union all select id, dt as tm from testDate where id = 2 union all select id, trim(Cast (dt as string)) as tm from testDate where id = 3 ) a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testdate
+POSTHOOK: Output: default@testdate
+POSTHOOK: Lineage: testdate.dt EXPRESSION []
+POSTHOOK: Lineage: testdate.id EXPRESSION [(testdate)testdate.FieldSchema(name:id, type:int, comment:null), (testdate)testdate.FieldSchema(name:id, type:int, comment:null), (testdate)testdate.FieldSchema(name:id, type:int, comment:null), ]