You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/01/27 02:05:23 UTC
svn commit: r1236479 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java
test/queries/clientpositive/union_lateralview.q
test/results/clientpositive/union_lateralview.q.out
Author: namit
Date: Fri Jan 27 01:05:22 2012
New Revision: 1236479
URL: http://svn.apache.org/viewvc?rev=1236479&view=rev
Log:
HIVE-2754 NPE in union with lateral view
(Yongqiang He via namit)
Added:
hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q
hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java?rev=1236479&r1=1236478&r2=1236479&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java Fri Jan 27 01:05:22 2012
@@ -63,7 +63,17 @@ public class GenMRRedSink3 implements No
Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx
.getMapCurrCtx();
GenMapRedCtx mapredCtx = mapCurrCtx.get(ctx.getCurrUnionOp());
- Task<? extends Serializable> unionTask = mapredCtx.getCurrTask();
+ Task<? extends Serializable> unionTask = null;
+ if(mapredCtx != null) {
+ unionTask = mapredCtx.getCurrTask();
+ } else {
+ Operator<? extends Serializable> topOp = ctx.getCurrTopOp();
+ if(topOp == null) {
+ return null;
+ }
+ unionTask = ctx.getCurrTask();
+ }
+
MapredWork plan = (MapredWork) unionTask.getWork();
HashMap<Operator<? extends Serializable>, Task<? extends Serializable>> opTaskMap = ctx
.getOpTaskMap();
Added: hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q?rev=1236479&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q Fri Jan 27 01:05:22 2012
@@ -0,0 +1,43 @@
+
+create table test_union_lateral_view(key int, arr_ele int, value string);
+
+EXPLAIN
+INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+ SELECT key, value, array(1,2,3) as arr
+ FROM src
+
+ UNION ALL
+
+ SELECT key, value, array(1,2,3) as arr
+ FROM srcpart
+ WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+;
+
+INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+ SELECT key, value, array(1,2,3) as arr
+ FROM src
+
+ UNION ALL
+
+ SELECT key, value, array(1,2,3) as arr
+ FROM srcpart
+ WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+;
+
+select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20;
Added: hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out?rev=1236479&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out Fri Jan 27 01:05:22 2012
@@ -0,0 +1,342 @@
+PREHOOK: query: create table test_union_lateral_view(key int, arr_ele int, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table test_union_lateral_view(key int, arr_ele int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_union_lateral_view
+PREHOOK: query: EXPLAIN
+INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+ SELECT key, value, array(1,2,3) as arr
+ FROM src
+
+ UNION ALL
+
+ SELECT key, value, array(1,2,3) as arr
+ FROM srcpart
+ WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+ SELECT key, value, array(1,2,3) as arr
+ FROM src
+
+ UNION ALL
+
+ SELECT key, value, array(1,2,3) as arr
+ FROM srcpart
+ WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION EXPLODE (TOK_TABLE_OR_COL arr)) arr_ele (TOK_TABALIAS c))) (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION array 1 2 3) arr)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION array 1 2 3) arr)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '12')))))) a))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) arr_ele) arr_ele) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_T
ABLE_OR_COL a) value) value)))) d) (TOK_TABREF (TOK_TABNAME src) b) (= (. (TOK_TABLE_OR_COL d) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME test_union_lateral_view))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL d) arr_ele)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL d) value)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b
+ TableScan
+ alias: b
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ d-subquery1:a-subquery1:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ expr: array(1,2,3)
+ type: array<int>
+ outputColumnNames: _col0, _col1, _col2
+ Union
+ Lateral View Forward
+ Select Operator
+ SELECT * : (no compute)
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col3
+ type: int
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1, _col2
+ Reduce Output Operator
+ key expressions:
+ expr: _col1
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col1
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: int
+ expr: _col2
+ type: string
+ Select Operator
+ expressions:
+ expr: _col2
+ type: array<int>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col3
+ type: int
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1, _col2
+ Reduce Output Operator
+ key expressions:
+ expr: _col1
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col1
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: int
+ expr: _col2
+ type: string
+ d-subquery2:a-subquery2:srcpart
+ TableScan
+ alias: srcpart
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ expr: array(1,2,3)
+ type: array<int>
+ outputColumnNames: _col0, _col1, _col2
+ Union
+ Lateral View Forward
+ Select Operator
+ SELECT * : (no compute)
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col3
+ type: int
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1, _col2
+ Reduce Output Operator
+ key expressions:
+ expr: _col1
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col1
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: int
+ expr: _col2
+ type: string
+ Select Operator
+ expressions:
+ expr: _col2
+ type: array<int>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col3
+ type: int
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1, _col2
+ Reduce Output Operator
+ key expressions:
+ expr: _col1
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col1
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: int
+ expr: _col2
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col2}
+ 1 {VALUE._col0}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col3
+ type: string
+ expr: _col0
+ type: int
+ expr: _col2
+ type: string
+ outputColumnNames: _col0, _col1, _col2
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: int
+ expr: _col2
+ type: string
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.test_union_lateral_view
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.test_union_lateral_view
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+
+PREHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+ SELECT key, value, array(1,2,3) as arr
+ FROM src
+
+ UNION ALL
+
+ SELECT key, value, array(1,2,3) as arr
+ FROM srcpart
+ WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@test_union_lateral_view
+POSTHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+ SELECT key, value, array(1,2,3) as arr
+ FROM src
+
+ UNION ALL
+
+ SELECT key, value, array(1,2,3) as arr
+ FROM srcpart
+ WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@test_union_lateral_view
+POSTHOOK: Lineage: test_union_lateral_view.arr_ele EXPRESSION []
+POSTHOOK: Lineage: test_union_lateral_view.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_union_lateral_view
+#### A masked pattern was here ####
+POSTHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_union_lateral_view
+#### A masked pattern was here ####
+POSTHOOK: Lineage: test_union_lateral_view.arr_ele EXPRESSION []
+POSTHOOK: Lineage: test_union_lateral_view.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ]
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 1 val_0
+0 2 val_0
+0 2 val_0