You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/09/24 05:16:26 UTC
svn commit: r1627210 [10/14] - in /hive/branches/spark:
itests/src/test/resources/ ql/src/test/results/clientpositive/spark/
Added: hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out?rev=1627210&view=auto
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out (added)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out Wed Sep 24 03:16:25 2014
@@ -0,0 +1,1736 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table src_10 as select * from src limit 10
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_10
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table src_10 as select * from src limit 10
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_10
+PREHOOK: query: create table src_lv1 (key string, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_lv1
+POSTHOOK: query: create table src_lv1 (key string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_lv1
+PREHOOK: query: create table src_lv2 (key string, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_lv2
+POSTHOOK: query: create table src_lv2 (key string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_lv2
+PREHOOK: query: create table src_lv3 (key string, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_lv3
+POSTHOOK: query: create table src_lv3 (key string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_lv3
+PREHOOK: query: -- 2LV
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-FS[12]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -LVF[6]-SEL[7]-LVJ[10]-SEL[13]-FS[14]
+-- -SEL[8]-UDTF[9]-LVJ[10]
+explain
+from src_10
+insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C
+insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C
+PREHOOK: type: QUERY
+POSTHOOK: query: -- 2LV
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-FS[12]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -LVF[6]-SEL[7]-LVJ[10]-SEL[13]-FS[14]
+-- -SEL[8]-UDTF[9]-LVJ[10]
+explain
+from src_10
+insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C
+insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-4 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-4, Stage-5
+ Stage-0 depends on stages: Stage-3
+ Stage-6 depends on stages: Stage-0
+ Stage-1 depends on stages: Stage-3
+ Stage-7 depends on stages: Stage-1
+ Stage-5 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: src_10
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ Lateral View Forward
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+ Select Operator
+ expressions: array((key + 1),(key + 2)) (type: array<double>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+
+ Stage: Stage-3
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+
+ Stage: Stage-6
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+
+ Stage: Stage-7
+ Stats-Aggr Operator
+
+ Stage: Stage-5
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ Lateral View Forward
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+ Select Operator
+ expressions: array((key + 3),(key + 4)) (type: array<double>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+
+PREHOOK: query: from src_10
+insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C
+insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_10
+PREHOOK: Output: default@src_lv1
+PREHOOK: Output: default@src_lv2
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+POSTHOOK: query: from src_10
+insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C
+insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_10
+POSTHOOK: Output: default@src_lv1
+POSTHOOK: Output: default@src_lv2
+POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv1.value SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv2.value SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+PREHOOK: query: select * from src_lv1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv1
+#### A masked pattern was here ####
+165 166.0
+165 167.0
+238 239.0
+238 240.0
+255 256.0
+255 257.0
+27 28.0
+27 29.0
+278 279.0
+278 280.0
+311 312.0
+311 313.0
+409 410.0
+409 411.0
+484 485.0
+484 486.0
+86 87.0
+86 88.0
+98 100.0
+98 99.0
+PREHOOK: query: select * from src_lv2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv2
+#### A masked pattern was here ####
+165 168.0
+165 169.0
+238 241.0
+238 242.0
+255 258.0
+255 259.0
+27 30.0
+27 31.0
+278 281.0
+278 282.0
+311 314.0
+311 315.0
+409 412.0
+409 413.0
+484 487.0
+484 488.0
+86 89.0
+86 90.0
+98 101.0
+98 102.0
+PREHOOK: query: -- 2(LV+GBY)
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22]
+-- -SEL[8]-UDTF[9]-LVJ[10]
+explain
+from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- 2(LV+GBY)
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22]
+-- -SEL[8]-UDTF[9]-LVJ[10]
+explain
+from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-4 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-4, Stage-5
+ Stage-0 depends on stages: Stage-3
+ Stage-6 depends on stages: Stage-0
+ Stage-1 depends on stages: Stage-3
+ Stage-7 depends on stages: Stage-1
+ Stage-5 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: src_10
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Spark
+ Edges:
+ Reducer 4 <- Map 1 (GROUP)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ Lateral View Forward
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col5)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double)
+ Select Operator
+ expressions: array((key + 1),(key + 2)) (type: array<double>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col5)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double)
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+
+ Stage: Stage-3
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+
+ Stage: Stage-6
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+
+ Stage: Stage-7
+ Stats-Aggr Operator
+
+ Stage: Stage-5
+ Spark
+ Edges:
+ Reducer 5 <- Map 2 (GROUP)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ Lateral View Forward
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col5)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double)
+ Select Operator
+ expressions: array((key + 3),(key + 4)) (type: array<double>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col5)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double)
+ Reducer 5
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+
+PREHOOK: query: from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_10
+PREHOOK: Output: default@src_lv1
+PREHOOK: Output: default@src_lv2
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+POSTHOOK: query: from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_10
+POSTHOOK: Output: default@src_lv1
+POSTHOOK: Output: default@src_lv2
+POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+PREHOOK: query: select * from src_lv1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv1
+#### A masked pattern was here ####
+165 333.0
+238 479.0
+255 513.0
+27 57.0
+278 559.0
+311 625.0
+409 821.0
+484 971.0
+86 175.0
+98 199.0
+PREHOOK: query: select * from src_lv2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv2
+#### A masked pattern was here ####
+165 337.0
+238 483.0
+255 517.0
+27 61.0
+278 563.0
+311 629.0
+409 825.0
+484 975.0
+86 179.0
+98 203.0
+PREHOOK: query: -- (LV+GBY) + RS:2GBY
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[6]-GBY[7]-RS[8]-GBY[9]-SEL[10]-FS[11]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -FIL[12]-SEL[13]-RS[14]-FOR[15]-FIL[16]-GBY[17]-SEL[18]-FS[19]
+-- -FIL[20]-GBY[21]-SEL[22]-FS[23]
+explain
+from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, count(value) where key > 200 group by key
+insert overwrite table src_lv3 select key, count(value) where key < 200 group by key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- (LV+GBY) + RS:2GBY
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[6]-GBY[7]-RS[8]-GBY[9]-SEL[10]-FS[11]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -FIL[12]-SEL[13]-RS[14]-FOR[15]-FIL[16]-GBY[17]-SEL[18]-FS[19]
+-- -FIL[20]-GBY[21]-SEL[22]-FS[23]
+explain
+from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, count(value) where key > 200 group by key
+insert overwrite table src_lv3 select key, count(value) where key < 200 group by key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-5 depends on stages: Stage-3
+ Stage-4 depends on stages: Stage-5, Stage-6
+ Stage-0 depends on stages: Stage-4
+ Stage-7 depends on stages: Stage-0
+ Stage-1 depends on stages: Stage-4
+ Stage-8 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-4
+ Stage-9 depends on stages: Stage-2
+ Stage-6 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-3
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: src_10
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-5
+ Spark
+ Edges:
+ Reducer 4 <- Map 1 (GROUP)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ Lateral View Forward
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col5)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col1 (type: double)
+ Select Operator
+ expressions: array((key + 1),(key + 2)) (type: array<double>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col5)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col1 (type: double)
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+
+ Stage: Stage-4
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+
+ Stage: Stage-7
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+
+ Stage: Stage-8
+ Stats-Aggr Operator
+
+ Stage: Stage-2
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv3
+
+ Stage: Stage-9
+ Stats-Aggr Operator
+
+ Stage: Stage-6
+ Spark
+ Edges:
+ Reducer 5 <- Map 2 (GROUP SORT)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ Filter Operator
+ predicate: ((key > 200) or (key < 200)) (type: boolean)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: key, value
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: string)
+ sort order: +
+ Map-reduce partition columns: key (type: string)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ value expressions: value (type: string)
+ Reducer 5
+ Reduce Operator Tree:
+ Forward
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Filter Operator
+ predicate: (KEY._col0 > 200) (type: boolean)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+ Filter Operator
+ predicate: (KEY._col0 < 200) (type: boolean)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv3
+
+PREHOOK: query: from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, count(value) where key > 200 group by key
+insert overwrite table src_lv3 select key, count(value) where key < 200 group by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_10
+PREHOOK: Output: default@src_lv1
+PREHOOK: Output: default@src_lv2
+PREHOOK: Output: default@src_lv3
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+POSTHOOK: query: from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, count(value) where key > 200 group by key
+insert overwrite table src_lv3 select key, count(value) where key < 200 group by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_10
+POSTHOOK: Output: default@src_lv1
+POSTHOOK: Output: default@src_lv2
+POSTHOOK: Output: default@src_lv3
+POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: select * from src_lv1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv1
+#### A masked pattern was here ####
+165 333.0
+238 479.0
+255 513.0
+27 57.0
+278 559.0
+311 625.0
+409 821.0
+484 971.0
+86 175.0
+98 199.0
+PREHOOK: query: select * from src_lv2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv2
+#### A masked pattern was here ####
+238 1
+255 1
+278 1
+311 1
+409 1
+484 1
+PREHOOK: query: select * from src_lv3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv3
+#### A masked pattern was here ####
+165 1
+27 1
+86 1
+98 1
+PREHOOK: query: -- todo: shared distinct columns (should work with hive.optimize.multigroupby.common.distincts)
+-- 2(LV+GBY) + RS:2GBY
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22]
+-- -SEL[8]-UDTF[9]-LVJ[10]
+-- -SEL[23]-GBY[24]-RS[25]-GBY[26]-SEL[27]-FS[28]
+explain
+from src_10
+insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C
+insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C
+insert overwrite table src_lv3 select value, sum(distinct key) group by value
+PREHOOK: type: QUERY
+POSTHOOK: query: -- todo: shared distinct columns (should work with hive.optimize.multigroupby.common.distincts)
+-- 2(LV+GBY) + RS:2GBY
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22]
+-- -SEL[8]-UDTF[9]-LVJ[10]
+-- -SEL[23]-GBY[24]-RS[25]-GBY[26]-SEL[27]-FS[28]
+explain
+from src_10
+insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C
+insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C
+insert overwrite table src_lv3 select value, sum(distinct key) group by value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-5 depends on stages: Stage-3
+ Stage-4 depends on stages: Stage-5, Stage-6, Stage-7
+ Stage-0 depends on stages: Stage-4
+ Stage-8 depends on stages: Stage-0
+ Stage-1 depends on stages: Stage-4
+ Stage-9 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-4
+ Stage-10 depends on stages: Stage-2
+ Stage-6 depends on stages: Stage-3
+ Stage-7 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-3
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: src_10
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-5
+ Spark
+ Edges:
+ Reducer 5 <- Map 1 (GROUP SORT)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ Lateral View Forward
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col5 (type: double), _col0 (type: string)
+ outputColumnNames: _col5, _col0
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT _col0)
+ keys: _col5 (type: double), _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: double), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: double)
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: array((key + 1),(key + 2)) (type: array<double>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col5 (type: double), _col0 (type: string)
+ outputColumnNames: _col5, _col0
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT _col0)
+ keys: _col5 (type: double), _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: double), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: double)
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reducer 5
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(DISTINCT KEY._col1:0._col0)
+ keys: KEY._col0 (type: double)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: double), _col1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+
+ Stage: Stage-4
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+
+ Stage: Stage-8
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+
+ Stage: Stage-9
+ Stats-Aggr Operator
+
+ Stage: Stage-2
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv3
+
+ Stage: Stage-10
+ Stats-Aggr Operator
+
+ Stage: Stage-6
+ Spark
+ Edges:
+ Reducer 6 <- Map 2 (GROUP SORT)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ Lateral View Forward
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col5 (type: double), _col0 (type: string)
+ outputColumnNames: _col5, _col0
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT _col0)
+ keys: _col5 (type: double), _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: double), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: double)
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: array((key + 3),(key + 4)) (type: array<double>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col5 (type: double), _col0 (type: string)
+ outputColumnNames: _col5, _col0
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT _col0)
+ keys: _col5 (type: double), _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: double), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: double)
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reducer 6
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(DISTINCT KEY._col1:0._col0)
+ keys: KEY._col0 (type: double)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: double), _col1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+
+ Stage: Stage-7
+ Spark
+ Edges:
+ Reducer 7 <- Map 3 (GROUP SORT)
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ Select Operator
+ expressions: value (type: string), key (type: string)
+ outputColumnNames: value, key
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT key)
+ keys: value (type: string), key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Reducer 7
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(DISTINCT KEY._col1:0._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv3
+
+PREHOOK: query: from src_10
+insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C
+insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C
+insert overwrite table src_lv3 select value, sum(distinct key) group by value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_10
+PREHOOK: Output: default@src_lv1
+PREHOOK: Output: default@src_lv2
+PREHOOK: Output: default@src_lv3
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+POSTHOOK: query: from src_10
+insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C
+insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C
+insert overwrite table src_lv3 select value, sum(distinct key) group by value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_10
+POSTHOOK: Output: default@src_lv1
+POSTHOOK: Output: default@src_lv2
+POSTHOOK: Output: default@src_lv3
+POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+PREHOOK: query: select * from src_lv1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv1
+#### A masked pattern was here ####
+100.0 98.0
+166.0 165.0
+167.0 165.0
+239.0 238.0
+240.0 238.0
+256.0 255.0
+257.0 255.0
+279.0 278.0
+28.0 27.0
+280.0 278.0
+29.0 27.0
+312.0 311.0
+313.0 311.0
+410.0 409.0
+411.0 409.0
+485.0 484.0
+486.0 484.0
+87.0 86.0
+88.0 86.0
+99.0 98.0
+PREHOOK: query: select * from src_lv2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv2
+#### A masked pattern was here ####
+101.0 98.0
+102.0 98.0
+168.0 165.0
+169.0 165.0
+241.0 238.0
+242.0 238.0
+258.0 255.0
+259.0 255.0
+281.0 278.0
+282.0 278.0
+30.0 27.0
+31.0 27.0
+314.0 311.0
+315.0 311.0
+412.0 409.0
+413.0 409.0
+487.0 484.0
+488.0 484.0
+89.0 86.0
+90.0 86.0
+PREHOOK: query: select * from src_lv3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv3
+#### A masked pattern was here ####
+val_165 165.0
+val_238 238.0
+val_255 255.0
+val_27 27.0
+val_278 278.0
+val_311 311.0
+val_409 409.0
+val_484 484.0
+val_86 86.0
+val_98 98.0
+PREHOOK: query: create table src_lv4 (key string, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_lv4
+POSTHOOK: query: create table src_lv4 (key string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_lv4
+PREHOOK: query: -- Common distincts optimization works across non-lateral view queries, but not across lateral view multi inserts
+explain
+from src_10
+insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key
+insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value
+insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Common distincts optimization works across non-lateral view queries, but not across lateral view multi inserts
+explain
+from src_10
+insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key
+insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value
+insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-4 is a root stage
+ Stage-6 depends on stages: Stage-4
+ Stage-5 depends on stages: Stage-6, Stage-7, Stage-8
+ Stage-0 depends on stages: Stage-5
+ Stage-9 depends on stages: Stage-0
+ Stage-1 depends on stages: Stage-5
+ Stage-10 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-5
+ Stage-11 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-5
+ Stage-12 depends on stages: Stage-3
+ Stage-7 depends on stages: Stage-4
+ Stage-8 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-4
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: src_10
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-6
+ Spark
+ Edges:
+ Reducer 5 <- Map 1 (GROUP SORT)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ Lateral View Forward
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT _col5)
+ keys: _col0 (type: string), _col5 (type: double)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: double)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: array((key + 1),(key + 2)) (type: array<double>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT _col5)
+ keys: _col0 (type: string), _col5 (type: double)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: double)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reducer 5
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(DISTINCT KEY._col1:0._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv1
+
+ Stage: Stage-9
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+
+ Stage: Stage-10
+ Stats-Aggr Operator
+
+ Stage: Stage-2
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv3
+
+ Stage: Stage-11
+ Stats-Aggr Operator
+
+ Stage: Stage-3
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv4
+
+ Stage: Stage-12
+ Stats-Aggr Operator
+
+ Stage: Stage-7
+ Spark
+ Edges:
+ Reducer 6 <- Map 2 (GROUP SORT)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ Lateral View Forward
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT _col5)
+ keys: _col0 (type: string), _col5 (type: double)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: double)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: array((key + 3),(key + 4)) (type: array<double>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col5 (type: double)
+ outputColumnNames: _col0, _col5
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT _col5)
+ keys: _col0 (type: string), _col5 (type: double)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: double)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE
+ Reducer 6
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(DISTINCT KEY._col1:0._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv2
+
+ Stage: Stage-8
+ Spark
+ Edges:
+ Reducer 7 <- Map 3 (GROUP SORT)
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ Filter Operator
+ predicate: ((key > 200) or (key < 200)) (type: boolean)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: key, value
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: string), key (type: string)
+ sort order: ++
+ Map-reduce partition columns: value (type: string)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Reducer 7
+ Reduce Operator Tree:
+ Forward
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Filter Operator
+ predicate: (KEY._col1:0._col0 > 200) (type: boolean)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT KEY._col1:0._col0)
+ keys: KEY._col0 (type: string)
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv3
+ Filter Operator
+ predicate: (KEY._col1:0._col0 < 200) (type: boolean)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Group By Operator
+ aggregations: sum(DISTINCT KEY._col1:0._col0)
+ keys: KEY._col0 (type: string)
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_lv4
+
+PREHOOK: query: from src_10
+insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key
+insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value
+insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_10
+PREHOOK: Output: default@src_lv1
+PREHOOK: Output: default@src_lv2
+PREHOOK: Output: default@src_lv3
+PREHOOK: Output: default@src_lv4
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
+POSTHOOK: query: from src_10
+insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key
+insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value
+insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_10
+POSTHOOK: Output: default@src_lv1
+POSTHOOK: Output: default@src_lv2
+POSTHOOK: Output: default@src_lv3
+POSTHOOK: Output: default@src_lv4
+POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.null, ]
+POSTHOOK: Lineage: src_lv4.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Lineage: src_lv4.value EXPRESSION [(src_10)src_10.null, ]
+PREHOOK: query: select * from src_lv1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv1
+#### A masked pattern was here ####
+165 333.0
+238 479.0
+255 513.0
+27 57.0
+278 559.0
+311 625.0
+409 821.0
+484 971.0
+86 175.0
+98 199.0
+PREHOOK: query: select * from src_lv2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv2
+#### A masked pattern was here ####
+165 337.0
+238 483.0
+255 517.0
+27 61.0
+278 563.0
+311 629.0
+409 825.0
+484 975.0
+86 179.0
+98 203.0
+PREHOOK: query: select * from src_lv3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv3
+#### A masked pattern was here ####
+val_238 238.0
+val_255 255.0
+val_278 278.0
+val_311 311.0
+val_409 409.0
+val_484 484.0
+PREHOOK: query: select * from src_lv4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_lv4
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_lv4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_lv4
+#### A masked pattern was here ####
+val_165 165.0
+val_27 27.0
+val_86 86.0
+val_98 98.0