You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/10/30 02:02:19 UTC
svn commit: r1536948 - in /hive/branches/tez: build-common.xml
ql/src/test/queries/clientpositive/mrr.q
ql/src/test/results/clientpositive/mrr.q.out
Author: gunther
Date: Wed Oct 30 01:02:18 2013
New Revision: 1536948
URL: http://svn.apache.org/r1536948
Log:
HIVE-5689: Add some simple MRR tests (Gunther Hagleitner)
Modified:
hive/branches/tez/build-common.xml
hive/branches/tez/ql/src/test/queries/clientpositive/mrr.q
hive/branches/tez/ql/src/test/results/clientpositive/mrr.q.out
Modified: hive/branches/tez/build-common.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/build-common.xml?rev=1536948&r1=1536947&r2=1536948&view=diff
==============================================================================
--- hive/branches/tez/build-common.xml (original)
+++ hive/branches/tez/build-common.xml Wed Oct 30 01:02:18 2013
@@ -61,7 +61,7 @@
<property name="test.junit.output.usefile" value="true"/>
<property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q"/>
<property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q" />
- <property name="minitez.query.files" value="tez_join_tests.q,tez_joins_explain.q"/>
+ <property name="minitez.query.files" value="tez_join_tests.q,tez_joins_explain.q,mrr.q"/>
<property name="test.silent" value="true"/>
<property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
<property name="test.serialize.qplan" value="false"/>
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/mrr.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/mrr.q?rev=1536948&r1=1536947&r2=1536948&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/mrr.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/mrr.q Wed Oct 30 01:02:18 2013
@@ -1,6 +1,61 @@
-set hive.auto.convert.join=true;
set hive.optimize.tez=true;
-EXPLAIN EXTENDED SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt;
-EXPLAIN EXTENDED SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt;
-SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt;
\ No newline at end of file
+-- simple query with multiple reduce stages
+EXPLAIN SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt;
+SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt;
+
+set hive.auto.convert.join=false;
+-- join query with multiple reduce stages;
+EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt;
+SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt;
+
+set hive.auto.convert.join=true;
+-- same query with broadcast join
+EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt;
+SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt;
+
+set hive.auto.convert.join=false;
+-- query with multiple branches in the task dag
+EXPLAIN
+SELECT *
+FROM
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s1
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s2
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s3
+ ON (s1.key = s2.key and s1.key = s3.key)
+WHERE
+ s1.cnt > 1
+ORDER BY s1.key;
+
+SELECT *
+FROM
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s1
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s2
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s3
+ ON (s1.key = s2.key and s1.key = s3.key)
+WHERE
+ s1.cnt > 1
+ORDER BY s1.key;
+
+set hive.auto.convert.join=true;
+-- query with broadcast join in the reduce stage
+EXPLAIN
+SELECT *
+FROM
+ (SELECT key, count(value) as cnt FROM src GROUP BY key) s1
+ JOIN src ON (s1.key = src.key);
+
+SELECT *
+FROM
+ (SELECT key, count(value) as cnt FROM src GROUP BY key) s1
+ JOIN src ON (s1.key = src.key);
Modified: hive/branches/tez/ql/src/test/results/clientpositive/mrr.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/mrr.q.out?rev=1536948&r1=1536947&r2=1536948&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/mrr.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/mrr.q.out Wed Oct 30 01:02:18 2013
@@ -1,6 +1,8 @@
-PREHOOK: query: EXPLAIN EXTENDED SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt
+PREHOOK: query: -- simple query with multiple reduce stages
+EXPLAIN SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt
+POSTHOOK: query: -- simple query with multiple reduce stages
+EXPLAIN SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt
POSTHOOK: type: QUERY
ABSTRACT SYNTAX TREE:
(TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count (TOK_TABLE_OR_COL value)) cnt)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL cnt)))))
@@ -16,7 +18,6 @@ STAGE PLANS:
src
TableScan
alias: src
- GatherStats: false
Select Operator
expressions:
expr: key
@@ -45,53 +46,6 @@ STAGE PLANS:
value expressions:
expr: _col1
type: bigint
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numPartitions 0
- numRows 0
- rawDataSize 0
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numPartitions 0
- numRows 0
- rawDataSize 0
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [src]
- Needs Tagging: false
Reduce Operator Tree:
Group By Operator
aggregations:
@@ -120,36 +74,343 @@ STAGE PLANS:
type: string
expr: _col1
type: bigint
- Needs Tagging: false
Reduce Operator Tree:
Extract
File Output Operator
compressed: false
GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
-#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- columns _col0,_col1
- columns.types string:bigint
- escape.delim \
- hive.serialization.extend.nesting.levels true
- serialization.format 1
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
limit: -1
-PREHOOK: query: EXPLAIN EXTENDED SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt
+PREHOOK: query: SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+490 1
+287 1
+286 1
+285 1
+284 1
+283 1
+114 1
+487 1
+485 1
+28 1
+484 1
+181 1
+275 1
+274 1
+183 1
+483 1
+27 1
+266 1
+482 1
+263 1
+262 1
+260 1
+481 1
+258 1
+257 1
+116 1
+479 1
+252 1
+249 1
+248 1
+247 1
+244 1
+92 1
+241 1
+477 1
+475 1
+472 1
+470 1
+235 1
+47 1
+186 1
+126 1
+228 1
+226 1
+131 1
+467 1
+222 1
+133 1
+82 1
+218 1
+80 1
+460 1
+214 1
+8 1
+78 1
+189 1
+457 1
+455 1
+136 1
+202 1
+201 1
+453 1
+20 1
+2 1
+19 1
+452 1
+196 1
+449 1
+194 1
+190 1
+192 1
+448 1
+446 1
+444 1
+443 1
+44 1
+77 1
+143 1
+437 1
+436 1
+435 1
+432 1
+145 1
+150 1
+43 1
+10 1
+427 1
+74 1
+421 1
+9 1
+419 1
+418 1
+153 1
+105 1
+69 1
+411 1
+41 1
+155 1
+407 1
+156 1
+87 1
+157 1
+402 1
+158 1
+400 1
+4 1
+66 1
+65 1
+160 1
+64 1
+394 1
+393 1
+392 1
+389 1
+386 1
+162 1
+86 1
+379 1
+378 1
+377 1
+375 1
+374 1
+373 1
+57 1
+163 1
+368 1
+54 1
+366 1
+365 1
+364 1
+362 1
+360 1
+356 1
+53 1
+351 1
+166 1
+168 1
+345 1
+85 1
+11 1
+341 1
+34 1
+339 1
+338 1
+336 1
+335 1
+111 1
+332 1
+497 1
+33 1
+17 1
+496 1
+323 1
+495 1
+494 1
+170 1
+493 1
+177 1
+315 1
+178 1
+310 1
+96 1
+308 1
+491 1
+306 1
+305 1
+302 1
+30 1
+180 1
+296 1
+292 1
+291 1
+289 1
+98 2
+97 2
+95 2
+84 2
+83 2
+76 2
+72 2
+67 2
+58 2
+51 2
+492 2
+478 2
+463 2
+462 2
+459 2
+458 2
+439 2
+429 2
+424 2
+42 2
+414 2
+413 2
+404 2
+399 2
+397 2
+395 2
+382 2
+37 2
+367 2
+353 2
+344 2
+342 2
+333 2
+331 2
+325 2
+322 2
+321 2
+317 2
+309 2
+307 2
+288 2
+282 2
+281 2
+280 2
+278 2
+272 2
+265 2
+26 2
+256 2
+255 2
+242 2
+24 2
+239 2
+238 2
+237 2
+233 2
+229 2
+224 2
+223 2
+221 2
+219 2
+217 2
+216 2
+213 2
+209 2
+207 2
+205 2
+203 2
+200 2
+197 2
+195 2
+191 2
+18 2
+179 2
+176 2
+175 2
+174 2
+172 2
+165 2
+164 2
+152 2
+15 2
+149 2
+146 2
+137 2
+134 2
+129 2
+125 2
+120 2
+12 2
+118 2
+113 2
+104 2
+103 2
+100 2
+498 3
+369 3
+384 3
+396 3
+403 3
+409 3
+417 3
+5 3
+430 3
+70 3
+119 3
+0 3
+431 3
+438 3
+480 3
+193 3
+199 3
+208 3
+187 3
+273 3
+298 3
+454 3
+311 3
+316 3
+466 3
+90 3
+128 3
+318 3
+327 3
+167 3
+35 3
+468 4
+489 4
+406 4
+169 4
+138 4
+277 4
+469 5
+401 5
+230 5
+348 5
+PREHOOK: query: -- join query with multiple reduce stages;
+EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt
+PREHOOK: type: QUERY
+POSTHOOK: query: -- join query with multiple reduce stages;
+EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt
POSTHOOK: type: QUERY
ABSTRACT SYNTAX TREE:
(TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) s1) (TOK_TABREF (TOK_TABNAME src) s2) (= (. (TOK_TABLE_OR_COL s1) key) (. (TOK_TABLE_OR_COL s2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL s2) key)) (TOK_SELEXPR (TOK_FUNCTIONDI count (. (TOK_TABLE_OR_COL s2) value)) cnt)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL s2) key)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL cnt)))))
@@ -165,7 +426,6 @@ STAGE PLANS:
s2
TableScan
alias: s2
- GatherStats: false
Reduce Output Operator
key expressions:
expr: key
@@ -180,57 +440,10 @@ STAGE PLANS:
type: string
expr: value
type: string
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numPartitions 0
- numRows 0
- rawDataSize 0
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numPartitions 0
- numRows 0
- rawDataSize 0
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [s2]
Alias -> Map Operator Tree:
s1
TableScan
alias: s1
- GatherStats: false
Reduce Output Operator
key expressions:
expr: key
@@ -240,53 +453,6 @@ STAGE PLANS:
expr: key
type: string
tag: 0
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numPartitions 0
- numRows 0
- rawDataSize 0
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numPartitions 0
- numRows 0
- rawDataSize 0
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [s1]
- Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
@@ -328,7 +494,6 @@ STAGE PLANS:
value expressions:
expr: _col2
type: bigint
- Needs Tagging: false
Reduce Operator Tree:
Group By Operator
aggregations:
@@ -357,30 +522,1750 @@ STAGE PLANS:
type: string
expr: _col1
type: bigint
- Needs Tagging: false
Reduce Operator Tree:
Extract
File Output Operator
compressed: false
GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
#### A masked pattern was here ####
- NumFilesPerFileSink: 1
+POSTHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
+98 1
+97 1
+96 1
+95 1
+92 1
+90 1
+9 1
+87 1
+86 1
+85 1
+84 1
+83 1
+82 1
+80 1
+8 1
+78 1
+77 1
+76 1
+74 1
+72 1
+70 1
+69 1
+67 1
+66 1
+65 1
+64 1
+58 1
+57 1
+54 1
+53 1
+51 1
+5 1
+498 1
+497 1
+496 1
+495 1
+494 1
+493 1
+492 1
+491 1
+490 1
+489 1
+487 1
+485 1
+484 1
+483 1
+482 1
+481 1
+480 1
+479 1
+478 1
+477 1
+475 1
+472 1
+470 1
+47 1
+469 1
+468 1
+467 1
+466 1
+463 1
+462 1
+460 1
+459 1
+458 1
+457 1
+455 1
+454 1
+453 1
+452 1
+449 1
+448 1
+446 1
+444 1
+443 1
+44 1
+439 1
+438 1
+437 1
+436 1
+435 1
+432 1
+431 1
+430 1
+43 1
+429 1
+427 1
+424 1
+421 1
+42 1
+419 1
+418 1
+417 1
+414 1
+413 1
+411 1
+41 1
+409 1
+407 1
+406 1
+404 1
+403 1
+402 1
+401 1
+400 1
+4 1
+399 1
+397 1
+396 1
+395 1
+394 1
+393 1
+392 1
+389 1
+386 1
+384 1
+382 1
+379 1
+378 1
+377 1
+375 1
+374 1
+373 1
+37 1
+369 1
+368 1
+367 1
+366 1
+365 1
+364 1
+362 1
+360 1
+356 1
+353 1
+351 1
+35 1
+348 1
+345 1
+344 1
+342 1
+341 1
+34 1
+339 1
+338 1
+336 1
+335 1
+333 1
+332 1
+331 1
+33 1
+327 1
+325 1
+323 1
+322 1
+321 1
+318 1
+317 1
+316 1
+315 1
+311 1
+310 1
+309 1
+308 1
+307 1
+306 1
+305 1
+302 1
+30 1
+298 1
+296 1
+292 1
+291 1
+289 1
+288 1
+287 1
+286 1
+285 1
+284 1
+283 1
+282 1
+281 1
+280 1
+28 1
+278 1
+277 1
+275 1
+274 1
+273 1
+272 1
+27 1
+266 1
+265 1
+263 1
+262 1
+260 1
+26 1
+258 1
+257 1
+256 1
+255 1
+252 1
+249 1
+248 1
+247 1
+244 1
+242 1
+241 1
+24 1
+239 1
+238 1
+237 1
+235 1
+233 1
+230 1
+229 1
+228 1
+226 1
+224 1
+223 1
+222 1
+221 1
+219 1
+218 1
+217 1
+216 1
+214 1
+213 1
+209 1
+208 1
+207 1
+205 1
+203 1
+202 1
+201 1
+200 1
+20 1
+2 1
+199 1
+197 1
+196 1
+195 1
+194 1
+193 1
+192 1
+191 1
+190 1
+19 1
+189 1
+187 1
+186 1
+183 1
+181 1
+180 1
+18 1
+179 1
+178 1
+177 1
+176 1
+175 1
+174 1
+172 1
+170 1
+17 1
+169 1
+168 1
+167 1
+166 1
+165 1
+164 1
+163 1
+162 1
+160 1
+158 1
+157 1
+156 1
+155 1
+153 1
+152 1
+150 1
+15 1
+149 1
+146 1
+145 1
+143 1
+138 1
+137 1
+136 1
+134 1
+133 1
+131 1
+129 1
+128 1
+126 1
+125 1
+120 1
+12 1
+119 1
+118 1
+116 1
+114 1
+113 1
+111 1
+11 1
+105 1
+104 1
+103 1
+100 1
+10 1
+0 1
+PREHOOK: query: -- same query with broadcast join
+EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt
+PREHOOK: type: QUERY
+POSTHOOK: query: -- same query with broadcast join
+EXPLAIN SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) s1) (TOK_TABREF (TOK_TABNAME src) s2) (= (. (TOK_TABLE_OR_COL s1) key) (. (TOK_TABLE_OR_COL s2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL s2) key)) (TOK_SELEXPR (TOK_FUNCTIONDI count (. (TOK_TABLE_OR_COL s2) value)) cnt)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL s2) key)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL cnt)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Alias -> Map Operator Tree:
+ s2
+ TableScan
+ alias: s2
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Alias -> Map Operator Tree:
+ s1
+ TableScan
+ alias: s1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col4, _col5
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col4, _col5
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT _col5)
+ bucketGroup: false
+ keys:
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col2
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT KEY._col1:0._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col1
+ type: bigint
+ sort order: +
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Extract
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- columns _col0,_col1
- columns.types string:bigint
- escape.delim \
- hive.serialization.extend.nesting.levels true
- serialization.format 1
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
limit: -1
+PREHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT s2.key, count(distinct s2.value) as cnt FROM src s1 join src s2 on (s1.key = s2.key) GROUP BY s2.key ORDER BY cnt
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+98 1
+97 1
+96 1
+95 1
+92 1
+90 1
+9 1
+87 1
+86 1
+85 1
+84 1
+83 1
+82 1
+80 1
+8 1
+78 1
+77 1
+76 1
+74 1
+72 1
+70 1
+69 1
+67 1
+66 1
+65 1
+64 1
+58 1
+57 1
+54 1
+53 1
+51 1
+5 1
+498 1
+497 1
+496 1
+495 1
+494 1
+493 1
+492 1
+491 1
+490 1
+489 1
+487 1
+485 1
+484 1
+483 1
+482 1
+481 1
+480 1
+479 1
+478 1
+477 1
+475 1
+472 1
+470 1
+47 1
+469 1
+468 1
+467 1
+466 1
+463 1
+462 1
+460 1
+459 1
+458 1
+457 1
+455 1
+454 1
+453 1
+452 1
+449 1
+448 1
+446 1
+444 1
+443 1
+44 1
+439 1
+438 1
+437 1
+436 1
+435 1
+432 1
+431 1
+430 1
+43 1
+429 1
+427 1
+424 1
+421 1
+42 1
+419 1
+418 1
+417 1
+414 1
+413 1
+411 1
+41 1
+409 1
+407 1
+406 1
+404 1
+403 1
+402 1
+401 1
+400 1
+4 1
+399 1
+397 1
+396 1
+395 1
+394 1
+393 1
+392 1
+389 1
+386 1
+384 1
+382 1
+379 1
+378 1
+377 1
+375 1
+374 1
+373 1
+37 1
+369 1
+368 1
+367 1
+366 1
+365 1
+364 1
+362 1
+360 1
+356 1
+353 1
+351 1
+35 1
+348 1
+345 1
+344 1
+342 1
+341 1
+34 1
+339 1
+338 1
+336 1
+335 1
+333 1
+332 1
+331 1
+33 1
+327 1
+325 1
+323 1
+322 1
+321 1
+318 1
+317 1
+316 1
+315 1
+311 1
+310 1
+309 1
+308 1
+307 1
+306 1
+305 1
+302 1
+30 1
+298 1
+296 1
+292 1
+291 1
+289 1
+288 1
+287 1
+286 1
+285 1
+284 1
+283 1
+282 1
+281 1
+280 1
+28 1
+278 1
+277 1
+275 1
+274 1
+273 1
+272 1
+27 1
+266 1
+265 1
+263 1
+262 1
+260 1
+26 1
+258 1
+257 1
+256 1
+255 1
+252 1
+249 1
+248 1
+247 1
+244 1
+242 1
+241 1
+24 1
+239 1
+238 1
+237 1
+235 1
+233 1
+230 1
+229 1
+228 1
+226 1
+224 1
+223 1
+222 1
+221 1
+219 1
+218 1
+217 1
+216 1
+214 1
+213 1
+209 1
+208 1
+207 1
+205 1
+203 1
+202 1
+201 1
+200 1
+20 1
+2 1
+199 1
+197 1
+196 1
+195 1
+194 1
+193 1
+192 1
+191 1
+190 1
+19 1
+189 1
+187 1
+186 1
+183 1
+181 1
+180 1
+18 1
+179 1
+178 1
+177 1
+176 1
+175 1
+174 1
+172 1
+170 1
+17 1
+169 1
+168 1
+167 1
+166 1
+165 1
+164 1
+163 1
+162 1
+160 1
+158 1
+157 1
+156 1
+155 1
+153 1
+152 1
+150 1
+15 1
+149 1
+146 1
+145 1
+143 1
+138 1
+137 1
+136 1
+134 1
+133 1
+131 1
+129 1
+128 1
+126 1
+125 1
+120 1
+12 1
+119 1
+118 1
+116 1
+114 1
+113 1
+111 1
+11 1
+105 1
+104 1
+103 1
+100 1
+10 1
+0 1
+PREHOOK: query: -- query with multiple branches in the task dag
+EXPLAIN
+SELECT *
+FROM
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s1
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s2
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s3
+ ON (s1.key = s2.key and s1.key = s3.key)
+WHERE
+ s1.cnt > 1
+ORDER BY s1.key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- query with multiple branches in the task dag
+EXPLAIN
+SELECT *
+FROM
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s1
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s2
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s3
+ ON (s1.key = s2.key and s1.key = s3.key)
+WHERE
+ s1.cnt > 1
+ORDER BY s1.key
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count (TOK_TABLE_OR_COL value)) cnt)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL cnt))))) s1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count (TOK_TABLE_OR_COL value)) cnt)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL cnt))))) s2)) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count (TOK_TABLE_OR_COL value)) cnt)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)) (TOK_O
RDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL cnt))))) s3) (and (= (. (TOK_TABLE_OR_COL s1) key) (. (TOK_TABLE_OR_COL s2) key)) (= (. (TOK_TABLE_OR_COL s1) key) (. (TOK_TABLE_OR_COL s3) key))))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (> (. (TOK_TABLE_OR_COL s1) cnt) 1)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL s1) key)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ aggregations:
+ expr: count(value)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col1
+ type: bigint
+ sort order: +
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Extract
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ aggregations:
+ expr: count(value)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col1
+ type: bigint
+ sort order: +
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Extract
+ Reduce Output Operator
+ sort order:
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ aggregations:
+ expr: count(value)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col1
+ type: bigint
+ sort order: +
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Extract
+ Reduce Output Operator
+ sort order:
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Filter Operator
+ predicate:
+ expr: (_col0 = _col2)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col2
+ type: string
+ expr: _col3
+ type: bigint
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Filter Operator
+ predicate:
+ expr: (_col3 > 1)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col3
+ type: bigint
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ expr: _col4
+ type: string
+ expr: _col5
+ type: bigint
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ expr: _col2
+ type: string
+ expr: _col3
+ type: bigint
+ expr: _col4
+ type: string
+ expr: _col5
+ type: bigint
+ Reduce Operator Tree:
+ Extract
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT *
+FROM
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s1
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s2
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s3
+ ON (s1.key = s2.key and s1.key = s3.key)
+WHERE
+ s1.cnt > 1
+ORDER BY s1.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s1
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s2
+ JOIN
+ (SELECT key, count(value) as cnt
+ FROM src GROUP BY key ORDER BY cnt) s3
+ ON (s1.key = s2.key and s1.key = s3.key)
+WHERE
+ s1.cnt > 1
+ORDER BY s1.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 3 0 3 0 3
+100 2 100 2 100 2
+103 2 103 2 103 2
+104 2 104 2 104 2
+113 2 113 2 113 2
+118 2 118 2 118 2
+119 3 119 3 119 3
+12 2 12 2 12 2
+120 2 120 2 120 2
+125 2 125 2 125 2
+128 3 128 3 128 3
+129 2 129 2 129 2
+134 2 134 2 134 2
+137 2 137 2 137 2
+138 4 138 4 138 4
+146 2 146 2 146 2
+149 2 149 2 149 2
+15 2 15 2 15 2
+152 2 152 2 152 2
+164 2 164 2 164 2
+165 2 165 2 165 2
+167 3 167 3 167 3
+169 4 169 4 169 4
+172 2 172 2 172 2
+174 2 174 2 174 2
+175 2 175 2 175 2
+176 2 176 2 176 2
+179 2 179 2 179 2
+18 2 18 2 18 2
+187 3 187 3 187 3
+191 2 191 2 191 2
+193 3 193 3 193 3
+195 2 195 2 195 2
+197 2 197 2 197 2
+199 3 199 3 199 3
+200 2 200 2 200 2
+203 2 203 2 203 2
+205 2 205 2 205 2
+207 2 207 2 207 2
+208 3 208 3 208 3
+209 2 209 2 209 2
+213 2 213 2 213 2
+216 2 216 2 216 2
+217 2 217 2 217 2
+219 2 219 2 219 2
+221 2 221 2 221 2
+223 2 223 2 223 2
+224 2 224 2 224 2
+229 2 229 2 229 2
+230 5 230 5 230 5
+233 2 233 2 233 2
+237 2 237 2 237 2
+238 2 238 2 238 2
+239 2 239 2 239 2
+24 2 24 2 24 2
+242 2 242 2 242 2
+255 2 255 2 255 2
+256 2 256 2 256 2
+26 2 26 2 26 2
+265 2 265 2 265 2
+272 2 272 2 272 2
+273 3 273 3 273 3
+277 4 277 4 277 4
+278 2 278 2 278 2
+280 2 280 2 280 2
+281 2 281 2 281 2
+282 2 282 2 282 2
+288 2 288 2 288 2
+298 3 298 3 298 3
+307 2 307 2 307 2
+309 2 309 2 309 2
+311 3 311 3 311 3
+316 3 316 3 316 3
+317 2 317 2 317 2
+318 3 318 3 318 3
+321 2 321 2 321 2
+322 2 322 2 322 2
+325 2 325 2 325 2
+327 3 327 3 327 3
+331 2 331 2 331 2
+333 2 333 2 333 2
+342 2 342 2 342 2
+344 2 344 2 344 2
+348 5 348 5 348 5
+35 3 35 3 35 3
+353 2 353 2 353 2
+367 2 367 2 367 2
+369 3 369 3 369 3
+37 2 37 2 37 2
+382 2 382 2 382 2
+384 3 384 3 384 3
+395 2 395 2 395 2
+396 3 396 3 396 3
+397 2 397 2 397 2
+399 2 399 2 399 2
+401 5 401 5 401 5
+403 3 403 3 403 3
+404 2 404 2 404 2
+406 4 406 4 406 4
+409 3 409 3 409 3
+413 2 413 2 413 2
+414 2 414 2 414 2
+417 3 417 3 417 3
+42 2 42 2 42 2
+424 2 424 2 424 2
+429 2 429 2 429 2
+430 3 430 3 430 3
+431 3 431 3 431 3
+438 3 438 3 438 3
+439 2 439 2 439 2
+454 3 454 3 454 3
+458 2 458 2 458 2
+459 2 459 2 459 2
+462 2 462 2 462 2
+463 2 463 2 463 2
+466 3 466 3 466 3
+468 4 468 4 468 4
+469 5 469 5 469 5
+478 2 478 2 478 2
+480 3 480 3 480 3
+489 4 489 4 489 4
+492 2 492 2 492 2
+498 3 498 3 498 3
+5 3 5 3 5 3
+51 2 51 2 51 2
+58 2 58 2 58 2
+67 2 67 2 67 2
+70 3 70 3 70 3
+72 2 72 2 72 2
+76 2 76 2 76 2
+83 2 83 2 83 2
+84 2 84 2 84 2
+90 3 90 3 90 3
+95 2 95 2 95 2
+97 2 97 2 97 2
+98 2 98 2 98 2
+PREHOOK: query: -- query with broadcast join in the reduce stage
+EXPLAIN
+SELECT *
+FROM
+ (SELECT key, count(value) as cnt FROM src GROUP BY key) s1
+ JOIN src ON (s1.key = src.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: -- query with broadcast join in the reduce stage
+EXPLAIN
+SELECT *
+FROM
+ (SELECT key, count(value) as cnt FROM src GROUP BY key) s1
+ JOIN src ON (s1.key = src.key)
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count (TOK_TABLE_OR_COL value)) cnt)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) s1) (TOK_TABREF (TOK_TABNAME src)) (= (. (TOK_TABLE_OR_COL s1) key) (. (TOK_TABLE_OR_COL src) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ aggregations:
+ expr: count(value)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT *
+FROM
+ (SELECT key, count(value) as cnt FROM src GROUP BY key) s1
+ JOIN src ON (s1.key = src.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM
+ (SELECT key, count(value) as cnt FROM src GROUP BY key) s1
+ JOIN src ON (s1.key = src.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 3 0 val_0
+10 1 10 val_10
+100 2 100 val_100
+103 2 103 val_103
+104 2 104 val_104
+105 1 105 val_105
+11 1 11 val_11
+111 1 111 val_111
+113 2 113 val_113
+114 1 114 val_114
+116 1 116 val_116
+118 2 118 val_118
+119 3 119 val_119
+12 2 12 val_12
+120 2 120 val_120
+125 2 125 val_125
+126 1 126 val_126
+128 3 128 val_128
+129 2 129 val_129
+131 1 131 val_131
+133 1 133 val_133
+134 2 134 val_134
+136 1 136 val_136
+137 2 137 val_137
+138 4 138 val_138
+143 1 143 val_143
+145 1 145 val_145
+146 2 146 val_146
+149 2 149 val_149
+15 2 15 val_15
+150 1 150 val_150
+152 2 152 val_152
+153 1 153 val_153
+155 1 155 val_155
+156 1 156 val_156
+157 1 157 val_157
+158 1 158 val_158
+160 1 160 val_160
+162 1 162 val_162
+163 1 163 val_163
+164 2 164 val_164
+165 2 165 val_165
+166 1 166 val_166
+167 3 167 val_167
+168 1 168 val_168
+169 4 169 val_169
+17 1 17 val_17
+170 1 170 val_170
+172 2 172 val_172
+174 2 174 val_174
+175 2 175 val_175
+176 2 176 val_176
+177 1 177 val_177
+178 1 178 val_178
+179 2 179 val_179
+18 2 18 val_18
+180 1 180 val_180
+181 1 181 val_181
+183 1 183 val_183
+186 1 186 val_186
+187 3 187 val_187
+189 1 189 val_189
+19 1 19 val_19
+190 1 190 val_190
+191 2 191 val_191
+192 1 192 val_192
+193 3 193 val_193
+194 1 194 val_194
+195 2 195 val_195
+196 1 196 val_196
+197 2 197 val_197
+199 3 199 val_199
+2 1 2 val_2
+20 1 20 val_20
+200 2 200 val_200
+201 1 201 val_201
+202 1 202 val_202
+203 2 203 val_203
+205 2 205 val_205
+207 2 207 val_207
+208 3 208 val_208
+209 2 209 val_209
+213 2 213 val_213
+214 1 214 val_214
+216 2 216 val_216
+217 2 217 val_217
+218 1 218 val_218
+219 2 219 val_219
+221 2 221 val_221
+222 1 222 val_222
+223 2 223 val_223
+224 2 224 val_224
+226 1 226 val_226
+228 1 228 val_228
+229 2 229 val_229
+230 5 230 val_230
+233 2 233 val_233
+235 1 235 val_235
+237 2 237 val_237
+238 2 238 val_238
+239 2 239 val_239
+24 2 24 val_24
+241 1 241 val_241
+242 2 242 val_242
+244 1 244 val_244
+247 1 247 val_247
+248 1 248 val_248
+249 1 249 val_249
+252 1 252 val_252
+255 2 255 val_255
+256 2 256 val_256
+257 1 257 val_257
+258 1 258 val_258
+26 2 26 val_26
+260 1 260 val_260
+262 1 262 val_262
+263 1 263 val_263
+265 2 265 val_265
+266 1 266 val_266
+27 1 27 val_27
+272 2 272 val_272
+273 3 273 val_273
+274 1 274 val_274
+275 1 275 val_275
+277 4 277 val_277
+278 2 278 val_278
+28 1 28 val_28
+280 2 280 val_280
+281 2 281 val_281
+282 2 282 val_282
+283 1 283 val_283
+284 1 284 val_284
+285 1 285 val_285
+286 1 286 val_286
+287 1 287 val_287
+288 2 288 val_288
+289 1 289 val_289
+291 1 291 val_291
+292 1 292 val_292
+296 1 296 val_296
+298 3 298 val_298
+30 1 30 val_30
+302 1 302 val_302
+305 1 305 val_305
+306 1 306 val_306
+307 2 307 val_307
+308 1 308 val_308
+309 2 309 val_309
+310 1 310 val_310
+311 3 311 val_311
+315 1 315 val_315
+316 3 316 val_316
+317 2 317 val_317
+318 3 318 val_318
+321 2 321 val_321
+322 2 322 val_322
+323 1 323 val_323
+325 2 325 val_325
+327 3 327 val_327
+33 1 33 val_33
+331 2 331 val_331
+332 1 332 val_332
+333 2 333 val_333
+335 1 335 val_335
+336 1 336 val_336
+338 1 338 val_338
+339 1 339 val_339
+34 1 34 val_34
+341 1 341 val_341
+342 2 342 val_342
+344 2 344 val_344
+345 1 345 val_345
+348 5 348 val_348
+35 3 35 val_35
+351 1 351 val_351
+353 2 353 val_353
+356 1 356 val_356
+360 1 360 val_360
+362 1 362 val_362
+364 1 364 val_364
+365 1 365 val_365
+366 1 366 val_366
+367 2 367 val_367
+368 1 368 val_368
+369 3 369 val_369
+37 2 37 val_37
+373 1 373 val_373
+374 1 374 val_374
+375 1 375 val_375
+377 1 377 val_377
+378 1 378 val_378
+379 1 379 val_379
+382 2 382 val_382
+384 3 384 val_384
+386 1 386 val_386
+389 1 389 val_389
+392 1 392 val_392
+393 1 393 val_393
+394 1 394 val_394
+395 2 395 val_395
+396 3 396 val_396
+397 2 397 val_397
+399 2 399 val_399
+4 1 4 val_4
+400 1 400 val_400
+401 5 401 val_401
+402 1 402 val_402
+403 3 403 val_403
+404 2 404 val_404
+406 4 406 val_406
+407 1 407 val_407
+409 3 409 val_409
+41 1 41 val_41
+411 1 411 val_411
+413 2 413 val_413
+414 2 414 val_414
+417 3 417 val_417
+418 1 418 val_418
+419 1 419 val_419
+42 2 42 val_42
+421 1 421 val_421
+424 2 424 val_424
+427 1 427 val_427
+429 2 429 val_429
+43 1 43 val_43
+430 3 430 val_430
+431 3 431 val_431
+432 1 432 val_432
+435 1 435 val_435
+436 1 436 val_436
+437 1 437 val_437
+438 3 438 val_438
+439 2 439 val_439
+44 1 44 val_44
+443 1 443 val_443
+444 1 444 val_444
+446 1 446 val_446
+448 1 448 val_448
+449 1 449 val_449
+452 1 452 val_452
+453 1 453 val_453
+454 3 454 val_454
+455 1 455 val_455
+457 1 457 val_457
+458 2 458 val_458
+459 2 459 val_459
+460 1 460 val_460
+462 2 462 val_462
+463 2 463 val_463
+466 3 466 val_466
+467 1 467 val_467
+468 4 468 val_468
+469 5 469 val_469
+47 1 47 val_47
+470 1 470 val_470
+472 1 472 val_472
+475 1 475 val_475
+477 1 477 val_477
+478 2 478 val_478
+479 1 479 val_479
+480 3 480 val_480
+481 1 481 val_481
+482 1 482 val_482
+483 1 483 val_483
+484 1 484 val_484
+485 1 485 val_485
+487 1 487 val_487
+489 4 489 val_489
+490 1 490 val_490
+491 1 491 val_491
+492 2 492 val_492
+493 1 493 val_493
+494 1 494 val_494
+495 1 495 val_495
+496 1 496 val_496
+497 1 497 val_497
+498 3 498 val_498
+5 3 5 val_5
+51 2 51 val_51
+53 1 53 val_53
+54 1 54 val_54
+57 1 57 val_57
+58 2 58 val_58
+64 1 64 val_64
+65 1 65 val_65
+66 1 66 val_66
+67 2 67 val_67
+69 1 69 val_69
+70 3 70 val_70
+72 2 72 val_72
+74 1 74 val_74
+76 2 76 val_76
+77 1 77 val_77
+78 1 78 val_78
+8 1 8 val_8
+80 1 80 val_80
+82 1 82 val_82
+83 2 83 val_83
+84 2 84 val_84
+85 1 85 val_85
+86 1 86 val_86
+87 1 87 val_87
+9 1 9 val_9
+90 3 90 val_90
+92 1 92 val_92
+95 2 95 val_95
+96 1 96 val_96
+97 2 97 val_97
+98 2 98 val_98