You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/08/28 16:58:18 UTC
svn commit: r1518234 [2/2] - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/io/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/java/org/a...
Added: hive/trunk/ql/src/test/results/clientpositive/limit_pushdown.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/limit_pushdown.q.out?rev=1518234&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/limit_pushdown.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/limit_pushdown.q.out Wed Aug 28 14:58:17 2013
@@ -0,0 +1,1499 @@
+PREHOOK: query: -- HIVE-3562 Some limit can be pushed down to map stage
+
+explain
+select key,value from src order by key limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: -- HIVE-3562 Some limit can be pushed down to map stage
+
+explain
+select key,value from src order by key limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ tag: -1
+ TopN: 20
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+
+PREHOOK: query: select key,value from src order by key limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key,value from src order by key limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
+100 val_100
+103 val_103
+103 val_103
+104 val_104
+104 val_104
+105 val_105
+11 val_11
+111 val_111
+113 val_113
+113 val_113
+114 val_114
+116 val_116
+118 val_118
+118 val_118
+119 val_119
+PREHOOK: query: explain
+select key,value from src order by key desc limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select key,value from src order by key desc limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEDESC (TOK_TABLE_OR_COL key))) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: -
+ tag: -1
+ TopN: 20
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+
+PREHOOK: query: select key,value from src order by key desc limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key,value from src order by key desc limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+98 val_98
+98 val_98
+97 val_97
+97 val_97
+96 val_96
+95 val_95
+95 val_95
+92 val_92
+90 val_90
+90 val_90
+90 val_90
+9 val_9
+87 val_87
+86 val_86
+85 val_85
+84 val_84
+84 val_84
+83 val_83
+83 val_83
+82 val_82
+PREHOOK: query: explain
+select value, sum(key + 1) as sum from src group by value limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select value, sum(key + 1) as sum from src group by value limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION sum (+ (TOK_TABLE_OR_COL key) 1)) sum)) (TOK_GROUPBY (TOK_TABLE_OR_COL value)) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: value
+ type: string
+ expr: key
+ type: string
+ outputColumnNames: value, key
+ Group By Operator
+ aggregations:
+ expr: sum((key + 1))
+ bucketGroup: false
+ keys:
+ expr: value
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ TopN: 20
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: _col1
+ type: double
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: double
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+
+PREHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+val_0 3.0
+val_10 11.0
+val_100 202.0
+val_103 208.0
+val_104 210.0
+val_105 106.0
+val_11 12.0
+val_111 112.0
+val_113 228.0
+val_114 115.0
+val_116 117.0
+val_118 238.0
+val_119 360.0
+val_12 26.0
+val_120 242.0
+val_125 252.0
+val_126 127.0
+val_128 387.0
+val_129 260.0
+val_131 132.0
+PREHOOK: query: -- deduped RS
+explain
+select value,avg(key + 1) from src group by value order by value limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: -- deduped RS
+explain
+select value,avg(key + 1) from src group by value order by value limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION avg (+ (TOK_TABLE_OR_COL key) 1)))) (TOK_GROUPBY (TOK_TABLE_OR_COL value)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: value
+ type: string
+ expr: key
+ type: string
+ outputColumnNames: value, key
+ Group By Operator
+ aggregations:
+ expr: avg((key + 1))
+ bucketGroup: false
+ keys:
+ expr: value
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ TopN: 20
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: _col1
+ type: struct<count:bigint,sum:double>
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: avg(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: double
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+
+PREHOOK: query: select value,avg(key + 1) from src group by value order by value limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select value,avg(key + 1) from src group by value order by value limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+val_0 1.0
+val_10 11.0
+val_100 101.0
+val_103 104.0
+val_104 105.0
+val_105 106.0
+val_11 12.0
+val_111 112.0
+val_113 114.0
+val_114 115.0
+val_116 117.0
+val_118 119.0
+val_119 120.0
+val_12 13.0
+val_120 121.0
+val_125 126.0
+val_126 127.0
+val_128 129.0
+val_129 130.0
+val_131 132.0
+PREHOOK: query: -- distincts
+explain
+select distinct(key) from src limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: -- distincts
+explain
+select distinct(key) from src limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECTDI (TOK_SELEXPR (TOK_TABLE_OR_COL key))) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ TopN: 20
+ TopN Hash Memory Usage: 0.3
+ Reduce Operator Tree:
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ outputColumnNames: _col0
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+
+PREHOOK: query: select distinct(key) from src limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select distinct(key) from src limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0
+10
+100
+103
+104
+105
+11
+111
+113
+114
+116
+118
+119
+12
+120
+125
+126
+128
+129
+131
+PREHOOK: query: explain
+select key, count(distinct(key)) from src group by key limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select key, count(distinct(key)) from src group by key limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_TABLE_OR_COL key)))) (TOK_GROUPBY (TOK_TABLE_OR_COL key)) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT key)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ TopN: 20
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT KEY._col1:0._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col1:0._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+
+PREHOOK: query: select key, count(distinct(key)) from src group by key limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key, count(distinct(key)) from src group by key limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 1
+10 1
+100 1
+103 1
+104 1
+105 1
+11 1
+111 1
+113 1
+114 1
+116 1
+118 1
+119 1
+12 1
+120 1
+125 1
+126 1
+128 1
+129 1
+131 1
+PREHOOK: query: -- limit zero
+explain
+select key,value from src order by key limit 0
+PREHOOK: type: QUERY
+POSTHOOK: query: -- limit zero
+explain
+select key,value from src order by key limit 0
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))) (TOK_LIMIT 0)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 0
+
+
+PREHOOK: query: select key,value from src order by key limit 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key,value from src order by key limit 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: -- 2MR (applied to last RS)
+explain
+select value, sum(key) as sum from src group by value order by sum limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: -- 2MR (applied to last RS)
+explain
+select value, sum(key) as sum from src group by value order by sum limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_TABLE_OR_COL key)) sum)) (TOK_GROUPBY (TOK_TABLE_OR_COL value)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL sum))) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: value
+ type: string
+ expr: key
+ type: string
+ outputColumnNames: value, key
+ Group By Operator
+ aggregations:
+ expr: sum(key)
+ bucketGroup: false
+ keys:
+ expr: value
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: double
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: double
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ key expressions:
+ expr: _col1
+ type: double
+ sort order: +
+ tag: -1
+ TopN: 20
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: double
+ Reduce Operator Tree:
+ Extract
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+
+PREHOOK: query: select value, sum(key) as sum from src group by value order by sum limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select value, sum(key) as sum from src group by value order by sum limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+val_0 0.0
+val_2 2.0
+val_4 4.0
+val_8 8.0
+val_9 9.0
+val_10 10.0
+val_11 11.0
+val_5 15.0
+val_17 17.0
+val_19 19.0
+val_20 20.0
+val_12 24.0
+val_27 27.0
+val_28 28.0
+val_30 30.0
+val_15 30.0
+val_33 33.0
+val_34 34.0
+val_18 36.0
+val_41 41.0
+PREHOOK: query: -- subqueries
+explain
+select * from
+(select key, count(1) from src group by key order by key limit 2) subq
+join
+(select key, count(1) from src group by key limit 3) subq2
+on subq.key=subq2.key limit 4
+PREHOOK: type: QUERY
+POSTHOOK: query: -- subqueries
+explain
+select * from
+(select key, count(1) from src group by key order by key limit 2) subq
+join
+(select key, count(1) from src group by key limit 3) subq2
+on subq.key=subq2.key limit 4
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_TABLE_OR_COL key)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))) (TOK_LIMIT 2))) subq) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_TABLE_OR_COL key)) (TOK_LIMIT 3))) subq2) (= (. (TOK_TABLE_OR_COL subq) key) (. (TOK_TABLE_OR_COL subq2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_LIMIT 4)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1, Stage-4
+ Stage-3 is a root stage
+ Stage-4 depends on stages: Stage-3
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
+ Group By Operator
+ aggregations:
+ expr: count(1)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ TopN: 2
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ $INTNAME1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ expr: _col2
+ type: string
+ expr: _col3
+ type: bigint
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
+ Group By Operator
+ aggregations:
+ expr: count(1)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ TopN: 3
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ TopN: 3
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Extract
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 4
+
+
+PREHOOK: query: -- map aggregation disabled
+explain
+select value, sum(key) as sum from src group by value limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: -- map aggregation disabled
+explain
+select value, sum(key) as sum from src group by value limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_TABLE_OR_COL key)) sum)) (TOK_GROUPBY (TOK_TABLE_OR_COL value)) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: value
+ type: string
+ expr: key
+ type: string
+ outputColumnNames: value, key
+ Reduce Output Operator
+ key expressions:
+ expr: value
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: value
+ type: string
+ tag: -1
+ TopN: 20
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: key
+ type: string
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: double
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+
+PREHOOK: query: select value, sum(key) as sum from src group by value limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select value, sum(key) as sum from src group by value limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+val_0 0.0
+val_10 10.0
+val_100 200.0
+val_103 206.0
+val_104 208.0
+val_105 105.0
+val_11 11.0
+val_111 111.0
+val_113 226.0
+val_114 114.0
+val_116 116.0
+val_118 236.0
+val_119 357.0
+val_12 24.0
+val_120 240.0
+val_125 250.0
+val_126 126.0
+val_128 384.0
+val_129 258.0
+val_131 131.0
+PREHOOK: query: -- flush for order-by
+explain
+select key,value,value,value,value,value,value,value,value from src order by key limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: -- flush for order-by
+explain
+select key,value,value,value,value,value,value,value,value from src order by key limit 100
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))) (TOK_LIMIT 100)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ expr: value
+ type: string
+ expr: value
+ type: string
+ expr: value
+ type: string
+ expr: value
+ type: string
+ expr: value
+ type: string
+ expr: value
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ tag: -1
+ TopN: 100
+ TopN Hash Memory Usage: 2.0E-5
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col6
+ type: string
+ expr: _col7
+ type: string
+ expr: _col8
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 100
+
+
+PREHOOK: query: select key,value,value,value,value,value,value,value,value from src order by key limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key,value,value,value,value,value,value,value,value from src order by key limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 val_0 val_0 val_0 val_0 val_0 val_0 val_0 val_0
+0 val_0 val_0 val_0 val_0 val_0 val_0 val_0 val_0
+0 val_0 val_0 val_0 val_0 val_0 val_0 val_0 val_0
+10 val_10 val_10 val_10 val_10 val_10 val_10 val_10 val_10
+100 val_100 val_100 val_100 val_100 val_100 val_100 val_100 val_100
+100 val_100 val_100 val_100 val_100 val_100 val_100 val_100 val_100
+103 val_103 val_103 val_103 val_103 val_103 val_103 val_103 val_103
+103 val_103 val_103 val_103 val_103 val_103 val_103 val_103 val_103
+104 val_104 val_104 val_104 val_104 val_104 val_104 val_104 val_104
+104 val_104 val_104 val_104 val_104 val_104 val_104 val_104 val_104
+105 val_105 val_105 val_105 val_105 val_105 val_105 val_105 val_105
+11 val_11 val_11 val_11 val_11 val_11 val_11 val_11 val_11
+111 val_111 val_111 val_111 val_111 val_111 val_111 val_111 val_111
+113 val_113 val_113 val_113 val_113 val_113 val_113 val_113 val_113
+113 val_113 val_113 val_113 val_113 val_113 val_113 val_113 val_113
+114 val_114 val_114 val_114 val_114 val_114 val_114 val_114 val_114
+116 val_116 val_116 val_116 val_116 val_116 val_116 val_116 val_116
+118 val_118 val_118 val_118 val_118 val_118 val_118 val_118 val_118
+118 val_118 val_118 val_118 val_118 val_118 val_118 val_118 val_118
+119 val_119 val_119 val_119 val_119 val_119 val_119 val_119 val_119
+119 val_119 val_119 val_119 val_119 val_119 val_119 val_119 val_119
+119 val_119 val_119 val_119 val_119 val_119 val_119 val_119 val_119
+12 val_12 val_12 val_12 val_12 val_12 val_12 val_12 val_12
+12 val_12 val_12 val_12 val_12 val_12 val_12 val_12 val_12
+120 val_120 val_120 val_120 val_120 val_120 val_120 val_120 val_120
+120 val_120 val_120 val_120 val_120 val_120 val_120 val_120 val_120
+125 val_125 val_125 val_125 val_125 val_125 val_125 val_125 val_125
+125 val_125 val_125 val_125 val_125 val_125 val_125 val_125 val_125
+126 val_126 val_126 val_126 val_126 val_126 val_126 val_126 val_126
+128 val_128 val_128 val_128 val_128 val_128 val_128 val_128 val_128
+128 val_128 val_128 val_128 val_128 val_128 val_128 val_128 val_128
+128 val_128 val_128 val_128 val_128 val_128 val_128 val_128 val_128
+129 val_129 val_129 val_129 val_129 val_129 val_129 val_129 val_129
+129 val_129 val_129 val_129 val_129 val_129 val_129 val_129 val_129
+131 val_131 val_131 val_131 val_131 val_131 val_131 val_131 val_131
+133 val_133 val_133 val_133 val_133 val_133 val_133 val_133 val_133
+134 val_134 val_134 val_134 val_134 val_134 val_134 val_134 val_134
+134 val_134 val_134 val_134 val_134 val_134 val_134 val_134 val_134
+136 val_136 val_136 val_136 val_136 val_136 val_136 val_136 val_136
+137 val_137 val_137 val_137 val_137 val_137 val_137 val_137 val_137
+137 val_137 val_137 val_137 val_137 val_137 val_137 val_137 val_137
+138 val_138 val_138 val_138 val_138 val_138 val_138 val_138 val_138
+138 val_138 val_138 val_138 val_138 val_138 val_138 val_138 val_138
+138 val_138 val_138 val_138 val_138 val_138 val_138 val_138 val_138
+138 val_138 val_138 val_138 val_138 val_138 val_138 val_138 val_138
+143 val_143 val_143 val_143 val_143 val_143 val_143 val_143 val_143
+145 val_145 val_145 val_145 val_145 val_145 val_145 val_145 val_145
+146 val_146 val_146 val_146 val_146 val_146 val_146 val_146 val_146
+146 val_146 val_146 val_146 val_146 val_146 val_146 val_146 val_146
+149 val_149 val_149 val_149 val_149 val_149 val_149 val_149 val_149
+149 val_149 val_149 val_149 val_149 val_149 val_149 val_149 val_149
+15 val_15 val_15 val_15 val_15 val_15 val_15 val_15 val_15
+15 val_15 val_15 val_15 val_15 val_15 val_15 val_15 val_15
+150 val_150 val_150 val_150 val_150 val_150 val_150 val_150 val_150
+152 val_152 val_152 val_152 val_152 val_152 val_152 val_152 val_152
+152 val_152 val_152 val_152 val_152 val_152 val_152 val_152 val_152
+153 val_153 val_153 val_153 val_153 val_153 val_153 val_153 val_153
+155 val_155 val_155 val_155 val_155 val_155 val_155 val_155 val_155
+156 val_156 val_156 val_156 val_156 val_156 val_156 val_156 val_156
+157 val_157 val_157 val_157 val_157 val_157 val_157 val_157 val_157
+158 val_158 val_158 val_158 val_158 val_158 val_158 val_158 val_158
+160 val_160 val_160 val_160 val_160 val_160 val_160 val_160 val_160
+162 val_162 val_162 val_162 val_162 val_162 val_162 val_162 val_162
+163 val_163 val_163 val_163 val_163 val_163 val_163 val_163 val_163
+164 val_164 val_164 val_164 val_164 val_164 val_164 val_164 val_164
+164 val_164 val_164 val_164 val_164 val_164 val_164 val_164 val_164
+165 val_165 val_165 val_165 val_165 val_165 val_165 val_165 val_165
+165 val_165 val_165 val_165 val_165 val_165 val_165 val_165 val_165
+166 val_166 val_166 val_166 val_166 val_166 val_166 val_166 val_166
+167 val_167 val_167 val_167 val_167 val_167 val_167 val_167 val_167
+167 val_167 val_167 val_167 val_167 val_167 val_167 val_167 val_167
+167 val_167 val_167 val_167 val_167 val_167 val_167 val_167 val_167
+168 val_168 val_168 val_168 val_168 val_168 val_168 val_168 val_168
+169 val_169 val_169 val_169 val_169 val_169 val_169 val_169 val_169
+169 val_169 val_169 val_169 val_169 val_169 val_169 val_169 val_169
+169 val_169 val_169 val_169 val_169 val_169 val_169 val_169 val_169
+169 val_169 val_169 val_169 val_169 val_169 val_169 val_169 val_169
+17 val_17 val_17 val_17 val_17 val_17 val_17 val_17 val_17
+170 val_170 val_170 val_170 val_170 val_170 val_170 val_170 val_170
+172 val_172 val_172 val_172 val_172 val_172 val_172 val_172 val_172
+172 val_172 val_172 val_172 val_172 val_172 val_172 val_172 val_172
+174 val_174 val_174 val_174 val_174 val_174 val_174 val_174 val_174
+174 val_174 val_174 val_174 val_174 val_174 val_174 val_174 val_174
+175 val_175 val_175 val_175 val_175 val_175 val_175 val_175 val_175
+175 val_175 val_175 val_175 val_175 val_175 val_175 val_175 val_175
+176 val_176 val_176 val_176 val_176 val_176 val_176 val_176 val_176
+176 val_176 val_176 val_176 val_176 val_176 val_176 val_176 val_176
+177 val_177 val_177 val_177 val_177 val_177 val_177 val_177 val_177
+178 val_178 val_178 val_178 val_178 val_178 val_178 val_178 val_178
+179 val_179 val_179 val_179 val_179 val_179 val_179 val_179 val_179
+179 val_179 val_179 val_179 val_179 val_179 val_179 val_179 val_179
+18 val_18 val_18 val_18 val_18 val_18 val_18 val_18 val_18
+18 val_18 val_18 val_18 val_18 val_18 val_18 val_18 val_18
+180 val_180 val_180 val_180 val_180 val_180 val_180 val_180 val_180
+181 val_181 val_181 val_181 val_181 val_181 val_181 val_181 val_181
+183 val_183 val_183 val_183 val_183 val_183 val_183 val_183 val_183
+186 val_186 val_186 val_186 val_186 val_186 val_186 val_186 val_186
+187 val_187 val_187 val_187 val_187 val_187 val_187 val_187 val_187
+187 val_187 val_187 val_187 val_187 val_187 val_187 val_187 val_187
+187 val_187 val_187 val_187 val_187 val_187 val_187 val_187 val_187
+PREHOOK: query: -- flush for group-by
+explain
+select sum(key) as sum from src group by concat(key,value,value,value,value,value,value,value,value,value) limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: -- flush for group-by
+explain
+select sum(key) as sum from src group by concat(key,value,value,value,value,value,value,value,value,value) limit 100
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_TABLE_OR_COL key)) sum)) (TOK_GROUPBY (TOK_FUNCTION concat (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value) (TOK_TABLE_OR_COL value) (TOK_TABLE_OR_COL value) (TOK_TABLE_OR_COL value) (TOK_TABLE_OR_COL value) (TOK_TABLE_OR_COL value) (TOK_TABLE_OR_COL value) (TOK_TABLE_OR_COL value) (TOK_TABLE_OR_COL value))) (TOK_LIMIT 100)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Reduce Output Operator
+ key expressions:
+ expr: concat(key, value, value, value, value, value, value, value, value, value)
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: concat(key, value, value, value, value, value, value, value, value, value)
+ type: string
+ tag: -1
+ TopN: 100
+ TopN Hash Memory Usage: 2.0E-5
+ value expressions:
+ expr: key
+ type: string
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col1
+ type: double
+ outputColumnNames: _col0
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 100
+
+
+PREHOOK: query: select sum(key) as sum from src group by concat(key,value,value,value,value,value,value,value,value,value) limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(key) as sum from src group by concat(key,value,value,value,value,value,value,value,value,value) limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0.0
+200.0
+206.0
+208.0
+105.0
+10.0
+111.0
+226.0
+114.0
+116.0
+236.0
+357.0
+11.0
+240.0
+250.0
+126.0
+384.0
+258.0
+24.0
+131.0
+133.0
+268.0
+136.0
+274.0
+552.0
+143.0
+145.0
+292.0
+298.0
+150.0
+304.0
+153.0
+155.0
+156.0
+157.0
+158.0
+30.0
+160.0
+162.0
+163.0
+328.0
+330.0
+166.0
+501.0
+168.0
+676.0
+170.0
+344.0
+348.0
+350.0
+352.0
+177.0
+178.0
+358.0
+17.0
+180.0
+181.0
+183.0
+186.0
+561.0
+189.0
+36.0
+190.0
+382.0
+192.0
+579.0
+194.0
+390.0
+196.0
+394.0
+597.0
+19.0
+400.0
+201.0
+202.0
+406.0
+410.0
+414.0
+624.0
+418.0
+20.0
+426.0
+214.0
+432.0
+434.0
+218.0
+438.0
+442.0
+222.0
+446.0
+448.0
+226.0
+228.0
+458.0
+1150.0
+466.0
+235.0
+474.0
+476.0
+478.0
Added: hive/trunk/ql/src/test/results/clientpositive/limit_pushdown_negative.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/limit_pushdown_negative.q.out?rev=1518234&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/limit_pushdown_negative.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/limit_pushdown_negative.q.out Wed Aug 28 14:58:17 2013
@@ -0,0 +1,504 @@
+PREHOOK: query: -- negative, RS + join
+explain select * from src a join src b on a.key=b.key limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: -- negative, RS + join
+explain select * from src a join src b on a.key=b.key limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_TABREF (TOK_TABNAME src) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ b
+ TableScan
+ alias: b
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+
+PREHOOK: query: -- negative, RS + filter
+explain select value, sum(key) as sum from src group by value having sum > 100 limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: -- negative, RS + filter
+explain select value, sum(key) as sum from src group by value having sum > 100 limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_TABLE_OR_COL key)) sum)) (TOK_GROUPBY (TOK_TABLE_OR_COL value)) (TOK_HAVING (> (TOK_TABLE_OR_COL sum) 100)) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: value
+ type: string
+ expr: key
+ type: string
+ outputColumnNames: value, key
+ Group By Operator
+ aggregations:
+ expr: sum(key)
+ bucketGroup: false
+ keys:
+ expr: value
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: double
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Filter Operator
+ predicate:
+ expr: (_col1 > 100.0)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: double
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+
+PREHOOK: query: -- negative, RS + lateral view
+explain select key, L.* from (select * from src order by key) a lateral view explode(array(value, value)) L as v limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: -- negative, RS + lateral view
+explain select key, L.* from (select * from src order by key) a lateral view explode(array(value, value)) L as v limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION explode (TOK_FUNCTION array (TOK_TABLE_OR_COL value) (TOK_TABLE_OR_COL value))) v (TOK_TABALIAS L))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))))) a))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME L)))) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Lateral View Forward
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ outputColumnNames: _col0
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col2
+ type: string
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Select Operator
+ expressions:
+ expr: array(_col1,_col1)
+ type: array<string>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col0, _col2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col2
+ type: string
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+
+
+PREHOOK: query: -- negative, RS + forward + multi-groupby
+CREATE TABLE dest_2(key STRING, c1 INT)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- negative, RS + forward + multi-groupby
+CREATE TABLE dest_2(key STRING, c1 INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest_2
+PREHOOK: query: CREATE TABLE dest_3(key STRING, c1 INT)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest_3(key STRING, c1 INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest_3
+PREHOOK: query: EXPLAIN FROM src
+INSERT OVERWRITE TABLE dest_2 SELECT value, sum(key) GROUP BY value
+INSERT OVERWRITE TABLE dest_3 SELECT value, sum(key) GROUP BY value limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN FROM src
+INSERT OVERWRITE TABLE dest_2 SELECT value, sum(key) GROUP BY value
+INSERT OVERWRITE TABLE dest_3 SELECT value, sum(key) GROUP BY value limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_2))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_TABLE_OR_COL key)))) (TOK_GROUPBY (TOK_TABLE_OR_COL value))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_3))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_TABLE_OR_COL key)))) (TOK_GROUPBY (TOK_TABLE_OR_COL value)) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-2
+ Stage-1 depends on stages: Stage-4
+ Stage-5 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: value
+ type: string
+ expr: key
+ type: string
+ outputColumnNames: value, key
+ Reduce Output Operator
+ key expressions:
+ expr: value
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: value
+ type: string
+ tag: -1
+ value expressions:
+ expr: key
+ type: string
+ Reduce Operator Tree:
+ Forward
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: UDFToInteger(_col1)
+ type: int
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_2
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: double
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_2
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ TopN: 20
+ TopN Hash Memory Usage: 0.3
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: double
+ Reduce Operator Tree:
+ Extract
+ Limit
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: UDFToInteger(_col1)
+ type: int
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 2
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_3
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_3
+
+ Stage: Stage-5
+ Stats-Aggr Operator
+
+
+PREHOOK: query: -- nagative, multi distinct
+explain
+select count(distinct key)+count(distinct value) from src limit 20
+PREHOOK: type: QUERY
+POSTHOOK: query: -- nagative, multi distinct
+explain
+select count(distinct key)+count(distinct value) from src limit 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ (TOK_FUNCTIONDI count (TOK_TABLE_OR_COL key)) (TOK_FUNCTIONDI count (TOK_TABLE_OR_COL value))))) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT key)
+ expr: count(DISTINCT value)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ sort order: ++
+ tag: -1
+ value expressions:
+ expr: _col2
+ type: bigint
+ expr: _col3
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT KEY._col0:0._col0)
+ expr: count(DISTINCT KEY._col0:1._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: (_col0 + _col1)
+ type: bigint
+ outputColumnNames: _col0
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+
+