You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/08/25 19:03:17 UTC
svn commit: r989224 - in /hadoop/hive/trunk: CHANGES.txt
ql/src/test/queries/clientpositive/udtf_explode.q
ql/src/test/results/clientpositive/udtf_explode.q.out
Author: namit
Date: Wed Aug 25 17:03:16 2010
New Revision: 989224
URL: http://svn.apache.org/viewvc?rev=989224&view=rev
Log:
HIVE-1593. Add test file udtf_explode
(Carl Steinbach via namit)
M CHANGES.txt
M ql/src/test/results/clientpositive/udtf_explode.q.out
M ql/src/test/queries/clientpositive/udtf_explode.q
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/test/queries/clientpositive/udtf_explode.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/udtf_explode.q.out
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=989224&r1=989223&r2=989224&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Aug 25 17:03:16 2010
@@ -204,6 +204,9 @@ Trunk - Unreleased
HIVE-741. NULL is not handled correctly in join
(Amareshwari Sriramadasu via Ning Zhang)
+ HIVE-1593. Add test file udtf_explode
+ (Carl Steinbach via namit)
+
TESTS
HIVE-1464. improve test query performance
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/udtf_explode.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/udtf_explode.q?rev=989224&r1=989223&r2=989224&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/udtf_explode.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/udtf_explode.q Wed Aug 25 17:03:16 2010
@@ -0,0 +1,9 @@
+DESCRIBE FUNCTION explode;
+DESCRIBE FUNCTION EXTENDED explode;
+
+EXPLAIN EXTENDED SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3;
+EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol;
+
+SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3;
+SELECT explode(array(1,2,3)) AS (myCol) FROM src LIMIT 3;
+SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol;
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/udtf_explode.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udtf_explode.q.out?rev=989224&r1=989223&r2=989224&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udtf_explode.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udtf_explode.q.out Wed Aug 25 17:03:16 2010
@@ -0,0 +1,307 @@
+PREHOOK: query: DESCRIBE FUNCTION explode
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION explode
+POSTHOOK: type: DESCFUNCTION
+explode(a) - separates the elements of array a into multiple rows
+PREHOOK: query: DESCRIBE FUNCTION EXTENDED explode
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION EXTENDED explode
+POSTHOOK: type: DESCFUNCTION
+explode(a) - separates the elements of array a into multiple rows
+PREHOOK: query: EXPLAIN EXTENDED SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN EXTENDED SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION explode (TOK_FUNCTION array 1 2 3)) myCol)) (TOK_LIMIT 3)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: array(1,2,3)
+ type: array<int>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: explode
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ directory: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-20_579_1032637901953811782/-ext-10001
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ columns col
+ columns.types int
+ serialization.format 1
+ TotalFiles: 1
+ MultiFileSpray: false
+ Needs Tagging: false
+ Path -> Alias:
+ pfile:/Users/carl/Projects/hive-base/build/ql/test/data/warehouse/src [src]
+ Path -> Partition:
+ pfile:/Users/carl/Projects/hive-base/build/ql/test/data/warehouse/src
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location pfile:/Users/carl/Projects/hive-base/build/ql/test/data/warehouse/src
+ name src
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ transient_lastDdlTime 1282715837
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location pfile:/Users/carl/Projects/hive-base/build/ql/test/data/warehouse/src
+ name src
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ transient_lastDdlTime 1282715837
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: src
+ name: src
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 3
+
+
+PREHOOK: query: EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION explode (TOK_FUNCTION array 1 2 3)) myCol)) (TOK_LIMIT 3))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) myCol)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (. (TOK_TABLE_OR_COL a) myCol))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: array(1,2,3)
+ type: array<int>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: explode
+ Limit
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: col
+ type: int
+ Needs Tagging: false
+ Path -> Alias:
+ pfile:/Users/carl/Projects/hive-base/build/ql/test/data/warehouse/src [a:src]
+ Path -> Partition:
+ pfile:/Users/carl/Projects/hive-base/build/ql/test/data/warehouse/src
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location pfile:/Users/carl/Projects/hive-base/build/ql/test/data/warehouse/src
+ name src
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ transient_lastDdlTime 1282715837
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location pfile:/Users/carl/Projects/hive-base/build/ql/test/data/warehouse/src
+ name src
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ transient_lastDdlTime 1282715837
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: src
+ name: src
+ Reduce Operator Tree:
+ Extract
+ Limit
+ Select Operator
+ expressions:
+ expr: _col0
+ type: int
+ outputColumnNames: _col0
+ Group By Operator
+ aggregations:
+ expr: count(1)
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: int
+ mode: hash
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ directory: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-22_070_3243714428136533506/-mr-10002
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types int,bigint
+ escape.delim \
+ TotalFiles: 1
+ MultiFileSpray: false
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-22_070_3243714428136533506/-mr-10002
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+ file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-22_070_3243714428136533506/-mr-10002 [file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-22_070_3243714428136533506/-mr-10002]
+ Path -> Partition:
+ file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-22_070_3243714428136533506/-mr-10002
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types int,bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types int,bigint
+ escape.delim \
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: int
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: int
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ directory: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-22_070_3243714428136533506/-ext-10001
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types int:bigint
+ serialization.format 1
+ TotalFiles: 1
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-22_203_3823746274402933485/-mr-10000
+POSTHOOK: query: SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-22_203_3823746274402933485/-mr-10000
+1
+2
+3
+PREHOOK: query: SELECT explode(array(1,2,3)) AS (myCol) FROM src LIMIT 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-30_768_8764239289780414796/-mr-10000
+POSTHOOK: query: SELECT explode(array(1,2,3)) AS (myCol) FROM src LIMIT 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-30_768_8764239289780414796/-mr-10000
+1
+2
+3
+PREHOOK: query: SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-34_517_4011176611215961822/-mr-10000
+POSTHOOK: query: SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/var/folders/b7/b7UUwNZdF1KKHtM+5la6f++++TI/-Tmp-/carl/hive_2010-08-24_22-57-34_517_4011176611215961822/-mr-10000
+1 1
+2 1
+3 1