You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ec...@apache.org on 2013/08/15 03:32:00 UTC
svn commit: r1514119 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/ java/org/apache/hadoop/hive/ql/parse/
test/queries/clientnegative/ test/queries/clientpositive/
test/results/clientnegative/ test/results/clientpositive/
Author: ecapriolo
Date: Thu Aug 15 01:32:00 2013
New Revision: 1514119
URL: http://svn.apache.org/r1514119
Log:
Hive-2608 Do not require AS a,b,c part in LATERAL VIEW (Navis Ryu via egc)
Submitted by: Navis Ryu
Reviewed by: Edward Capriolo
Added:
hive/trunk/ql/src/test/queries/clientpositive/lateral_view_noalias.q
hive/trunk/ql/src/test/results/clientpositive/lateral_view_noalias.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/test/queries/clientnegative/udtf_not_supported2.q
hive/trunk/ql/src/test/results/clientnegative/lateral_view_join.q.out
hive/trunk/ql/src/test/results/clientnegative/udtf_not_supported2.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java?rev=1514119&r1=1514118&r2=1514119&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java Thu Aug 15 01:32:00 2013
@@ -144,7 +144,7 @@ public enum ErrorMsg {
COLUMN_ALIAS_ALREADY_EXISTS(10074, "Column alias already exists:", "42S02"),
UDTF_MULTIPLE_EXPR(10075, "Only a single expression in the SELECT clause is "
+ "supported with UDTF's"),
- UDTF_REQUIRE_AS(10076, "UDTF's require an AS clause"),
+ @Deprecated UDTF_REQUIRE_AS(10076, "UDTF's require an AS clause"),
UDTF_NO_GROUP_BY(10077, "GROUP BY is not supported with a UDTF in the SELECT clause"),
UDTF_NO_SORT_BY(10078, "SORT BY is not supported with a UDTF in the SELECT clause"),
UDTF_NO_CLUSTER_BY(10079, "CLUSTER BY is not supported with a UDTF in the SELECT clause"),
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g?rev=1514119&r1=1514118&r2=1514119&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g Thu Aug 15 01:32:00 2013
@@ -125,11 +125,11 @@ lateralView
@init {gParent.msgs.push("lateral view"); }
@after {gParent.msgs.pop(); }
:
- KW_LATERAL KW_VIEW KW_OUTER function tableAlias KW_AS identifier (COMMA identifier)*
- -> ^(TOK_LATERAL_VIEW_OUTER ^(TOK_SELECT ^(TOK_SELEXPR function identifier+ tableAlias)))
+ KW_LATERAL KW_VIEW KW_OUTER function tableAlias (KW_AS identifier (COMMA identifier)*)?
+ -> ^(TOK_LATERAL_VIEW_OUTER ^(TOK_SELECT ^(TOK_SELEXPR function identifier* tableAlias)))
|
- KW_LATERAL KW_VIEW function tableAlias KW_AS identifier (COMMA identifier)*
- -> ^(TOK_LATERAL_VIEW ^(TOK_SELECT ^(TOK_SELEXPR function identifier+ tableAlias)))
+ KW_LATERAL KW_VIEW function tableAlias (KW_AS identifier (COMMA identifier)*)?
+ -> ^(TOK_LATERAL_VIEW ^(TOK_SELECT ^(TOK_SELEXPR function identifier* tableAlias)))
;
tableAlias
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1514119&r1=1514118&r2=1514119&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Aug 15 01:32:00 2013
@@ -2493,14 +2493,12 @@ public class SemanticAnalyzer extends Ba
(ASTNode) selExprList.getChild(1),
ErrorMsg.UDTF_MULTIPLE_EXPR.getMsg()));
}
- // Require an AS for UDTFs for column aliases
+
ASTNode selExpr = (ASTNode) selExprList.getChild(posn);
- if (selExpr.getChildCount() < 2) {
- throw new SemanticException(generateErrorMessage(udtfExpr,
- ErrorMsg.UDTF_REQUIRE_AS.getMsg()));
- }
+
// Get the column / table aliases from the expression. Start from 1 as
// 0 is the TOK_FUNCTION
+ // column names also can be inferred from result of UDTF
for (int i = 1; i < selExpr.getChildCount(); i++) {
ASTNode selExprChild = (ASTNode) selExpr.getChild(i);
switch (selExprChild.getType()) {
@@ -5490,9 +5488,15 @@ public class SemanticAnalyzer extends Ba
}
StructObjectInspector outputOI = genericUDTF.initialize(colOIs);
+ int numUdtfCols = outputOI.getAllStructFieldRefs().size();
+ if (colAliases.isEmpty()) {
+ // user did not specfied alias names, infer names from outputOI
+ for (StructField field : outputOI.getAllStructFieldRefs()) {
+ colAliases.add(field.getFieldName());
+ }
+ }
// Make sure that the number of column aliases in the AS clause matches
// the number of columns output by the UDTF
- int numUdtfCols = outputOI.getAllStructFieldRefs().size();
int numSuppliedAliases = colAliases.size();
if (numUdtfCols != numSuppliedAliases) {
throw new SemanticException(ErrorMsg.UDTF_ALIAS_MISMATCH
Modified: hive/trunk/ql/src/test/queries/clientnegative/udtf_not_supported2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/udtf_not_supported2.q?rev=1514119&r1=1514118&r2=1514119&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/udtf_not_supported2.q (original)
+++ hive/trunk/ql/src/test/queries/clientnegative/udtf_not_supported2.q Thu Aug 15 01:32:00 2013
@@ -1 +0,0 @@
-SELECT explode(array(1,2,3)) FROM src;
Added: hive/trunk/ql/src/test/queries/clientpositive/lateral_view_noalias.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/lateral_view_noalias.q?rev=1514119&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/lateral_view_noalias.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/lateral_view_noalias.q Thu Aug 15 01:32:00 2013
@@ -0,0 +1,12 @@
+--HIVE-2608 Do not require AS a,b,c part in LATERAL VIEW
+EXPLAIN SELECT myTab.* from src LATERAL VIEW explode(map('key1', 100, 'key2', 200)) myTab limit 2;
+SELECT myTab.* from src LATERAL VIEW explode(map('key1', 100, 'key2', 200)) myTab limit 2;
+
+EXPLAIN SELECT explode(map('key1', 100, 'key2', 200)) from src limit 2;
+SELECT explode(map('key1', 100, 'key2', 200)) from src limit 2;
+
+-- view
+create view lv_noalias as SELECT myTab.* from src LATERAL VIEW explode(map('key1', 100, 'key2', 200)) myTab limit 2;
+
+explain select * from lv_noalias a join lv_noalias b on a.key=b.key;
+select * from lv_noalias a join lv_noalias b on a.key=b.key;
\ No newline at end of file
Modified: hive/trunk/ql/src/test/results/clientnegative/lateral_view_join.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/lateral_view_join.q.out?rev=1514119&r1=1514118&r2=1514119&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/lateral_view_join.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/lateral_view_join.q.out Thu Aug 15 01:32:00 2013
@@ -1,2 +1,2 @@
-FAILED: ParseException line 1:62 missing AS at 'myTable' near '<EOF>'
+FAILED: ParseException line 1:62 missing EOF at 'myTable' near 'AS'
Modified: hive/trunk/ql/src/test/results/clientnegative/udtf_not_supported2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/udtf_not_supported2.q.out?rev=1514119&r1=1514118&r2=1514119&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/udtf_not_supported2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/udtf_not_supported2.q.out Thu Aug 15 01:32:00 2013
@@ -1 +0,0 @@
-FAILED: SemanticException 1:7 UDTF's require an AS clause. Error encountered near token '3'
Added: hive/trunk/ql/src/test/results/clientpositive/lateral_view_noalias.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/lateral_view_noalias.q.out?rev=1514119&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/lateral_view_noalias.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/lateral_view_noalias.q.out Thu Aug 15 01:32:00 2013
@@ -0,0 +1,345 @@
+PREHOOK: query: --HIVE-2608 Do not require AS a,b,c part in LATERAL VIEW
+EXPLAIN SELECT myTab.* from src LATERAL VIEW explode(map('key1', 100, 'key2', 200)) myTab limit 2
+PREHOOK: type: QUERY
+POSTHOOK: query: --HIVE-2608 Do not require AS a,b,c part in LATERAL VIEW
+EXPLAIN SELECT myTab.* from src LATERAL VIEW explode(map('key1', 100, 'key2', 200)) myTab limit 2
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION explode (TOK_FUNCTION map 'key1' 100 'key2' 200)) (TOK_TABALIAS myTab))) (TOK_TABREF (TOK_TABNAME src)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME myTab)))) (TOK_LIMIT 2)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Lateral View Forward
+ Select Operator
+ Lateral View Join Operator
+ outputColumnNames: _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Select Operator
+ expressions:
+ expr: map('key1':100,'key2':200)
+ type: map<string,int>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 2
+
+
+PREHOOK: query: SELECT myTab.* from src LATERAL VIEW explode(map('key1', 100, 'key2', 200)) myTab limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT myTab.* from src LATERAL VIEW explode(map('key1', 100, 'key2', 200)) myTab limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+key1 100
+key2 200
+PREHOOK: query: EXPLAIN SELECT explode(map('key1', 100, 'key2', 200)) from src limit 2
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT explode(map('key1', 100, 'key2', 200)) from src limit 2
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION explode (TOK_FUNCTION map 'key1' 100 'key2' 200)))) (TOK_LIMIT 2)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: map('key1':100,'key2':200)
+ type: map<string,int>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: explode
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 2
+
+
+PREHOOK: query: SELECT explode(map('key1', 100, 'key2', 200)) from src limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT explode(map('key1', 100, 'key2', 200)) from src limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+key1 100
+key2 200
+PREHOOK: query: -- view
+create view lv_noalias as SELECT myTab.* from src LATERAL VIEW explode(map('key1', 100, 'key2', 200)) myTab limit 2
+PREHOOK: type: CREATEVIEW
+POSTHOOK: query: -- view
+create view lv_noalias as SELECT myTab.* from src LATERAL VIEW explode(map('key1', 100, 'key2', 200)) myTab limit 2
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Output: default@lv_noalias
+PREHOOK: query: explain select * from lv_noalias a join lv_noalias b on a.key=b.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from lv_noalias a join lv_noalias b on a.key=b.key
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME lv_noalias) a) (TOK_TABREF (TOK_TABNAME lv_noalias) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1, Stage-3
+ Stage-3 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src
+ TableScan
+ alias: src
+ Lateral View Forward
+ Select Operator
+ Lateral View Join Operator
+ outputColumnNames: _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ outputColumnNames: _col0, _col1
+ Limit
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: int
+ Select Operator
+ expressions:
+ expr: map('key1':100,'key2':200)
+ type: map<string,int>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ outputColumnNames: _col0, _col1
+ Limit
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: int
+ Reduce Operator Tree:
+ Extract
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: int
+ $INTNAME1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: int
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: int
+ expr: _col2
+ type: string
+ expr: _col3
+ type: int
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b:src
+ TableScan
+ alias: src
+ Lateral View Forward
+ Select Operator
+ Lateral View Join Operator
+ outputColumnNames: _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ outputColumnNames: _col0, _col1
+ Limit
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: int
+ Select Operator
+ expressions:
+ expr: map('key1':100,'key2':200)
+ type: map<string,int>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ outputColumnNames: _col0, _col1
+ Limit
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: int
+ Reduce Operator Tree:
+ Extract
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select * from lv_noalias a join lv_noalias b on a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@lv_noalias
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select * from lv_noalias a join lv_noalias b on a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@lv_noalias
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+key1 100 key1 100
+key2 200 key2 200