You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by px...@apache.org on 2017/01/25 19:07:06 UTC
hive git commit: HIVE-15578: Simplify IdentifiersParser (Pengcheng
Xiong, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 608194fc5 -> 92f31d07a
HIVE-15578: Simplify IdentifiersParser (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/92f31d07
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/92f31d07
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/92f31d07
Branch: refs/heads/master
Commit: 92f31d07aa988d4a460aac56e369bfa386361776
Parents: 608194f
Author: Pengcheng Xiong <px...@apache.org>
Authored: Wed Jan 25 11:06:52 2017 -0800
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Wed Jan 25 11:06:52 2017 -0800
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/IdentifiersParser.g | 86 ++++++++------------
.../clientnegative/char_pad_convert_fail2.q.out | 2 +-
.../invalid_select_expression.q.out | 2 +-
.../ptf_negative_DistributeByOrderBy.q.out | 3 +-
.../ptf_negative_PartitionBySortBy.q.out | 3 +-
.../clientnegative/ptf_window_boundaries.q.out | 2 +-
.../clientnegative/ptf_window_boundaries2.q.out | 2 +-
.../llap/offset_limit_ppd_optimizer.q.out | 2 +-
8 files changed, 42 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
index b90210c..43fd69e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
@@ -76,15 +76,14 @@ rollupOldSyntax
@init { gParent.pushMsg("rollup old syntax", state); }
@after { gParent.popMsg(state); }
:
- expression
- ( COMMA expression)*
+ expressionsNotInParenthese
((rollup=KW_WITH KW_ROLLUP) | (cube=KW_WITH KW_CUBE)) ?
(sets=KW_GROUPING KW_SETS
LPAREN groupingSetExpression ( COMMA groupingSetExpression)* RPAREN ) ?
- -> {rollup != null}? ^(TOK_ROLLUP_GROUPBY expression+)
- -> {cube != null}? ^(TOK_CUBE_GROUPBY expression+)
- -> {sets != null}? ^(TOK_GROUPING_SETS expression+ groupingSetExpression+)
- -> ^(TOK_GROUPBY expression+)
+ -> {rollup != null}? ^(TOK_ROLLUP_GROUPBY expressionsNotInParenthese)
+ -> {cube != null}? ^(TOK_CUBE_GROUPBY expressionsNotInParenthese)
+ -> {sets != null}? ^(TOK_GROUPING_SETS expressionsNotInParenthese groupingSetExpression+)
+ -> ^(TOK_GROUPBY expressionsNotInParenthese)
;
@@ -92,7 +91,7 @@ groupingSetExpression
@init {gParent.pushMsg("grouping set expression", state); }
@after {gParent.popMsg(state); }
:
- (LPAREN) => groupingSetExpressionMultiple
+ (groupingSetExpressionMultiple) => groupingSetExpressionMultiple
|
groupingExpressionSingle
;
@@ -130,7 +129,7 @@ havingCondition
expressionsInParenthese
:
- LPAREN expression (COMMA expression)* RPAREN -> expression+
+ LPAREN! expressionsNotInParenthese RPAREN!
;
expressionsNotInParenthese
@@ -138,6 +137,13 @@ expressionsNotInParenthese
expression (COMMA expression)* -> expression+
;
+expressions
+ :
+ (expressionsInParenthese) => expressionsInParenthese
+ |
+ expressionsNotInParenthese
+ ;
+
columnRefOrderInParenthese
:
LPAREN columnRefOrder (COMMA columnRefOrder)* RPAREN -> columnRefOrder+
@@ -160,36 +166,21 @@ clusterByClause
@init { gParent.pushMsg("cluster by clause", state); }
@after { gParent.popMsg(state); }
:
- KW_CLUSTER KW_BY
- (
- (LPAREN) => expressionsInParenthese -> ^(TOK_CLUSTERBY expressionsInParenthese)
- |
- expressionsNotInParenthese -> ^(TOK_CLUSTERBY expressionsNotInParenthese)
- )
+ KW_CLUSTER KW_BY expressions -> ^(TOK_CLUSTERBY expressions)
;
partitionByClause
@init { gParent.pushMsg("partition by clause", state); }
@after { gParent.popMsg(state); }
:
- KW_PARTITION KW_BY
- (
- (LPAREN) => expressionsInParenthese -> ^(TOK_DISTRIBUTEBY expressionsInParenthese)
- |
- expressionsNotInParenthese -> ^(TOK_DISTRIBUTEBY expressionsNotInParenthese)
- )
+ KW_PARTITION KW_BY expressions -> ^(TOK_DISTRIBUTEBY expressions)
;
distributeByClause
@init { gParent.pushMsg("distribute by clause", state); }
@after { gParent.popMsg(state); }
:
- KW_DISTRIBUTE KW_BY
- (
- (LPAREN) => expressionsInParenthese -> ^(TOK_DISTRIBUTEBY expressionsInParenthese)
- |
- expressionsNotInParenthese -> ^(TOK_DISTRIBUTEBY expressionsNotInParenthese)
- )
+ KW_DISTRIBUTE KW_BY expressions -> ^(TOK_DISTRIBUTEBY expressions)
;
sortByClause
@@ -225,9 +216,7 @@ functionName
@init { gParent.pushMsg("function name", state); }
@after { gParent.popMsg(state); }
: // Keyword IF is also a function name
- (KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE) => (KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE)
- |
- (functionIdentifier) => functionIdentifier
+ functionIdentifier
|
sql11ReservedKeywordsUsedAsFunctionName -> Identifier[$sql11ReservedKeywordsUsedAsFunctionName.start]
;
@@ -323,6 +312,7 @@ constant
| NumberLiteral
| charSetStringLiteral
| booleanValue
+ | KW_NULL -> TOK_NULL
;
stringLiteralSequence
@@ -389,17 +379,16 @@ expression
atomExpression
:
- (KW_NULL) => KW_NULL -> TOK_NULL
- | (intervalExpression)=>intervalExpression
+ (intervalExpression)=>intervalExpression
| (constant) => constant
| castExpression
| extractExpression
| floorExpression
| caseExpression
| whenExpression
- | (LPAREN KW_SELECT)=> (subQueryExpression)
+ | (subQueryExpression)=> (subQueryExpression)
-> ^(TOK_SUBQUERY_EXPR TOK_SUBQUERY_OP subQueryExpression)
- | (functionName LPAREN) => function
+ | (function) => function
| tableOrColumn
| LPAREN! expression RPAREN!
;
@@ -519,40 +508,35 @@ subQueryExpression
precedenceEqualExpression
:
- (LPAREN precedenceBitwiseOrExpression COMMA) => precedenceEqualExpressionMutiple
+ (precedenceEqualExpressionSingle) => precedenceEqualExpressionSingle
|
- precedenceEqualExpressionSingle
+ precedenceEqualExpressionMutiple
;
-
+
precedenceEqualExpressionSingle
:
(left=precedenceBitwiseOrExpression -> $left)
(
- (KW_NOT precedenceEqualNegatableOperator notExpr=precedenceBitwiseOrExpression)
+ (KW_NOT precedenceEqualNegatableOperator notExpr=precedenceBitwiseOrExpression)
-> ^(KW_NOT ^(precedenceEqualNegatableOperator $precedenceEqualExpressionSingle $notExpr))
| (precedenceEqualOperator equalExpr=precedenceBitwiseOrExpression)
-> ^(precedenceEqualOperator $precedenceEqualExpressionSingle $equalExpr)
| (KW_NOT KW_IN LPAREN KW_SELECT)=> (KW_NOT KW_IN subQueryExpression)
-> ^(KW_NOT ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_IN) subQueryExpression $precedenceEqualExpressionSingle))
- | (KW_NOT KW_IN expressions)
- -> ^(KW_NOT ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionSingle expressions))
- | (KW_IN LPAREN KW_SELECT)=> (KW_IN subQueryExpression)
+ | (KW_NOT KW_IN expressionsInParenthese)
+ -> ^(KW_NOT ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionSingle expressionsInParenthese))
+ | (KW_IN LPAREN KW_SELECT)=> (KW_IN subQueryExpression)
-> ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_IN) subQueryExpression $precedenceEqualExpressionSingle)
- | (KW_IN expressions)
- -> ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionSingle expressions)
+ | (KW_IN expressionsInParenthese)
+ -> ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionSingle expressionsInParenthese)
| ( KW_NOT KW_BETWEEN (min=precedenceBitwiseOrExpression) KW_AND (max=precedenceBitwiseOrExpression) )
- -> ^(TOK_FUNCTION Identifier["between"] KW_TRUE $left $min $max)
+ -> ^(TOK_FUNCTION Identifier["between"] KW_TRUE $precedenceEqualExpressionSingle $min $max)
| ( KW_BETWEEN (min=precedenceBitwiseOrExpression) KW_AND (max=precedenceBitwiseOrExpression) )
- -> ^(TOK_FUNCTION Identifier["between"] KW_FALSE $left $min $max)
+ -> ^(TOK_FUNCTION Identifier["between"] KW_FALSE $precedenceEqualExpressionSingle $min $max)
)*
- | (KW_EXISTS LPAREN KW_SELECT)=> (KW_EXISTS subQueryExpression)
- -> ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_EXISTS) subQueryExpression)
+ | KW_EXISTS subQueryExpression -> ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_EXISTS) subQueryExpression)
;
-expressions
- :
- LPAREN expression (COMMA expression)* RPAREN -> expression+
- ;
//we transform the (col0, col1) in ((v00,v01),(v10,v11)) into struct(col0, col1) in (struct(v00,v01),struct(v10,v11))
precedenceEqualExpressionMutiple
@@ -770,5 +754,5 @@ nonReserved
//The following SQL2011 reserved keywords are used as function name only, but not as identifiers.
sql11ReservedKeywordsUsedAsFunctionName
:
- KW_BIGINT | KW_BINARY | KW_BOOLEAN | KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP | KW_DATE | KW_DOUBLE | KW_FLOAT | KW_GROUPING | KW_INT | KW_SMALLINT | KW_TIMESTAMP
+ KW_IF | KW_ARRAY | KW_MAP | KW_BIGINT | KW_BINARY | KW_BOOLEAN | KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP | KW_DATE | KW_DOUBLE | KW_FLOAT | KW_GROUPING | KW_INT | KW_SMALLINT | KW_TIMESTAMP
;
http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/char_pad_convert_fail2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/char_pad_convert_fail2.q.out b/ql/src/test/results/clientnegative/char_pad_convert_fail2.q.out
index 90f9356..2d08fb3 100644
--- a/ql/src/test/results/clientnegative/char_pad_convert_fail2.q.out
+++ b/ql/src/test/results/clientnegative/char_pad_convert_fail2.q.out
@@ -40,4 +40,4 @@ POSTHOOK: query: load data local inpath '../../data/files/over1k' into table ove
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@over1k
-FAILED: ParseException line 7:12 cannot recognize input near '{' '"key1"' ':' in function specification
+FAILED: ParseException line 7:7 cannot recognize input near 'lpad' '(' '{' in expression specification
http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/invalid_select_expression.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/invalid_select_expression.q.out b/ql/src/test/results/clientnegative/invalid_select_expression.q.out
index 63a682a..3650ca7 100644
--- a/ql/src/test/results/clientnegative/invalid_select_expression.q.out
+++ b/ql/src/test/results/clientnegative/invalid_select_expression.q.out
@@ -1 +1 @@
-FAILED: ParseException line 1:32 cannot recognize input near '.' 'foo' '<EOF>' in expression specification
+FAILED: ParseException line 1:26 cannot recognize input near 'foo' '>' '.' in expression specification
http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out b/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out
index cb7f154..a0f3fe2 100644
--- a/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out
+++ b/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out
@@ -1,2 +1 @@
-FAILED: ParseException line 3:46 missing ) at 'order' near 'p_mfgr'
-line 3:61 missing EOF at ')' near 'p_mfgr'
+FAILED: ParseException line 3:0 cannot recognize input near 'sum' '(' 'p_retailprice' in expression specification
http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out b/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out
index 5daf86c..a0f3fe2 100644
--- a/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out
+++ b/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out
@@ -1,2 +1 @@
-FAILED: ParseException line 3:45 missing ) at 'sort' near 'p_mfgr'
-line 3:59 missing EOF at ')' near 'p_mfgr'
+FAILED: ParseException line 3:0 cannot recognize input near 'sum' '(' 'p_retailprice' in expression specification
http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out b/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out
index c76feee..3808f2c 100644
--- a/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out
+++ b/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out
@@ -1 +1 @@
-FAILED: ParseException line 2:44 mismatched input 'following' expecting KW_PRECEDING near 'unbounded' in windowframestartboundary
+FAILED: ParseException line 2:4 cannot recognize input near 'sum' '(' 'p_retailprice' in expression specification
http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out b/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out
index 9ed8be5..3808f2c 100644
--- a/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out
+++ b/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out
@@ -1 +1 @@
-FAILED: ParseException line 2:45 mismatched input 'following' expecting KW_PRECEDING near 'unbounded' in windowframestartboundary
+FAILED: ParseException line 2:4 cannot recognize input near 'sum' '(' 'p_retailprice' in expression specification
http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
index 26ee6f4..fb8e3ad 100644
--- a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
+++ b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
@@ -1301,7 +1301,7 @@ STAGE PLANS:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
Reducer 5 <- Map 4 (SIMPLE_EDGE)
- Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+ Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1