You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/09/09 04:18:06 UTC
[02/11] hive git commit: HIVE-11600 : Hive Parser to Support multi
col in clause (x, y..) in ((..), ..., ()) (Pengcheng Xiong,
reviewed by Laljo John Pullokkaran)
HIVE-11600 : Hive Parser to Support multi col in clause (x,y..) in ((..),..., ()) (Pengcheng Xiong, reviewed by Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8bed378e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8bed378e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8bed378e
Branch: refs/heads/llap
Commit: 8bed378eac538a1bf1f4599b50929abedc735891
Parents: 730a404
Author: Pengcheng Xiong <px...@apache.org>
Authored: Fri Sep 4 10:13:49 2015 -0700
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Fri Sep 4 10:13:49 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/IdentifiersParser.g | 46 ++-
.../TestSQL11ReservedKeyWordsNegative.java | 32 +-
.../TestSQL11ReservedKeyWordsPositive.java | 23 +-
ql/src/test/queries/clientpositive/char_udf1.q | 9 +-
ql/src/test/queries/clientpositive/keyword_2.q | 14 +
.../queries/clientpositive/multi_column_in.q | 71 ++++
.../test/queries/clientpositive/varchar_udf1.q | 6 +-
.../clientpositive/char_udf1.q.java1.7.out | 22 +-
.../test/results/clientpositive/keyword_2.q.out | 51 +++
.../clientpositive/multi_column_in.q.out | 410 +++++++++++++++++++
.../clientpositive/varchar_udf1.q.java1.7.out | 12 +-
11 files changed, 663 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
index 64af7d1..bac0d22 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
@@ -430,24 +430,31 @@ precedenceEqualOperator
subQueryExpression
:
LPAREN! selectStatement[true] RPAREN!
- ;
-
+ ;
+
precedenceEqualExpression
:
+ (LPAREN precedenceBitwiseOrExpression COMMA) => precedenceEqualExpressionMutiple
+ |
+ precedenceEqualExpressionSingle
+ ;
+
+precedenceEqualExpressionSingle
+ :
(left=precedenceBitwiseOrExpression -> $left)
(
(KW_NOT precedenceEqualNegatableOperator notExpr=precedenceBitwiseOrExpression)
- -> ^(KW_NOT ^(precedenceEqualNegatableOperator $precedenceEqualExpression $notExpr))
+ -> ^(KW_NOT ^(precedenceEqualNegatableOperator $precedenceEqualExpressionSingle $notExpr))
| (precedenceEqualOperator equalExpr=precedenceBitwiseOrExpression)
- -> ^(precedenceEqualOperator $precedenceEqualExpression $equalExpr)
+ -> ^(precedenceEqualOperator $precedenceEqualExpressionSingle $equalExpr)
| (KW_NOT KW_IN LPAREN KW_SELECT)=> (KW_NOT KW_IN subQueryExpression)
- -> ^(KW_NOT ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_IN) subQueryExpression $precedenceEqualExpression))
+ -> ^(KW_NOT ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_IN) subQueryExpression $precedenceEqualExpressionSingle))
| (KW_NOT KW_IN expressions)
- -> ^(KW_NOT ^(TOK_FUNCTION KW_IN $precedenceEqualExpression expressions))
+ -> ^(KW_NOT ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionSingle expressions))
| (KW_IN LPAREN KW_SELECT)=> (KW_IN subQueryExpression)
- -> ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_IN) subQueryExpression $precedenceEqualExpression)
+ -> ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_IN) subQueryExpression $precedenceEqualExpressionSingle)
| (KW_IN expressions)
- -> ^(TOK_FUNCTION KW_IN $precedenceEqualExpression expressions)
+ -> ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionSingle expressions)
| ( KW_NOT KW_BETWEEN (min=precedenceBitwiseOrExpression) KW_AND (max=precedenceBitwiseOrExpression) )
-> ^(TOK_FUNCTION Identifier["between"] KW_TRUE $left $min $max)
| ( KW_BETWEEN (min=precedenceBitwiseOrExpression) KW_AND (max=precedenceBitwiseOrExpression) )
@@ -458,7 +465,22 @@ precedenceEqualExpression
expressions
:
- LPAREN expression (COMMA expression)* RPAREN -> expression*
+ LPAREN expression (COMMA expression)* RPAREN -> expression+
+ ;
+
+//we transform the (col0, col1) in ((v00,v01),(v10,v11)) into struct(col0, col1) in (struct(v00,v01),struct(v10,v11))
+precedenceEqualExpressionMutiple
+ :
+ (LPAREN precedenceBitwiseOrExpression (COMMA precedenceBitwiseOrExpression)+ RPAREN -> ^(TOK_FUNCTION Identifier["struct"] precedenceBitwiseOrExpression+))
+ ( (KW_IN LPAREN expressionsToStruct (COMMA expressionsToStruct)+ RPAREN)
+ -> ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionMutiple expressionsToStruct+)
+ | (KW_NOT KW_IN LPAREN expressionsToStruct (COMMA expressionsToStruct)+ RPAREN)
+ -> ^(KW_NOT ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionMutiple expressionsToStruct+)))
+ ;
+
+expressionsToStruct
+ :
+ LPAREN expression (COMMA expression)* RPAREN -> ^(TOK_FUNCTION Identifier["struct"] expression+)
;
precedenceNotOperator
@@ -635,7 +657,7 @@ nonReserved
| KW_MAPJOIN | KW_MATERIALIZED | KW_METADATA | KW_MINUS | KW_MINUTE | KW_MONTH | KW_MSCK | KW_NOSCAN | KW_NO_DROP | KW_OFFLINE
| KW_OPTION | KW_OUTPUTDRIVER | KW_OUTPUTFORMAT | KW_OVERWRITE | KW_OWNER | KW_PARTITIONED | KW_PARTITIONS | KW_PLUS | KW_PRETTY
| KW_PRINCIPALS | KW_PROTECTION | KW_PURGE | KW_READ | KW_READONLY | KW_REBUILD | KW_RECORDREADER | KW_RECORDWRITER
- | KW_REGEXP | KW_RELOAD | KW_RENAME | KW_REPAIR | KW_REPLACE | KW_REPLICATION | KW_RESTRICT | KW_REWRITE | KW_RLIKE
+ | KW_RELOAD | KW_RENAME | KW_REPAIR | KW_REPLACE | KW_REPLICATION | KW_RESTRICT | KW_REWRITE
| KW_ROLE | KW_ROLES | KW_SCHEMA | KW_SCHEMAS | KW_SECOND | KW_SEMI | KW_SERDE | KW_SERDEPROPERTIES | KW_SERVER | KW_SETS | KW_SHARED
| KW_SHOW | KW_SHOW_DATABASE | KW_SKEWED | KW_SORT | KW_SORTED | KW_SSL | KW_STATISTICS | KW_STORED
| KW_STREAMTABLE | KW_STRING | KW_STRUCT | KW_TABLES | KW_TBLPROPERTIES | KW_TEMPORARY | KW_TERMINATED
@@ -668,5 +690,7 @@ sql11ReservedKeywordsUsedAsIdentifier
| KW_LEFT | KW_LIKE | KW_LOCAL | KW_NONE | KW_NULL | KW_OF | KW_ORDER | KW_OUT | KW_OUTER | KW_PARTITION
| KW_PERCENT | KW_PROCEDURE | KW_RANGE | KW_READS | KW_REVOKE | KW_RIGHT
| KW_ROLLUP | KW_ROW | KW_ROWS | KW_SET | KW_SMALLINT | KW_TABLE | KW_TIMESTAMP | KW_TO | KW_TRIGGER | KW_TRUE
- | KW_TRUNCATE | KW_UNION | KW_UPDATE | KW_USER | KW_USING | KW_VALUES | KW_WITH
+ | KW_TRUNCATE | KW_UNION | KW_UPDATE | KW_USER | KW_USING | KW_VALUES | KW_WITH
+//The following two keywords come from MySQL. Although they are not keywords in SQL2011, they are reserved keywords in MySQL.
+ | KW_REGEXP | KW_RLIKE
;
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsNegative.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsNegative.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsNegative.java
index 61b5892..97ae0d9 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsNegative.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsNegative.java
@@ -30,7 +30,7 @@ import org.junit.Test;
/**
* Parser tests for SQL11 Reserved KeyWords. Please find more information in
- * HIVE-6617. Total number : 74
+ * HIVE-6617. Total number : 74 + 2 (MySQL)
*/
public class TestSQL11ReservedKeyWordsNegative {
private static HiveConf conf;
@@ -1070,4 +1070,34 @@ public class TestSQL11ReservedKeyWordsNegative {
ex.getMessage());
}
}
+
+ // MySQL reserved keywords.
+ @Test
+ public void testSQL11ReservedKeyWords_RLIKE() {
+ try {
+ parse("CREATE TABLE RLIKE (col STRING)");
+ Assert.assertFalse("Expected ParseException", true);
+ } catch (ParseException ex) {
+ Assert
+ .assertEquals(
+ "Failure didn't match.",
+ "line 1:13 Failed to recognize predicate 'RLIKE'. Failed rule: 'identifier' in table name",
+ ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testSQL11ReservedKeyWords_REGEXP() {
+ try {
+ parse("CREATE TABLE REGEXP (col STRING)");
+ Assert.assertFalse("Expected ParseException", true);
+ } catch (ParseException ex) {
+ Assert
+ .assertEquals(
+ "Failure didn't match.",
+ "line 1:13 Failed to recognize predicate 'REGEXP'. Failed rule: 'identifier' in table name",
+ ex.getMessage());
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsPositive.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsPositive.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsPositive.java
index 4c84e91..2a68899 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsPositive.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsPositive.java
@@ -30,7 +30,7 @@ import org.junit.Test;
/**
* Parser tests for SQL11 Reserved KeyWords. Please find more information in
- * HIVE-6617. Total number : 74
+ * HIVE-6617. Total number : 74 + 2 (MySQL)
*/
public class TestSQL11ReservedKeyWordsPositive {
private static HiveConf conf;
@@ -798,4 +798,25 @@ public class TestSQL11ReservedKeyWordsPositive {
"(TOK_CREATETABLE (TOK_TABNAME WITH) TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL col TOK_STRING)))",
ast.toStringTree());
}
+
+ // MySQL reserved keywords.
+ @Test
+ public void testSQL11ReservedKeyWords_RLIKE() throws ParseException {
+ ASTNode ast = parse("CREATE TABLE RLIKE (col STRING)");
+ Assert
+ .assertEquals(
+ "AST doesn't match",
+ "(TOK_CREATETABLE (TOK_TABNAME RLIKE) TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL col TOK_STRING)))",
+ ast.toStringTree());
+ }
+
+ @Test
+ public void testSQL11ReservedKeyWords_REGEXP() throws ParseException {
+ ASTNode ast = parse("CREATE TABLE REGEXP (col STRING)");
+ Assert
+ .assertEquals(
+ "AST doesn't match",
+ "(TOK_CREATETABLE (TOK_TABNAME REGEXP) TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL col TOK_STRING)))",
+ ast.toStringTree());
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/test/queries/clientpositive/char_udf1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/char_udf1.q b/ql/src/test/queries/clientpositive/char_udf1.q
index 8848609..09012b4 100644
--- a/ql/src/test/queries/clientpositive/char_udf1.q
+++ b/ql/src/test/queries/clientpositive/char_udf1.q
@@ -74,10 +74,13 @@ select
ltrim(c2) = ltrim(c4)
from char_udf_1 limit 1;
+-- In hive wiki page https://cwiki.apache.org/confluence/display/Hive/LanguageManual+UDF
+-- we only allow A regexp B, not regexp (A,B).
+
select
- regexp(c2, 'val'),
- regexp(c4, 'val'),
- regexp(c2, 'val') = regexp(c4, 'val')
+ c2 regexp 'val',
+ c4 regexp 'val',
+ (c2 regexp 'val') = (c4 regexp 'val')
from char_udf_1 limit 1;
select
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/test/queries/clientpositive/keyword_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/keyword_2.q b/ql/src/test/queries/clientpositive/keyword_2.q
new file mode 100644
index 0000000..054e26a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/keyword_2.q
@@ -0,0 +1,14 @@
+set hive.support.sql11.reserved.keywords=false;
+drop table varchar_udf_1;
+
+create table varchar_udf_1 (c1 string, c2 string, c3 varchar(10), c4 varchar(20));
+insert overwrite table varchar_udf_1
+ select key, value, key, value from src where key = '238' limit 1;
+
+select
+ regexp(c2, 'val'),
+ regexp(c4, 'val'),
+ regexp(c2, 'val') = regexp(c4, 'val')
+from varchar_udf_1 limit 1;
+
+drop table varchar_udf_1;
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/test/queries/clientpositive/multi_column_in.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/multi_column_in.q b/ql/src/test/queries/clientpositive/multi_column_in.q
new file mode 100644
index 0000000..18a56cc
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/multi_column_in.q
@@ -0,0 +1,71 @@
+drop table emps;
+
+create table emps (empno int, deptno int, empname string);
+
+insert into table emps values (1,2,"11"),(1,2,"11"),(3,4,"33"),(1,3,"11"),(2,5,"22"),(2,5,"22");
+
+select * from emps;
+
+select * from emps where (int(empno+deptno/2), int(deptno/3)) in ((2,0),(3,2));
+
+select * from emps where (int(empno+deptno/2), int(deptno/3)) not in ((2,0),(3,2));
+
+select * from emps where (empno,deptno) in ((1,2),(3,2));
+
+select * from emps where (empno,deptno) not in ((1,2),(3,2));
+
+select * from emps where (empno,deptno) in ((1,2),(1,3));
+
+select * from emps where (empno,deptno) not in ((1,2),(1,3));
+
+explain
+select * from emps where (empno+1,deptno) in ((1,2),(3,2));
+
+explain
+select * from emps where (empno+1,deptno) not in ((1,2),(3,2));
+
+select * from emps where empno in (1,2);
+
+select * from emps where empno in (1,2) and deptno > 2;
+
+select * from emps where (empno) in (1,2) and deptno > 2;
+
+select * from emps where ((empno) in (1,2) and deptno > 2);
+
+explain select * from emps where ((empno*2)|1,deptno) in ((empno+1,2),(empno+2,2));
+
+select * from emps where ((empno*2)|1,deptno) in ((empno+1,2),(empno+2,2));
+
+select (empno*2)|1,substr(empname,1,1) from emps;
+
+select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+2,'2'));
+
+select * from emps where ((empno*2)|1,substr(empname,1,1)) not in ((empno+1,'2'),(empno+2,'2'));
+
+select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'));
+
+select * from emps where ((empno*2)|1,substr(empname,1,1)) not in ((empno+1,'2'),(empno+3,'2'));
+
+
+select sum(empno), empname from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+group by empname;
+
+select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+union
+select * from emps where (empno,deptno) in ((1,2),(3,2));
+
+drop view v;
+
+create view v as
+select * from(
+select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+union
+select * from emps where (empno,deptno) in ((1,2),(3,2)))subq order by empno desc;
+
+select * from v;
+
+select subq.e1 from
+(select (empno*2)|1 as e1, substr(empname,1,1) as n1 from emps)subq
+join
+(select empno as e2 from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2')))subq2
+on e1=e2+1;
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/test/queries/clientpositive/varchar_udf1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/varchar_udf1.q b/ql/src/test/queries/clientpositive/varchar_udf1.q
index 395fb12..ff40b31 100644
--- a/ql/src/test/queries/clientpositive/varchar_udf1.q
+++ b/ql/src/test/queries/clientpositive/varchar_udf1.q
@@ -75,9 +75,9 @@ select
from varchar_udf_1 limit 1;
select
- regexp(c2, 'val'),
- regexp(c4, 'val'),
- regexp(c2, 'val') = regexp(c4, 'val')
+ c2 regexp 'val',
+ c4 regexp 'val',
+ (c2 regexp 'val') = (c4 regexp 'val')
from varchar_udf_1 limit 1;
select
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/test/results/clientpositive/char_udf1.q.java1.7.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/char_udf1.q.java1.7.out b/ql/src/test/results/clientpositive/char_udf1.q.java1.7.out
index ced0132..bfed116 100644
--- a/ql/src/test/results/clientpositive/char_udf1.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/char_udf1.q.java1.7.out
@@ -219,18 +219,24 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@char_udf_1
#### A masked pattern was here ####
val_238 val_238 true
-PREHOOK: query: select
- regexp(c2, 'val'),
- regexp(c4, 'val'),
- regexp(c2, 'val') = regexp(c4, 'val')
+PREHOOK: query: -- In hive wiki page https://cwiki.apache.org/confluence/display/Hive/LanguageManual+UDF
+-- we only allow A regexp B, not regexp (A,B).
+
+select
+ c2 regexp 'val',
+ c4 regexp 'val',
+ (c2 regexp 'val') = (c4 regexp 'val')
from char_udf_1 limit 1
PREHOOK: type: QUERY
PREHOOK: Input: default@char_udf_1
#### A masked pattern was here ####
-POSTHOOK: query: select
- regexp(c2, 'val'),
- regexp(c4, 'val'),
- regexp(c2, 'val') = regexp(c4, 'val')
+POSTHOOK: query: -- In hive wiki page https://cwiki.apache.org/confluence/display/Hive/LanguageManual+UDF
+-- we only allow A regexp B, not regexp (A,B).
+
+select
+ c2 regexp 'val',
+ c4 regexp 'val',
+ (c2 regexp 'val') = (c4 regexp 'val')
from char_udf_1 limit 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@char_udf_1
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/test/results/clientpositive/keyword_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/keyword_2.q.out b/ql/src/test/results/clientpositive/keyword_2.q.out
new file mode 100644
index 0000000..3d248fb
--- /dev/null
+++ b/ql/src/test/results/clientpositive/keyword_2.q.out
@@ -0,0 +1,51 @@
+PREHOOK: query: drop table varchar_udf_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table varchar_udf_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table varchar_udf_1 (c1 string, c2 string, c3 varchar(10), c4 varchar(20))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@varchar_udf_1
+POSTHOOK: query: create table varchar_udf_1 (c1 string, c2 string, c3 varchar(10), c4 varchar(20))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@varchar_udf_1
+PREHOOK: query: insert overwrite table varchar_udf_1
+ select key, value, key, value from src where key = '238' limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@varchar_udf_1
+POSTHOOK: query: insert overwrite table varchar_udf_1
+ select key, value, key, value from src where key = '238' limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@varchar_udf_1
+POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select
+ regexp(c2, 'val'),
+ regexp(c4, 'val'),
+ regexp(c2, 'val') = regexp(c4, 'val')
+from varchar_udf_1 limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@varchar_udf_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ regexp(c2, 'val'),
+ regexp(c4, 'val'),
+ regexp(c2, 'val') = regexp(c4, 'val')
+from varchar_udf_1 limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@varchar_udf_1
+#### A masked pattern was here ####
+true true true
+PREHOOK: query: drop table varchar_udf_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@varchar_udf_1
+PREHOOK: Output: default@varchar_udf_1
+POSTHOOK: query: drop table varchar_udf_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@varchar_udf_1
+POSTHOOK: Output: default@varchar_udf_1
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/test/results/clientpositive/multi_column_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/multi_column_in.q.out b/ql/src/test/results/clientpositive/multi_column_in.q.out
new file mode 100644
index 0000000..e0ec848
--- /dev/null
+++ b/ql/src/test/results/clientpositive/multi_column_in.q.out
@@ -0,0 +1,410 @@
+PREHOOK: query: drop table emps
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table emps
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table emps (empno int, deptno int, empname string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@emps
+POSTHOOK: query: create table emps (empno int, deptno int, empname string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@emps
+PREHOOK: query: insert into table emps values (1,2,"11"),(1,2,"11"),(3,4,"33"),(1,3,"11"),(2,5,"22"),(2,5,"22")
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@emps
+POSTHOOK: query: insert into table emps values (1,2,"11"),(1,2,"11"),(3,4,"33"),(1,3,"11"),(2,5,"22"),(2,5,"22")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@emps
+POSTHOOK: Lineage: emps.deptno EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: emps.empname SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+POSTHOOK: Lineage: emps.empno EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: select * from emps
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 2 11
+1 2 11
+3 4 33
+1 3 11
+2 5 22
+2 5 22
+PREHOOK: query: select * from emps where (int(empno+deptno/2), int(deptno/3)) in ((2,0),(3,2))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where (int(empno+deptno/2), int(deptno/3)) in ((2,0),(3,2))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 2 11
+1 2 11
+PREHOOK: query: select * from emps where (int(empno+deptno/2), int(deptno/3)) not in ((2,0),(3,2))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where (int(empno+deptno/2), int(deptno/3)) not in ((2,0),(3,2))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+3 4 33
+1 3 11
+2 5 22
+2 5 22
+PREHOOK: query: select * from emps where (empno,deptno) in ((1,2),(3,2))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where (empno,deptno) in ((1,2),(3,2))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 2 11
+1 2 11
+PREHOOK: query: select * from emps where (empno,deptno) not in ((1,2),(3,2))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where (empno,deptno) not in ((1,2),(3,2))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+3 4 33
+1 3 11
+2 5 22
+2 5 22
+PREHOOK: query: select * from emps where (empno,deptno) in ((1,2),(1,3))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where (empno,deptno) in ((1,2),(1,3))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 2 11
+1 2 11
+1 3 11
+PREHOOK: query: select * from emps where (empno,deptno) not in ((1,2),(1,3))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where (empno,deptno) not in ((1,2),(1,3))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+3 4 33
+2 5 22
+2 5 22
+PREHOOK: query: explain
+select * from emps where (empno+1,deptno) in ((1,2),(3,2))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select * from emps where (empno+1,deptno) in ((1,2),(3,2))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: emps
+ Statistics: Num rows: 6 Data size: 36 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (struct((empno + 1),deptno)) IN (const struct(1,2), const struct(3,2)) (type: boolean)
+ Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: empno (type: int), deptno (type: int), empname (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain
+select * from emps where (empno+1,deptno) not in ((1,2),(3,2))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select * from emps where (empno+1,deptno) not in ((1,2),(3,2))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: emps
+ Statistics: Num rows: 6 Data size: 36 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (not (struct((empno + 1),deptno)) IN (const struct(1,2), const struct(3,2))) (type: boolean)
+ Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: empno (type: int), deptno (type: int), empname (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from emps where empno in (1,2)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where empno in (1,2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 2 11
+1 2 11
+1 3 11
+2 5 22
+2 5 22
+PREHOOK: query: select * from emps where empno in (1,2) and deptno > 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where empno in (1,2) and deptno > 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 3 11
+2 5 22
+2 5 22
+PREHOOK: query: select * from emps where (empno) in (1,2) and deptno > 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where (empno) in (1,2) and deptno > 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 3 11
+2 5 22
+2 5 22
+PREHOOK: query: select * from emps where ((empno) in (1,2) and deptno > 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where ((empno) in (1,2) and deptno > 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 3 11
+2 5 22
+2 5 22
+PREHOOK: query: explain select * from emps where ((empno*2)|1,deptno) in ((empno+1,2),(empno+2,2))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from emps where ((empno*2)|1,deptno) in ((empno+1,2),(empno+2,2))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: emps
+ Statistics: Num rows: 6 Data size: 36 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (struct(((empno * 2) | 1),deptno)) IN (struct((empno + 1),2), struct((empno + 2),2)) (type: boolean)
+ Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: empno (type: int), deptno (type: int), empname (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from emps where ((empno*2)|1,deptno) in ((empno+1,2),(empno+2,2))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where ((empno*2)|1,deptno) in ((empno+1,2),(empno+2,2))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 2 11
+1 2 11
+PREHOOK: query: select (empno*2)|1,substr(empname,1,1) from emps
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select (empno*2)|1,substr(empname,1,1) from emps
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+3 1
+3 1
+7 3
+3 1
+5 2
+5 2
+PREHOOK: query: select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+2,'2'))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+2,'2'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+PREHOOK: query: select * from emps where ((empno*2)|1,substr(empname,1,1)) not in ((empno+1,'2'),(empno+2,'2'))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where ((empno*2)|1,substr(empname,1,1)) not in ((empno+1,'2'),(empno+2,'2'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 2 11
+1 2 11
+3 4 33
+1 3 11
+2 5 22
+2 5 22
+PREHOOK: query: select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+2 5 22
+2 5 22
+PREHOOK: query: select * from emps where ((empno*2)|1,substr(empname,1,1)) not in ((empno+1,'2'),(empno+3,'2'))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where ((empno*2)|1,substr(empname,1,1)) not in ((empno+1,'2'),(empno+3,'2'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 2 11
+1 2 11
+3 4 33
+1 3 11
+PREHOOK: query: select sum(empno), empname from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+group by empname
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(empno), empname from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+group by empname
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+4 22
+PREHOOK: query: select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+union
+select * from emps where (empno,deptno) in ((1,2),(3,2))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+union
+select * from emps where (empno,deptno) in ((1,2),(3,2))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+1 2 11
+2 5 22
+PREHOOK: query: drop view v
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: drop view v
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: create view v as
+select * from(
+select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+union
+select * from emps where (empno,deptno) in ((1,2),(3,2)))subq order by empno desc
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@emps
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as
+select * from(
+select * from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2'))
+union
+select * from emps where (empno,deptno) in ((1,2),(3,2)))subq order by empno desc
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@emps
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: select * from v
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+PREHOOK: Input: default@v
+#### A masked pattern was here ####
+POSTHOOK: query: select * from v
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+POSTHOOK: Input: default@v
+#### A masked pattern was here ####
+2 5 22
+1 2 11
+PREHOOK: query: select subq.e1 from
+(select (empno*2)|1 as e1, substr(empname,1,1) as n1 from emps)subq
+join
+(select empno as e2 from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2')))subq2
+on e1=e2+1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emps
+#### A masked pattern was here ####
+POSTHOOK: query: select subq.e1 from
+(select (empno*2)|1 as e1, substr(empname,1,1) as n1 from emps)subq
+join
+(select empno as e2 from emps where ((empno*2)|1,substr(empname,1,1)) in ((empno+1,'2'),(empno+3,'2')))subq2
+on e1=e2+1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emps
+#### A masked pattern was here ####
+3
+3
+3
+3
+3
+3
http://git-wip-us.apache.org/repos/asf/hive/blob/8bed378e/ql/src/test/results/clientpositive/varchar_udf1.q.java1.7.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/varchar_udf1.q.java1.7.out b/ql/src/test/results/clientpositive/varchar_udf1.q.java1.7.out
index 96ba06e..853bc4a 100644
--- a/ql/src/test/results/clientpositive/varchar_udf1.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/varchar_udf1.q.java1.7.out
@@ -220,17 +220,17 @@ POSTHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
val_238 val_238 true
PREHOOK: query: select
- regexp(c2, 'val'),
- regexp(c4, 'val'),
- regexp(c2, 'val') = regexp(c4, 'val')
+ c2 regexp 'val',
+ c4 regexp 'val',
+ (c2 regexp 'val') = (c4 regexp 'val')
from varchar_udf_1 limit 1
PREHOOK: type: QUERY
PREHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
POSTHOOK: query: select
- regexp(c2, 'val'),
- regexp(c4, 'val'),
- regexp(c2, 'val') = regexp(c4, 'val')
+ c2 regexp 'val',
+ c4 regexp 'val',
+ (c2 regexp 'val') = (c4 regexp 'val')
from varchar_udf_1 limit 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@varchar_udf_1