You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2015/03/11 22:50:24 UTC
svn commit: r1666021 [1/7] - in /hive/branches/cbo: ./
common/src/java/org/apache/hadoop/hive/conf/
contrib/src/test/queries/clientnegative/
contrib/src/test/queries/clientpositive/
contrib/src/test/results/clientnegative/ contrib/src/test/results/clie...
Author: hashutosh
Date: Wed Mar 11 21:50:22 2015
New Revision: 1666021
URL: http://svn.apache.org/r1666021
Log:
Merged latest trunk into branch (Ashutosh Chauhan)
Added:
hive/branches/cbo/dev-support/jenkins-execute-hms-test.sh
- copied unchanged from r1666019, hive/trunk/dev-support/jenkins-execute-hms-test.sh
hive/branches/cbo/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.12.0-to-0.13.0.mssql.sql
- copied unchanged from r1666019, hive/trunk/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.12.0-to-0.13.0.mssql.sql
hive/branches/cbo/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.13.0-to-0.14.0.mssql.sql
- copied unchanged from r1666019, hive/trunk/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.13.0-to-0.14.0.mssql.sql
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsNegative.java
- copied unchanged from r1666019, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsNegative.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsPositive.java
- copied unchanged from r1666019, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSQL11ReservedKeyWordsPositive.java
hive/branches/cbo/ql/src/test/queries/clientpositive/ambiguitycheck.q
- copied unchanged from r1666019, hive/trunk/ql/src/test/queries/clientpositive/ambiguitycheck.q
hive/branches/cbo/ql/src/test/queries/clientpositive/tez_join.q
- copied unchanged from r1666019, hive/trunk/ql/src/test/queries/clientpositive/tez_join.q
hive/branches/cbo/ql/src/test/results/clientpositive/ambiguitycheck.q.out
- copied unchanged from r1666019, hive/trunk/ql/src/test/results/clientpositive/ambiguitycheck.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/tez_join.q.out
- copied unchanged from r1666019, hive/trunk/ql/src/test/results/clientpositive/tez/tez_join.q.out
hive/branches/cbo/testutils/metastore/
- copied from r1666019, hive/trunk/testutils/metastore/
Modified:
hive/branches/cbo/ (props changed)
hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/cbo/contrib/src/test/queries/clientnegative/serde_regex.q
hive/branches/cbo/contrib/src/test/queries/clientpositive/serde_regex.q
hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out
hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out
hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationReads.java
hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
hive/branches/cbo/itests/src/test/resources/testconfiguration.properties
hive/branches/cbo/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql
hive/branches/cbo/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql
hive/branches/cbo/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java
hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex.q
hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex2.q
hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex3.q
hive/branches/cbo/ql/src/test/queries/clientpositive/array_map_access_nonconstant.q
hive/branches/cbo/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q
hive/branches/cbo/ql/src/test/queries/clientpositive/decimal_10_0.q
hive/branches/cbo/ql/src/test/queries/clientpositive/dynamic_partition_pruning.q
hive/branches/cbo/ql/src/test/queries/clientpositive/innerjoin.q
hive/branches/cbo/ql/src/test/queries/clientpositive/keyword_1.q
hive/branches/cbo/ql/src/test/queries/clientpositive/nonreserved_keywords_input37.q
hive/branches/cbo/ql/src/test/queries/clientpositive/nonreserved_keywords_insert_into1.q
hive/branches/cbo/ql/src/test/queries/clientpositive/partition_wise_fileformat17.q
hive/branches/cbo/ql/src/test/queries/clientpositive/partition_wise_fileformat18.q
hive/branches/cbo/ql/src/test/queries/clientpositive/ppd_field_garbage.q
hive/branches/cbo/ql/src/test/queries/clientpositive/serde_regex.q
hive/branches/cbo/ql/src/test/queries/clientpositive/tez_union_group_by.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_1.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_10.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_11.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_12.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_13.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_14.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_15.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_16.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_17.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_18.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_19.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_2.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_20.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_21.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_22.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_23.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_24.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_25.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_3.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_4.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_5.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_6.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_6_subq.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_7.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_8.q
hive/branches/cbo/ql/src/test/queries/clientpositive/union_remove_9.q
hive/branches/cbo/ql/src/test/queries/clientpositive/vector_decimal_10_0.q
hive/branches/cbo/ql/src/test/queries/clientpositive/vectorized_dynamic_partition_pruning.q
hive/branches/cbo/ql/src/test/results/clientnegative/authorization_cannot_create_all_role.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/authorization_cannot_create_none_role.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/lateral_view_join.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/select_charliteral.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/serde_regex.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/serde_regex2.q.out
hive/branches/cbo/ql/src/test/results/clientnegative/serde_regex3.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/array_map_access_nonconstant.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/decimal_10_0.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/innerjoin.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/keyword_1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/nonreserved_keywords_input37.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/partition_wise_fileformat17.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/partition_wise_fileformat18.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/ppd_field_garbage.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/serde_regex.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/innerjoin.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_10.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_11.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_15.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_16.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_17.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_18.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_20.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_21.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_24.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_25.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_3.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_4.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_5.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_6.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_7.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_8.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/spark/union_remove_9.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_decimal_10_0.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_1.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_10.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_11.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_12.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_13.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_14.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_15.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_16.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_17.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_18.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_19.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_2.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_20.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_21.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_22.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_23.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_24.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_25.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_3.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_4.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_5.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_6.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_6_subq.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_7.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_8.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/union_remove_9.q.out
hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_10_0.q.out
hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/branches/cbo/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java
Propchange: hive/branches/cbo/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Mar 11 21:50:22 2015
@@ -3,4 +3,4 @@
/hive/branches/spark:1608589-1660298
/hive/branches/tez:1494760-1622766
/hive/branches/vectorization:1466908-1527856
-/hive/trunk:1605012-1665317
+/hive/trunk:1605012-1666019
Modified: hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Mar 11 21:50:22 2015
@@ -1955,7 +1955,9 @@ public class HiveConf extends Configurat
" none: default(past) behavior. Implies only alphaNumeric and underscore are valid characters in identifiers.\n" +
" column: implies column names can contain any character."
),
-
+ HIVE_SUPPORT_SQL11_RESERVED_KEYWORDS("hive.support.sql11.reserved.keywords", true,
+ "This flag should be set to true to enable support for SQL2011 reserved keywords.\n" +
+ "The default value is true."),
// role names are case-insensitive
USERS_IN_ADMIN_ROLE("hive.users.in.admin.role", "", false,
"Comma separated list of users who are in admin role for bootstrapping.\n" +
Modified: hive/branches/cbo/contrib/src/test/queries/clientnegative/serde_regex.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/contrib/src/test/queries/clientnegative/serde_regex.q?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/contrib/src/test/queries/clientnegative/serde_regex.q (original)
+++ hive/branches/cbo/contrib/src/test/queries/clientnegative/serde_regex.q Wed Mar 11 21:50:22 2015
@@ -7,7 +7,7 @@ EXPLAIN
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status INT,
@@ -24,7 +24,7 @@ STORED AS TEXTFILE;
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status INT,
Modified: hive/branches/cbo/contrib/src/test/queries/clientpositive/serde_regex.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/contrib/src/test/queries/clientpositive/serde_regex.q?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/contrib/src/test/queries/clientpositive/serde_regex.q (original)
+++ hive/branches/cbo/contrib/src/test/queries/clientpositive/serde_regex.q Wed Mar 11 21:50:22 2015
@@ -4,7 +4,7 @@ EXPLAIN
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status STRING,
@@ -21,7 +21,7 @@ STORED AS TEXTFILE;
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status STRING,
Modified: hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out (original)
+++ hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out Wed Mar 11 21:50:22 2015
@@ -9,7 +9,7 @@ EXPLAIN
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status INT,
@@ -28,7 +28,7 @@ EXPLAIN
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status INT,
@@ -61,7 +61,7 @@ STAGE PLANS:
PREHOOK: query: CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status INT,
Modified: hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out (original)
+++ hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out Wed Mar 11 21:50:22 2015
@@ -2,7 +2,7 @@ PREHOOK: query: EXPLAIN
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status STRING,
@@ -20,7 +20,7 @@ POSTHOOK: query: EXPLAIN
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status STRING,
@@ -53,7 +53,7 @@ STAGE PLANS:
PREHOOK: query: CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status STRING,
@@ -72,7 +72,7 @@ PREHOOK: Output: default@serde_regex
POSTHOOK: query: CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status STRING,
Modified: hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationReads.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationReads.java?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationReads.java (original)
+++ hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationReads.java Wed Mar 11 21:50:22 2015
@@ -60,7 +60,7 @@ public class TestStorageBasedMetastoreAu
setPermissions(db.getLocationUri(), "-rwxrwxrwx");
String dbDotTable = dbName + "." + tblName;
- resp = driver.run("create table " + dbDotTable + "(i int) partitioned by (date string)");
+ resp = driver.run("create table " + dbDotTable + "(i int) partitioned by (`date` string)");
Assert.assertEquals(0, resp.getResponseCode());
Table tab = msc.getTable(dbName, tblName);
setPermissions(tab.getSd().getLocation(), perm);
Modified: hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java (original)
+++ hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java Wed Mar 11 21:50:22 2015
@@ -94,7 +94,7 @@ public class TestHiveAuthorizerCheckInvo
SessionState.start(conf);
driver = new Driver(conf);
runCmd("create table " + tableName
- + " (i int, j int, k string) partitioned by (city string, date string) ");
+ + " (i int, j int, k string) partitioned by (city string, `date` string) ");
runCmd("create database " + dbName);
// Need a separate table for ACID testing since it has to be bucketed and it has to be Acid
runCmd("create table " + acidTableName + " (i int, j int) clustered by (i) into 2 buckets " +
Modified: hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java (original)
+++ hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java Wed Mar 11 21:50:22 2015
@@ -123,7 +123,7 @@ public class TestHiveAuthorizerShowFilte
SessionState.start(conf);
driver = new Driver(conf);
runCmd("create table " + tableName1
- + " (i int, j int, k string) partitioned by (city string, date string) ");
+ + " (i int, j int, k string) partitioned by (city string, `date` string) ");
runCmd("create table " + tableName2 + "(i int)");
runCmd("create database " + dbName1);
Modified: hive/branches/cbo/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/src/test/resources/testconfiguration.properties?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/cbo/itests/src/test/resources/testconfiguration.properties Wed Mar 11 21:50:22 2015
@@ -305,7 +305,8 @@ minitez.query.files=bucket_map_join_tez1
tez_smb_main.q,\
tez_smb_1.q,\
vectorized_dynamic_partition_pruning.q,\
- tez_multi_union.q
+ tez_multi_union.q,\
+ tez_join.q
encrypted.query.files=encryption_join_unencrypted_tbl.q,\
encryption_insert_partition_static.q,\
Modified: hive/branches/cbo/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql (original)
+++ hive/branches/cbo/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql Wed Mar 11 21:50:22 2015
@@ -1,32 +1 @@
---
--- Create the table if it doesn't exist.
---
-if not exists (SELECT 1 FROM INFORMATION_SCHEMA.TABLES
- WHERE TABLE_NAME='PART_COL_STATS')
-CREATE TABLE PART_COL_STATS
-(
- CS_ID bigint NOT NULL,
- AVG_COL_LEN float NULL,
- "COLUMN_NAME" varchar(128) NOT NULL,
- COLUMN_TYPE varchar(128) NOT NULL,
- DB_NAME varchar(128) NOT NULL,
- BIG_DECIMAL_HIGH_VALUE varchar(255) NULL,
- BIG_DECIMAL_LOW_VALUE varchar(255) NULL,
- DOUBLE_HIGH_VALUE float NULL,
- DOUBLE_LOW_VALUE float NULL,
- LAST_ANALYZED bigint NOT NULL,
- LONG_HIGH_VALUE bigint NULL,
- LONG_LOW_VALUE bigint NULL,
- MAX_COL_LEN bigint NULL,
- NUM_DISTINCTS bigint NULL,
- NUM_FALSES bigint NULL,
- NUM_NULLS bigint NOT NULL,
- NUM_TRUES bigint NULL,
- PART_ID bigint NULL,
- PARTITION_NAME varchar(767) NOT NULL,
- "TABLE_NAME" varchar(128) NOT NULL
-)
-go
-
-
CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (DB_NAME,TABLE_NAME,COLUMN_NAME,PARTITION_NAME);
Modified: hive/branches/cbo/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql (original)
+++ hive/branches/cbo/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql Wed Mar 11 21:50:22 2015
@@ -2,12 +2,9 @@
DROP INDEX PARTITIONS.UNIQUEPARTITION;
ALTER TABLE PARTITIONS ALTER COLUMN PART_NAME NVARCHAR(767) NULL;
CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (PART_NAME,TBL_ID);
-GO
--ALTER SDS.LOCATION from varchar to nvarchar
ALTER TABLE SDS ALTER COLUMN LOCATION NVARCHAR(4000) NULL;
-GO
--ALTER PARTITION_KEY_VALS.PART_KEY_VAL from varchar to nvarchar
ALTER TABLE PARTITION_KEY_VALS ALTER COLUMN PART_KEY_VAL NVARCHAR(255) NULL;
-GO
\ No newline at end of file
Modified: hive/branches/cbo/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql (original)
+++ hive/branches/cbo/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql Wed Mar 11 21:50:22 2015
@@ -8,19 +8,18 @@ CREATE TABLE NOTIFICATION_LOG
TBL_NAME varchar(128) NULL,
MESSAGE text NULL
);
-GO
+
ALTER TABLE NOTIFICATION_LOG ADD CONSTRAINT NOTIFICATION_LOG_PK PRIMARY KEY (NL_ID);
-GO
CREATE TABLE NOTIFICATION_SEQUENCE
(
NNI_ID bigint NOT NULL,
NEXT_EVENT_ID bigint NOT NULL
);
-GO
+
ALTER TABLE NOTIFICATION_SEQUENCE ADD CONSTRAINT NOTIFICATION_SEQUENCE_PK PRIMARY KEY (NNI_ID);
-GO
+
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Wed Mar 11 21:50:22 2015
@@ -111,7 +111,7 @@ public class ConvertJoinMapJoin implemen
}
if (parentOp instanceof ReduceSinkOperator) {
- ReduceSinkOperator rs = (ReduceSinkOperator)parentOp;
+ ReduceSinkOperator rs = (ReduceSinkOperator) parentOp;
estimatedBuckets = (estimatedBuckets < rs.getConf().getNumReducers()) ?
rs.getConf().getNumReducers() : estimatedBuckets;
}
@@ -133,10 +133,10 @@ public class ConvertJoinMapJoin implemen
if (retval == null) {
return retval;
} else {
- // only case is full outer join with SMB enabled which is not possible. Convert to regular
- // join.
- convertJoinSMBJoin(joinOp, context, 0, 0, false, false);
- return null;
+ // only case is full outer join with SMB enabled which is not possible. Convert to regular
+ // join.
+ convertJoinSMBJoin(joinOp, context, 0, 0, false, false);
+ return null;
}
}
@@ -160,8 +160,10 @@ public class ConvertJoinMapJoin implemen
}
MapJoinOperator mapJoinOp = convertJoinMapJoin(joinOp, context, mapJoinConversionPos);
- // map join operator by default has no bucket cols
- mapJoinOp.setOpTraits(new OpTraits(null, -1, null));
+ // map join operator by default has no bucket cols and num of reduce sinks
+ // reduced by 1
+ mapJoinOp
+ .setOpTraits(new OpTraits(null, -1, null, joinOp.getOpTraits().getNumReduceSinks()));
mapJoinOp.setStatistics(joinOp.getStatistics());
// propagate this change till the next RS
for (Operator<? extends OperatorDesc> childOp : mapJoinOp.getChildOperators()) {
@@ -176,7 +178,8 @@ public class ConvertJoinMapJoin implemen
TezBucketJoinProcCtx tezBucketJoinProcCtx) throws SemanticException {
// we cannot convert to bucket map join, we cannot convert to
// map join either based on the size. Check if we can convert to SMB join.
- if (context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) == false) {
+ if ((context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) == false)
+ || (joinOp.getOpTraits().getNumReduceSinks() >= 2)) {
convertJoinSMBJoin(joinOp, context, 0, 0, false, false);
return null;
}
@@ -221,7 +224,7 @@ public class ConvertJoinMapJoin implemen
convertJoinSMBJoin(joinOp, context, pos, 0, false, false);
}
return null;
-}
+ }
// replaces the join operator with a new CommonJoinOperator, removes the
// parent reduce sinks
@@ -240,9 +243,9 @@ public class ConvertJoinMapJoin implemen
new MapJoinDesc(
MapJoinProcessor.getKeys(joinOp.getConf().isLeftInputJoin(),
joinOp.getConf().getBaseSrc(), joinOp).getSecond(),
- null, joinDesc.getExprs(), null, null,
- joinDesc.getOutputColumnNames(), mapJoinConversionPos, joinDesc.getConds(),
- joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null);
+ null, joinDesc.getExprs(), null, null,
+ joinDesc.getOutputColumnNames(), mapJoinConversionPos, joinDesc.getConds(),
+ joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null);
mapJoinDesc.setNullSafes(joinDesc.getNullSafes());
mapJoinDesc.setFilterMap(joinDesc.getFilterMap());
mapJoinDesc.resetOrder();
@@ -251,9 +254,9 @@ public class ConvertJoinMapJoin implemen
CommonMergeJoinOperator mergeJoinOp =
(CommonMergeJoinOperator) OperatorFactory.get(new CommonMergeJoinDesc(numBuckets,
isSubQuery, mapJoinConversionPos, mapJoinDesc), joinOp.getSchema());
- OpTraits opTraits =
- new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, joinOp.getOpTraits()
- .getSortCols());
+ int numReduceSinks = joinOp.getOpTraits().getNumReduceSinks();
+ OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, joinOp
+ .getOpTraits().getSortCols(), numReduceSinks);
mergeJoinOp.setOpTraits(opTraits);
mergeJoinOp.setStatistics(joinOp.getStatistics());
@@ -289,8 +292,7 @@ public class ConvertJoinMapJoin implemen
if (adjustParentsChildren) {
mergeJoinOp.getConf().setGenJoinKeys(true);
- List<Operator<? extends OperatorDesc>> newParentOpList =
- new ArrayList<Operator<? extends OperatorDesc>>();
+ List<Operator<? extends OperatorDesc>> newParentOpList = new ArrayList<Operator<? extends OperatorDesc>>();
for (Operator<? extends OperatorDesc> parentOp : mergeJoinOp.getParentOperators()) {
for (Operator<? extends OperatorDesc> grandParentOp : parentOp.getParentOperators()) {
grandParentOp.getChildOperators().remove(parentOp);
@@ -328,7 +330,8 @@ public class ConvertJoinMapJoin implemen
if (currentOp instanceof ReduceSinkOperator) {
return;
}
- currentOp.setOpTraits(new OpTraits(null, -1, null));
+ currentOp.setOpTraits(new OpTraits(null, -1, null,
+ currentOp.getOpTraits().getNumReduceSinks()));
for (Operator<? extends OperatorDesc> childOp : currentOp.getChildOperators()) {
if ((childOp instanceof ReduceSinkOperator) || (childOp instanceof GroupByOperator)) {
break;
@@ -351,7 +354,7 @@ public class ConvertJoinMapJoin implemen
// we can set the traits for this join operator
OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(),
- tezBucketJoinProcCtx.getNumBuckets(), null);
+ tezBucketJoinProcCtx.getNumBuckets(), null, joinOp.getOpTraits().getNumReduceSinks());
mapJoinOp.setOpTraits(opTraits);
mapJoinOp.setStatistics(joinOp.getStatistics());
setNumberOfBucketsOnChildren(mapJoinOp);
@@ -377,8 +380,7 @@ public class ConvertJoinMapJoin implemen
ReduceSinkOperator bigTableRS =
(ReduceSinkOperator) joinOp.getParentOperators().get(bigTablePosition);
- int numBuckets = bigTableRS.getParentOperators().get(0).getOpTraits()
- .getNumBuckets();
+ int numBuckets = bigTableRS.getParentOperators().get(0).getOpTraits().getNumBuckets();
// the sort and bucket cols have to match on both sides for this
// transformation of the join operation
@@ -425,13 +427,12 @@ public class ConvertJoinMapJoin implemen
}
/*
- * If the parent reduce sink of the big table side has the same emit key cols
- * as its parent, we can create a bucket map join eliminating the reduce sink.
+ * If the parent reduce sink of the big table side has the same emit key cols as its parent, we
+ * can create a bucket map join eliminating the reduce sink.
*/
private boolean checkConvertJoinBucketMapJoin(JoinOperator joinOp,
OptimizeTezProcContext context, int bigTablePosition,
- TezBucketJoinProcCtx tezBucketJoinProcCtx)
- throws SemanticException {
+ TezBucketJoinProcCtx tezBucketJoinProcCtx) throws SemanticException {
// bail on mux-operator because mux operator masks the emit keys of the
// constituent reduce sinks
if (!(joinOp.getParentOperators().get(0) instanceof ReduceSinkOperator)) {
@@ -453,8 +454,8 @@ public class ConvertJoinMapJoin implemen
}
/*
- * this is the case when the big table is a sub-query and is probably
- * already bucketed by the join column in say a group by operation
+ * this is the case when the big table is a sub-query and is probably already bucketed by the
+ * join column in say a group by operation
*/
boolean isSubQuery = false;
if (numBuckets < 0) {
@@ -484,10 +485,16 @@ public class ConvertJoinMapJoin implemen
int colCount = 0;
// parent op is guaranteed to have a single list because it is a reduce sink
for (String colName : parentColNames.get(0)) {
+ if (listBucketCols.size() <= colCount) {
+ // can happen with virtual columns. RS would add the column to its output columns
+ // but it would not exist in the grandparent output columns or exprMap.
+ return false;
+ }
// all columns need to be at least a subset of the parentOfParent's bucket cols
ExprNodeDesc exprNodeDesc = colExprMap.get(colName);
if (exprNodeDesc instanceof ExprNodeColumnDesc) {
- if (((ExprNodeColumnDesc)exprNodeDesc).getColumn().equals(listBucketCols.get(colCount))) {
+ if (((ExprNodeColumnDesc) exprNodeDesc).getColumn()
+ .equals(listBucketCols.get(colCount))) {
colCount++;
} else {
break;
@@ -557,14 +564,13 @@ public class ConvertJoinMapJoin implemen
Statistics currInputStat = parentOp.getStatistics();
if (currInputStat == null) {
- LOG.warn("Couldn't get statistics from: "+parentOp);
+ LOG.warn("Couldn't get statistics from: " + parentOp);
return -1;
}
long inputSize = currInputStat.getDataSize();
- if ((bigInputStat == null) ||
- ((bigInputStat != null) &&
- (inputSize > bigInputStat.getDataSize()))) {
+ if ((bigInputStat == null)
+ || ((bigInputStat != null) && (inputSize > bigInputStat.getDataSize()))) {
if (bigTableFound) {
// cannot convert to map join; we've already chosen a big table
@@ -634,11 +640,11 @@ public class ConvertJoinMapJoin implemen
}
}
- //can safely convert the join to a map join.
+ // can safely convert the join to a map join.
MapJoinOperator mapJoinOp =
MapJoinProcessor.convertJoinOpMapJoinOp(context.conf, joinOp,
- joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(),
- joinOp.getConf().getMapAliases(), bigTablePosition, true);
+ joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(),
+ joinOp.getConf().getMapAliases(), bigTablePosition, true);
Operator<? extends OperatorDesc> parentBigTableOp =
mapJoinOp.getParentOperators().get(bigTablePosition);
@@ -662,7 +668,7 @@ public class ConvertJoinMapJoin implemen
parentBigTableOp.getParentOperators().get(0));
}
parentBigTableOp.getParentOperators().get(0).removeChild(parentBigTableOp);
- for (Operator<? extends OperatorDesc> op : mapJoinOp.getParentOperators()) {
+ for (Operator<? extends OperatorDesc>op : mapJoinOp.getParentOperators()) {
if (!(op.getChildOperators().contains(mapJoinOp))) {
op.getChildOperators().add(mapJoinOp);
}
@@ -676,7 +682,7 @@ public class ConvertJoinMapJoin implemen
private boolean hasDynamicPartitionBroadcast(Operator<?> parent) {
boolean hasDynamicPartitionPruning = false;
- for (Operator<?> op: parent.getChildOperators()) {
+ for (Operator<?> op : parent.getChildOperators()) {
while (op != null) {
if (op instanceof AppMasterEventOperator && op.getConf() instanceof DynamicPruningEventDesc) {
// found dynamic partition pruning operator
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java Wed Mar 11 21:50:22 2015
@@ -82,7 +82,7 @@ public class OpTraitsRulesProcFactory {
}
/*
- * Reduce sink operator is the de-facto operator
+ * Reduce sink operator is the de-facto operator
* for determining keyCols (emit keys of a map phase)
*/
public static class ReduceSinkRule implements NodeProcessor {
@@ -106,24 +106,25 @@ public class OpTraitsRulesProcFactory {
List<List<String>> listBucketCols = new ArrayList<List<String>>();
listBucketCols.add(bucketCols);
int numBuckets = -1;
+ int numReduceSinks = 1;
OpTraits parentOpTraits = rs.getParentOperators().get(0).getConf().getOpTraits();
if (parentOpTraits != null) {
numBuckets = parentOpTraits.getNumBuckets();
+ numReduceSinks += parentOpTraits.getNumReduceSinks();
}
- OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listBucketCols);
+ OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listBucketCols, numReduceSinks);
rs.setOpTraits(opTraits);
return null;
}
}
/*
- * Table scan has the table object and pruned partitions that has information such as
- * bucketing, sorting, etc. that is used later for optimization.
+ * Table scan has the table object and pruned partitions that has information
+ * such as bucketing, sorting, etc. that is used later for optimization.
*/
public static class TableScanRule implements NodeProcessor {
- public boolean checkBucketedTable(Table tbl,
- ParseContext pGraphContext,
+ public boolean checkBucketedTable(Table tbl, ParseContext pGraphContext,
PrunedPartitionList prunedParts) throws SemanticException {
if (tbl.isPartitioned()) {
@@ -131,9 +132,11 @@ public class OpTraitsRulesProcFactory {
// construct a mapping of (Partition->bucket file names) and (Partition -> bucket number)
if (!partitions.isEmpty()) {
for (Partition p : partitions) {
- List<String> fileNames =
- AbstractBucketJoinProc.getBucketFilePathsOfPartition(p.getDataLocation(), pGraphContext);
- // The number of files for the table should be same as number of buckets.
+ List<String> fileNames =
+ AbstractBucketJoinProc.getBucketFilePathsOfPartition(p.getDataLocation(),
+ pGraphContext);
+ // The number of files for the table should be same as number of
+ // buckets.
int bucketCount = p.getBucketCount();
if (fileNames.size() != 0 && fileNames.size() != bucketCount) {
@@ -143,8 +146,9 @@ public class OpTraitsRulesProcFactory {
}
} else {
- List<String> fileNames =
- AbstractBucketJoinProc.getBucketFilePathsOfPartition(tbl.getDataLocation(), pGraphContext);
+ List<String> fileNames =
+ AbstractBucketJoinProc.getBucketFilePathsOfPartition(tbl.getDataLocation(),
+ pGraphContext);
Integer num = new Integer(tbl.getNumBuckets());
// The number of files for the table should be same as number of buckets.
@@ -183,7 +187,8 @@ public class OpTraitsRulesProcFactory {
}
sortedColsList.add(sortCols);
}
- OpTraits opTraits = new OpTraits(bucketColsList, numBuckets, sortedColsList);
+ // num reduce sinks hardcoded to 0 because TS has no parents
+ OpTraits opTraits = new OpTraits(bucketColsList, numBuckets, sortedColsList, 0);
ts.setOpTraits(opTraits);
return null;
}
@@ -208,8 +213,13 @@ public class OpTraitsRulesProcFactory {
}
List<List<String>> listBucketCols = new ArrayList<List<String>>();
+ int numReduceSinks = 0;
+ OpTraits parentOpTraits = gbyOp.getParentOperators().get(0).getOpTraits();
+ if (parentOpTraits != null) {
+ numReduceSinks = parentOpTraits.getNumReduceSinks();
+ }
listBucketCols.add(gbyKeys);
- OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols);
+ OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols, numReduceSinks);
gbyOp.setOpTraits(opTraits);
return null;
}
@@ -217,8 +227,8 @@ public class OpTraitsRulesProcFactory {
public static class SelectRule implements NodeProcessor {
- public List<List<String>> getConvertedColNames(List<List<String>> parentColNames,
- SelectOperator selOp) {
+ public List<List<String>> getConvertedColNames(
+ List<List<String>> parentColNames, SelectOperator selOp) {
List<List<String>> listBucketCols = new ArrayList<List<String>>();
if (selOp.getColumnExprMap() != null) {
if (parentColNames != null) {
@@ -244,8 +254,8 @@ public class OpTraitsRulesProcFactory {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
- SelectOperator selOp = (SelectOperator)nd;
- List<List<String>> parentBucketColNames =
+ SelectOperator selOp = (SelectOperator) nd;
+ List<List<String>> parentBucketColNames =
selOp.getParentOperators().get(0).getOpTraits().getBucketColNames();
List<List<String>> listBucketCols = null;
@@ -254,18 +264,21 @@ public class OpTraitsRulesProcFactory {
if (parentBucketColNames != null) {
listBucketCols = getConvertedColNames(parentBucketColNames, selOp);
}
- List<List<String>> parentSortColNames = selOp.getParentOperators().get(0).getOpTraits()
- .getSortCols();
+ List<List<String>> parentSortColNames =
+ selOp.getParentOperators().get(0).getOpTraits().getSortCols();
if (parentSortColNames != null) {
listSortCols = getConvertedColNames(parentSortColNames, selOp);
}
}
int numBuckets = -1;
- if (selOp.getParentOperators().get(0).getOpTraits() != null) {
- numBuckets = selOp.getParentOperators().get(0).getOpTraits().getNumBuckets();
+ int numReduceSinks = 0;
+ OpTraits parentOpTraits = selOp.getParentOperators().get(0).getOpTraits();
+ if (parentOpTraits != null) {
+ numBuckets = parentOpTraits.getNumBuckets();
+ numReduceSinks = parentOpTraits.getNumReduceSinks();
}
- OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols);
+ OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols, numReduceSinks);
selOp.setOpTraits(opTraits);
return null;
}
@@ -276,26 +289,31 @@ public class OpTraitsRulesProcFactory {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
- JoinOperator joinOp = (JoinOperator)nd;
+ JoinOperator joinOp = (JoinOperator) nd;
List<List<String>> bucketColsList = new ArrayList<List<String>>();
List<List<String>> sortColsList = new ArrayList<List<String>>();
byte pos = 0;
+ int numReduceSinks = 0; // will be set to the larger of the parents
for (Operator<? extends OperatorDesc> parentOp : joinOp.getParentOperators()) {
if (!(parentOp instanceof ReduceSinkOperator)) {
// can be mux operator
break;
}
- ReduceSinkOperator rsOp = (ReduceSinkOperator)parentOp;
+ ReduceSinkOperator rsOp = (ReduceSinkOperator) parentOp;
if (rsOp.getOpTraits() == null) {
ReduceSinkRule rsRule = new ReduceSinkRule();
rsRule.process(rsOp, stack, procCtx, nodeOutputs);
}
- bucketColsList.add(getOutputColNames(joinOp, rsOp.getOpTraits().getBucketColNames(), pos));
- sortColsList.add(getOutputColNames(joinOp, rsOp.getOpTraits().getSortCols(), pos));
+ OpTraits parentOpTraits = rsOp.getOpTraits();
+ bucketColsList.add(getOutputColNames(joinOp, parentOpTraits.getBucketColNames(), pos));
+ sortColsList.add(getOutputColNames(joinOp, parentOpTraits.getSortCols(), pos));
+ if (parentOpTraits.getNumReduceSinks() > numReduceSinks) {
+ numReduceSinks = parentOpTraits.getNumReduceSinks();
+ }
pos++;
}
- joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList));
+ joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList, numReduceSinks));
return null;
}
@@ -311,7 +329,7 @@ public class OpTraitsRulesProcFactory {
for (String colName : colNames) {
for (ExprNodeDesc exprNode : joinOp.getConf().getExprs().get(pos)) {
if (exprNode instanceof ExprNodeColumnDesc) {
- if(((ExprNodeColumnDesc)(exprNode)).getColumn().equals(colName)) {
+ if (((ExprNodeColumnDesc) (exprNode)).getColumn().equals(colName)) {
for (Entry<String, ExprNodeDesc> entry : joinOp.getColumnExprMap().entrySet()) {
if (entry.getValue().isSame(exprNode)) {
bucketColNames.add(entry.getKey());
@@ -338,20 +356,30 @@ public class OpTraitsRulesProcFactory {
}
/*
- * When we have operators that have multiple parents, it is not
- * clear which parent's traits we need to propagate forward.
+ * When we have operators that have multiple parents, it is not clear which
+ * parent's traits we need to propagate forward.
*/
public static class MultiParentRule implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
- OpTraits opTraits = new OpTraits(null, -1, null);
@SuppressWarnings("unchecked")
- Operator<? extends OperatorDesc> operator = (Operator<? extends OperatorDesc>)nd;
+ Operator<? extends OperatorDesc> operator = (Operator<? extends OperatorDesc>) nd;
+
+ int numReduceSinks = 0;
+ for (Operator<?> parentOp : operator.getParentOperators()) {
+ if (parentOp.getOpTraits() == null) {
+ continue;
+ }
+ if (parentOp.getOpTraits().getNumReduceSinks() > numReduceSinks) {
+ numReduceSinks = parentOp.getOpTraits().getNumReduceSinks();
+ }
+ }
+ OpTraits opTraits = new OpTraits(null, -1, null, numReduceSinks);
operator.setOpTraits(opTraits);
return null;
- }
+ }
}
public static NodeProcessor getTableScanRule() {
@@ -361,7 +389,7 @@ public class OpTraitsRulesProcFactory {
public static NodeProcessor getReduceSinkRule() {
return new ReduceSinkRule();
}
-
+
public static NodeProcessor getSelectRule() {
return new SelectRule();
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java Wed Mar 11 21:50:22 2015
@@ -103,7 +103,7 @@ public class SparkMapJoinOptimizer imple
}
// we can set the traits for this join operator
- OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null);
+ OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null, joinOp.getOpTraits().getNumReduceSinks());
mapJoinOp.setOpTraits(opTraits);
mapJoinOp.setStatistics(joinOp.getStatistics());
setNumberOfBucketsOnChildren(mapJoinOp);
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g Wed Mar 11 21:50:22 2015
@@ -35,6 +35,9 @@ k=3;
RecognitionException e) {
gParent.errors.add(new ParseError(gParent, e, tokenNames));
}
+ protected boolean useSQL11ReservedKeywordsForIdentifier() {
+ return gParent.useSQL11ReservedKeywordsForIdentifier();
+ }
}
@rulecatch {
@@ -126,7 +129,7 @@ lateralView
@init {gParent.pushMsg("lateral view", state); }
@after {gParent.popMsg(state); }
:
- KW_LATERAL KW_VIEW KW_OUTER function tableAlias (KW_AS identifier ((COMMA)=> COMMA identifier)*)?
+ (KW_LATERAL KW_VIEW KW_OUTER) => KW_LATERAL KW_VIEW KW_OUTER function tableAlias (KW_AS identifier ((COMMA)=> COMMA identifier)*)?
-> ^(TOK_LATERAL_VIEW_OUTER ^(TOK_SELECT ^(TOK_SELEXPR function identifier* tableAlias)))
|
KW_LATERAL KW_VIEW function tableAlias (KW_AS identifier ((COMMA)=> COMMA identifier)*)?
@@ -177,7 +180,12 @@ tableSample
tableSource
@init { gParent.pushMsg("table source", state); }
@after { gParent.popMsg(state); }
- : tabname=tableName (props=tableProperties)? (ts=tableSample)? (KW_AS? alias=Identifier)?
+ : tabname=tableName
+ ((tableProperties) => props=tableProperties)?
+ ((tableSample) => ts=tableSample)?
+ ((KW_AS) => (KW_AS alias=Identifier)
+ |
+ (Identifier) => (alias=Identifier))?
-> ^(TOK_TABREF $tabname $props? $ts? $alias?)
;
@@ -232,11 +240,11 @@ partitionedTableFunction
@init { gParent.pushMsg("ptf clause", state); }
@after { gParent.popMsg(state); }
:
- name=Identifier
- LPAREN KW_ON ptfsrc=partitionTableFunctionSource partitioningSpec?
- ((Identifier LPAREN expression RPAREN ) => Identifier LPAREN expression RPAREN ( COMMA Identifier LPAREN expression RPAREN)*)?
- RPAREN alias=Identifier?
- -> ^(TOK_PTBLFUNCTION $name $alias? partitionTableFunctionSource partitioningSpec? expression*)
+ name=Identifier LPAREN KW_ON
+ ((partitionTableFunctionSource) => (ptfsrc=partitionTableFunctionSource spec=partitioningSpec?))
+ ((Identifier LPAREN expression RPAREN ) => Identifier LPAREN expression RPAREN ( COMMA Identifier LPAREN expression RPAREN)*)?
+ ((RPAREN) => (RPAREN)) ((Identifier) => alias=Identifier)?
+ -> ^(TOK_PTBLFUNCTION $name $alias? $ptfsrc $spec? expression*)
;
//----------------------- Rules for parsing whereClause -----------------------------
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Wed Mar 11 21:50:22 2015
@@ -42,7 +42,6 @@ KW_TRUE : 'TRUE';
KW_FALSE : 'FALSE';
KW_ALL : 'ALL';
KW_NONE: 'NONE';
-KW_DEFAULT : 'DEFAULT';
KW_AND : 'AND';
KW_OR : 'OR';
KW_NOT : 'NOT' | '!';
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Wed Mar 11 21:50:22 2015
@@ -354,6 +354,8 @@ package org.apache.hadoop.hive.ql.parse;
import java.util.Collection;
import java.util.HashMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
}
@@ -370,7 +372,6 @@ import java.util.HashMap;
xlateMap.put("KW_FALSE", "FALSE");
xlateMap.put("KW_ALL", "ALL");
xlateMap.put("KW_NONE", "NONE");
- xlateMap.put("KW_DEFAULT", "DEFAULT");
xlateMap.put("KW_AND", "AND");
xlateMap.put("KW_OR", "OR");
xlateMap.put("KW_NOT", "NOT");
@@ -620,6 +621,13 @@ import java.util.HashMap;
private CommonTree throwSetOpException() throws RecognitionException {
throw new FailedPredicateException(input, "orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.", "");
}
+ private Configuration hiveConf;
+ public void setHiveConf(Configuration hiveConf) {
+ this.hiveConf = hiveConf;
+ }
+ protected boolean useSQL11ReservedKeywordsForIdentifier() {
+ return !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_SUPPORT_SQL11_RESERVED_KEYWORDS);
+ }
}
@rulecatch {
@@ -712,8 +720,8 @@ ddlStatement
| unlockDatabase
| createRoleStatement
| dropRoleStatement
- | grantPrivileges
- | revokePrivileges
+ | (grantPrivileges) => grantPrivileges
+ | (revokePrivileges) => revokePrivileges
| showGrants
| showRoleGrants
| showRolePrincipals
@@ -955,8 +963,7 @@ alterStatement
alterTableStatementSuffix
@init { pushMsg("alter table statement", state); }
@after { popMsg(state); }
- : alterStatementSuffixRename[true]
- | alterStatementSuffixUpdateStatsCol
+ : (alterStatementSuffixRename[true]) => alterStatementSuffixRename[true]
| alterStatementSuffixDropPartitions[true]
| alterStatementSuffixAddPartitions[true]
| alterStatementSuffixTouch
@@ -1297,15 +1304,15 @@ fileFormat
tabTypeExpr
@init { pushMsg("specifying table types", state); }
@after { popMsg(state); }
-
- : identifier (DOT^ (KW_ELEM_TYPE | KW_KEY_TYPE | KW_VALUE_TYPE | identifier))*
- ;
-
-descTabTypeExpr
-@init { pushMsg("specifying describe table types", state); }
-@after { popMsg(state); }
-
- : identifier (DOT^ (KW_ELEM_TYPE | KW_KEY_TYPE | KW_VALUE_TYPE | identifier))* identifier?
+ : identifier (DOT^
+ (
+ (KW_ELEM_TYPE) => KW_ELEM_TYPE
+ |
+ (KW_KEY_TYPE) => KW_KEY_TYPE
+ |
+ (KW_VALUE_TYPE) => KW_VALUE_TYPE
+ | identifier
+ ))* identifier?
;
partTypeExpr
@@ -1314,21 +1321,22 @@ partTypeExpr
: tabTypeExpr partitionSpec? -> ^(TOK_TABTYPE tabTypeExpr partitionSpec?)
;
-descPartTypeExpr
-@init { pushMsg("specifying describe table partitions", state); }
-@after { popMsg(state); }
- : descTabTypeExpr partitionSpec? -> ^(TOK_TABTYPE descTabTypeExpr partitionSpec?)
- ;
-
descStatement
@init { pushMsg("describe statement", state); }
@after { popMsg(state); }
- : (KW_DESCRIBE|KW_DESC) (KW_DATABASE|KW_SCHEMA) KW_EXTENDED? (dbName=identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?)
- | (KW_DESCRIBE|KW_DESC) (descOptions=KW_FORMATTED|descOptions=KW_EXTENDED|descOptions=KW_PRETTY)? (parttype=descPartTypeExpr) -> ^(TOK_DESCTABLE $parttype $descOptions?)
- | (KW_DESCRIBE|KW_DESC) KW_FUNCTION KW_EXTENDED? (name=descFuncNames) -> ^(TOK_DESCFUNCTION $name KW_EXTENDED?)
+ :
+ (KW_DESCRIBE|KW_DESC)
+ (
+ (KW_DATABASE|KW_SCHEMA) => (KW_DATABASE|KW_SCHEMA) KW_EXTENDED? (dbName=identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?)
+ |
+ (KW_FUNCTION) => KW_FUNCTION KW_EXTENDED? (name=descFuncNames) -> ^(TOK_DESCFUNCTION $name KW_EXTENDED?)
+ |
+ (KW_FORMATTED|KW_EXTENDED|KW_PRETTY) => ((descOptions=KW_FORMATTED|descOptions=KW_EXTENDED|descOptions=KW_PRETTY) parttype=partTypeExpr) -> ^(TOK_DESCTABLE $parttype $descOptions)
+ |
+ parttype=partTypeExpr -> ^(TOK_DESCTABLE $parttype)
+ )
;
-
analyzeStatement
@init { pushMsg("analyze statement", state); }
@after { popMsg(state); }
@@ -1350,8 +1358,12 @@ showStatement
| KW_SHOW KW_TABLE KW_EXTENDED ((KW_FROM|KW_IN) db_name=identifier)? KW_LIKE showStmtIdentifier partitionSpec?
-> ^(TOK_SHOW_TABLESTATUS showStmtIdentifier $db_name? partitionSpec?)
| KW_SHOW KW_TBLPROPERTIES tableName (LPAREN prptyName=StringLiteral RPAREN)? -> ^(TOK_SHOW_TBLPROPERTIES tableName $prptyName?)
- | KW_SHOW KW_LOCKS (parttype=partTypeExpr)? (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWLOCKS $parttype? $isExtended?)
- | KW_SHOW KW_LOCKS (KW_DATABASE|KW_SCHEMA) (dbName=Identifier) (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWDBLOCKS $dbName $isExtended?)
+ | KW_SHOW KW_LOCKS
+ (
+ (KW_DATABASE|KW_SCHEMA) => (KW_DATABASE|KW_SCHEMA) (dbName=Identifier) (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWDBLOCKS $dbName $isExtended?)
+ |
+ (parttype=partTypeExpr)? (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWLOCKS $parttype? $isExtended?)
+ )
| KW_SHOW (showOptions=KW_FORMATTED)? (KW_INDEX|KW_INDEXES) KW_ON showStmtIdentifier ((KW_FROM|KW_IN) db_name=identifier)?
-> ^(TOK_SHOWINDEXES showStmtIdentifier $showOptions? $db_name?)
| KW_SHOW KW_COMPACTIONS -> ^(TOK_SHOW_COMPACTIONS)
@@ -1459,8 +1471,12 @@ showCurrentRole
setRole
@init {pushMsg("set role", state);}
@after {popMsg(state);}
- : KW_SET KW_ROLE roleName=identifier
- -> ^(TOK_SHOW_SET_ROLE $roleName)
+ : KW_SET KW_ROLE
+ (
+ (KW_ALL) => (all=KW_ALL) -> ^(TOK_SHOW_SET_ROLE Identifier[$all.text])
+ |
+ identifier -> ^(TOK_SHOW_SET_ROLE identifier)
+ )
;
showGrants
@@ -1481,7 +1497,7 @@ showRolePrincipals
privilegeIncludeColObject
@init {pushMsg("privilege object including columns", state);}
@after {popMsg(state);}
- : KW_ALL -> ^(TOK_RESOURCE_ALL)
+ : (KW_ALL) => KW_ALL -> ^(TOK_RESOURCE_ALL)
| privObjectCols -> ^(TOK_PRIV_OBJECT_COL privObjectCols)
;
@@ -1720,7 +1736,7 @@ tableSkewed
@init { pushMsg("table skewed specification", state); }
@after { popMsg(state); }
:
- KW_SKEWED KW_BY LPAREN skewedCols=columnNameList RPAREN KW_ON LPAREN (skewedValues=skewedValueElement) RPAREN (storedAsDirs)?
+ KW_SKEWED KW_BY LPAREN skewedCols=columnNameList RPAREN KW_ON LPAREN (skewedValues=skewedValueElement) RPAREN ((storedAsDirs) => storedAsDirs)?
-> ^(TOK_TABLESKEWED $skewedCols $skewedValues storedAsDirs?)
;
@@ -1851,7 +1867,7 @@ tableFileFormat
@init { pushMsg("table file format specification", state); }
@after { popMsg(state); }
:
- KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)?
+ (KW_STORED KW_AS KW_INPUTFORMAT) => KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)?
-> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt $inDriver? $outDriver?)
| KW_STORED KW_BY storageHandler=StringLiteral
(KW_WITH KW_SERDEPROPERTIES serdeprops=tableProperties)?
@@ -2231,7 +2247,7 @@ simpleSelectStatement
whereClause?
groupByClause?
havingClause?
- window_clause?
+ ((window_clause) => window_clause)?
-> ^(TOK_QUERY fromClause? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
selectClause whereClause? groupByClause? havingClause? window_clause?))
;
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Wed Mar 11 21:50:22 2015
@@ -35,6 +35,9 @@ k=3;
RecognitionException e) {
gParent.errors.add(new ParseError(gParent, e, tokenNames));
}
+ protected boolean useSQL11ReservedKeywordsForIdentifier() {
+ return gParent.useSQL11ReservedKeywordsForIdentifier();
+ }
}
@rulecatch {
@@ -51,40 +54,41 @@ groupByClause
@after { gParent.popMsg(state); }
:
KW_GROUP KW_BY
- groupByExpression
- ( COMMA groupByExpression )*
+ expression
+ ( COMMA expression)*
((rollup=KW_WITH KW_ROLLUP) | (cube=KW_WITH KW_CUBE)) ?
(sets=KW_GROUPING KW_SETS
LPAREN groupingSetExpression ( COMMA groupingSetExpression)* RPAREN ) ?
- -> {rollup != null}? ^(TOK_ROLLUP_GROUPBY groupByExpression+)
- -> {cube != null}? ^(TOK_CUBE_GROUPBY groupByExpression+)
- -> {sets != null}? ^(TOK_GROUPING_SETS groupByExpression+ groupingSetExpression+)
- -> ^(TOK_GROUPBY groupByExpression+)
+ -> {rollup != null}? ^(TOK_ROLLUP_GROUPBY expression+)
+ -> {cube != null}? ^(TOK_CUBE_GROUPBY expression+)
+ -> {sets != null}? ^(TOK_GROUPING_SETS expression+ groupingSetExpression+)
+ -> ^(TOK_GROUPBY expression+)
;
groupingSetExpression
@init {gParent.pushMsg("grouping set expression", state); }
@after {gParent.popMsg(state); }
:
- groupByExpression
- -> ^(TOK_GROUPING_SETS_EXPRESSION groupByExpression)
+ (LPAREN) => groupingSetExpressionMultiple
|
+ groupingExpressionSingle
+ ;
+
+groupingSetExpressionMultiple
+@init {gParent.pushMsg("grouping set part expression", state); }
+@after {gParent.popMsg(state); }
+ :
LPAREN
- groupByExpression (COMMA groupByExpression)*
+ expression? (COMMA expression)*
RPAREN
- -> ^(TOK_GROUPING_SETS_EXPRESSION groupByExpression+)
- |
- LPAREN
- RPAREN
- -> ^(TOK_GROUPING_SETS_EXPRESSION)
+ -> ^(TOK_GROUPING_SETS_EXPRESSION expression*)
;
-
-groupByExpression
-@init { gParent.pushMsg("group by expression", state); }
+groupingExpressionSingle
+@init { gParent.pushMsg("groupingExpression expression", state); }
@after { gParent.popMsg(state); }
:
- expression
+ expression -> ^(TOK_GROUPING_SETS_EXPRESSION expression)
;
havingClause
@@ -101,6 +105,26 @@ havingCondition
expression
;
+expressionsInParenthese
+ :
+ LPAREN expression (COMMA expression)* RPAREN -> expression+
+ ;
+
+expressionsNotInParenthese
+ :
+ expression (COMMA expression)* -> expression+
+ ;
+
+columnRefOrderInParenthese
+ :
+ LPAREN columnRefOrder (COMMA columnRefOrder)* RPAREN -> columnRefOrder+
+ ;
+
+columnRefOrderNotInParenthese
+ :
+ columnRefOrder (COMMA columnRefOrder)* -> columnRefOrder+
+ ;
+
// order by a,b
orderByClause
@init { gParent.pushMsg("order by clause", state); }
@@ -108,17 +132,17 @@ orderByClause
:
KW_ORDER KW_BY columnRefOrder ( COMMA columnRefOrder)* -> ^(TOK_ORDERBY columnRefOrder+)
;
-
+
clusterByClause
@init { gParent.pushMsg("cluster by clause", state); }
@after { gParent.popMsg(state); }
:
KW_CLUSTER KW_BY
- LPAREN expression (COMMA expression)* RPAREN -> ^(TOK_CLUSTERBY expression+)
+ (
+ (LPAREN) => expressionsInParenthese -> ^(TOK_CLUSTERBY expressionsInParenthese)
|
- KW_CLUSTER KW_BY
- expression
- ( (COMMA)=>COMMA expression )* -> ^(TOK_CLUSTERBY expression+)
+ expressionsNotInParenthese -> ^(TOK_CLUSTERBY expressionsNotInParenthese)
+ )
;
partitionByClause
@@ -126,10 +150,11 @@ partitionByClause
@after { gParent.popMsg(state); }
:
KW_PARTITION KW_BY
- LPAREN expression (COMMA expression)* RPAREN -> ^(TOK_DISTRIBUTEBY expression+)
+ (
+ (LPAREN) => expressionsInParenthese -> ^(TOK_DISTRIBUTEBY expressionsInParenthese)
|
- KW_PARTITION KW_BY
- expression ((COMMA)=> COMMA expression)* -> ^(TOK_DISTRIBUTEBY expression+)
+ expressionsNotInParenthese -> ^(TOK_DISTRIBUTEBY expressionsNotInParenthese)
+ )
;
distributeByClause
@@ -137,10 +162,11 @@ distributeByClause
@after { gParent.popMsg(state); }
:
KW_DISTRIBUTE KW_BY
- LPAREN expression (COMMA expression)* RPAREN -> ^(TOK_DISTRIBUTEBY expression+)
+ (
+ (LPAREN) => expressionsInParenthese -> ^(TOK_DISTRIBUTEBY expressionsInParenthese)
|
- KW_DISTRIBUTE KW_BY
- expression ((COMMA)=> COMMA expression)* -> ^(TOK_DISTRIBUTEBY expression+)
+ expressionsNotInParenthese -> ^(TOK_DISTRIBUTEBY expressionsNotInParenthese)
+ )
;
sortByClause
@@ -148,12 +174,11 @@ sortByClause
@after { gParent.popMsg(state); }
:
KW_SORT KW_BY
- LPAREN columnRefOrder
- ( COMMA columnRefOrder)* RPAREN -> ^(TOK_SORTBY columnRefOrder+)
+ (
+ (LPAREN) => columnRefOrderInParenthese -> ^(TOK_SORTBY columnRefOrderInParenthese)
|
- KW_SORT KW_BY
- columnRefOrder
- ( (COMMA)=> COMMA columnRefOrder)* -> ^(TOK_SORTBY columnRefOrder+)
+ columnRefOrderNotInParenthese -> ^(TOK_SORTBY columnRefOrderNotInParenthese)
+ )
;
// fun(par1, par2, par3)
@@ -164,7 +189,7 @@ function
functionName
LPAREN
(
- (star=STAR)
+ (STAR) => (star=STAR)
| (dist=KW_DISTINCT)? (selectExpression (COMMA selectExpression)*)?
)
RPAREN (KW_OVER ws=window_specification)?
@@ -173,29 +198,15 @@ function
-> ^(TOK_FUNCTIONDI functionName (selectExpression+)?)
;
-nonParenthesizedFunction
-@init { gParent.pushMsg("non-parenthesized function name", state); }
-@after { gParent.popMsg(state); }
- :
- nonParenthesizedFunctionName
- -> ^(TOK_FUNCTION nonParenthesizedFunctionName)
- ;
-
-nonParenthesizedFunctionName
-@init { gParent.pushMsg("non-parenthesized function name", state); }
-@after { gParent.popMsg(state); }
- :
- KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP
- ;
-
functionName
@init { gParent.pushMsg("function name", state); }
@after { gParent.popMsg(state); }
: // Keyword IF is also a function name
- KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE | functionIdentifier
+ (KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE) => (KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE)
+ |
+ (functionIdentifier) => functionIdentifier
|
- // This allows current_timestamp() to work as well as current_timestamp
- nonParenthesizedFunctionName
+ {!useSQL11ReservedKeywordsForIdentifier()}? sql11ReservedKeywordsUsedAsCastFunctionName -> Identifier[$sql11ReservedKeywordsUsedAsCastFunctionName.text]
;
castExpression
@@ -267,6 +278,8 @@ dateLiteral
// This makes the dateLiteral more consistent with the other type literals.
adaptor.create(TOK_DATELITERAL, $StringLiteral.text)
}
+ |
+ KW_CURRENT_DATE -> ^(TOK_FUNCTION KW_CURRENT_DATE)
;
timestampLiteral
@@ -275,6 +288,8 @@ timestampLiteral
{
adaptor.create(TOK_TIMESTAMPLITERAL, $StringLiteral.text)
}
+ |
+ KW_CURRENT_TIMESTAMP -> ^(TOK_FUNCTION KW_CURRENT_TIMESTAMP)
;
expression
@@ -286,12 +301,11 @@ expression
atomExpression
:
- KW_NULL -> TOK_NULL
- | constant
+ (KW_NULL) => KW_NULL -> TOK_NULL
+ | (constant) => constant
| castExpression
| caseExpression
| whenExpression
- | nonParenthesizedFunction
| (functionName LPAREN) => function
| tableOrColumn
| LPAREN! expression RPAREN!
@@ -543,7 +557,7 @@ sysFuncNames
descFuncNames
:
- sysFuncNames
+ (sysFuncNames) => sysFuncNames
| StringLiteral
| functionIdentifier
;
@@ -552,6 +566,9 @@ identifier
:
Identifier
| nonReserved -> Identifier[$nonReserved.text]
+ // If it decides to support SQL11 reserved keywords, i.e., useSQL11ReservedKeywordsForIdentifier()=false,
+ // the sql11keywords in existing q tests will NOT be added back.
+ | {useSQL11ReservedKeywordsForIdentifier()}? sql11ReservedKeywordsUsedAsIdentifier -> Identifier[$sql11ReservedKeywordsUsedAsIdentifier.text]
;
functionIdentifier
@@ -570,7 +587,44 @@ principalIdentifier
| QuotedIdentifier
;
+//the new version of nonReserved + sql11ReservedKeywordsUsedAsIdentifier = old version of nonReserved
nonReserved
:
- KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS |
KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_
BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES | KW_URI | KW_SERVER | KW_RELOAD
+ KW_ADD | KW_ADMIN | KW_AFTER | KW_ANALYZE | KW_ARCHIVE | KW_ASC | KW_BEFORE | KW_BUCKET | KW_BUCKETS
+ | KW_CASCADE | KW_CHANGE | KW_CLUSTER | KW_CLUSTERED | KW_CLUSTERSTATUS | KW_COLLECTION | KW_COLUMNS
+ | KW_COMMENT | KW_COMPACT | KW_COMPACTIONS | KW_COMPUTE | KW_CONCATENATE | KW_CONTINUE | KW_DATA
+ | KW_DATABASES | KW_DATETIME | KW_DBPROPERTIES | KW_DEFERRED | KW_DEFINED | KW_DELIMITED | KW_DEPENDENCY
+ | KW_DESC | KW_DIRECTORIES | KW_DIRECTORY | KW_DISABLE | KW_DISTRIBUTE | KW_ELEM_TYPE
+ | KW_ENABLE | KW_ESCAPED | KW_EXCLUSIVE | KW_EXPLAIN | KW_EXPORT | KW_FIELDS | KW_FILE | KW_FILEFORMAT
+ | KW_FIRST | KW_FORMAT | KW_FORMATTED | KW_FUNCTIONS | KW_HOLD_DDLTIME | KW_IDXPROPERTIES | KW_IGNORE
+ | KW_INDEX | KW_INDEXES | KW_INPATH | KW_INPUTDRIVER | KW_INPUTFORMAT | KW_ITEMS | KW_JAR
+ | KW_KEYS | KW_KEY_TYPE | KW_LIMIT | KW_LINES | KW_LOAD | KW_LOCATION | KW_LOCK | KW_LOCKS | KW_LOGICAL | KW_LONG
+ | KW_MAPJOIN | KW_MATERIALIZED | KW_MINUS | KW_MSCK | KW_NOSCAN | KW_NO_DROP | KW_OFFLINE | KW_OPTION
+ | KW_OUTPUTDRIVER | KW_OUTPUTFORMAT | KW_OVERWRITE | KW_OWNER | KW_PARTITIONED | KW_PARTITIONS | KW_PLUS | KW_PRETTY | KW_PRINCIPALS
+ | KW_PROTECTION | KW_PURGE | KW_READ | KW_READONLY | KW_REBUILD | KW_RECORDREADER | KW_RECORDWRITER
+ | KW_REGEXP | KW_RELOAD | KW_RENAME | KW_REPAIR | KW_REPLACE | KW_RESTRICT | KW_REWRITE | KW_RLIKE
+ | KW_ROLE | KW_ROLES | KW_SCHEMA | KW_SCHEMAS | KW_SEMI | KW_SERDE | KW_SERDEPROPERTIES | KW_SERVER | KW_SETS | KW_SHARED
+ | KW_SHOW | KW_SHOW_DATABASE | KW_SKEWED | KW_SORT | KW_SORTED | KW_SSL | KW_STATISTICS | KW_STORED
+ | KW_STREAMTABLE | KW_STRING | KW_STRUCT | KW_TABLES | KW_TBLPROPERTIES | KW_TEMPORARY | KW_TERMINATED
+ | KW_TINYINT | KW_TOUCH | KW_TRANSACTIONS | KW_UNARCHIVE | KW_UNDO | KW_UNIONTYPE | KW_UNLOCK | KW_UNSET
+ | KW_UNSIGNED | KW_URI | KW_USE | KW_UTC | KW_UTCTIMESTAMP | KW_VALUE_TYPE | KW_VIEW | KW_WHILE
+ ;
+
+//The following SQL2011 reserved keywords are used as cast function name only, it is a subset of the sql11ReservedKeywordsUsedAsIdentifier.
+sql11ReservedKeywordsUsedAsCastFunctionName
+ :
+ KW_BIGINT | KW_BINARY | KW_BOOLEAN | KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP | KW_DATE | KW_DOUBLE | KW_FLOAT | KW_INT | KW_SMALLINT | KW_TIMESTAMP
+ ;
+
+//The following SQL2011 reserved keywords are used as identifiers in many q tests, they may be added back due to backward compatibility.
+sql11ReservedKeywordsUsedAsIdentifier
+ :
+ KW_ALL | KW_ALTER | KW_ARRAY | KW_AS | KW_AUTHORIZATION | KW_BETWEEN | KW_BIGINT | KW_BINARY | KW_BOOLEAN
+ | KW_BOTH | KW_BY | KW_CREATE | KW_CUBE | KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP | KW_CURSOR | KW_DATE | KW_DECIMAL | KW_DELETE | KW_DESCRIBE
+ | KW_DOUBLE | KW_DROP | KW_EXISTS | KW_EXTERNAL | KW_FALSE | KW_FETCH | KW_FLOAT | KW_FOR | KW_FULL | KW_GRANT
+ | KW_GROUP | KW_GROUPING | KW_IMPORT | KW_IN | KW_INNER | KW_INSERT | KW_INT | KW_INTERSECT | KW_INTO | KW_IS | KW_LATERAL
+ | KW_LEFT | KW_LIKE | KW_LOCAL | KW_NONE | KW_NULL | KW_OF | KW_ORDER | KW_OUT | KW_OUTER | KW_PARTITION
+ | KW_PERCENT | KW_PROCEDURE | KW_RANGE | KW_READS | KW_REVOKE | KW_RIGHT
+ | KW_ROLLUP | KW_ROW | KW_ROWS | KW_SET | KW_SMALLINT | KW_TABLE | KW_TIMESTAMP | KW_TO | KW_TRIGGER | KW_TRUE
+ | KW_TRUNCATE | KW_UNION | KW_UPDATE | KW_USER | KW_USING | KW_VALUES | KW_WITH
;
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java Wed Mar 11 21:50:22 2015
@@ -193,6 +193,9 @@ public class ParseDriver {
lexer.setHiveConf(ctx.getConf());
}
HiveParser parser = new HiveParser(tokens);
+ if (ctx != null) {
+ parser.setHiveConf(ctx.getConf());
+ }
parser.setTreeAdaptor(adaptor);
HiveParser.statement_return r = null;
try {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g Wed Mar 11 21:50:22 2015
@@ -35,6 +35,9 @@ k=3;
RecognitionException e) {
gParent.errors.add(new ParseError(gParent, e, tokenNames));
}
+ protected boolean useSQL11ReservedKeywordsForIdentifier() {
+ return gParent.useSQL11ReservedKeywordsForIdentifier();
+ }
}
@rulecatch {
@@ -125,10 +128,11 @@ selectItem
@init { gParent.pushMsg("selection target", state); }
@after { gParent.popMsg(state); }
:
+ (tableAllColumns) => tableAllColumns -> ^(TOK_SELEXPR tableAllColumns)
+ |
( expression
((KW_AS? identifier) | (KW_AS LPAREN identifier (COMMA identifier)* RPAREN))?
) -> ^(TOK_SELEXPR expression identifier*)
- | tableAllColumns -> ^(TOK_SELEXPR tableAllColumns)
;
trfmClause
@@ -148,7 +152,9 @@ selectExpression
@init { gParent.pushMsg("select expression", state); }
@after { gParent.popMsg(state); }
:
- expression | tableAllColumns
+ (tableAllColumns) => tableAllColumns
+ |
+ expression
;
selectExpressionList
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java Wed Mar 11 21:50:22 2015
@@ -25,11 +25,14 @@ public class OpTraits {
List<List<String>> bucketColNames;
List<List<String>> sortColNames;
int numBuckets;
+ int numReduceSinks;
- public OpTraits(List<List<String>> bucketColNames, int numBuckets, List<List<String>> sortColNames) {
+ public OpTraits(List<List<String>> bucketColNames, int numBuckets,
+ List<List<String>> sortColNames, int numReduceSinks) {
this.bucketColNames = bucketColNames;
this.numBuckets = numBuckets;
this.sortColNames = sortColNames;
+ this.numReduceSinks = numReduceSinks;
}
public List<List<String>> getBucketColNames() {
@@ -55,4 +58,12 @@ public class OpTraits {
public List<List<String>> getSortCols() {
return sortColNames;
}
+
+ public void setNumReduceSinks(int numReduceSinks) {
+ this.numReduceSinks = numReduceSinks;
+ }
+
+ public int getNumReduceSinks() {
+ return this.numReduceSinks;
+ }
}
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java Wed Mar 11 21:50:22 2015
@@ -17,7 +17,10 @@
*/
package org.apache.hadoop.hive.ql.parse;
+import java.io.IOException;
+
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.junit.Assert;
import org.junit.Before;
@@ -31,7 +34,6 @@ public class TestIUD {
private static HiveConf conf;
private ParseDriver pd;
- private SemanticAnalyzer sA;
@BeforeClass
public static void initialize() {
@@ -40,15 +42,20 @@ public class TestIUD {
}
@Before
- public void setup() throws SemanticException {
+ public void setup() throws SemanticException, IOException {
pd = new ParseDriver();
- sA = new CalcitePlanner(conf);
}
ASTNode parse(String query) throws ParseException {
- ASTNode nd = pd.parse(query);
+ ASTNode nd = null;
+ try {
+ nd = pd.parse(query, new Context(conf));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
return (ASTNode) nd.getChild(0);
}
+
@Test
public void testDeleteNoWhere() throws ParseException {
ASTNode ast = parse("DELETE FROM src");
@@ -147,11 +154,11 @@ public class TestIUD {
@Test
public void testSelectStarFromAnonymousVirtTable1Row() throws ParseException {
try {
- parse("select * from values (3,4)");
+ parse("select * from `values` (3,4)");
Assert.assertFalse("Expected ParseException", true);
}
catch(ParseException ex) {
- Assert.assertEquals("Failure didn't match.", "line 1:21 missing EOF at '(' near 'values'",ex.getMessage());
+ Assert.assertEquals("Failure didn't match.", "line 1:23 missing EOF at '(' near 'values'",ex.getMessage());
}
}
@Test
Modified: hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex.q?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex.q Wed Mar 11 21:50:22 2015
@@ -3,7 +3,7 @@ USE default;
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time TIMESTAMP,
request STRING,
status INT,
Modified: hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex2.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex2.q?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex2.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex2.q Wed Mar 11 21:50:22 2015
@@ -3,7 +3,7 @@ USE default;
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status STRING,
Modified: hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex3.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex3.q?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex3.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientnegative/serde_regex3.q Wed Mar 11 21:50:22 2015
@@ -3,7 +3,7 @@ USE default;
CREATE TABLE serde_regex(
host STRING,
identity STRING,
- user STRING,
+ `user` STRING,
time STRING,
request STRING,
status STRING,
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/array_map_access_nonconstant.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/array_map_access_nonconstant.q?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/array_map_access_nonconstant.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/array_map_access_nonconstant.q Wed Mar 11 21:50:22 2015
@@ -1,11 +1,11 @@
set hive.fetch.task.conversion=more;
-create table array_table (array array<string>, index int );
+create table array_table (`array` array<string>, index int );
insert into table array_table select array('first', 'second', 'third'), key%3 from src tablesample (4 rows);
explain
-select index, array[index] from array_table;
-select index, array[index] from array_table;
+select index, `array`[index] from array_table;
+select index, `array`[index] from array_table;
create table map_table (data map<string,string>, key int );
insert into table map_table select map('1','one','2','two','3','three'), cast((key%3+1) as int) from src tablesample (4 rows);
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q Wed Mar 11 21:50:22 2015
@@ -82,4 +82,6 @@ from tab1 a join tab_part b on a.key = b
explain select a.key, b.key from tab_part a join tab_part c on a.key = c.key join tab_part b on a.value = b.value;
-
+explain
+select a.key, a.value, b.value
+from tab a join tab_part b on a.key = b.key and a.ds = b.ds;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/decimal_10_0.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/decimal_10_0.q?rev=1666021&r1=1666020&r2=1666021&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/decimal_10_0.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/decimal_10_0.q Wed Mar 11 21:50:22 2015
@@ -1,9 +1,9 @@
-DROP TABLE IF EXISTS DECIMAL;
+DROP TABLE IF EXISTS `DECIMAL`;
-CREATE TABLE DECIMAL (dec decimal);
+CREATE TABLE `DECIMAL` (dec decimal);
-LOAD DATA LOCAL INPATH '../../data/files/decimal_10_0.txt' OVERWRITE INTO TABLE DECIMAL;
+LOAD DATA LOCAL INPATH '../../data/files/decimal_10_0.txt' OVERWRITE INTO TABLE `DECIMAL`;
-SELECT dec FROM DECIMAL;
+SELECT dec FROM `DECIMAL`;
-DROP TABLE DECIMAL;
\ No newline at end of file
+DROP TABLE `DECIMAL`;
\ No newline at end of file