You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by px...@apache.org on 2015/12/29 19:46:30 UTC
hive git commit: HIVE-12744: GROUPING__ID failed to be recognized in
multiple insert (Pengcheng Xiong via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-2.0 06902462f -> 7cbbfa777
HIVE-12744: GROUPING__ID failed to be recognized in multiple insert (Pengcheng Xiong via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7cbbfa77
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7cbbfa77
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7cbbfa77
Branch: refs/heads/branch-2.0
Commit: 7cbbfa77754a7cb19af3a8e1c9425ce94cf64938
Parents: 0690246
Author: Pengcheng Xiong <px...@apache.org>
Authored: Tue Dec 29 10:28:00 2015 -0800
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Tue Dec 29 10:46:25 2015 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 3 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +
.../clientnegative/groupby_cube_multi_gby.q | 10 ++
.../clientpositive/groupby_cube_multi_gby.q | 12 ++
.../clientnegative/groupby_cube_multi_gby.q.out | 17 +++
.../clientpositive/groupby_cube_multi_gby.q.out | 146 +++++++++++++++++++
6 files changed, 196 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7cbbfa77/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 7585bad..6a62592 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -433,6 +433,9 @@ public enum ErrorMsg {
CANNOT_CHANGE_COLUMN_TYPE(10312, "Changing from type {0} to {1} is not supported for column {2}. SerDe may be incompatible", true),
REPLACE_CANNOT_DROP_COLUMNS(10313, "Replacing columns cannot drop columns for table {0}. SerDe may be incompatible", true),
REPLACE_UNSUPPORTED_TYPE_CONVERSION(10314, "Replacing columns with unsupported type conversion (from {0} to {1}) for column {2}. SerDe may be incompatible", true),
+ HIVE_GROUPING_SETS_AGGR_NOMAPAGGR_MULTIGBY(10315,
+ "Grouping sets aggregations (with rollups or cubes) are not allowed when " +
+ "HIVEMULTIGROUPBYSINGLEREDUCER is turned on. Set hive.multigroupby.singlereducer=false if you want to use grouping sets"),
//========================== 20000 range starts here ========================//
SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. "
http://git-wip-us.apache.org/repos/asf/hive/blob/7cbbfa77/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 2433cae..197e6da 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -5373,6 +5373,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
List<ExprNodeDesc.ExprNodeDescEqualityWrapper> whereExpressions =
new ArrayList<ExprNodeDesc.ExprNodeDescEqualityWrapper>();
for (String dest : dests) {
+ ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets =
+ getGroupByGroupingSetsForClause(parseInfo, dest);
+
+ List<Integer> groupingSets = grpByExprsGroupingSets.getSecond();
+ if (!groupingSets.isEmpty()) {
+ throw new SemanticException(ErrorMsg.HIVE_GROUPING_SETS_AGGR_NOMAPAGGR_MULTIGBY.getMsg());
+ }
+
ASTNode whereExpr = parseInfo.getWhrForClause(dest);
if (whereExpr != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/7cbbfa77/ql/src/test/queries/clientnegative/groupby_cube_multi_gby.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/groupby_cube_multi_gby.q b/ql/src/test/queries/clientnegative/groupby_cube_multi_gby.q
new file mode 100644
index 0000000..cddbe1a
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/groupby_cube_multi_gby.q
@@ -0,0 +1,10 @@
+create table t1 like src;
+create table t2 like src;
+
+explain from src
+insert into table t1 select
+key, GROUPING__ID
+group by key, value with cube
+insert into table t2 select
+key, value
+group by key, value grouping sets ((key), (key, value));
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/7cbbfa77/ql/src/test/queries/clientpositive/groupby_cube_multi_gby.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/groupby_cube_multi_gby.q b/ql/src/test/queries/clientpositive/groupby_cube_multi_gby.q
new file mode 100644
index 0000000..80022bb
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/groupby_cube_multi_gby.q
@@ -0,0 +1,12 @@
+set hive.multigroupby.singlereducer=false;
+
+create table t1 like src;
+create table t2 like src;
+
+explain from src
+insert into table t1 select
+key, GROUPING__ID
+group by key, value with cube
+insert into table t2 select
+key, value
+group by key, value grouping sets ((key), (key, value));
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/7cbbfa77/ql/src/test/results/clientnegative/groupby_cube_multi_gby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/groupby_cube_multi_gby.q.out b/ql/src/test/results/clientnegative/groupby_cube_multi_gby.q.out
new file mode 100644
index 0000000..88b87bb
--- /dev/null
+++ b/ql/src/test/results/clientnegative/groupby_cube_multi_gby.q.out
@@ -0,0 +1,17 @@
+PREHOOK: query: create table t1 like src
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: create table t1 like src
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+PREHOOK: query: create table t2 like src
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t2
+POSTHOOK: query: create table t2 like src
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t2
+FAILED: SemanticException [Error 10315]: Grouping sets aggregations (with rollups or cubes) are not allowed when HIVEMULTIGROUPBYSINGLEREDUCER is turned on. Set hive.multigroupby.singlereducer=false if you want to use grouping sets
http://git-wip-us.apache.org/repos/asf/hive/blob/7cbbfa77/ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out b/ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out
new file mode 100644
index 0000000..992fd2d
--- /dev/null
+++ b/ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out
@@ -0,0 +1,146 @@
+PREHOOK: query: create table t1 like src
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: create table t1 like src
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+PREHOOK: query: create table t2 like src
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t2
+POSTHOOK: query: create table t2 like src
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t2
+PREHOOK: query: explain from src
+insert into table t1 select
+key, GROUPING__ID
+group by key, value with cube
+insert into table t2 select
+key, value
+group by key, value grouping sets ((key), (key, value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain from src
+insert into table t1 select
+key, GROUPING__ID
+group by key, value with cube
+insert into table t2 select
+key, value
+group by key, value grouping sets ((key), (key, value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-2
+ Stage-1 depends on stages: Stage-4
+ Stage-5 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: key, value
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: key (type: string), value (type: string), '0' (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string)
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: key, value
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: key (type: string), value (type: string), '0' (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t1
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string)
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ pruneGroupingSetId: true
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t2
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t2
+
+ Stage: Stage-5
+ Stats-Aggr Operator
+